diff --git a/.claude/skills/quest-finish/SKILL.md b/.claude/skills/quest-complete/SKILL.md similarity index 71% rename from .claude/skills/quest-finish/SKILL.md rename to .claude/skills/quest-complete/SKILL.md index 4583b29d666..6c6bd4e2dd5 100644 --- a/.claude/skills/quest-finish/SKILL.md +++ b/.claude/skills/quest-complete/SKILL.md @@ -1,7 +1,7 @@ --- -name: quest-finish +name: quest-complete description: Decide which open PRs to merge, then merge them in parallel. --- -Run `quest skill finish` and follow its output. +Run `quest skill complete` and follow its output. If `quest` is not installed, follow https://github.com/kixelated/quest/blob/main/SETUP.md first. diff --git a/.claude/skills/quest-convert/SKILL.md b/.claude/skills/quest-convert/SKILL.md deleted file mode 100644 index bb2e8cbeab2..00000000000 --- a/.claude/skills/quest-convert/SKILL.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -name: quest-convert -description: Convert Github issues to quests. ---- - -Run `quest skill convert` and follow its output. -If `quest` is not installed, follow https://github.com/kixelated/quest/blob/main/SETUP.md first. diff --git a/.claude/skills/quest-export/SKILL.md b/.claude/skills/quest-export/SKILL.md new file mode 100644 index 00000000000..7f52f470a6f --- /dev/null +++ b/.claude/skills/quest-export/SKILL.md @@ -0,0 +1,7 @@ +--- +name: quest-export +description: Turn active quests into GitHub issues before leaving Quest. Use when the user invokes /quest-export or asks to export quests before removing Quest. +--- + +Run `quest skill export` and follow its output. +If `quest` is not installed, follow https://github.com/kixelated/quest/blob/main/SETUP.md first. diff --git a/.claude/skills/quest-import/SKILL.md b/.claude/skills/quest-import/SKILL.md new file mode 100644 index 00000000000..fbd48cc9e91 --- /dev/null +++ b/.claude/skills/quest-import/SKILL.md @@ -0,0 +1,7 @@ +--- +name: quest-import +description: Import GitHub issues as quests. +--- + +Run `quest skill import` and follow its output. +If `quest` is not installed, follow https://github.com/kixelated/quest/blob/main/SETUP.md first. diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index e18364366f7..6203c4ff0ba 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -39,6 +39,7 @@ on: - Nightly - OBS - Platform + - Quest - release-brew - Release Dart - Release Dart FFI diff --git a/.github/workflows/audio-quality.yml b/.github/workflows/audio-quality.yml new file mode 100644 index 00000000000..1ad4af43cc9 --- /dev/null +++ b/.github/workflows/audio-quality.yml @@ -0,0 +1,63 @@ +name: Audio quality + +# Grades the recorded-trace replays of test/audio-quality against their exact +# budgets on every PR that can move them. They replay the player's container +# consumer and rings on a simulated clock, so they are deterministic and take +# seconds, unlike the browser rows, which stay in nightly.yml. +on: + pull_request: + paths: + - "js/watch/**" + # The container consumer decides what a replay delivers, waits for, and skips, on the + # groups, tracks, and signals underneath it. + - "js/hang/**" + - "js/net/**" + - "js/signals/**" + - "test/audio-quality/**" + - "test/lib/**" + - "test/justfile" + - ".github/workflows/audio-quality.yml" + +concurrency: + group: audio-quality-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + replay: + name: Replay + runs-on: ubuntu-24.04-arm + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter; see check.yml. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + - name: Replay + run: nix develop --command just test audio-quality --replays --enforce + shell: bash -leo pipefail {0} + env: + # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + + # The harness keeps a failing run's directory: every row's samples and summary. + - name: Keep the failing run + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: audio-quality-replay + path: ${{ runner.temp }}/moq-test-runs/ + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9012fda13e4..476dfd8fd48 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,6 +18,15 @@ name: Nightly # the wheel builds in a manylinux container, the Kotlin/Swift/Go/Dart packages # link the FFI on every platform) and nothing notices until the tag is pushed. # +# `audio-quality` is here because of what it measures. It plays twenty-four rows of +# audio in real time, a minute each, which no merge gate should carry, and every +# number it grades is a timing one that moves with whatever else the runner is +# doing. A per-PR arm would fail on unrelated load often enough to be ignored; +# nightly it competes with nothing, and a regression surfaces a day late against +# a budget that still means something. The recorded-trace replays are the +# exception: deterministic and seconds long, so audio-quality.yml also gates +# PRs on them. +# # A failure here lands on main rather than being caught in review, which is the # accepted trade. Anything that must block a merge belongs in check.yml. @@ -170,8 +179,11 @@ jobs: matrix: # `rs uring` is a separate feature compile (off the default set) and # kernel-gated below 6.12. The embedding tests for io_uring live there. + # `test drain` stands up two relays and drains one under a live JS + # viewer: diff-independent, since a break can arrive through the relay, + # js/net, or the wire between them. # `gh test` exercises the release tooling, which no pull request runs. - recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "gh test"] + recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "test drain", "gh test"] steps: - name: Free disk space uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main @@ -197,6 +209,54 @@ jobs: NEXTEST_PROFILE: ci MOQ_STRICT: 1 + # Plays the matrix in headless Chromium through moq-shaper and grades what the + # listener heard against test/audio-quality/budgets.json. ARM64 like wasm.yml, + # the only architecture cache.yml warms a Rust store for. + audio-quality: + name: Audio quality + runs-on: ubuntu-24.04-arm + # Twenty-four rows at a minute each, plus the relay, CLI, and shaper built from source. + timeout-minutes: 60 + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - name: Rust cache + uses: ./.github/actions/rust-cache + # Playwright's Chromium isn't in the nix devShell; `--with-deps` installs + # its apt runtime libs, and run.sh's own install then no-ops. + - name: Playwright Chromium + run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium" + shell: bash -leo pipefail {0} + - name: Audio quality + run: nix develop --command just test audio-quality --enforce + shell: bash -leo pipefail {0} + env: + # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The run directory is the diagnosis: each process's log, the shaper's + # counters, the raw samples, and every row's summary. The harness keeps it + # on a failure without being asked. + - name: Keep the failing run + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: audio-quality-run + path: ${{ runner.temp }}/moq-test-runs/ + retention-days: 7 + if-no-files-found: ignore + # Every tag-driven release workflow, run against main with publishing skipped: # the workflows gate their publish jobs on `github.ref_type == 'tag'`, and no # secrets are passed, so a dry-run cannot publish even by accident. diff --git a/.github/workflows/quest.yml b/.github/workflows/quest.yml new file mode 100644 index 00000000000..6118d649ddb --- /dev/null +++ b/.github/workflows/quest.yml @@ -0,0 +1,62 @@ +name: Quest + +# Validates the quest tree on every branch that carries one. check.yml already +# runs `quest check` on a pull request into any branch, but a direct push (a +# merge to `main` or `dev`, or `main` merged into a questline) is nobody's pull +# request, so a broken tree could land there unseen. +# +# Push only: a pull request is check.yml's job, and a pull_request trigger here +# would spawn a skipped Alert run on every push to every pull request. +# +# The paths mirror the quest module in sh/dispatch.sh's impact map: the +# documents, and flake.lock, which pins the validator. + +permissions: + contents: read + +on: + push: + branches: + - main + - dev + - "quest/**" + paths: + - "quest/**" + - "flake.lock" + - ".github/workflows/quest.yml" + +# Only the newest push to a branch matters; Alert ignores the cancelled runs. +concurrency: + group: quest-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + # Not `Check`: that is check.yml's required status, and a second job with + # the same name would make it ambiguous on a head that runs both. + name: Quest + runs-on: ubuntu-24.04-arm + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter, which holds the quest build + # as part of the dev shell closure; see check.yml. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + # The whole tree, unscoped: a push has no base to diff against. + # + # Only the quest package, not `nix develop`: the dev shell compiles + # uniffi-bindgen-go from source whenever cache.yml hasn't pushed it for + # this flake.lock yet, which ran past the timeout. + - name: Check + run: nix run .#quest -- check diff --git a/AGENTS.md b/AGENTS.md index e97c9d5a700..0222c104d21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -92,7 +92,7 @@ just fix # Auto-fix lint/formatting, same scope These diff the branch against its base and only run the affected packages. -When work mentions a quest, run `quest guide` and follow it. +Quests: when work mentions a quest, run `quest guide` and follow it. The `quest` binary comes from the kixelated/quest flake input and serves the quest skills; change them upstream and bump the input. A quest deleted on `dev` is done, even while `main` still lists it. diff --git a/Cargo.lock b/Cargo.lock index 4b62ea25e83..7ce0681dc7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3036,6 +3036,16 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + [[package]] name = "hybrid-array" version = "0.4.15" @@ -4454,6 +4464,7 @@ dependencies = [ "anyhow", "base64 0.23.1", "bytes", + "crc", "futures", "h264-parser", "hang", @@ -4614,6 +4625,7 @@ dependencies = [ "futures", "http-cache-reqwest", "humantime", + "hyper-util", "libc", "moq-auth", "moq-net", @@ -4711,8 +4723,12 @@ dependencies = [ "anyhow", "clap", "humantime", + "humantime-serde", "rand 0.10.3", + "serde", + "serde_json", "tokio", + "toml 1.1.6+spec-1.1.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 43beff973af..3b0fe9f1fe7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -123,6 +123,8 @@ hex = "0.4" # HMAC-SHA256 for the mDNS membership proofs (moq-tokio's `mdns` feature). hmac = "0.13" humantime = "2.3" +humantime-serde = "1.1.1" +hyper-util = { version = "0.1.21", features = ["tokio"] } io-uring = "0.7.14" jsonwebtoken = "11" kio = { version = "0.6.2", path = "rs/kio" } diff --git a/bun.lock b/bun.lock index be29b2b19bb..34791e0f8db 100644 --- a/bun.lock +++ b/bun.lock @@ -331,6 +331,28 @@ "vite": "^8.3.0", }, }, + "test/audio-quality/clients/js": { + "name": "@moq/audio-quality", + "dependencies": { + "@moq/hang": "workspace:*", + "@moq/net": "workspace:*", + "@moq/publish": "workspace:*", + "@moq/signals": "workspace:*", + "@moq/watch": "workspace:*", + }, + "devDependencies": { + "@types/audioworklet": "^0.0.100", + "esbuild": "^0.28.2", + "playwright": "^1.63.0", + "vite": "^8.3.0", + }, + }, + "test/drain": { + "name": "@moq/drain-test", + "dependencies": { + "@moq/net": "workspace:*", + }, + }, "test/interop/clients/js": { "name": "@moq/interop-browser", "dependencies": { @@ -628,6 +650,8 @@ "@libav.js/variant-opus-af": ["@libav.js/variant-opus-af@6.10.9", "", {}, "sha512-yULJUCQIHDtwDPbT38t9pP/7EGgqwynVcOau46hZjtCe8TWMp67DTi2X0Emr2M4Ci3nzxNPf25IzO+IXSmHDBg=="], + "@moq/audio-quality": ["@moq/audio-quality@workspace:test/audio-quality/clients/js"], + "@moq/auth": ["@moq/auth@workspace:js/auth"], "@moq/boy": ["@moq/boy@workspace:js/moq-boy"], @@ -638,6 +662,8 @@ "@moq/demo-boy": ["@moq/demo-boy@workspace:demo/boy"], + "@moq/drain-test": ["@moq/drain-test@workspace:test/drain"], + "@moq/flate": ["@moq/flate@workspace:js/flate"], "@moq/hang": ["@moq/hang@workspace:js/hang"], @@ -1982,6 +2008,8 @@ "@types/concat-stream/@types/node": ["@types/node@26.0.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw=="], + "@vitejs/plugin-vue/vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="], + "@vue/compiler-core/@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="], "@vue/compiler-core/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], @@ -2158,6 +2186,10 @@ "@types/concat-stream/@types/node/undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + "@vitejs/plugin-vue/vite/esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], + + "@vitejs/plugin-vue/vite/postcss": ["postcss@8.5.16", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg=="], + "@vue/compiler-core/@babel/parser/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], "@vue/compiler-sfc/@babel/parser/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], @@ -2268,6 +2300,54 @@ "@npmcli/package-json/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "@vitejs/plugin-vue/vite/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="], + + "@vitejs/plugin-vue/vite/postcss/nanoid": ["nanoid@3.3.15", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA=="], + "@vue/compiler-core/@babel/parser/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], "@vue/compiler-core/@babel/parser/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 3f22497a27f..4ed16dcdd9a 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -91,6 +91,11 @@ the first PAT lists as its own broadcast, with its own clock and catalog, keepin the catalog suffix last: `--broadcast event.hang` publishes `event/1.hang`, `event/2.hang`, and so on. `export ts` writes one program per broadcast. `import srt` takes the same `--program`. +A selected program's SI describes that service alone: its SDT lists only the +selected service, and other services' EIT is dropped. Network-wide tables (NIT, +BAT, TDT/TOT, and the SDT and EIT of other transport streams) pass through. +SI matches the selection by DVB `service_id`, which is assumed to equal the PAT +`program_number`. ```bash moq --connect https://relay.example.com/anon --broadcast event.hang import ts --program all < mux.ts @@ -184,6 +189,16 @@ id. Requires the `capture` feature; on Linux that needs the ALSA headers for the microphone, and `--display` and `pipewire:` cameras also need the `pipewire` feature (links libpipewire). +On Windows, display and window capture use Windows.Graphics.Capture and +require Windows 10 2004 (build 19041) or newer. Cursor capture follows the +capture configuration. The system capture border stays visible unless the OS +supports borderless capture and grants access. Frames are converted to NV12 +on the GPU; software encoding reads them back. Windows application capture +and system audio are separate capabilities, not enabled by this backend. +Windows `display:N` selectors are enumeration indices; switching from Desktop +Duplication to WGC can change which monitor a saved selector names. Run +`moq devices` again and reselect the intended display after upgrading. + ## Transcode ```bash diff --git a/doc/bin/relay/cluster.md b/doc/bin/relay/cluster.md index d64da538e70..dfdc203ba25 100644 --- a/doc/bin/relay/cluster.md +++ b/doc/bin/relay/cluster.md @@ -185,6 +185,20 @@ for loop detection but makes a restarted relay look like a new node. Set `cluster.id` to a stable non-zero integer to pin it, below 2^53 if browser clients decode it. +## Failure detection + +A peer that crashes or drops off the network sends no goodbye, so a relay only +learns it is gone when the link goes quiet for [`quic.idle_timeout`](/bin/relay/config#quic) +(10s by default). Until then its routes stay in place and subscribes through +them go nowhere. Lower it to fail over faster; raise it if a lossy long-haul +link drops while the peer is still alive, and keep `quic.keep_alive` well under +it. + +QUIC uses the smaller of the two endpoints' idle timeouts +([RFC 9000 section 10.1](https://www.rfc-editor.org/rfc/rfc9000#section-10.1)), +so either relay on a link can shorten it for both. iroh links use the same +timeout; WebSocket links keep their own 30s deadline. + ## Authentication Peers dial with **mTLS** (recommended: `listen.tls.root` on the listener, diff --git a/doc/bin/relay/config.md b/doc/bin/relay/config.md index 209b8032e3c..841d8bb13b6 100644 --- a/doc/bin/relay/config.md +++ b/doc/bin/relay/config.md @@ -19,6 +19,7 @@ written (an empty list, a `false` boolean) overrides the built-in default. [listen] bind = "[::]:443" # QUIC (UDP), as --listen. Omit for a stream-only relay. version = ["moq-lite-05"] # Restrict accepted versions. Omit for all. +timeout = "10s" # Handshake deadline. "0" waits forever. [listen.tls] cert = "cert.pem" # Certificate chain and key. Reloaded on change. @@ -34,6 +35,14 @@ bind = "/run/moq/internal.sock" allow.uid = [1001] ``` +`timeout` bounds how long an accepted connection has to finish its handshake: +the QUIC, WebTransport, WebSocket, or qmux one, then the MoQ SETUP, through the +relay accepting the session. After that it is an ordinary session. A peer that +connects and never speaks is closed instead of being held open by keep-alives, +with the MoQ timeout code once its transport is up. The `[web]` listeners apply it to reading HTTP request +headers and, for the WebSocket fallback, to the SETUP after the upgrade. The +`io_uring` workers do not apply it yet. + ## \[quic] Transport tuning, applied to accepted and dialed connections alike. @@ -42,8 +51,8 @@ Transport tuning, applied to accepted and dialed connections alike. [quic] congestion_control = "delay" # "delay" (BBR, the default) or "loss" (CUBIC). max_streams = 10000 # Concurrent streams per connection, bidi and uni. Default. -idle_timeout = "30s" # Drop a connection after this long with nothing on it. -keep_alive = "5s" # Ping interval; "0s" disables it. Ignored by iroh. +idle_timeout = "10s" # Drop a connection after this long with nothing on it. Default. +keep_alive = "3s" # Ping interval; "0s" disables it. Ignored by iroh. Default. gso = true # UDP segmentation offload. iroh cannot turn it off. mtu_discovery = false # Path MTU discovery. Default. receive_window = 67108864 # Flow-control windows, in bytes. Omit for the backend default. @@ -55,6 +64,14 @@ qlog = "/var/log/moq/qlog" # Existing directory. Needs the `qlog` buil The native QUIC stack uses BBRv3 for delay-based congestion control. Iroh also uses noq and the same congestion controller. +`idle_timeout` is how long a peer that vanished without a close keeps its +sessions, and so its [cluster routes](/bin/relay/cluster#failure-detection). +QUIC uses the smaller of the two endpoints' values +([RFC 9000 section 10.1](https://www.rfc-editor.org/rfc/rfc9000#section-10.1)), +so this also bounds the clients and peers on the other end. Keep `keep_alive` +under a third of it, so a quiet connection that loses one ping still pings +again before the deadline. + Raise the receive windows when a fat, long path idles below the link rate: a window under the bandwidth-delay product stalls the sender waiting for credit. Keep `stream_receive_window` well under `receive_window` so one slow group @@ -162,9 +179,11 @@ A draining upstream may name a replacement URI. `same-host` follows it only onto the host we already dialed, so a peer moves us between ports and schemes; `follow` also lets it choose the host, which means trusting it not to point us into the local network, since a name it controls resolves wherever it likes; -`ignore` keeps the current address list. Empty, malformed, or refused redirects -also preserve caller-configured fallbacks; only an accepted redirect replaces -the list with the peer's URI. `handover` is a cap: a shorter deadline on +`ignore` keeps the current address list. An empty URI also keeps it, including +caller-configured fallbacks; an accepted redirect replaces the list with the +peer's URI. A malformed or refused redirect ends the connection with an error +rather than redialing the old address or a fallback, and so does one leaving +the host a `tls.fingerprint` pin verifies. `handover` is a cap: a shorter deadline on the received GOAWAY wins, a longer one does not extend it. ## \[cache] @@ -221,6 +240,28 @@ secret = "./iroh-secret.key" # Persist the key so the endpoint id surviv See [Transport](/concept/transport#iroh-peer-to-peer-experimental). +## Shutdown + +```toml +drain_timeout = "10s" # Top-level key, as --drain-timeout / MOQ_DRAIN_TIMEOUT. +``` + +The first SIGTERM or SIGINT starts a drain: every session is sent a GOAWAY +asking it to reconnect, and is force-closed if it is still connected when the +window ends. A session that connects during the drain, such as a client with a +cached DNS answer, is sent a GOAWAY immediately, with only the time left in +the window. The relay exits as soon as every session has left, when the window +ends, or immediately on a second signal. `0` skips the GOAWAY and closes every +session at once. + +The exit is logged with how long the drain took, as either +`drain complete: every session left` or `drain deadline force-closed sessions` +with the number `forced`. A session still in its handshake when the last one +leaves is not waited for. +Only moq-lite-04+ and moq-transport clients act on a GOAWAY; older ones are +closed when the window ends. An embedder can take over the signals and start +the drain itself; see [Embed](/bin/relay/#embed). + ## \[log] ```toml diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index 4863714326b..68ffbe1d3c0 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -55,7 +55,9 @@ split by `tier` and `role`, plus accept-loop counters per TCP listener. Alert on `moq_relay_accept_failures_total{class="exhausted"}`, which means the process ran out of a resource `accept` needs. Content dropped for drifting past a subscriber's budget is counted separately as `moq_relay_stale_bytes_total` -and friends. Host CPU and memory belong to a node exporter. +and friends. During a [shutdown drain](/bin/relay/config#shutdown), +`moq_relay_draining_sessions` counts the sessions sent a GOAWAY that have not +left yet. Host CPU and memory belong to a node exporter. Traffic and session counters accumulate for the node's lifetime, including broadcasts and sessions that have ended. The stats publishing prefix (normally diff --git a/doc/bin/relay/index.md b/doc/bin/relay/index.md index a2af14c1061..a897c95c67a 100644 --- a/doc/bin/relay/index.md +++ b/doc/bin/relay/index.md @@ -81,12 +81,18 @@ TLS, and a certificate fingerprint for client pinning. The accessors borrow and `run` consumes the relay, so clone `cluster`, `auth`, `client`, `stats`, `shutdown`, and `shutdown_trigger` for application -tasks before calling it. `trigger.start()` drains every session with a GOAWAY -and `run` returns once the drain window elapses, with the listeners released -and the workers joined. Build routes from `web().routes()` (or +tasks before calling it. `trigger.start()` drains every session with a GOAWAY, +including any that connect afterwards, and `run` returns once every session +has left or the drain window elapses, with the listeners released and the +workers joined. `run` also starts +the drain on SIGTERM or SIGINT. An application that owns those signals, for +example to withdraw the node from DNS and wait out the TTL before draining, +calls `with_signals(false)` and fires the trigger itself. Build routes from `web().routes()` (or `internal().routes()`): `with_web` replaces the router, so `Router::new()` drops the built-in routes. Extra listeners (RTMP, SRT, ...) sit beside `run` -in the application's `select!`. `runtime.workers` and `runtime.io_uring` stay +in the application's `select!`. The drain reaches only MoQ sessions: an +extra listener has no GOAWAY, so the application stops it on its own deadline +and leaves the encoder to reconnect. `runtime.workers` and `runtime.io_uring` stay inside the owner; do not split the worker group yourself. An application that decides admissions itself leaves `[auth]` empty and answers `relay.admissions()`; see [Authentication](/bin/relay/auth#in-process). See diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 6d70ac61376..04ae5ff2ecc 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -57,6 +57,19 @@ a group at or past `SUBSCRIBE_END`, or a `SUBSCRIBE_END` below a group already received. moq-lite 05 specified an inclusive end, so there it only drops that group or the early boundary. +## Group reads across failover + +Rust origin readers can keep reading an in-flight group after its source fails. +A group reader waiting on a replacement copy subscribes to it from the frame it +needs, even when the caller is not polling for the next group, and holds that +subscription until the group ends. If that copy refuses the subscription, the +read ends with an error. The copy's declared start says whether the group is +still coming, so a newer group overtaking it on the wire does not end the wait. +A copy that declares no start (a local track) and has advanced past the group +ends the read with an error. This also applies when waiting for the group's +completion; no FETCH is issued. A replacement that later drops the group it is +serving still needs SUBSCRIBE\_DROP support to resolve that wait. + ## Discovery A session can ask for announcements matching a path prefix. The peer replies @@ -326,7 +339,9 @@ codes as `moq_net::Error::Session(SessionError)` or `Error::Stream(StreamError)` JavaScript exposes `SessionError` and `StreamError`. Match the registry before interpreting the number. Native bindings expose scope, code, kind, and a diagnostic message; unknown and application codes retain their numeric value. Transport -failures without a protocol code remain separate. +failures without a protocol code remain separate. A deliberate local close ends +received tracks cleanly after their delivered groups; a peer close ends tracks +and open group readers with the session error. ## Local read limits diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 2d88de25676..4a4ddc2ddf7 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -31,6 +31,11 @@ This is an implementation limit, not a limit in the IETF draft. A larger declared block stops its subgroup stream with `MALFORMED_TRACK` before reading the block; other groups and the session stay open. +Rust and JavaScript read an incoming padding stream (draft-18 and later) to +the end and discard it, without sending `STOP_SENDING`. A unidirectional stream type the negotiated draft does not +define, or a `SUBGROUP_HEADER` type it marks invalid, closes the session with +`PROTOCOL_VIOLATION`, as the draft requires. + An IETF publisher declares the track's default priority in `SUBSCRIBE_OK` or `PUBLISH` when that draft carries track properties. Groups without a priority flag inherit it. If the property is absent, the IETF wire default of 128 maps @@ -51,7 +56,7 @@ On drafts 14–19, the Rust publisher also serves relative and absolute joining saved prefix only, while the subscription delivers later objects. One reaching back to earlier groups is refused with `NOT_SUPPORTED`. Draft-20 uses subscription fills instead. JavaScript -publishing does not yet serve `FETCH`; +publishing refuses every `FETCH` with `NOT_SUPPORTED`; Rust and JavaScript subscribers request unfiltered delivery on older drafts because they do not issue joining fetches. Other publishers may replay a cached backlog for that filter; selecting the next group instead would leave static @@ -70,7 +75,18 @@ and bytes to the application unverified; a relay forwards them to its [auth server](/bin/relay/auth#the-contract). An alias reference (`DELETE`, `USE_ALIAS`) closes the session with `PROTOCOL_VIOLATION`, a structure that does not decode with `KEY_VALUE_FORMATTING_ERROR`, and a second token is -refused. +refused. An `AUTHORIZATION TOKEN` parameter on a request is read and ignored: +the session's credential is what authorizes it. + +A legal request that is not served is refused on its own with `NOT_SUPPORTED`, +leaving the session open: a `SUBSCRIBE` with `FORWARD=0`, a `SUBSCRIBE` or +`FETCH` carrying Range Filters (no `MAX_FILTER_RANGES` is advertised), a +`FETCH` carrying `FILL_TIMEOUT` (Timed-Out gaps are not written), +`TRACK_STATUS`, `SUBSCRIBE_TRACKS` (draft-18 and later), and the `FETCH` +forms above. `NEW_GROUP_REQUEST` is ignored, as +the draft allows a publisher without dynamic groups to do. A parameter the +negotiated draft does not define still closes the session with +`PROTOCOL_VIOLATION`, as the draft requires. Several project drafts extend the IETF wire without breaking it, since `SETUP` ignores unknown parameters: [cluster](/draft/moq-cluster) routing hop lists, diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index 4e77fa26593..9680d539ef0 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -48,14 +48,14 @@ for (;;) { ``` - **Origins** hold the broadcasts, not the connection: closing a session unannounces them but leaves them created for the next one. `origin.request(path)` resolves an announced local broadcast with no round trip, so a page that watches what it publishes reads its own copy, unless a cheaper route announces the same path. A refusal from the most specific route ends the request: `request.closed` settles with the handler's error and no broader route is asked. Create, populate, then `announce()` for an exact path; use `dynamic(prefix, route)` when the set of paths is not known: an exact-path subscribe before the tracks exist is refused, and nobody, local or remote, can see or reach a broadcast until it announces. -- **Connections** race WebTransport against WebSocket. `new Connection({ url })` pools one connection per relay URL and reconnects with backoff, which the elements use. Supplying WebTransport/WebSocket options, discovery, delay, or a caller-owned origin selects a private loop; explicit `share: true` refuses those options. `closed` settles when the handle is released (`null` on a clean close); the failure that stopped retrying the current URL is `error`, and a new URL recovers the same handle. A connection owns one send-rate sampler and one `Bandwidth.Allocator`; publishers reserve against it so their encoder targets sum to the estimate instead of each matching it. +- **Connections** race WebTransport against WebSocket. `new Connection({ url })` pools one connection per relay URL and reconnects with backoff, which the elements use. Supplying WebTransport/WebSocket options, discovery, delay, `goaway`, or a caller-owned origin selects a private loop; explicit `share: true` refuses those options. On a relay's GOAWAY the connection dials the replacement at once while the old session keeps serving its groups until it closes or the `goaway.handover` cap (default 10s, lowered to the relay's deadline) passes; the handle and its origin stay the same. An empty GOAWAY redials the same URL through a fresh DNS resolve. A redirect is followed onto the same host by default (`goaway.redirect: "follow"` lets the relay pick the host, `"ignore"` never moves), and moves the pooled entry to the new URL; a malformed or refused one, including a host change under a pinned certificate, ends the connection with `Error.RefusedRedirect` instead of reconnecting. `closed` settles when the handle is released (`null` on a clean close); the failure that stopped retrying the current URL is `error`, and a new URL recovers the same handle. A connection owns one send-rate sampler and one `Bandwidth.Allocator`; publishers reserve against it so their encoder targets sum to the estimate instead of each matching it. - **Bandwidth** (`Bandwidth.Allocator`) divides the connection's send-rate estimate by track priority, max-min fair within a tier. An idle track claims nothing. The receive side is untouched. - **Discovery** by any pattern scope (`origin.announced(scope)`, such as `room/*/chat`; default everything). A route event's `kind` says whether the route started, updated, or ended; its `prefix` is the covered prefix relative to the origin, and `captures` reports what the scope's wildcards matched when the prefix pins them. One `{ kind: "live" }` marker, with no prefix, follows the routes live at subscribe time. The consumer is an async iterable. `origin.broadcasts(scope)` is a live `Getter>` of the same covered prefixes for UIs that need the current set. A borrowed `Connection.origin` also exposes `dynamic(prefix, route)` for serving paths on demand. - **Publisher retention** is optional `Track.Info.maxAge`: omission sets no limit, while zero keeps the live edge. No implicit five-second window is applied. - **Subscriptions** carry a priority, a `Time.Milli` max age, and optional `groups` bounds. Groups arrive out of order and are read frame by frame, with `Error.TooFarBehind` when a reader asks for a frame the group never held and `Error.GroupTooLarge` when a write exceeds the cache budget and aborts the group. - **Track ends**: `close()` ends a track at its live edge, while `finishAt(n)` declares the exclusive end ahead of it and still accepts the groups below. A subscriber reads the end with `final()` or awaits `finished()`. A remote track ends only once every group below its end has arrived or was dropped; one reset before its header arrived is skipped after the subscription's max age on moq-lite (one second without one), or after one second on IETF. - **Datagrams** on moq-lite 05+ and fetch-by-sequence for history. `track.fetchGroup(sequence)` on moq-lite resolves when the publisher sends the first response byte or finishes an empty group. A missing group rejects the fetch with `StreamCode.NotFound`, including every concurrent caller sharing that fetch. `fetchGroup(sequence, { signal })` abandons a pending fetch with the signal's reason; the shared stream is cancelled only once every caller has left. Once resolved, close the group instead. -- **Errors** live under one namespace: a stream reset throws `Error.Stream` with a `StreamCode`, while a session close gives `Error.Session` with a `SessionCode`. The registries are disjoint, so the same number means different things in each, and 64+ is yours. Named conditions such as `Error.TooFarBehind`, `Error.FrameTooLarge`, and `Error.GroupTooLarge` subclass `Error.Stream`, so one `code` check handles a condition raised here or reported by the peer. IETF streams use their own mapping: cancellation sends CANCELLED, other local failures send INTERNAL\_ERROR, and received codes remain opaque. +- **Errors**: deliberately closing a connection ends its received tracks cleanly. A peer close ends received tracks and open group readers with the session error and its close code. Errors live under one namespace: a stream reset throws `Error.Stream` with a `StreamCode`, while a session close gives `Error.Session` with a `SessionCode`. The registries are disjoint, so the same number means different things in each, and 64+ is yours. Named conditions such as `Error.TooFarBehind`, `Error.FrameTooLarge`, and `Error.GroupTooLarge` subclass `Error.Stream`, so one `code` check handles a condition raised here or reported by the peer. IETF streams use their own mapping: cancellation sends CANCELLED, other local failures send INTERNAL\_ERROR, and received codes remain opaque. - **Paths** with `Path.relative` for the cross-broadcast catalog references hang uses. A `Broadcast.Consumer` names its broadcast by `path`: the path it was requested at relative to the origin handle's root, or empty for a standalone broadcast. Those references resolve against it. Path patterns (`Path.Pattern`, `Path.Patterns`) are re-exported from [`@moq/pattern`](https://www.npmjs.com/package/@moq/pattern). Literal `Path` stays a coordinate. The [path pattern](/concept/moq-lite#path-patterns) grammar lives on the diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index cb2975a2063..f3de4af5b5a 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -66,7 +66,9 @@ Both protocols withdraw the session's announcements and wait for delivery under that same deadline. IETF drafts 14 through 16 send withdrawals without waiting, and IETF media streams are not drained yet. Dropping the driver cancels the session. `moq-tokio` and `moq-wasm` drive sessions for their -callers. +callers. A deliberate local session close ends received tracks cleanly after +their delivered groups. A peer close ends received tracks and open group +readers with the session error, preserving its close code. `origin::Producer::new` returns a driver with the same `time::Driver` interface. It calls `cache::Pool::gc(now)` after each poll and folds the next diff --git a/doc/lib/rs/moq-video.md b/doc/lib/rs/moq-video.md index 20bfaaec78b..1053b5257dd 100644 --- a/doc/lib/rs/moq-video.md +++ b/doc/lib/rs/moq-video.md @@ -13,7 +13,7 @@ ffmpeg, no GStreamer, no system codec to install. | Module | Does | Backends | | --- | --- | --- | -| `capture` | Camera, display, window, or application frames | AVFoundation + ScreenCaptureKit (macOS), V4L2 + X11/portal + PipeWire (Linux), Media Foundation + DXGI (Windows) | +| `capture` | Camera, display, window, or application frames | AVFoundation + ScreenCaptureKit (macOS), V4L2 + X11/portal + PipeWire (Linux), Media Foundation + Windows.Graphics.Capture (Windows) | | `encode` | Frames to H.264/H.265, published as a hang track | VideoToolbox, Media Foundation, NVENC, VAAPI, V4L2 M2M, MediaCodec (Android), openh264 | | `decode` | A subscribed track back to frames | VideoToolbox, Media Foundation/DXVA, NVDEC, VAAPI, V4L2 M2M, MediaCodec (Android), openh264 | | `render` | A frame as a `wgpu` texture | wgpu, with zero-copy Metal and Vulkan imports | @@ -39,6 +39,29 @@ and an empty rate list means no discrete intervals were reported. Device errors are returned rather than treated as an empty list. Other platforms return `Error::Unsupported`. +Windows display and window capture require Windows 10 2004 (build 19041) or +newer and use Windows.Graphics.Capture without a Desktop Duplication or GDI +fallback. `Config::cursor` controls cursor capture. Frames leave the capture +pool as owned GPU NV12 textures; odd native edges are cropped to even sizes. +The conversion declares BT.601 limited range through 576 lines and BT.709 +limited range above it, preserving that color through resize and CPU download. +Media Foundation encodes on the capture device; openh264 uses the download path. +Display enumeration covers all adapters, and window enumeration excludes +DWM-cloaked windows and reports visible frame bounds. Windows `display:N` +selectors are enumeration indices, not persistent monitor identities. Switching +from Desktop Duplication to WGC can change their mapping; enumerate displays +again and reselect the intended monitor after upgrading. Enumeration does not +start capture. A settled source resize ends the stream so callers can reopen; +closing the captured item returns `Error::SourceUnavailable`. Dropping the +stream releases its session even when no new frames arrive. +If no usable first frame arrives within five seconds after capture starts, +opening fails with `Error::SourceUnavailable` and releases the session. + +Windows shows a capture border by default. On build 20348 or newer the backend +requests borderless access; denial keeps the border and does not fail capture. +Older supported builds keep the border. Application capture and system audio +are not provided by this backend. + With `pipewire` enabled, `capture::cameras` also lists PipeWire camera nodes as `pipewire:` after the V4L2 devices. V4L2 lists only devices offering YUYV or MJPEG. A PipeWire V4L2 node is hidden only when its device path was diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index 149cb99f86f..ac775aaace0 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -37,6 +37,10 @@ These land with the next breaking release, not the 2026-09-23 train. In TypeScript, `Track.Producer`'s `used` and `unused()` are `producer.demand().used` and `.unused()`, and `Allocator.reserve` takes `producer.demand()`, replacing the `Bandwidth.Demand` interface. +- **moq-mux has no clock translators.** `clock::Anchor`, `clock::Lane`, and + `SourceMap` (#4667) are gone, along with the importers' `live()`. Publish the + source's own timestamps and let the catalog clock map them to wall time; + pin that mapping with `Config::with_clock` when the source's zero is known. ## Wire diff --git a/flake.lock b/flake.lock index 85980a8b8f8..dcc6cd7be3d 100644 --- a/flake.lock +++ b/flake.lock @@ -65,17 +65,17 @@ ] }, "locked": { - "lastModified": 1790635831, - "narHash": "sha256-K/X+stFBmlCylJ9wiMCXxKOckoHeSninMvBb0wZa1rM=", + "lastModified": 1790797838, + "narHash": "sha256-/WmkmY7+FJSNGXR6+1i0sV2xHLRnWUxAVC0xogJdqlo=", "owner": "kixelated", "repo": "quest", - "rev": "8590d2a1ddd91c2f499adf37b78aad0d673e3228", + "rev": "362489bcf02833d8674cff339463b086442cf92d", "type": "github" }, "original": { "owner": "kixelated", "repo": "quest", - "rev": "8590d2a1ddd91c2f499adf37b78aad0d673e3228", + "rev": "362489bcf02833d8674cff339463b086442cf92d", "type": "github" } }, diff --git a/flake.nix b/flake.nix index f60e34bbcbb..b1cfb8013f6 100644 --- a/flake.nix +++ b/flake.nix @@ -27,7 +27,7 @@ # The quest CLI, which also serves the quest guide and skills the stubs in # .claude/skills call. Bump the rev to upgrade them. quest = { - url = "github:kixelated/quest/8590d2a1ddd91c2f499adf37b78aad0d673e3228"; + url = "github:kixelated/quest/362489bcf02833d8674cff339463b086442cf92d"; inputs.nixpkgs.follows = "nixpkgs"; inputs.flake-utils.follows = "flake-utils"; inputs.crane.follows = "crane"; @@ -427,6 +427,8 @@ # Apply our overlay to get the package definitions overlayPkgs = pkgs.extend self.overlays.default; + + quest-cli = quest.packages.${system}.default; in { packages = (rec { @@ -460,6 +462,10 @@ inherit uniffi-bindgen-dart; + # The quest CLI alone, so quest.yml can validate the tree without + # realising the whole dev shell. + quest = quest-cli; + # Bundle of packaging + repo-publish tooling, pinned via flake.lock. # CI builds this and prepends its bin/ to $PATH so subsequent steps # use the same versions a local `nix develop` user would. @@ -504,7 +510,7 @@ ++ goDeps ++ dartDeps ++ devTools - ++ [ quest.packages.${system}.default ]; + ++ [ quest-cli ]; # jemalloc's configure uses -O0 test builds, which conflict with # Nix's _FORTIFY_SOURCE hardening (requires -O). diff --git a/js/net/src/connection/accept.ts b/js/net/src/connection/accept.ts index 86a9ac33ad3..4a6ea050efc 100644 --- a/js/net/src/connection/accept.ts +++ b/js/net/src/connection/accept.ts @@ -60,8 +60,9 @@ async function acceptInner( url: URL, props: Omit, ): Promise { - // @ts-expect-error - TODO: add protocol to WebTransport - const protocol: string | undefined = transport.protocol; + // The DOM lib has no `protocol` property yet. It is "" when none was negotiated, and + // undefined in a browser that predates subprotocols (Firefox before 155). + const protocol = (transport as { protocol?: string }).protocol; const wiring: SessionProps = { discovery: props.discovery ?? true, @@ -114,7 +115,7 @@ async function acceptAlpn( version: Ietf.IetfVersion, wiring: SessionProps, ): Promise { - const { control, solicit, hidden, cluster } = await exchangeSetup(transport, version, "moq-lite-js"); + const { control, early, solicit, hidden, cluster } = await exchangeSetup(transport, version, "moq-lite-js"); return new Ietf.Connection({ ...wiring, @@ -122,6 +123,7 @@ async function acceptAlpn( url, quic: transport, control, + early, solicit, hidden, cluster, diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index fc79b2454ea..9f2bcccf815 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -254,8 +254,8 @@ async function connectTransport(url: URL, session: WebTransport, wiring: Session // Negotiate the MoQ protocol over an established transport. The caller races this against // the session closing so a close code is not lost behind a failed or stalled SETUP stream. async function negotiate(url: URL, session: WebTransport, wiring: SessionProps): Promise { - // qmux Session exposes the negotiated protocol directly (as "" when there is none); - // native WebTransport doesn't have a standard .protocol property yet. + // The DOM lib has no `protocol` property yet. It is "" when none was negotiated, and + // undefined in a browser that predates subprotocols (Firefox before 155). const protocol: string | undefined = (session as { protocol?: string }).protocol || undefined; if (dev()) console.debug(redact(url), "negotiated ALPN:", protocol ?? "(none)"); @@ -362,7 +362,7 @@ async function handshakeAlpn( version: Ietf.IetfVersion, wiring: SessionProps, ): Promise { - const { control, solicit, hidden, cluster } = await exchangeSetup(session, version, "moq-lite-js"); + const { control, early, solicit, hidden, cluster } = await exchangeSetup(session, version, "moq-lite-js"); return new Ietf.Connection({ ...wiring, @@ -370,6 +370,7 @@ async function handshakeAlpn( url, quic: session, control, + early, solicit, hidden, cluster, diff --git a/js/net/src/connection/forward.test.ts b/js/net/src/connection/forward.test.ts index 844a50718ab..932a7031944 100644 --- a/js/net/src/connection/forward.test.ts +++ b/js/net/src/connection/forward.test.ts @@ -1,4 +1,5 @@ import { expect, test } from "bun:test"; +import { Once } from "@moq/signals"; import * as Announce from "../announced.ts"; import { type Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "../broadcast.ts"; import { Route } from "../hop.ts"; @@ -33,7 +34,7 @@ class FakeSession { constructor(discovery = true) { this.discovery = discovery; - registerWire(this, { consume: (path) => this.consume(path) }); + registerWire(this, { consume: (path) => this.consume(path), goaway: new Once() }); this.closed = new Promise((resolve) => { this.#die = resolve; }); diff --git a/js/net/src/connection/goaway.test.ts b/js/net/src/connection/goaway.test.ts new file mode 100644 index 00000000000..fb99bbe1c20 --- /dev/null +++ b/js/net/src/connection/goaway.test.ts @@ -0,0 +1,110 @@ +import { expect, test } from "bun:test"; +import { RefusedRedirect } from "../error.ts"; +import * as Time from "../time.ts"; +import { dialed, handover, isLocal, pinnedTransport, type Redirect, target } from "./goaway.ts"; + +const current = new URL("https://relay.example/room?jwt=secret"); + +test("no URI, or a policy that ignores it, keeps the current URL", () => { + expect(target("follow", "", current, false)).toBeUndefined(); + expect(target("same-host", "", current, false)).toBeUndefined(); + expect(target("ignore", "https://other.example/", current, false)).toBeUndefined(); +}); + +test("an explicit URI the policy will not follow is refused, not ignored", () => { + const refused: [Redirect, string][] = [ + ["same-host", "https://other.example/"], + ["follow", "not a url"], + ["ignore", "not a url"], + ["follow", "http://relay.example/"], + ["follow", "unix:///tmp/moq.sock"], + ["follow", "https://127.0.0.1/"], + ["follow", "https://[::ffff:127.0.0.1]/"], + ["follow", "https://[::ffff:7f00:1]/"], + ["follow", "moqt://169.254.169.254/"], + ]; + for (const [policy, uri] of refused) { + expect(() => target(policy, uri, current, false), `${policy}: ${uri}`).toThrow(RefusedRedirect); + } +}); + +test("a refusal never repeats the URI, which can carry credentials", () => { + try { + target("same-host", "https://other.example/?jwt=leaked", current, false); + throw new Error("not refused"); + } catch (err) { + expect(err).toBeInstanceOf(RefusedRedirect); + expect(String(err)).not.toContain("leaked"); + } +}); + +test("same-host follows a port or scheme move on the host already dialed", () => { + expect(target("same-host", "https://relay.example:5443/", current, false)?.port).toBe("5443"); + // An explicit assignment is still one when it names the current URL. + expect(target("same-host", current.href, current, false)?.href).toBe(current.href); + // An upgrade is fine; only a downgrade is refused. + const plain = new URL("http://relay.example/"); + expect(target("same-host", "https://relay.example/", plain, false)?.protocol).toBe("https:"); +}); + +test("follow lets the peer name another public host", () => { + expect(target("follow", "https://other.example/next", current, false)?.href).toBe("https://other.example/next"); +}); + +test("a local endpoint may redirect to another local one", () => { + const local = new URL("https://localhost:4443/"); + expect(target("follow", "https://127.0.0.1:9999/", local, false)?.port).toBe("9999"); +}); + +test("a certificate pin holds the host even under follow", () => { + expect(() => target("follow", "https://other.example/", current, true)).toThrow(RefusedRedirect); + expect(target("follow", "https://relay.example:5443/", current, true)?.port).toBe("5443"); +}); + +test("local literals are recognized in every spelling", () => { + const local = [ + "https://127.0.0.1/", + "https://localhost/", + "https://a.localhost/", + "https://[::1]/", + "https://[::]/", + "https://10.0.0.1/", + "https://172.16.0.1/", + "https://192.168.1.1/", + "https://169.254.1.1/", + "https://0.0.0.0/", + "https://[::ffff:10.0.0.1]/", + "https://[fe80::1]/", + "https://[fc00::1]/", + "moqt://127.0.0.1/", + "moqt://[::ffff:127.0.0.1]/", + "unix:///tmp/moq.sock", + ]; + for (const url of local) expect(isLocal(new URL(url)), url).toBe(true); + + for (const url of ["https://example.com/", "https://8.8.8.8/", "https://172.32.0.1/", "https://[2606:4700::1]/"]) { + expect(isLocal(new URL(url)), url).toBe(false); + } +}); + +test("a WebSocket fallback that connected is the host a redirect is judged against", () => { + const primary = new URL("https://relay.example/"); + const socket = new URL("wss://edge.example/moq"); + expect(dialed(primary, "websocket", socket).href).toBe(socket.href); + expect(dialed(primary, "webtransport", socket).href).toBe(primary.href); + expect(dialed(primary, "websocket").href).toBe(primary.href); +}); + +test("a certificate pin holds only the WebTransport session that used it", () => { + expect(pinnedTransport("webtransport", true)).toBe(true); + expect(pinnedTransport("websocket", true)).toBe(false); + expect(pinnedTransport("webtransport", false)).toBe(false); +}); + +test("the handover is the cap, lowered only by a positive peer deadline", () => { + const cap = Time.Milli(10_000); + expect(handover(cap)).toBe(cap); + expect(handover(cap, Time.Milli(0))).toBe(cap); + expect(handover(cap, Time.Milli(3_000))).toBe(Time.Milli(3_000)); + expect(handover(cap, Time.Milli(3_600_000))).toBe(cap); +}); diff --git a/js/net/src/connection/goaway.ts b/js/net/src/connection/goaway.ts new file mode 100644 index 00000000000..a304e458383 --- /dev/null +++ b/js/net/src/connection/goaway.ts @@ -0,0 +1,224 @@ +/** + * GOAWAY handling for the reconnect loop: the peer's drain signal and the policy for the + * redirect it may name. Mirrors `Goaway` and `Redirect` in `moq_tokio::connection`. + * + * @module + */ +import { RefusedRedirect } from "../error.ts"; +import * as Time from "../time.ts"; + +/** A peer's GOAWAY as a session received it. */ +export interface Drain { + /** Where to reconnect, including any credentials it needs. Empty means the same endpoint. */ + readonly uri: string; + /** When the peer force-closes the session. Undefined when the wire carried none. */ + readonly timeout?: Time.Milli; +} + +/** + * What to do with the URI a peer names in its GOAWAY. + * + * - `same-host` (the default) follows it only onto the host already dialed, so a peer can move + * the connection between ports or schemes but not to another host. + * - `follow` also lets the peer name the host. The name is dialed as written, so only use it + * with a relay trusted not to point the connection into the local network. + * - `ignore` reconnects to the current URL whatever the peer names. + * + * A malformed or refused URI ends the connection with {@link RefusedRedirect}. + */ +export type Redirect = "follow" | "same-host" | "ignore"; + +/** How a reconnecting connection reacts to a peer's GOAWAY. */ +export interface GoawayProps { + /** What to do with the URI the peer names (default: `same-host`). */ + redirect?: Redirect; + + /** + * How long the old session keeps serving after the GOAWAY while the replacement dials + * (default: 10000ms). A cap: a shorter deadline from the peer wins, a longer one does not + * extend it. + */ + handover?: Time.Milli; +} + +/** The handover cap when the caller names none. */ +export const DEFAULT_HANDOVER = Time.Milli(10_000); + +/** + * How long a drained session may keep serving: the cap, lowered to the peer's deadline when + * it named a positive one. Absence is not a zero-length handover. + * + * @internal + */ +export function handover(cap: Time.Milli, timeout?: Time.Milli): Time.Milli { + return timeout !== undefined && timeout > 0 ? Time.Milli(Math.min(cap, timeout)) : cap; +} + +/** + * The URL a redirect is judged against. A WebSocket fallback that won the race is the host + * we dialed; the primary URL was not, so same-host must not treat it as the current peer. + * + * @internal + */ +export function dialed(primary: URL, transport: "webtransport" | "websocket", websocket?: URL): URL { + return transport === "websocket" && websocket ? websocket : primary; +} + +/** + * Whether a certificate pin constrains this session. The pin is a WebTransport option, so it + * holds the host only when that transport is the one that connected. + * + * @internal + */ +export function pinnedTransport(transport: "webtransport" | "websocket", configured: boolean): boolean { + return configured && transport === "webtransport"; +} + +/** + * The URL a GOAWAY assigns: `undefined` keeps the current URL (the peer named none, or the + * policy ignores a URI it could parse), and a URL replaces it. Throws {@link RefusedRedirect} + * for an explicit URI the policy will not follow, including a malformed one under `ignore`. + * + * `pinned` is a certificate pin on the connection, which can only verify the host it was + * configured for, so it refuses a host change even under `follow`. + * + * @internal + */ +export function target(policy: Redirect, uri: string, current: URL, pinned: boolean): URL | undefined { + if (uri === "") return undefined; + + // The URI can carry credentials, so the error names the reason, never the URI. + // Parse before `ignore`: a malformed redirect is terminal even when the policy + // would otherwise stay on the current URL. + let next: URL; + try { + next = new URL(uri); + } catch { + throw new RefusedRedirect("the GOAWAY URI is malformed"); + } + if (policy === "ignore") return undefined; + + if (schemeTier(next.protocol) < schemeTier(current.protocol)) { + throw new RefusedRedirect("the GOAWAY redirect downgrades the scheme"); + } + + // Only as far as the URL itself says: a name is dialed, never resolved here, so this + // refuses a peer naming a local address outright, not one hiding it behind a hostname. + // That gap is why `same-host` is the default. + if (isLocal(next) && !isLocal(current)) { + throw new RefusedRedirect("the GOAWAY redirect widens reachability to a local address"); + } + + // Host only, not the full authority: the port is what a peer legitimately moves us + // across when it hands off to a sibling process on the same box. + const sameHost = next.hostname === current.hostname; + if (policy === "same-host" && !sameHost) { + throw new RefusedRedirect("the GOAWAY redirect leaves the current host"); + } + if (pinned && !sameHost) { + throw new RefusedRedirect("the GOAWAY redirect leaves the host a certificate pin verifies"); + } + + return next; +} + +/** Rank a scheme so a redirect cannot silently drop encryption; unknown schemes rank lowest. */ +function schemeTier(protocol: string): number { + switch (protocol) { + case "https:": + case "wss:": + case "moqt:": + case "moql:": + return 2; + case "http:": + case "ws:": + case "tcp:": + return 1; + default: + return 0; + } +} + +/** + * Whether a URL says it names something only reachable from this host or network. A judgement + * about the URL, not about where a dial lands: `false` means "not local on its face". + * + * @internal + */ +export function isLocal(url: URL): boolean { + const host = url.hostname.toLowerCase(); + // No host at all, e.g. a `unix:` socket path. + if (host === "") return true; + if (host === "localhost" || host.endsWith(".localhost")) return true; + + if (host.startsWith("[") && host.endsWith("]")) { + const segments = parseIpv6(host.slice(1, -1)); + return segments !== undefined && isLocalV6(segments); + } + + const v4 = parseIpv4(host); + return v4 !== undefined && isLocalV4(v4); +} + +function parseIpv4(host: string): number[] | undefined { + const parts = host.split("."); + if (parts.length !== 4) return undefined; + const octets = parts.map((part) => (/^\d{1,3}$/.test(part) ? Number(part) : Number.NaN)); + return octets.every((octet) => octet <= 255) ? octets : undefined; +} + +function isLocalV4([a, b, c, d]: number[]): boolean { + return ( + a === 127 || + a === 10 || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) || + (a === 169 && b === 254) || + (a === 0 && b === 0 && c === 0 && d === 0) + ); +} + +/** Parse an IPv6 literal (without brackets) into its eight 16-bit segments. */ +function parseIpv6(host: string): number[] | undefined { + // Drop a zone id; it scopes the address without changing it. + let text = host.split("%")[0] ?? ""; + + // A trailing dotted quad (`::ffff:127.0.0.1`) spells the last two segments. + const lastColon = text.lastIndexOf(":"); + const quad = parseIpv4(text.slice(lastColon + 1)); + if (quad) { + const [a = 0, b = 0, c = 0, d = 0] = quad; + const word = (hi: number, lo: number) => ((hi << 8) | lo).toString(16); + text = `${text.slice(0, lastColon + 1)}${word(a, b)}:${word(c, d)}`; + } + + const halves = text.split("::"); + if (halves.length > 2) return undefined; + const words = (part: string) => (part === "" ? [] : part.split(":")); + const head = words(halves[0] ?? ""); + const rest = halves.length === 2 ? words(halves[1] ?? "") : []; + const missing = 8 - head.length - rest.length; + if (halves.length === 2 ? missing < 0 : missing !== 0) return undefined; + + const all = [...head, ...Array(missing).fill("0"), ...rest]; + const segments = all.map((word) => (/^[0-9a-f]{1,4}$/i.test(word) ? Number.parseInt(word, 16) : Number.NaN)); + return segments.some(Number.isNaN) ? undefined : segments; +} + +function isLocalV6(segments: number[]): boolean { + // An IPv4-mapped address reaches the same host as the v4 it wraps. + if (segments.slice(0, 5).every((s) => s === 0) && segments[5] === 0xffff) { + const [hi = 0, lo = 0] = segments.slice(6); + return isLocalV4([hi >> 8, hi & 0xff, lo >> 8, lo & 0xff]); + } + const first = segments[0] ?? 0; + const zeroPrefix = segments.slice(0, 7).every((s) => s === 0); + const last = segments[7] ?? 0; + return ( + // Loopback (::1) and unspecified (::). + (zeroPrefix && (last === 1 || last === 0)) || + // Unique local (fc00::/7) and link local (fe80::/10). + (first & 0xfe00) === 0xfc00 || + (first & 0xffc0) === 0xfe80 + ); +} diff --git a/js/net/src/connection/handshake.ts b/js/net/src/connection/handshake.ts index 05cd214c8b8..81bb2052a9e 100644 --- a/js/net/src/connection/handshake.ts +++ b/js/net/src/connection/handshake.ts @@ -1,6 +1,7 @@ +import { StreamError } from "../error.ts"; import { type Hop, randomHop } from "../hop.ts"; import * as Ietf from "../ietf/index.ts"; -import { Reader, Stream, Writer } from "../stream.ts"; +import { Reader, Stream, UnexpectedEnd, Writer } from "../stream.ts"; /** * Draft-17+ SETUP exchange. Each side opens a uni stream, writes its Setup @@ -8,7 +9,8 @@ import { Reader, Stream, Writer } from "../stream.ts"; * halves run in parallel and the protocol is symmetric, so both `connect` * (client) and `accept` (server) use this same function. * - * Returns the control stream plus what the peer's SETUP declared: whether it requires + * Returns the control stream, the uni streams that arrived before it (see + * {@link receiveSetup}), plus what the peer's SETUP declared: whether it requires * solicitation, which decides whether we announce namespaces unprompted (see the MoQ Solicit * extension), and its Hop ID (see the MoQ Cluster extension). We declare both ourselves on * every session: we send SUBSCRIBE_NAMESPACE for each prefix we want, so an unsolicited @@ -19,7 +21,13 @@ export async function exchangeSetup( transport: WebTransport, version: Ietf.IetfVersion, implementation: string, -): Promise<{ control: Stream; solicit: boolean | undefined; hidden: boolean; cluster: Ietf.Cluster.Hops }> { +): Promise<{ + control: Stream; + early: Reader[]; + solicit: boolean | undefined; + hidden: boolean; + cluster: Ietf.Cluster.Hops; +}> { const encoder = new TextEncoder(); const params = new Ietf.SetupOptions(); params.setBytes(Ietf.SetupOption.Implementation, encoder.encode(implementation)); @@ -40,6 +48,7 @@ export async function exchangeSetup( return { control: new Stream({ writer, reader: received.reader }), + early: received.early, solicit: received.solicit, hidden: received.hidden, cluster: { self, peer: received.cluster }, @@ -55,27 +64,59 @@ async function sendSetup(transport: WebTransport, version: Ietf.IetfVersion, set return writer; } +/** + * Read the peer's SETUP off its uni stream. Any other uni stream that beats it (padding, + * or group data for a subscription the peer already holds) is held, type unread, for the + * session to classify once it starts: the drafts say to buffer early data rather than + * reject it. QUIC stream credit bounds how many can pile up. + */ async function receiveSetup( transport: WebTransport, version: Ietf.IetfVersion, -): Promise<{ reader: Reader; solicit: boolean | undefined; hidden: boolean; cluster: Hop | undefined }> { +): Promise<{ + reader: Reader; + early: Reader[]; + solicit: boolean | undefined; + hidden: boolean; + cluster: Hop | undefined; +}> { const uniReader = transport.incomingUnidirectionalStreams.getReader() as ReadableStreamDefaultReader< ReadableStream >; - const next = await uniReader.read(); - uniReader.releaseLock(); - if (next.done) throw new Error("no incoming uni stream for SETUP"); + const early: Reader[] = []; - const reader = new Reader(next.value, undefined, version); + let reader: Reader; + try { + for (;;) { + const next = await uniReader.read(); + if (next.done) throw new Error("no incoming uni stream for SETUP"); - const streamType = await reader.u53(); - if (streamType !== Ietf.Setup.id) { - throw new Error(`unexpected stream type on setup uni: 0x${streamType.toString(16)}`); + const stream = new Reader(next.value, undefined, version); + // A stream that ended or reset before its full type is that stream's failure, not the session's. + // Malformed bytes are still the peer's fault, so a bad type fails the handshake. + if (await stream.done().catch(() => true)) continue; + // Full width, so an unknown type past 2^53 is held for the classifier to refuse. + const streamType = await stream.peekU62().catch((err: unknown) => { + if (err instanceof StreamError || err instanceof UnexpectedEnd) return undefined; + throw err; + }); + if (streamType === undefined) continue; + if (streamType === BigInt(Ietf.Setup.id)) { + reader = stream; + break; + } + early.push(stream); + } + } finally { + uniReader.releaseLock(); } + + await reader.u53(); // the SETUP type, peeked above const setup = await Ietf.Setup.decode(reader, version); return { reader, + early, solicit: Ietf.solicitFromSetup(setup.parameters), hidden: Ietf.hiddenFromSetup(setup.parameters), cluster: Ietf.Cluster.fromSetup(setup.parameters, version), diff --git a/js/net/src/connection/migrate.test.ts b/js/net/src/connection/migrate.test.ts new file mode 100644 index 00000000000..2d6969764c0 --- /dev/null +++ b/js/net/src/connection/migrate.test.ts @@ -0,0 +1,277 @@ +import { afterEach, expect, test } from "bun:test"; +import { RefusedRedirect } from "../error.ts"; +import * as Lite from "../lite/index.ts"; +import { createMockTransportPair } from "../mock.ts"; +import { Producer as OriginProducer } from "../origin.ts"; +import * as Path from "../path.ts"; +import { Stream } from "../stream.ts"; +import * as Time from "../time.ts"; +import { accept } from "./accept.ts"; +import { Connection, resetShared } from "./pool.ts"; +import { Reload } from "./reload.ts"; + +const original = globalThis.WebTransport; + +afterEach(() => { + resetShared(); + globalThis.WebTransport = original; +}); + +async function settle() { + await new Promise((resolve) => setTimeout(resolve, 0)); +} + +// Polls until `pred` holds, so a regression fails the test instead of hanging it. +async function waitUntil(pred: () => boolean, ms = 2000): Promise { + const deadline = Date.now() + ms; + for (;;) { + if (pred()) return; + if (Date.now() > deadline) throw new Error("timed out waiting for condition"); + await settle(); + } +} + +/** One accepted session on the fake fleet: the URL it was dialed at, and its server side. */ +interface Dial { + url: string; + server: WebTransport; + closed: boolean; +} + +/** + * Stand in for a fleet of relays behind any URL: every dial gets a fresh session serving + * `origin`, so each one publishes the same broadcasts the way siblings of a fleet do. + */ +function fleet(origin = new OriginProducer()): { dials: Dial[]; origin: OriginProducer } { + const dials: Dial[] = []; + const stub = function StubWebTransport(url: string | URL) { + const pair = createMockTransportPair(Lite.ALPN_06); + const dial: Dial = { url: new URL(url).href, server: pair.server, closed: false }; + dials.push(dial); + void pair.server.closed.then( + () => { + dial.closed = true; + }, + () => { + dial.closed = true; + }, + ); + void accept({ transport: pair.server, url: new URL(url), publish: origin.consume() }); + return pair.client; + }; + globalThis.WebTransport = stub as unknown as typeof WebTransport; + return { dials, origin }; +} + +/** Send a moq-lite GOAWAY from the server side of a session. */ +async function goaway(server: WebTransport, uri: string): Promise { + const stream = await Stream.open(server, { version: Lite.Version.DRAFT_06 }); + await stream.writer.u53(Lite.StreamId.Goaway); + await new Lite.Goaway(uri).encode(stream.writer, Lite.Version.DRAFT_06); +} + +const url = new URL("https://relay.example/room"); + +// Short enough to watch, long enough that a handover visibly overlaps the replacement. +const handover = Time.Milli(200); + +test("an empty GOAWAY migrates without unrouting the path", async () => { + const { dials, origin } = fleet(); + const broadcast = origin.createBroadcast(Path.from("cam")); + broadcast.announce(); + + const consume = new OriginProducer(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + consume, + goaway: { handover }, + // A session younger than this counts as redirected immediately and backs off first. + delay: { initial: Time.Milli(1) }, + }); + const watched = consume.request(Path.from("cam"), { announced: true }); + + try { + await waitUntil(() => watched.active.peek() !== undefined); + const first = reload.established.peek(); + + // Record every moment the path had no route, from here on. + let gaps = 0; + const stop = watched.active.subscribe((active) => { + if (active === undefined) gaps++; + }); + + const drained = dials[0]; + if (!drained) throw new Error("no first dial"); + await goaway(drained.server, ""); + + // The replacement dials the configured URL at once, while the old session still serves. + await waitUntil(() => dials.length === 2); + expect(dials[1]?.url).toBe(url.href); + await waitUntil(() => reload.established.peek() !== first && reload.established.peek() !== undefined); + expect(drained.closed).toBe(false); + expect(reload.status.peek()).toBe("connected"); + + // The old session closes at the handover cap, and the path never went unrouted. + await waitUntil(() => drained.closed, handover * 10); + await settle(); + expect(watched.active.peek()).not.toBeUndefined(); + expect(gaps).toBe(0); + expect(dials.length).toBe(2); + stop(); + } finally { + watched.close(); + reload.close(); + consume.close(); + broadcast.close(); + } +}); + +test("a GOAWAY without a timeout hands over at the configured cap", async () => { + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + goaway: { handover }, + delay: { initial: Time.Milli(1) }, + }); + + try { + await waitUntil(() => reload.status.peek() === "connected"); + const drained = dials[0]; + if (!drained) throw new Error("no first dial"); + + const sent = performance.now(); + await goaway(drained.server, ""); + await waitUntil(() => dials.length === 2 && reload.status.peek() === "connected"); + + // Lite carries no deadline, which must read as "the cap", never as a zero handover. + await waitUntil(() => drained.closed, handover * 10); + expect(performance.now() - sent).toBeGreaterThanOrEqual(handover * 0.9); + } finally { + reload.close(); + } +}); + +test("a refused redirect ends the connection instead of reconnecting", async () => { + const refused = ["https://other.example/", "not a url", "http://relay.example/", "https://127.0.0.1/"]; + for (const uri of refused) { + const { dials } = fleet(); + const reload = new Reload({ url, websocket: { enabled: false }, delay: { initial: Time.Milli(1) } }); + + try { + await waitUntil(() => reload.status.peek() === "connected"); + const drained = dials[0]; + if (!drained) throw new Error("no first dial"); + + await goaway(drained.server, uri); + await waitUntil(() => reload.error.peek() !== undefined); + expect(reload.error.peek(), uri).toBeInstanceOf(RefusedRedirect); + expect(reload.status.peek()).toBe("disconnected"); + await waitUntil(() => drained.closed); + + // Nothing redials: not the original URL, not anything else. + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(dials.length, uri).toBe(1); + } finally { + reload.close(); + } + } +}); + +test("a certificate pin refuses a redirect to another host even under follow", async () => { + const { dials } = fleet(); + const reload = new Reload({ + url, + websocket: { enabled: false }, + webtransport: { serverCertificateHashes: [{ value: "00".repeat(32) }] }, + goaway: { redirect: "follow" }, + delay: { initial: Time.Milli(1) }, + }); + + try { + await waitUntil(() => reload.status.peek() === "connected"); + await goaway(dials[0]?.server as WebTransport, "https://other.example/"); + await waitUntil(() => reload.error.peek() !== undefined); + expect(reload.error.peek()).toBeInstanceOf(RefusedRedirect); + expect(dials.length).toBe(1); + } finally { + reload.close(); + } +}); + +// A same-host move to another port: what `same-host` exists to allow. +const moved = new URL("https://relay.example:5443/room"); + +test("a redirect moves the pool key while the handle keeps its origin", async () => { + const { dials } = fleet(); + + const handle = new Connection({ url }); + try { + await waitUntil(() => handle.status.peek() === "connected"); + const origin = handle.origin.peek(); + + await goaway(dials[0]?.server as WebTransport, moved.href); + await waitUntil(() => dials.length === 2); + expect(dials[1]?.url).toBe(moved.href); + expect(handle.origin.peek()).toBe(origin); + + // A caller configured with the target shares the migrated connection. + const joined = new Connection({ url: moved }); + await waitUntil(() => joined.status.peek() === "connected"); + expect(joined.origin.peek()).toBe(origin); + expect(dials.length).toBe(2); + + // One still asking for the original URL gets a fresh entry. + const fresh = new Connection({ url }); + await waitUntil(() => fresh.status.peek() === "connected"); + expect(fresh.origin.peek()).not.toBe(origin); + expect(dials.length).toBe(3); + + joined.close(); + fresh.close(); + } finally { + handle.close(); + } +}); + +test("a redirect onto an already pooled key leaves both entries to their handles", async () => { + const { dials } = fleet(); + + const redirected = new Connection({ url }); + const resident = new Connection({ url: moved }); + try { + await waitUntil(() => redirected.status.peek() === "connected" && resident.status.peek() === "connected"); + const migrated = redirected.origin.peek(); + const target = resident.origin.peek(); + expect(migrated).not.toBe(target); + + const drained = dials.find((dial) => dial.url === url.href); + await goaway(drained?.server as WebTransport, moved.href); + await waitUntil(() => dials.filter((dial) => dial.url === moved.href).length === 2); + await waitUntil(() => redirected.status.peek() === "connected"); + + // Existing handles keep their own entries. + expect(redirected.origin.peek()).toBe(migrated); + expect(resident.origin.peek()).toBe(target); + + // The target key still belongs to the entry that was there. + const joined = new Connection({ url: moved }); + await waitUntil(() => joined.origin.peek() !== undefined); + expect(joined.origin.peek()).toBe(target); + + // The original key was vacated, so a new caller dials fresh. + const before = dials.length; + const fresh = new Connection({ url }); + await waitUntil(() => fresh.status.peek() === "connected"); + expect(fresh.origin.peek()).not.toBe(migrated); + expect(fresh.origin.peek()).not.toBe(target); + expect(dials.length).toBe(before + 1); + + joined.close(); + fresh.close(); + } finally { + redirected.close(); + resident.close(); + } +}); diff --git a/js/net/src/connection/pool.ts b/js/net/src/connection/pool.ts index 3b35c13468b..5acbf59babd 100644 --- a/js/net/src/connection/pool.ts +++ b/js/net/src/connection/pool.ts @@ -20,6 +20,7 @@ import { type WebTransportProps as WebTransportPropsType, } from "./connect.ts"; import type { Established as EstablishedType } from "./established.ts"; +import type { GoawayProps, Redirect as RedirectType } from "./goaway.ts"; import { Reload, type ReloadDelay, type ReloadStatus } from "./reload.ts"; import type { Probe as ProbeType, Stats as StatsType } from "./stats.ts"; import type { Transport as TransportType } from "./transport.ts"; @@ -69,6 +70,13 @@ export interface ConnectionProps { /** Backoff settings for the reconnect loop; an unset field uses its default. */ delay?: ReloadDelay; + /** + * How to react to the relay's GOAWAY; an unset field uses its default. Every connection + * migrates on GOAWAY, dialing the replacement while the old session finishes its groups. + * This only tunes where it may go and for how long the old session serves. + */ + goaway?: GoawayProps; + /** A Connection owns the abort signal for each connection attempt. */ signal?: never; @@ -93,7 +101,11 @@ export interface ConnectionProps { * connection so the next handle dials fresh; a new URL on this handle starts another * sequence. {@link closed} settles only when this handle is released. * - * Options the pool cannot honor (transport options, discovery, delay, a pinned + * A relay's GOAWAY migrates rather than drops: the replacement dials at once while the old + * session finishes its groups, and the handle and its origin carry across. A redirect the + * {@link ConnectionProps.goaway} policy refuses stops the loop like an auth rejection does. + * + * Options the pool cannot honor (transport options, discovery, delay, goaway, a pinned * certificate, caller-owned origins) use a private loop. An explicit `share: true` * refuses them. A supplied transport cannot reconnect at all; pass it to {@link Connection.connect} * instead. @@ -246,6 +258,7 @@ export class Connection { // A handle nobody watches wants unlimited retries; an auth rejection still // stops this URL, and a new one starts another sequence. delay: { timeout: Time.Milli(0), ...props.delay }, + goaway: props.goaway, }); this.#signals.cleanup(() => loop.close()); @@ -341,6 +354,10 @@ export namespace Connection { export type AcceptProps = AcceptPropsType; /** Backoff settings for a private reconnect loop. */ export type Backoff = ReloadDelay; + /** How a connection reacts to the relay's GOAWAY. */ + export type Goaway = GoawayProps; + /** What to do with the URI a relay names in its GOAWAY. */ + export type Redirect = RedirectType; /** Current state of a {@link Connection}. */ export type Status = ReloadStatus; /** The current connection's PROBE estimates. */ @@ -391,6 +408,9 @@ function refuse(props?: ConnectionProps): void { if (props.delay !== undefined) { throw new Error("delay cannot be shared; pass share: false"); } + if (props.goaway !== undefined) { + throw new Error("goaway cannot be shared; pass share: false"); + } } /** Options tied to one handle cannot be represented by a URL-keyed shared entry. */ @@ -400,6 +420,7 @@ function requiresPrivate(props: ConnectionProps): boolean { props.websocket !== undefined || props.discovery !== undefined || props.delay !== undefined || + props.goaway !== undefined || props.publish !== undefined || props.consume !== undefined ); @@ -407,6 +428,8 @@ function requiresPrivate(props: ConnectionProps): boolean { /** One shared connection and the handles keeping it alive. */ interface Entry { + /** The pool key: the dialed URL, which a GOAWAY redirect moves. */ + key: string; origin: Origin.Producer; connection: Reload; refs: number; @@ -433,14 +456,25 @@ function acquire(key: string, linger?: Time.Milli): Entry & { release: () => voi delay: { timeout: Time.Milli(0) }, }); - const created: Entry = { origin, connection, refs: 0, linger: linger ?? LINGER_MS }; + const created: Entry = { key, origin, connection, refs: 0, linger: linger ?? LINGER_MS }; - // The loop only stops on a peer saying these credentials will never work. Drop the - // entry so a later handle dials fresh rather than joining a loop that has stopped; - // handles already on it keep it until they release, since a redial would be refused - // the same way. + // The loop only stops on a peer saying these credentials will never work, or on a + // GOAWAY redirect it refused. Drop the entry so a later handle dials fresh rather + // than joining a loop that has stopped; handles already on it keep it until they + // release. connection.error.subscribe((err) => { - if (err !== undefined && pool.get(key) === created) pool.delete(key); + if (err !== undefined && pool.get(created.key) === created) pool.delete(created.key); + }); + + // An accepted redirect moves the entry to the URL it now dials, so a later handle + // configured with that URL shares it, and one asking for the old URL dials fresh. + // A live entry already at the target wins: this one leaves the pool and serves only + // the handles it has until they release it. + connection.redirect.subscribe((redirect) => { + if (!redirect || redirect.href === created.key) return; + if (pool.get(created.key) === created) pool.delete(created.key); + created.key = redirect.href; + if (!pool.has(created.key)) pool.set(created.key, created); }); entry = created; @@ -466,7 +500,7 @@ function acquire(key: string, linger?: Time.Milli): Entry & { release: () => voi if (taken.refs > 0) return; taken.timer = setTimeout(() => { - if (pool.get(key) === taken) pool.delete(key); + if (pool.get(taken.key) === taken) pool.delete(taken.key); taken.connection.close(); taken.origin.close(); }, taken.linger); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 28bc386d6ac..f597485a6b8 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -8,6 +8,7 @@ import * as Time from "../time.ts"; import { wireOf } from "../wire.ts"; import { type ConnectProps, connect, type WebSocketProps, type WebTransportProps } from "./connect.ts"; import type { Established } from "./established.ts"; +import { DEFAULT_HANDOVER, type Drain, dialed, type GoawayProps, handover, pinnedTransport, target } from "./goaway.ts"; import type { Probe, Stats } from "./stats.ts"; /** @@ -59,6 +60,9 @@ export type ReloadProps = Omit & { /** Backoff settings for the reconnect loop; every field falls back to its default. */ delay?: ReloadDelay; + + /** How to react to the peer's GOAWAY; every field falls back to its default. */ + goaway?: GoawayProps; }; /** @@ -162,6 +166,16 @@ export class Reload { /** Backoff settings for the reconnect loop; an unset field uses its default. */ delay: ReloadDelay; + /** How to react to the peer's GOAWAY (not reactive). */ + goaway: GoawayProps; + + /** + * The URL an accepted GOAWAY redirect assigned, or undefined while the loop dials + * {@link Reload.url}. Sticky across reconnects: a redirect is an assignment, not a + * detour. Cleared when a new URL or a disable/re-enable starts another sequence. + */ + readonly redirect: Getter; + /** The reactive effect scope driving the connect loop; closed by {@link Reload.close}. */ #signals = new Effect(); @@ -179,6 +193,16 @@ export class Reload { #closed = new Once(); #error = new Signal(undefined); + #redirect = new Signal(undefined); + // The configured href the redirect was assigned for. + #redirectHref: string | undefined; + + // A session the peer sent GOAWAY on, serving its groups in flight while the replacement + // dials. Retired when it closes, at its handover cap, or when the sequence ends. + #draining: Draining | undefined; + + // Whether a connect attempt is in flight, so a retiring predecessor reports the right status. + #dialing = false; // The current wait between attempts, doubling per failure, and when the retry window expires. // Both are undefined between sequences, so a later edit to `delay` applies to the next one. @@ -207,6 +231,8 @@ export class Reload { this.url = Signal.from(props?.url); this.enabled = Signal.from(props?.enabled ?? true); this.delay = props?.delay ?? {}; + this.goaway = props?.goaway ?? {}; + this.redirect = this.#redirect; this.webtransport = props?.webtransport; this.websocket = props?.websocket; this.discovery = props?.discovery; @@ -279,6 +305,7 @@ export class Reload { if (!enabled) { this.#givenUpHref = undefined; this.#resetSequence(); + this.#redirect.set(undefined); return; } @@ -293,6 +320,7 @@ export class Reload { if (!href) { this.#givenUpHref = undefined; this.#resetSequence(); + this.#redirect.set(undefined); return; } const url = new URL(href); @@ -303,49 +331,100 @@ export class Reload { if (this.#sequenceHref !== href) { this.#resetSequence(); + // A redirect was assigned for another URL; this one starts from itself. A page + // hide/show resumes the same URL, so it keeps the assignment. + if (this.#redirectHref !== href) this.#redirect.set(undefined); this.#sequenceHref = href; this.#error.set(undefined); this.#givenUpHref = undefined; } - effect.set(this.status, "connecting", "disconnected"); + // A drained predecessor still serves while its replacement dials. + if (!this.#draining) this.status.set("connecting"); // This run's teardown, handed to connect() so a rerun cancels the attempt in flight. const signal = effect.abort; + // The session this run serves, closed with the run. A drained one leaves this slot + // for #draining, which outlives the run. + let current: Established | undefined; + effect.cleanup(() => { + if (current) { + current.abort(); + if (this.established.peek() === current) this.established.set(undefined); + current = undefined; + } + if (this.established.peek() === undefined) this.status.set("disconnected"); + }); + effect.spawn(async () => { // Set once the session is live, so #retry can tell a healthy session that // later dropped from a connect failure or a peer that flaps immediately. let connected: DOMHighResTimeStamp | undefined; try { - const connection = await connect({ - url, - websocket: this.websocket, - webtransport: this.webtransport, - discovery: this.discovery, - publish: this.publish, - consume: this.consume, - signal, - }); - - // Hand the connection to the effect, which closes it now if this run is already over. - effect.cleanup(() => connection.abort()); - if (signal.aborted) return; - - effect.set(this.established, connection); - effect.set(this.status, "connected", "disconnected"); - - connected = performance.now(); + // Loops only to migrate: a GOAWAY off a healthy session dials its replacement + // straight away, with no backoff. + for (;;) { + const dialing = this.#redirect.peek() ?? url; + + this.#dialing = true; + let connection: Established; + try { + connection = await connect({ + url: dialing, + // A redirect names the relay; a fallback URL pinned for the old one does not follow. + websocket: this.#redirect.peek() ? { ...this.websocket, url: undefined } : this.websocket, + webtransport: this.webtransport, + discovery: this.discovery, + publish: this.publish, + consume: this.consume, + signal, + }); + } finally { + this.#dialing = false; + } - // A cancelled effect resolves undefined, so the sentinel tells the session - // closing (null for clean, an Error otherwise) apart from this run being - // torn down. - const closed = await effect.race(connection.closed); - if (closed === undefined) return; + // Hand the connection to the effect, which closes it now if this run is already over. + if (signal.aborted) { + connection.abort(); + return; + } + current = connection; + + // The replacement serves now; a predecessor keeps draining its groups in flight. + this.established.set(connection); + this.status.set("connected"); + connected = performance.now(); + + // A cancelled effect resolves undefined, so the sentinel tells the session + // closing (null for clean, an Error otherwise) apart from this run being + // torn down. Anything else is the peer's GOAWAY. + const ended = await effect.race(connection.closed, wireOf(connection).goaway); + if (ended === undefined) return; + if (ended === null || ended instanceof Error) { + console.warn("connection closed, reconnecting"); + if (this.established.peek() === connection) this.established.set(undefined); + this.#retry(effect, connected, ended ?? undefined); + return; + } - console.warn("connection closed, reconnecting"); - this.#retry(effect, connected, closed ?? undefined); + current = undefined; + // A pinned WebSocket URL can win the race against the primary. Judge the + // redirect against that endpoint, not the primary we never reached. + const socket = this.#redirect.peek() ? undefined : this.websocket?.url; + if (!this.#migrate(connection, dialed(dialing, connection.transport, socket), ended)) return; + + // A session that outlived the initial delay was healthy, so its handover is not a + // failure. One redirected almost at once still migrates, but through the backoff, + // so two peers bouncing us between them escalate and eventually give up. + if (performance.now() - connected < this.#initial()) { + this.#retry(effect, connected, new Error("peer redirected immediately")); + return; + } + this.#delay = undefined; + this.#deadline = undefined; + } } catch (err) { // Treat teardown as cancellation, not a connection failure. if (signal.aborted) return; @@ -356,6 +435,56 @@ export class Reload { }); } + /** + * Act on the peer's GOAWAY for `connection`, which was dialed at `dialing`: resolve where + * to go next and leave the old session serving until it drains. Returns false when the + * redirect is refused, which ends the sequence rather than redialing. + */ + #migrate(connection: Established, dialing: URL, drain: Drain): boolean { + const hashes = this.webtransport?.serverCertificateHashes?.length ?? 0; + const configured = hashes > 0 || this.webtransport?.serverCertificate !== undefined; + // The pin is a WebTransport option. A WebSocket that won the race never used it. + const pinned = pinnedTransport(connection.transport, configured); + + let next: URL | undefined; + try { + next = target(this.goaway.redirect ?? "same-host", drain.uri, dialing, pinned); + } catch (err) { + // The peer is leaving and named somewhere we won't go: redialing the old address + // would ignore it, so stop here. + console.warn("GOAWAY redirect refused:", err); + connection.abort(); + this.established.set(undefined); + this.status.set("disconnected"); + this.#giveUp(error(err)); + return false; + } + + // Only an accepted redirect replaces the URL; an empty one keeps it. + if (next) { + this.#redirect.set(next); + this.#redirectHref = this.#sequenceHref; + } + + console.info("GOAWAY received; migrating"); + // A newer GOAWAY retires an older predecessor rather than holding two open. + this.#draining?.retire(); + const cap = handover(this.goaway.handover ?? DEFAULT_HANDOVER, drain.timeout); + const draining = new Draining(connection, cap, () => { + if (this.#draining === draining) this.#draining = undefined; + // If nothing replaced it yet, nothing is serving. + if (this.established.peek() !== connection) return; + this.established.set(undefined); + this.status.set(this.#dialing ? "connecting" : "disconnected"); + }); + this.#draining = draining; + return true; + } + + #initial(): Time.Milli { + return this.delay?.initial ?? DEFAULT_DELAY.initial; + } + /** * Schedule the next connect attempt after the current backoff, or stop once the retry window * has expired. `connected` is when the dead session was established, if it ever was, and @@ -368,15 +497,14 @@ export class Reload { // optional value passes an explicit undefined, which a spread would take as the // answer, turning the backoff into NaN or the window into forever. const delay = this.delay ?? {}; - const initial = delay.initial ?? DEFAULT_DELAY.initial; + const initial = this.#initial(); const multiplier = delay.multiplier ?? DEFAULT_DELAY.multiplier; const max = delay.max ?? DEFAULT_DELAY.max; const timeout = delay.timeout ?? DEFAULT_DELAY.timeout; - // Any session is dead now: report disconnected during the backoff rather than - // when the retry reruns the effect. - this.established.set(undefined); - this.status.set("disconnected"); + // Report disconnected during the backoff rather than when the retry reruns the + // effect, unless a drained predecessor still serves until it retires. + if (this.established.peek() === undefined) this.status.set("disconnected"); // A session that outlived the initial delay was healthy, so clear the backoff and // start a fresh retry window: a one-off drop should reconnect promptly. Anything @@ -432,6 +560,7 @@ export class Reload { this.#delay = undefined; this.#deadline = undefined; this.#sequenceHref = undefined; + this.#draining?.retire(); } /** @@ -509,6 +638,37 @@ export class Reload { /** Stop reconnecting, close the current connection, and settle {@link Reload.closed}. Idempotent. */ close(abort?: Error) { this.#signals.close(); + this.#draining?.retire(); if (this.#closed.peek() === undefined) this.#closed.set(abort ?? null); } } + +/** + * A session the peer sent GOAWAY on, left serving so its groups in flight finish. It retires + * when it closes on its own or overstays its handover window, and `onRetire` runs once either way. + */ +class Draining { + #connection: Established; + #timer: ReturnType; + #onRetire: () => void; + #retired = false; + + constructor(connection: Established, handover: Time.Milli, onRetire: () => void) { + this.#connection = connection; + this.#onRetire = onRetire; + this.#timer = setTimeout(() => { + console.warn("old session did not drain in time; closing"); + this.retire(); + }, handover); + void connection.closed.then(() => this.retire()); + } + + /** Close the old session now, whatever remains of its window. Idempotent. */ + retire(): void { + if (this.#retired) return; + this.#retired = true; + clearTimeout(this.#timer); + this.#connection.abort(); + this.#onRetire(); + } +} diff --git a/js/net/src/error.ts b/js/net/src/error.ts index 04ad2b84c2f..9912a457ef1 100644 --- a/js/net/src/error.ts +++ b/js/net/src/error.ts @@ -246,6 +246,21 @@ export class NotFound extends Stream { } } +/** + * A peer's GOAWAY named a redirect the connection refuses, or one it could not parse. + * + * Terminal: the peer is leaving, so the connection stops rather than redialing the old + * address. Mirrors the Rust `Error::RefusedRedirect`. + * + * @public + */ +export class RefusedRedirect extends Error { + constructor(reason: string) { + super(`GOAWAY redirect refused: ${reason}`); + this.name = "RefusedRedirect"; + } +} + /** * A peer broke the protocol in a way the spec says must end the session. * diff --git a/js/net/src/errors.ts b/js/net/src/errors.ts index 09cc4dd6065..d84bf02d1e9 100644 --- a/js/net/src/errors.ts +++ b/js/net/src/errors.ts @@ -8,6 +8,7 @@ export { GroupTooLarge, NotFound, ProtocolViolation, + RefusedRedirect, Session, Stream, type StreamOptions, diff --git a/js/net/src/ietf/adapter.test.ts b/js/net/src/ietf/adapter.test.ts index 1945cef825c..ebde36a91f1 100644 --- a/js/net/src/ietf/adapter.test.ts +++ b/js/net/src/ietf/adapter.test.ts @@ -4,6 +4,7 @@ import * as Path from "../path.ts"; import { Stream } from "../stream.ts"; import { ControlStreamAdapter } from "./adapter.ts"; import { toRequestCode } from "./error.ts"; +import { GoAway } from "./goaway.ts"; import { PublishNamespace, PublishNamespaceCancel, PublishNamespaceDone } from "./publish_namespace.ts"; import { RequestError } from "./request.ts"; import { ALPN, Version } from "./version.ts"; @@ -184,3 +185,51 @@ test("withdrawals distinguish the same namespace by direction", async () => { await cancel(peer, namespace); expect(await closed(outgoing)).toBe(true); }); + +/** + * Draft-14 to -16 carry GOAWAY on the shared control stream. The adapter decodes it and keeps + * routing, so the session serves its groups in flight while the caller migrates. + */ +test("the control stream adapter decodes a GOAWAY and keeps running", async () => { + const { adapter, peer } = await connect(); + + await peer.writer.u53(GoAway.id); + await new GoAway({ newSessionUri: "https://relay.example/next" }).encode(peer.writer, VERSION); + + const drain = await adapter.goaway; + expect(drain.uri).toBe("https://relay.example/next"); + // These drafts carry no timeout: absence means the caller's cap, never a zero handover. + expect(drain.timeout).toBeUndefined(); + + // Still routing: a later announcement opens its virtual stream. + await announce(peer, 1n, Path.from("still")); + await accept(adapter); +}); + +test("a server adapter rejects a client GOAWAY that names a redirect", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_15); + const control = await Stream.open(pair.server, { version: VERSION }); + const adapter = new ControlStreamAdapter(pair.server, control, VERSION, 100n, false); + const running = adapter.run(); + const peer = await Stream.accept(pair.client, VERSION); + if (!peer) throw new Error("no control stream"); + + await peer.writer.u53(GoAway.id); + await new GoAway({ newSessionUri: "https://other.example/" }).encode(peer.writer, VERSION); + await expect(running).rejects.toThrow("client GOAWAY must not name a redirect"); +}); + +test("a second GOAWAY on the control stream closes the session", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_15); + const control = await Stream.open(pair.server, { version: VERSION }); + const adapter = new ControlStreamAdapter(pair.server, control, VERSION, 100n, true); + const running = adapter.run(); + const peer = await Stream.accept(pair.client, VERSION); + if (!peer) throw new Error("no control stream"); + + for (let i = 0; i < 2; i++) { + await peer.writer.u53(GoAway.id); + await new GoAway({ newSessionUri: "" }).encode(peer.writer, VERSION); + } + await expect(running).rejects.toThrow("duplicate GOAWAY"); +}); diff --git a/js/net/src/ietf/adapter.ts b/js/net/src/ietf/adapter.ts index 780b6a1b39c..fa9671cfcab 100644 --- a/js/net/src/ietf/adapter.ts +++ b/js/net/src/ietf/adapter.ts @@ -1,7 +1,10 @@ +import { Once } from "@moq/signals"; import { Mutex } from "async-mutex"; +import type { Drain } from "../connection/goaway.ts"; import { error, ProtocolViolation } from "../error.ts"; import { Reader, Stream, type Writer } from "../stream.ts"; import * as Varint from "../varint.ts"; +import { GoAway } from "./goaway.ts"; import * as Namespace from "./namespace.ts"; import { type IetfVersion, Version } from "./version.ts"; @@ -91,6 +94,9 @@ export class ControlStreamAdapter implements Session { #writeMutex = new Mutex(); readonly version: IetfVersion; + /** The peer's GOAWAY, which draft-14 to -16 carry on the shared control stream. */ + readonly goaway = new Once(); + // Virtual streams keyed by requestId #streams = new Map(); @@ -124,6 +130,9 @@ export class ControlStreamAdapter implements Session { #closed = false; + // Whether this side opened the session. Only a server may name a redirect. + #client: boolean; + constructor( quic: WebTransport, controlStream: Stream, @@ -139,6 +148,7 @@ export class ControlStreamAdapter implements Session { this.version = version; this.#maxRequestId = maxRequestId; this.#requestId = client ? 0n : 1n; + this.#client = client; } /** @@ -267,8 +277,15 @@ export class ControlStreamAdapter implements Session { const classified = await this.#classify(typeId, body); if (classified.route === Route.GoAway) { - console.warn("received GOAWAY on control stream"); - return; + // The session keeps serving: a GOAWAY asks us to migrate, not to stop reading. + const msg = await GoAway.decodeBody(body, this.version); + if (this.goaway.peek() !== undefined) throw new ProtocolViolation("duplicate GOAWAY"); + // A client may leave, but only the server may name where to go. + if (!this.#client && msg.newSessionUri !== "") { + throw new ProtocolViolation("client GOAWAY must not name a redirect"); + } + this.goaway.set(msg.drain()); + continue; } const { route, requestId } = classified; diff --git a/js/net/src/ietf/connection.test.ts b/js/net/src/ietf/connection.test.ts new file mode 100644 index 00000000000..3d12bf2758e --- /dev/null +++ b/js/net/src/ietf/connection.test.ts @@ -0,0 +1,191 @@ +import { expect, spyOn, test } from "bun:test"; +import { exchangeSetup } from "../connection/handshake.ts"; +import { SessionCode } from "../error.ts"; +import { createMockTransportPair } from "../mock.ts"; +import { Stream, Writer } from "../stream.ts"; +import { Connection } from "./connection.ts"; +import { Group } from "./object.ts"; +import { SetupOption, SetupOptions } from "./parameters.ts"; +import { Setup } from "./setup.ts"; +import { ALPN, type IetfVersion, Version } from "./version.ts"; + +const PADDING = 0x132b3e28n; + +/** A server-side session over `version`, and a client uni stream that sends only `type`. */ +async function openUni( + version: IetfVersion, + alpn: string, + type: bigint, +): Promise<{ pair: ReturnType; connection: Connection; writer: Writer }> { + const pair = createMockTransportPair(alpn); + const control = await Stream.open(pair.server, { version }); + const connection = new Connection({ + url: new URL("https://example.com"), + quic: pair.server, + control, + maxRequestId: 100n, + version, + client: false, + }); + + const writer = new Writer(await pair.client.createUnidirectionalStream(), version); + await writer.u62(type); + + return { pair, connection, writer }; +} + +/** Padding is read to the end and dropped: no STOP_SENDING, and the session stays up. */ +test("a padding stream is discarded", async () => { + const { pair, connection, writer } = await openUni(Version.DRAFT_19, ALPN.DRAFT_19, PADDING); + let closed = false; + void pair.server.closed.then(() => { + closed = true; + }); + + try { + await writer.write(new Uint8Array(4096)); + writer.close(); + + // A STOP_SENDING would reject this; a clean close means every byte was read. + await writer.closed; + + // A session close would come from the same handler that read the stream. + await Bun.sleep(0); + expect(closed).toBe(false); + } finally { + connection.abort(); + } +}); + +/** An unknown or invalid stream type MUST close the session with PROTOCOL_VIOLATION. */ +test("an unknown uni stream type closes the session", async () => { + for (const [version, alpn, type] of [ + [Version.DRAFT_19, ALPN.DRAFT_19, 0n], + // Past 2^53, so it cannot be read as a number. + [Version.DRAFT_19, ALPN.DRAFT_19, 2n ** 53n], + // Padding arrived in draft-18. + [Version.DRAFT_17, ALPN.DRAFT_17, PADDING], + // A SUBGROUP_HEADER with the reserved SUBGROUP_ID_MODE (0b11). + [Version.DRAFT_19, ALPN.DRAFT_19, 0x56n], + ] as const) { + const logged = spyOn(console, "error").mockImplementation(() => void 0); + const { pair, connection } = await openUni(version, alpn, type); + + try { + const info = await pair.client.closed; + expect(info.closeCode).toBe(SessionCode.ProtocolViolation); + } finally { + logged.mockRestore(); + connection.abort(); + } + } +}); + +/** + * Padding and group streams that beat the peer's SETUP are held and classified once it + * lands. The drafts say to buffer early data; failing the handshake over it broke a peer + * that already held a subscription or probed for bandwidth straight away. + */ +test("uni streams before SETUP are held until it lands", async () => { + const version = Version.DRAFT_19; + const pair = createMockTransportPair(ALPN.DRAFT_19); + + const padding = new Writer(await pair.client.createUnidirectionalStream(), version); + await padding.u62(PADDING); + await padding.write(new Uint8Array(4096)); + padding.close(); + + const group = new Writer(await pair.client.createUnidirectionalStream(), version); + await new Group({ + trackAlias: 7n, + groupId: 0, + subGroupId: 0, + publisherPriority: 128, + flags: { + hasExtensions: false, + hasSubgroup: false, + hasSubgroupObject: false, + hasEnd: false, + hasPriority: true, + firstObject: true, + }, + }).encode(group, version); + + const setup = new Writer(await pair.client.createUnidirectionalStream(), version); + await setup.u53(Setup.id); + const parameters = new SetupOptions(); + parameters.setBytes(SetupOption.Implementation, new TextEncoder().encode("test")); + await new Setup({ parameters }).encode(setup, version); + + const { control, early, solicit, hidden, cluster } = await exchangeSetup(pair.server, version, "test"); + expect(early.length).toBe(2); + + let closed = false; + void pair.server.closed.then(() => { + closed = true; + }); + + const connection = new Connection({ + url: new URL("https://example.com"), + quic: pair.server, + control, + early, + solicit, + hidden, + cluster, + maxRequestId: 0n, + version, + client: false, + }); + + try { + // A STOP_SENDING would reject this; a clean close means the padding was read to its end. + await padding.closed; + + await Bun.sleep(0); + expect(closed).toBe(false); + } finally { + connection.abort(); + } +}); + +/** An early stream that dies before its type is skipped, but a malformed type is still fatal. */ +test("malformed uni stream before SETUP fails the handshake", async () => { + const version = Version.DRAFT_17; + const pair = createMockTransportPair(ALPN.DRAFT_17); + + const empty = new Writer(await pair.client.createUnidirectionalStream(), version); + empty.close(); + + // 0b1111110x is a reserved varint prefix on draft-17. + const malformed = new Writer(await pair.client.createUnidirectionalStream(), version); + await malformed.write(new Uint8Array([0xfc, 0, 0, 0, 0, 0, 0, 0, 0])); + + const setup = new Writer(await pair.client.createUnidirectionalStream(), version); + await setup.u53(Setup.id); + const parameters = new SetupOptions(); + parameters.setBytes(SetupOption.Implementation, new TextEncoder().encode("test")); + await new Setup({ parameters }).encode(setup, version); + + await expect(exchangeSetup(pair.server, version, "test")).rejects.toThrow("leading-ones varint"); +}); + +/** An early stream that ends partway through its type died; it is skipped like an empty one. */ +test("truncated uni stream before SETUP is skipped", async () => { + const version = Version.DRAFT_17; + const pair = createMockTransportPair(ALPN.DRAFT_17); + + // A two-byte varint prefix, then FIN. + const truncated = new Writer(await pair.client.createUnidirectionalStream(), version); + await truncated.write(new Uint8Array([0x80])); + truncated.close(); + + const setup = new Writer(await pair.client.createUnidirectionalStream(), version); + await setup.u53(Setup.id); + const parameters = new SetupOptions(); + parameters.setBytes(SetupOption.Implementation, new TextEncoder().encode("test")); + await new Setup({ parameters }).encode(setup, version); + + const { early } = await exchangeSetup(pair.server, version, "test"); + expect(early.length).toBe(0); +}); diff --git a/js/net/src/ietf/connection.ts b/js/net/src/ietf/connection.ts index b957ac5a308..e324be9a7fc 100644 --- a/js/net/src/ietf/connection.ts +++ b/js/net/src/ietf/connection.ts @@ -1,9 +1,10 @@ -import { type Getter, Signal } from "@moq/signals"; +import { type Getter, Once, Signal } from "@moq/signals"; import type * as announce from "../announced.ts"; import type { Established } from "../connection/established.ts"; +import type { Drain } from "../connection/goaway.ts"; import { type Probe, type Stats, transportStats } from "../connection/stats.ts"; import { type Transport, transportOf } from "../connection/transport.ts"; -import { error, fromClose, ProtocolViolation, StreamCode, StreamError } from "../error.ts"; +import { error, fromClose, ProtocolViolation, SessionCode, StreamCode, StreamError } from "../error.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import type * as Path from "../path.ts"; import { type Reader, Readers, type Stream } from "../stream.ts"; @@ -11,17 +12,21 @@ import { withTimeout } from "../util/timeout.ts"; import { registerWire } from "../wire.ts"; import { ControlStreamAdapter, NativeSession, type Session } from "./adapter.ts"; import * as Cluster from "./cluster.ts"; +import { Fetch, FetchHeader } from "./fetch.ts"; import { GoAway } from "./goaway.ts"; import { Group } from "./object.ts"; import { Publish } from "./publish.ts"; import { PublishNamespace } from "./publish_namespace.ts"; import { Publisher } from "./publisher.ts"; import { Subscribe, SubscribeUpdate } from "./subscribe.ts"; -import { SubscribeNamespace, SubscribeNamespaceLegacy } from "./subscribe_namespace.ts"; +import { SUBSCRIBE_TRACKS_ID, SubscribeNamespace, SubscribeNamespaceLegacy } from "./subscribe_namespace.ts"; import { Subscriber } from "./subscriber.ts"; import { TrackStatusRequest } from "./track.ts"; import { type IetfVersion, Version, versionName } from "./version.ts"; +// The PADDING stream type (draft-18+): bytes a peer sends to probe for bandwidth. +const PADDING = 0x132b3e28n; + /** * Represents a connection to a MoQ server using moq-transport protocol. * @@ -47,6 +52,9 @@ export class Connection implements Established { // The established WebTransport session. #quic: WebTransport; + // Whether this side opened the session. Only a server may name a redirect. + #client: boolean; + // Session abstraction: adapter for v14-v16, native for v17. #session: Session; @@ -62,6 +70,9 @@ export class Connection implements Established { // The Hop IDs this session declared; see {@link Cluster}. #cluster?: Cluster.Hops; + // The peer's GOAWAY: read here on v17+, by the control stream adapter before that. + #goaway: Once; + // Just to avoid logging when `close()` is called. #closed = false; @@ -74,6 +85,7 @@ export class Connection implements Established { * @param version - The negotiated protocol version * @param solicit - What the peer's SETUP declared (undefined when it declared nothing) * @param cluster - The Hop IDs the SETUP exchange settled, on the versions that negotiate them + * @param early - Uni streams that arrived before the peer's SETUP * * @internal */ @@ -89,6 +101,7 @@ export class Connection implements Established { solicit, hidden = false, cluster, + early = [], }: { url: URL; quic: WebTransport; @@ -112,25 +125,30 @@ export class Connection implements Established { * cannot negotiate the extension, as is a `peer` the peer never declared. */ cluster?: Cluster.Hops; + /** Uni streams that arrived before the peer's SETUP, type unread (v17+). */ + early?: Reader[]; }) { this.url = url; this.discovery = discovery; this.version = versionName(version); this.transport = transportOf(quic); this.#quic = quic; + this.#client = client; // Two-path dispatch: v14-v16 uses adapter, v17+ uses native bidi streams if (version >= Version.DRAFT_17) { this.#session = new NativeSession(quic, version, client); + this.#goaway = new Once(); // v17+: control/setup stream only carries GoAway void this.#runGoAway(control, version); } else { const adapter = new ControlStreamAdapter(quic, control, version, maxRequestId, client); this.#session = adapter; + this.#goaway = adapter.goaway; // Start the adapter read loop (routes control messages to virtual streams) void adapter.run().catch((err: unknown) => { if (!this.#closed) console.error("adapter error", err); - this.abort(); + this.#close(); }); } @@ -145,9 +163,9 @@ export class Connection implements Established { this.#solicit = solicit; this.#cluster = cluster; this.#subscriber = new Subscriber({ session: this.#session, quic, cluster, hidden }); - registerWire(this, { consume: (path) => this.#subscriber.consume(path) }); + registerWire(this, { consume: (path) => this.#subscriber.consume(path), goaway: this.#goaway }); - void this.#run(); + void this.#run(early); } /** Snapshot the transport's counters; see {@link Established.stats}. */ @@ -166,21 +184,35 @@ export class Connection implements Established { /** End the session immediately without waiting for delivery. */ abort(): void { if (this.#closed) return; + this.#subscriber.close(); + this.#close(); + } - this.#closed = true; + // Close with the session code the peer should see, a clean close by default. + #close(info?: WebTransportCloseInfo) { + if (this.#closed) return; - this.#session.close(); + this.#closed = true; + // Before the session, whose own close would send a clean code first. try { - this.#quic.close(); + this.#quic.close(info); } catch { // ignore } + + this.#session.close(); + } + + // The peer broke the protocol, so losing the stream is not enough: nothing stops it + // repeating the violation on the next one. + #violated(err: ProtocolViolation) { + this.#close({ closeCode: SessionCode.ProtocolViolation, reason: err.message }); } - async #run(): Promise { + async #run(early: Reader[]): Promise { try { - await Promise.all([this.#runBidis(), this.#runUnis(), this.#publisher.runPublishNamespaces()]); + await Promise.all([this.#runBidis(), this.#runUnis(early), this.#publisher.runPublishNamespaces()]); } catch (err) { if (!this.#closed) { console.error("fatal error running connection", err); @@ -188,9 +220,9 @@ export class Connection implements Established { } finally { // A graceful close owns the teardown while it drains. runPublishNamespaces is // a tracked withdrawal, so a failure in it ends this driver while close() is - // still waiting on the sibling loops; aborting here would drop their + // still waiting on the sibling loops; closing here would drop their // withdrawals. close() aborts once its barrier settles or the deadline hits. - if (!this.#closing) this.abort(); + if (!this.#closing) this.#close(); } } @@ -210,10 +242,7 @@ export class Connection implements Established { void this.#runBidi(stream).catch((err: unknown) => { console.error("error processing bidi stream", err); stream.abort(new Error("bidi stream error")); - - // The peer broke the protocol, so losing the stream is not enough: nothing - // stops it repeating the violation on the next one. - if (err instanceof ProtocolViolation) this.abort(); + if (err instanceof ProtocolViolation) this.#violated(err); }); } } @@ -254,11 +283,24 @@ export class Connection implements Established { await this.#publisher.runSubscribe(msg, stream); break; } + case SUBSCRIBE_TRACKS_ID: { + // 0x51 is only a message from draft-18 on. + if (this.#session.version < Version.DRAFT_18) { + throw new ProtocolViolation("SUBSCRIBE_TRACKS before draft-18"); + } + await this.#publisher.runSubscribeTracks(stream); + break; + } case TrackStatusRequest.id: { const msg = await TrackStatusRequest.decode(stream.reader, this.#session.version); await this.#publisher.runTrackStatusRequest(msg, stream); break; } + case Fetch.id: { + const msg = await Fetch.decode(stream.reader, this.#session.version); + await this.#publisher.runFetch(msg, stream); + break; + } // Subscriber handles incoming notifications case PublishNamespace.id: { @@ -283,7 +325,7 @@ export class Connection implements Established { console.error( `unsolicited publish_namespace from a peer that implements MoQ Solicit: broadcast=${msg.trackNamespace}`, ); - this.abort(); + this.#close(); break; } @@ -305,50 +347,90 @@ export class Connection implements Established { /** * Handles unidirectional streams for media delivery (groups). */ - async #runUnis() { - const readers = new Readers(this.#quic, this.#session.version); + async #runUnis(early: Reader[]) { + // Streams that beat the SETUP go first, in arrival order. + for (const stream of early) this.#spawnUni(stream); + const readers = new Readers(this.#quic, this.#session.version); for (;;) { const stream = await readers.next(); if (!stream) break; - - this.#runUni(stream) - .then(() => { - stream.stop(new StreamError(StreamCode.Cancel, { message: "cancel" })); - }) - .catch((err: unknown) => { - console.error("error processing object stream", err); - stream.stop(err); - }); + this.#spawnUni(stream); } } + #spawnUni(stream: Reader) { + this.#runUni(stream) + .then(() => { + stream.stop(new StreamError(StreamCode.Cancel, { message: "cancel" })); + }) + .catch((err: unknown) => { + console.error("error processing object stream", err); + stream.stop(err); + + // An unknown or invalid stream type MUST close the session, not just the stream. + if (err instanceof ProtocolViolation) this.#violated(err); + }); + } + async #runUni(stream: Reader) { - const header = await Group.decode(stream, this.#session.version); - await this.#subscriber.handleGroup(header, stream); + const version = this.#session.version; + // Full width, so an unknown type past 2^53 is still classified rather than thrown. + const type = await stream.u62(); + + // SUBGROUP_HEADER types match 0b0XX1XXXX; Group.decode validates the bits per draft. + if (type <= 0xffn && (type & 0x90n) === 0x10n) { + const header = await Group.decode(stream, version, Number(type)); + await this.#subscriber.handleGroup(header, stream); + return; + } + + // The receiver MUST discard padding. We read it to the end rather than cancel, + // so a peer probing for bandwidth gets the throughput it is measuring. + if (type === PADDING && version >= Version.DRAFT_18) { + await stream.discard(); + return; + } + + // We never FETCH, so a fetch response answers nothing of ours. + if (type === BigInt(FetchHeader.type)) throw new Error("unexpected fetch stream"); + + // Anything else is unknown, and a second SETUP is a violation too. + throw new ProtocolViolation(`unknown uni stream type: 0x${type.toString(16)}`); } /** * v17+ only: reads GoAway from the setup/control stream. + * + * The session keeps serving after a GOAWAY so its groups in flight can finish while the + * caller migrates; only the stream ending, or a second GOAWAY, closes it here. */ async #runGoAway(controlStream: Stream, version: IetfVersion) { try { - const done = await controlStream.reader.done(); - if (done) return; + for (;;) { + const done = await controlStream.reader.done(); + if (done) return; + + const typeId = await controlStream.reader.u53(); + if (typeId !== GoAway.id) { + console.warn(`unexpected message on setup stream: 0x${typeId.toString(16)}`); + return; + } - const typeId = await controlStream.reader.u53(); - if (typeId === GoAway.id) { const msg = await GoAway.decode(controlStream.reader, version); - console.warn(`received GOAWAY with redirect URI: ${msg.newSessionUri}`); - } else { - console.warn(`unexpected message on setup stream: 0x${typeId.toString(16)}`); + if (this.#goaway.peek() !== undefined) throw new ProtocolViolation("duplicate GOAWAY"); + // A client may leave, but only the server may name where to go. + if (!this.#client && msg.newSessionUri !== "") { + throw new ProtocolViolation("client GOAWAY must not name a redirect"); + } + this.#goaway.set(msg.drain()); } } catch (err) { if (!this.#closed) { console.error("error reading setup stream", err); } } finally { - this.abort(); + this.#close(); } } diff --git a/js/net/src/ietf/control.ts b/js/net/src/ietf/control.ts deleted file mode 100644 index 6a3c65657c8..00000000000 --- a/js/net/src/ietf/control.ts +++ /dev/null @@ -1,294 +0,0 @@ -import { Mutex } from "async-mutex"; -import type { Stream as StreamInner, Writer } from "../stream.ts"; -import { Fetch, FetchCancel, FetchError, FetchOk } from "./fetch.ts"; -import { GoAway } from "./goaway.ts"; -import { Publish, PublishDone, PublishError, PublishOk } from "./publish.ts"; -import { - PublishNamespace, - PublishNamespaceCancel, - PublishNamespaceDone, - PublishNamespaceError, - PublishNamespaceOk, -} from "./publish_namespace.ts"; -import { MaxRequestId, RequestError, RequestOk, RequestsBlocked } from "./request.ts"; -import * as Setup from "./setup.ts"; -import { Subscribe, SubscribeError, SubscribeOk, SubscribeUpdate, Unsubscribe } from "./subscribe.ts"; -import { - SUBSCRIBE_TRACKS_ID, - SubscribeNamespaceError, - SubscribeNamespaceLegacy, - SubscribeNamespaceOk, - UnsubscribeNamespace, -} from "./subscribe_namespace.ts"; -import { TrackStatus, TrackStatusRequest } from "./track.ts"; -import { type IetfVersion, Version } from "./version.ts"; - -// v14 message map — IDs that have different meanings in v15 are handled specially -const MessagesV14 = { - [Setup.ClientSetup.id]: Setup.ClientSetup, - [Setup.ServerSetup.id]: Setup.ServerSetup, - [SubscribeUpdate.id]: SubscribeUpdate, - [Subscribe.id]: Subscribe, - [SubscribeOk.id]: SubscribeOk, - [SubscribeError.id]: SubscribeError, - [PublishNamespace.id]: PublishNamespace, - [PublishNamespaceOk.id]: PublishNamespaceOk, - [PublishNamespaceError.id]: PublishNamespaceError, - [PublishNamespaceDone.id]: PublishNamespaceDone, - [Unsubscribe.id]: Unsubscribe, - [PublishDone.id]: PublishDone, - [PublishNamespaceCancel.id]: PublishNamespaceCancel, - [TrackStatusRequest.id]: TrackStatusRequest, - [TrackStatus.id]: TrackStatus, - [GoAway.id]: GoAway, - [Fetch.id]: Fetch, - [FetchCancel.id]: FetchCancel, - [FetchOk.id]: FetchOk, - [FetchError.id]: FetchError, - [SubscribeNamespaceLegacy.id]: SubscribeNamespaceLegacy, - [SubscribeNamespaceOk.id]: SubscribeNamespaceOk, - [SubscribeNamespaceError.id]: SubscribeNamespaceError, - [UnsubscribeNamespace.id]: UnsubscribeNamespace, - [Publish.id]: Publish, - [PublishOk.id]: PublishOk, - [PublishError.id]: PublishError, - [MaxRequestId.id]: MaxRequestId, - [RequestsBlocked.id]: RequestsBlocked, -} as const; - -// v15 message map — 0x05 → RequestError, 0x07 → RequestOk (different wire format) -// Messages removed in v15 (0x08, 0x0E, 0x12, 0x13, 0x19, 0x1E, 0x1F) are excluded and will be rejected -const MessagesV15 = { - [Setup.ClientSetup.id]: Setup.ClientSetup, - [Setup.ServerSetup.id]: Setup.ServerSetup, - [SubscribeUpdate.id]: SubscribeUpdate, - [Subscribe.id]: Subscribe, - [SubscribeOk.id]: SubscribeOk, - [RequestError.id]: RequestError, // 0x05 → RequestError instead of SubscribeError - [PublishNamespace.id]: PublishNamespace, - [RequestOk.id]: RequestOk, // 0x07 → RequestOk instead of PublishNamespaceOk - [PublishNamespaceDone.id]: PublishNamespaceDone, - [Unsubscribe.id]: Unsubscribe, - [PublishDone.id]: PublishDone, - [PublishNamespaceCancel.id]: PublishNamespaceCancel, - [TrackStatusRequest.id]: TrackStatusRequest, - [GoAway.id]: GoAway, - [Fetch.id]: Fetch, - [FetchCancel.id]: FetchCancel, - [FetchOk.id]: FetchOk, - [SubscribeNamespaceLegacy.id]: SubscribeNamespaceLegacy, - [UnsubscribeNamespace.id]: UnsubscribeNamespace, - [Publish.id]: Publish, - [MaxRequestId.id]: MaxRequestId, - [RequestsBlocked.id]: RequestsBlocked, -} as const; - -// v16 message map — SubscribeNamespace (0x11) and UnsubscribeNamespace (0x14) move to bidi streams -const MessagesV16 = { - [Setup.ClientSetup.id]: Setup.ClientSetup, - [Setup.ServerSetup.id]: Setup.ServerSetup, - [SubscribeUpdate.id]: SubscribeUpdate, - [Subscribe.id]: Subscribe, - [SubscribeOk.id]: SubscribeOk, - [RequestError.id]: RequestError, // 0x05 → RequestError - [PublishNamespace.id]: PublishNamespace, - [RequestOk.id]: RequestOk, // 0x07 → RequestOk - [PublishNamespaceDone.id]: PublishNamespaceDone, - [Unsubscribe.id]: Unsubscribe, - [PublishDone.id]: PublishDone, - [PublishNamespaceCancel.id]: PublishNamespaceCancel, - [TrackStatusRequest.id]: TrackStatusRequest, - [GoAway.id]: GoAway, - [Fetch.id]: Fetch, - [FetchCancel.id]: FetchCancel, - [FetchOk.id]: FetchOk, - // SubscribeNamespace (0x11) removed — now on bidi stream - // UnsubscribeNamespace (0x14) removed — now use stream close - [Publish.id]: Publish, - [MaxRequestId.id]: MaxRequestId, - [RequestsBlocked.id]: RequestsBlocked, -} as const; - -// v17 message map — uses unified Setup (0x2F00), removes several messages -const MessagesV17 = { - [Setup.Setup.id]: Setup.Setup, // 0x2F00: unified SETUP - [SubscribeUpdate.id]: SubscribeUpdate, // 0x02: REQUEST_UPDATE - [Subscribe.id]: Subscribe, - [SubscribeOk.id]: SubscribeOk, - [RequestError.id]: RequestError, // 0x05 - [PublishNamespace.id]: PublishNamespace, - [RequestOk.id]: RequestOk, // 0x07 - // 0x08: NAMESPACE (bidi stream only) - // 0x09: removed in d17 - // 0x0a: removed in d17 - [PublishDone.id]: PublishDone, - // 0x0c: removed in d17 - [TrackStatusRequest.id]: TrackStatusRequest, - // 0x0e: NAMESPACE_DONE (bidi stream only), NOT TrackStatus - // 0x0f: PUBLISH_BLOCKED (bidi stream only) - [GoAway.id]: GoAway, - [Fetch.id]: Fetch, - // FetchCancel (0x17) removed in d17 - [FetchOk.id]: FetchOk, - [SubscribeNamespaceLegacy.id]: SubscribeNamespaceLegacy, - [Publish.id]: Publish, - // MaxRequestId (0x15) removed in d17 - // RequestsBlocked (0x1a) removed in d17 -} as const; - -// v18 message map — same as v17 with these spec changes: -// * #1615 removes Required Request ID (transparent on the wire for messages here) -// * #1542 splits SUBSCRIBE_NAMESPACE into SUBSCRIBE_NAMESPACE + SUBSCRIBE_TRACKS (0x51). -// moq-lite does not implement track subscription via PUBLISH replication, so 0x51 -// is rejected as an explicit error rather than silently ignored. -// -// draft-19 (#1765 range filters, #1613 MAX_REQUEST_UPDATES, GOAWAY request-id -// removal, etc.) makes no wire changes to the messages moq-lite exchanges, so it -// reuses the same map. -const MessagesV18 = MessagesV17; - -type V14MessageType = (typeof MessagesV14)[keyof typeof MessagesV14]; -type V15MessageType = (typeof MessagesV15)[keyof typeof MessagesV15]; -type V16MessageType = (typeof MessagesV16)[keyof typeof MessagesV16]; -type V17MessageType = (typeof MessagesV17)[keyof typeof MessagesV17]; -type MessageType = V14MessageType | V15MessageType | V16MessageType | V17MessageType; - -// Type for control message instances (not constructors) -export type Message = InstanceType; - -export class Stream { - stream: StreamInner; - version: IetfVersion; - - // The client always starts at 0. - #requestId = 0n; - - #maxRequestId: bigint; - - #maxRequestIdUpdate?: PromiseWithResolvers; - - #writeLock = new Mutex(); - #readLock = new Mutex(); - - constructor({ - stream, - maxRequestId, - version = Version.DRAFT_14, - }: { - stream: StreamInner; - maxRequestId: bigint; - version?: IetfVersion; - }) { - this.stream = stream; - this.version = version; - // Set version on reader/writer so varint encoding is version-aware - this.stream.reader.version = version; - this.stream.writer.version = version; - this.#maxRequestId = maxRequestId; - this.#maxRequestIdUpdate = Promise.withResolvers(); - } - - /** - * Writes a control message to the control stream with proper framing. - * Format: Message Type (varint) + Message Length (u16) + Message Payload - */ - async write(message: T): Promise { - console.debug("message write", message); - - await this.#writeLock.runExclusive(async () => { - // Write message type - await this.stream.writer.u53((message.constructor as MessageType).id); - - // Write message payload with u16 size prefix - await (message.encode as (w: Writer, v: IetfVersion) => Promise)(this.stream.writer, this.version); - }); - } - - /** - * Reads a control message from the control stream. - * Returns the message type and a reader for the payload. - */ - async read(): Promise { - return await this.#readLock.runExclusive(async () => { - const messageType = await this.stream.reader.u53(); - - // Draft-18+ (#1542) splits SUBSCRIBE_NAMESPACE into SUBSCRIBE_NAMESPACE + - // SUBSCRIBE_TRACKS (0x51). moq-lite does not support track subscription - // via PUBLISH replication, so reject loudly rather than wait forever. - if (this.version >= Version.DRAFT_18 && messageType === SUBSCRIBE_TRACKS_ID) { - throw new Error("SUBSCRIBE_TRACKS (0x51) is not supported in moq-lite (no PUBLISH replication)"); - } - - let messages: Record; - if (this.version === Version.DRAFT_14) { - messages = MessagesV14 as unknown as Record; - } else if (this.version === Version.DRAFT_15) { - messages = MessagesV15 as unknown as Record; - } else if (this.version === Version.DRAFT_16) { - messages = MessagesV16 as unknown as Record; - } else if (this.version === Version.DRAFT_17) { - messages = MessagesV17 as unknown as Record; - } else { - messages = MessagesV18 as unknown as Record; - } - - if (!(messageType in messages)) { - throw new Error(`Unknown control message type: ${messageType}`); - } - - try { - const msgClass = messages[messageType]; - const msg = await msgClass.decode(this.stream.reader, this.version); - return msg; - } catch (err) { - console.error("failed to decode message", messageType, err); - throw err; - } - }); - } - - maxRequestId(max: bigint): void { - if (max <= this.#maxRequestId) { - throw new Error( - `max request id must be greater than current max request id: max=${max} current=${this.#maxRequestId}`, - ); - } - - this.#maxRequestId = max; - this.#maxRequestIdUpdate?.resolve(); - this.#maxRequestIdUpdate = Promise.withResolvers(); - } - - async nextRequestId(): Promise { - while (true) { - const id = this.#requestId; - - // d14-d16 use MAX_REQUEST_ID flow control; d17+ removed it. - if ( - this.version === Version.DRAFT_14 || - this.version === Version.DRAFT_15 || - this.version === Version.DRAFT_16 - ) { - if (id < this.#maxRequestId) { - this.#requestId += 2n; - return id; - } - } else { - this.#requestId += 2n; - return id; - } - - if (!this.#maxRequestIdUpdate) { - return undefined; - } - - console.warn("blocked on max request id"); - await this.#maxRequestIdUpdate.promise; - } - } - - close(): void { - this.#maxRequestIdUpdate?.resolve(); - this.#maxRequestIdUpdate = undefined; - } -} diff --git a/js/net/src/ietf/fetch.ts b/js/net/src/ietf/fetch.ts index d4a9f8c92af..e0a5159873f 100644 --- a/js/net/src/ietf/fetch.ts +++ b/js/net/src/ietf/fetch.ts @@ -1,7 +1,9 @@ -import type * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; +import { hasRangeFilters, isDraft20 } from "./filter.ts"; import * as Message from "./message.ts"; -import type { IetfVersion } from "./version.ts"; +import * as Namespace from "./namespace.ts"; +import { Parameters } from "./parameters.ts"; +import { type IetfVersion, Version } from "./version.ts"; /** * The header that begins a fetch stream (draft-20 section 11.4.4), naming the request it @@ -29,49 +31,17 @@ export class FetchHeader { } } +/** + * A FETCH request. We serve none, so only what a refusal needs is kept; the rest is + * decoded so a legal request is refused rather than breaking the stream. + */ export class Fetch { static id = 0x16; requestId: bigint; - trackNamespace: Path.Valid; - trackName: string; - subscriberPriority: number; - groupOrder: number; - startGroup: bigint; - startObject: bigint; - endGroup: bigint; - endObject: bigint; - constructor({ - requestId, - trackNamespace, - trackName, - subscriberPriority, - groupOrder, - startGroup, - startObject, - endGroup, - endObject, - }: { - requestId: bigint; - trackNamespace: Path.Valid; - trackName: string; - subscriberPriority: number; - groupOrder: number; - startGroup: bigint; - startObject: bigint; - endGroup: bigint; - endObject: bigint; - }) { + constructor({ requestId }: { requestId: bigint }) { this.requestId = requestId; - this.trackNamespace = trackNamespace; - this.trackName = trackName; - this.subscriberPriority = subscriberPriority; - this.groupOrder = groupOrder; - this.startGroup = startGroup; - this.startObject = startObject; - this.endGroup = endGroup; - this.endObject = endObject; } async #encode(_w: Writer): Promise { @@ -82,12 +52,52 @@ export class Fetch { return Message.encode(w, this.#encode.bind(this)); } - static async decode(r: Reader, _version: IetfVersion): Promise { - return Message.decode(r, Fetch.#decode); - } - - static async #decode(_r: Reader): Promise { - throw new Error("FETCH messages are not supported"); + static async decode(r: Reader, version: IetfVersion): Promise { + return Message.decode(r, (mr) => Fetch.#decode(mr, version)); + } + + static async #decode(r: Reader, version: IetfVersion): Promise { + const requestId = await r.u62(); + if (version === Version.DRAFT_17) { + await r.u62(); // required_request_id_delta + } + + if (version === Version.DRAFT_14) { + await r.u8(); // subscriber_priority + await r.u8(); // group_order + } + + if (isDraft20(version)) { + // Draft-20 names the track up front and moves the range into LOCATION_FILTER. + await Namespace.decode(r); + await r.string(); + } else { + const fetchType = await r.u53(); + switch (fetchType) { + case 0x1: // Standalone: namespace, name, start and end Locations + await Namespace.decode(r); + await r.string(); + for (let i = 0; i < 4; i++) await r.u62(); + break; + case 0x2: // Relative Joining: subscription, group offset + case 0x3: // Absolute Joining: subscription, group + await r.u62(); + await r.u62(); + break; + default: + throw new Error(`unknown fetch type: ${fetchType}`); + } + } + + const params = await Parameters.decode(r, version); + if (params.rangeFilters && !hasRangeFilters(version)) { + throw new Error("Range Filters need draft-19"); + } + if (params.trackPropertyFilter) { + throw new Error("TRACK_PROPERTY_FILTER is not allowed on FETCH"); + } + + return new Fetch({ requestId }); } } diff --git a/js/net/src/ietf/filter.ts b/js/net/src/ietf/filter.ts index 91129903723..d3157b1a7fc 100644 --- a/js/net/src/ietf/filter.ts +++ b/js/net/src/ietf/filter.ts @@ -48,6 +48,11 @@ export function isDraft20(version: IetfVersion): boolean { ); } +/** Whether the Range Filters (draft-19) exist on this draft. */ +export function hasRangeFilters(version: IetfVersion): boolean { + return isDraft20(version) || version === Version.DRAFT_19; +} + /** * Draft-17 replaced QUIC's two-bit-length varint with a leading-1-bits one. The two agree * below 64 and diverge above it, so getting this wrong is invisible until group or object diff --git a/js/net/src/ietf/goaway.test.ts b/js/net/src/ietf/goaway.test.ts new file mode 100644 index 00000000000..cfa5e19cce0 --- /dev/null +++ b/js/net/src/ietf/goaway.test.ts @@ -0,0 +1,82 @@ +import { expect, test } from "bun:test"; +import { createMockTransportPair } from "../mock.ts"; +import { Stream } from "../stream.ts"; +import * as Time from "../time.ts"; +import { wireOf } from "../wire.ts"; +import { Connection } from "./connection.ts"; +import { GoAway } from "./goaway.ts"; +import { ALPN, Version } from "./version.ts"; + +// Draft-17+ carry GOAWAY on the setup stream, with the peer's deadline. The session keeps +// serving afterwards so its groups in flight finish while the caller migrates. +test("a draft-17 GOAWAY surfaces its URI and deadline without closing the session", async () => { + const version = Version.DRAFT_17; + const pair = createMockTransportPair(ALPN.DRAFT_17); + const control = await Stream.open(pair.client, { version }); + const peer = await Stream.accept(pair.server, version); + if (!peer) throw new Error("no setup stream"); + + const connection = new Connection({ + url: new URL("https://relay.example/"), + quic: pair.client, + control, + maxRequestId: 100n, + version, + client: true, + }); + + let closed = false; + void connection.closed.then(() => { + closed = true; + }); + + try { + await peer.writer.u53(GoAway.id); + await new GoAway({ newSessionUri: "", timeout: 5000n }).encode(peer.writer, version); + + const drain = await wireOf(connection).goaway; + expect(drain.uri).toBe(""); + expect(drain.timeout).toBe(Time.Milli(5000)); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(closed).toBe(false); + } finally { + connection.abort(); + } +}); + +test("a server rejects a client GOAWAY that names a redirect", async () => { + const version = Version.DRAFT_17; + const pair = createMockTransportPair(ALPN.DRAFT_17); + const control = await Stream.open(pair.client, { version }); + const peer = await Stream.accept(pair.server, version); + if (!peer) throw new Error("no setup stream"); + + const connection = new Connection({ + url: new URL("https://relay.example/"), + quic: pair.client, + control, + maxRequestId: 100n, + version, + client: false, + }); + + let closed = false; + void connection.closed.then(() => { + closed = true; + }); + + try { + await peer.writer.u53(GoAway.id); + await new GoAway({ newSessionUri: "https://other.example/", timeout: 0n }).encode(peer.writer, version); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(closed).toBe(true); + } finally { + connection.abort(); + } +}); + +test("a zero GOAWAY timeout reads as no deadline", async () => { + const msg = new GoAway({ newSessionUri: "https://relay.example/next", timeout: 0n }); + expect(msg.drain()).toEqual({ uri: "https://relay.example/next", timeout: undefined }); +}); diff --git a/js/net/src/ietf/goaway.ts b/js/net/src/ietf/goaway.ts index 9619b6ab4a0..d77f188cfe5 100644 --- a/js/net/src/ietf/goaway.ts +++ b/js/net/src/ietf/goaway.ts @@ -1,4 +1,6 @@ -import type { Reader, Writer } from "../stream.ts"; +import type { Drain } from "../connection/goaway.ts"; +import { Reader, type Writer } from "../stream.ts"; +import * as Time from "../time.ts"; import * as Message from "./message.ts"; import { type IetfVersion, Version } from "./version.ts"; @@ -35,6 +37,25 @@ export class GoAway { return Message.decode(r, (mr) => GoAway.#decode(mr, version)); } + /** + * Decode a body the caller already unframed, as the draft-14 to -16 control stream + * adapter does before routing a message. + */ + static async decodeBody(body: Uint8Array, version: IetfVersion): Promise { + const r = new Reader(undefined, body, version); + const msg = await GoAway.#decode(r, version); + if (!(await r.done())) throw new Error("GOAWAY has trailing bytes"); + return msg; + } + + /** The drain signal this message carries. A zero timeout is the wire saying "none". */ + drain(): Drain { + return { + uri: this.newSessionUri, + timeout: this.timeout > 0n ? Time.Milli(Number(this.timeout)) : undefined, + }; + } + static async #decode(r: Reader, version: IetfVersion): Promise { const newSessionUri = await r.string(); // All drafts cap the New Session URI at 8,192 bytes; a longer one is a diff --git a/js/net/src/ietf/ietf.test.ts b/js/net/src/ietf/ietf.test.ts index cfd7d29525d..93ef4815a7f 100644 --- a/js/net/src/ietf/ietf.test.ts +++ b/js/net/src/ietf/ietf.test.ts @@ -3,6 +3,7 @@ import * as Path from "../path.ts"; import { type Cursor, Reader, Writer } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import * as Varint from "../varint.ts"; +import { Fetch } from "./fetch.ts"; import * as GoAway from "./goaway.ts"; import * as Namespace from "./namespace.ts"; import { FetchFrame, type FetchPosition, Frame, Group, type GroupFlags } from "./object.ts"; @@ -1797,3 +1798,123 @@ test("FetchFrame: an unstamped first object keeps its ids", async () => { 0x01, ]); }); + +/** Frame a control message body with its 16-bit Length, as every draft does. */ +function framed(body: number[]): Uint8Array { + return new Uint8Array([body.length >> 8, body.length & 0xff, ...body]); +} + +/** Request ID 1, Track Namespace ("live"), Track Name ("video"): the head of a SUBSCRIBE or draft-20 FETCH. */ +const TRACK_HEAD = [0x01, 0x01, 0x04, ...new TextEncoder().encode("live"), 0x05, ...new TextEncoder().encode("video")]; + +// Every parameter draft-20 lets a SUBSCRIBE carry that we don't act on still decodes, with +// what we have to refuse recorded rather than failing the stream. +test("Subscribe v20: decodes every legal parameter", async () => { + const body = framed([ + ...TRACK_HEAD, + 0x06, // Number of Parameters + ...[0x03, 0x03, 0x03, 0x00, 0xaa], // AUTHORIZATION TOKEN + ...[0x00, 0x03, 0x03, 0x00, 0xbb], // a second one, which the draft allows + ...[0x0d, 0x00], // FORWARD (0x10) = 0 + ...[0x15, 0x03, 0x00, 0x00, 0x01], // SUBGROUP_FILTER (0x25) + ...[0x0d, 0x07], // NEW_GROUP_REQUEST (0x32) + ...[0x03, 0x00], // INCLUDE_PROPERTIES (0x35) = 0, a uint8 + ]); + + const msg = await decodeVersioned(body, Subscribe.Subscribe.decode, Version.DRAFT_20); + expect(msg.forward).toBe(false); + expect(msg.rangeFilters).toBe(true); + expect(msg.propertiesWanted).toBe(false); +}); + +// INCLUDE_PROPERTIES is a uint8: one raw byte with no Length. +test("Subscribe v20: INCLUDE_PROPERTIES is a uint8", async () => { + const msg = new Subscribe.Subscribe({ + requestId: 1n, + trackNamespace: Path.from("live"), + trackName: "video", + subscriberPriority: 128, + propertiesWanted: false, + }); + const encoded = await encodeVersioned(msg, Version.DRAFT_20); + expect(Array.from(encoded.slice(-2))).toEqual([0x13, 0x00]); + expect(Array.from(encoded.slice(-3, -2))).not.toEqual([0x01]); +}); + +// FORWARD=0 decodes on every draft, so the request can be refused rather than failing. +test("Subscribe: FORWARD=0 decodes", async () => { + for (const version of [Version.DRAFT_14, Version.DRAFT_15, Version.DRAFT_17, Version.DRAFT_20] as const) { + const msg = new Subscribe.Subscribe({ + requestId: 1n, + trackNamespace: Path.from("live"), + trackName: "video", + subscriberPriority: 128, + forward: false, + }); + const decoded = await decodeVersioned(await encodeVersioned(msg, version), Subscribe.Subscribe.decode, version); + expect(decoded.forward).toBe(false); + } +}); + +// A draft-20 FETCH names the track up front and carries its range in LOCATION_FILTER. +test("Fetch v20: decodes every legal parameter", async () => { + const body = framed([ + ...TRACK_HEAD, + 0x07, // Number of Parameters + ...[0x03, 0x03, 0x03, 0x00, 0xaa], // AUTHORIZATION TOKEN + ...[0x07, 0x64], // FILL_TIMEOUT (0x0A) + ...[0x16, 0x40], // SUBSCRIBER_PRIORITY (0x20) + ...[0x01, 0x03, 0x04, 0x00, 0x02], // LOCATION_FILTER (0x21) + ...[0x01, 0x01], // GROUP_ORDER (0x22) + ...[0x04, 0x02, 0x00, 0x05], // OBJECTID_FILTER (0x26) + ...[0x0f, 0x00], // INCLUDE_PROPERTIES (0x35) + ]); + + const msg = await decodeVersioned(body, Fetch.decode, Version.DRAFT_20); + expect(msg.requestId).toBe(1n); +}); + +// The tagged forms through draft-19, with a token. +test("Fetch v15: decodes a joining FETCH", async () => { + const body = framed([ + 0x01, // Request ID + ...[0x02, 0x03, 0x00], // Relative Joining: subscription 3, offset 0 + 0x01, // Number of Parameters + ...[0x03, 0x03, 0x03, 0x00, 0xaa], // AUTHORIZATION TOKEN + ]); + + const msg = await decodeVersioned(body, Fetch.decode, Version.DRAFT_15); + expect(msg.requestId).toBe(1n); +}); + +// TRACK_STATUS is identical to SUBSCRIBE on every draft, fields and parameters included. +test("TrackStatusRequest: carries SUBSCRIBE fields", async () => { + const cases: [IetfVersion, number[]][] = [ + // Priority, group order, forward, AbsoluteStart {5, 1}, no parameters. + [Version.DRAFT_14, [0x80, 0x02, 0x00, 0x03, 0x05, 0x01, 0x00]], + // AUTHORIZATION TOKEN and INCLUDE_PROPERTIES. + [Version.DRAFT_20, [0x02, 0x03, 0x03, 0x03, 0x00, 0xaa, 0x32, 0x00]], + ]; + for (const [version, rest] of cases) { + const msg = await decodeVersioned(framed([...TRACK_HEAD, ...rest]), Track.TrackStatusRequest.decode, version); + expect(msg.trackName).toBe("video"); + } +}); + +// TRACK_PROPERTY_FILTER is legal only on SUBSCRIBE_TRACKS and its updates, so on a +// SUBSCRIBE or FETCH it stays a protocol violation rather than a per-request refusal. +test("TRACK_PROPERTY_FILTER is rejected on SUBSCRIBE and FETCH", async () => { + // One parameter: TRACK_PROPERTY_FILTER (0x29), SetID 0, property type 2, from 0. + const params = [0x01, 0x29, 0x03, 0x00, 0x02, 0x00]; + const body = framed([...TRACK_HEAD, ...params]); + await expect(decodeVersioned(body, Subscribe.Subscribe.decode, Version.DRAFT_20)).rejects.toThrow(); + await expect(decodeVersioned(body, Fetch.decode, Version.DRAFT_20)).rejects.toThrow(); +}); + +// INCLUDE_PROPERTIES postdates draft-16, where it arrives as a Key-Value-Pair; it must +// still read as present so the draft gate refuses it. +test("Subscribe v16: rejects INCLUDE_PROPERTIES", async () => { + // One parameter: INCLUDE_PROPERTIES (0x35), length 1, value 0. + const body = framed([...TRACK_HEAD, 0x01, 0x35, 0x01, 0x00]); + await expect(decodeVersioned(body, Subscribe.Subscribe.decode, Version.DRAFT_16)).rejects.toThrow(); +}); diff --git a/js/net/src/ietf/index.ts b/js/net/src/ietf/index.ts index 9c895e9e294..97df0698fcb 100644 --- a/js/net/src/ietf/index.ts +++ b/js/net/src/ietf/index.ts @@ -1,7 +1,6 @@ export * from "./adapter.ts"; export * as Cluster from "./cluster.ts"; export * from "./connection.ts"; -export * from "./control.ts"; export * from "./fetch.ts"; export * from "./goaway.ts"; export * from "./hidden.ts"; diff --git a/js/net/src/ietf/object.ts b/js/net/src/ietf/object.ts index 9d6a63b883d..769c9332526 100644 --- a/js/net/src/ietf/object.ts +++ b/js/net/src/ietf/object.ts @@ -1,4 +1,4 @@ -import { StreamCode, Stream as StreamError } from "../error.ts"; +import { ProtocolViolation, StreamCode, Stream as StreamError } from "../error.ts"; import { asIetf, type Cursor, type Reader, Writer } from "../stream.ts"; import { Timescale, Timestamp } from "../time.ts"; import { type IetfVersion, Version } from "./version.ts"; @@ -213,8 +213,9 @@ export class Group { } } - static async decode(r: Reader, version: IetfVersion): Promise { - const raw = await r.u53(); + /** Decode a SUBGROUP_HEADER. Pass `type` when the caller already read it to classify the stream. */ + static async decode(r: Reader, version: IetfVersion, type?: number): Promise { + const raw = type ?? (await r.u53()); // Strip the draft-18 FIRST_OBJECT bit before the range check, but keep the value: // it is the only signal that a subgroup starts partway through. Drafts that predate // it carry no such signal, so they are taken at their word. @@ -222,18 +223,17 @@ export class Group { const firstObject = legacy || (raw & FIRST_OBJECT_BIT) !== 0; const id = legacy ? raw : raw & ~FIRST_OBJECT_BIT; - let hasPriority: boolean; - let baseId: number; - if (id >= 0x10 && id <= 0x1f) { - hasPriority = true; - baseId = id; - } else if (id >= 0x30 && id <= 0x3f) { - hasPriority = false; - baseId = id - (0x30 - 0x10); - } else { - throw new Error(`Unsupported group type: ${id}`); + // An invalid type MUST close the session, including the reserved SUBGROUP_ID_MODE + // 0b11 (draft-21 section 11.3.1). + const known = (id >= 0x10 && id <= 0x1f) || (id >= 0x30 && id <= 0x3f); + if (!known || (id & 0x06) === 0x06) { + throw new ProtocolViolation(`Unsupported group type: ${raw}`); } + // 0x30-0x3F omit the priority and inherit it from the control message. + const hasPriority = id < 0x30; + const baseId = hasPriority ? id : id - (0x30 - 0x10); + const flags: GroupFlags = { hasExtensions: (baseId & 0x01) !== 0, hasSubgroupObject: (baseId & 0x02) !== 0, diff --git a/js/net/src/ietf/parameters.ts b/js/net/src/ietf/parameters.ts index f2a6a3120b2..0530e279173 100644 --- a/js/net/src/ietf/parameters.ts +++ b/js/net/src/ietf/parameters.ts @@ -194,6 +194,10 @@ export class SetupOptions { // Varint parameter IDs (even) const MSG_PARAM_DELIVERY_TIMEOUT = 0x02n; +/// FILL_TIMEOUT, on a FETCH. Ignored: a refusal doesn't wait. +const MSG_PARAM_FILL_TIMEOUT = 0x0an; +/// NEW_GROUP_REQUEST. Ignored, as the draft lets a publisher without dynamic groups do. +const MSG_PARAM_NEW_GROUP_REQUEST = 0x32n; /// SUBGROUP_DELIVERY_TIMEOUT, alongside the per-object one above. const MSG_PARAM_SUBGROUP_DELIVERY_TIMEOUT = 0x06n; const MSG_PARAM_MAX_CACHE_DURATION = 0x04n; @@ -202,22 +206,36 @@ const MSG_PARAM_PUBLISHER_PRIORITY = 0x0en; const MSG_PARAM_FORWARD = 0x10n; const MSG_PARAM_SUBSCRIBER_PRIORITY = 0x20n; const MSG_PARAM_GROUP_ORDER = 0x22n; +/// INCLUDE_PROPERTIES, draft-20's opt-out from Track Properties. +const MSG_PARAM_INCLUDE_PROPERTIES = 0x35n; /// ROUTE_COST, from the MoQ Cluster extension. See `cluster.ts`. const MSG_PARAM_ROUTE_COST = 0x40b58n; /// HIDDEN, from the MoQ Hidden extension. See `hidden.ts`. const MSG_PARAM_HIDDEN = 0x40b5en; // Bytes parameter IDs (odd) +/// AUTHORIZATION TOKEN. Ignored: the session's grant is what authorizes a request. +const MSG_PARAM_AUTHORIZATION_TOKEN = 0x03n; const MSG_PARAM_LARGEST_OBJECT = 0x09n; const MSG_PARAM_SUBSCRIPTION_FILTER = 0x21n; /// FILL_PARAMETERS, draft-20's request for a backfill. const MSG_PARAM_FILL_PARAMETERS = 0x23n; -/// INCLUDE_PROPERTIES, draft-20's opt-out from Track Properties. Its own section calls the -/// value a uint8, but 0x35 is odd, so the Key-Value-Pair rule length prefixes it. -const MSG_PARAM_INCLUDE_PROPERTIES = 0x35n; /// HOP_PATH, from the MoQ Cluster extension. See `cluster.ts`. const MSG_PARAM_HOP_PATH = 0x40b57n; +/// The object Range Filters (draft-19): SUBGROUP, OBJECTID, PRIORITY and OBJECT_PROPERTY. +/// Each is length prefixed whatever the parity of its id. +const MSG_PARAM_RANGE_FILTERS: readonly bigint[] = [0x25n, 0x26n, 0x27n, 0x28n]; +/// TRACK_PROPERTY_FILTER, the Range Filter legal only on SUBSCRIBE_TRACKS and its updates. +const MSG_PARAM_TRACK_PROPERTY_FILTER = 0x29n; + +/// The parameters whose definitions let them repeat within one message. +const MSG_PARAM_REPEATABLE: readonly bigint[] = [ + MSG_PARAM_AUTHORIZATION_TOKEN, + ...MSG_PARAM_RANGE_FILTERS, + MSG_PARAM_TRACK_PROPERTY_FILTER, +]; + type MessageParamKind = "varint" | "uint8" | "bool" | "location" | "bytes"; /** A `{Group, Object}` pair carried by a message parameter, such as LARGEST_OBJECT. */ export type MessageLocation = { groupId: bigint; objectId: bigint }; @@ -225,6 +243,8 @@ export type MessageLocation = { groupId: bigint; objectId: bigint }; function getMessageParamKind(id: bigint): MessageParamKind { switch (id) { case MSG_PARAM_DELIVERY_TIMEOUT: + case MSG_PARAM_FILL_TIMEOUT: + case MSG_PARAM_NEW_GROUP_REQUEST: case MSG_PARAM_SUBGROUP_DELIVERY_TIMEOUT: case MSG_PARAM_MAX_CACHE_DURATION: case MSG_PARAM_EXPIRES: @@ -234,17 +254,19 @@ function getMessageParamKind(id: bigint): MessageParamKind { case MSG_PARAM_PUBLISHER_PRIORITY: case MSG_PARAM_SUBSCRIBER_PRIORITY: case MSG_PARAM_GROUP_ORDER: + case MSG_PARAM_INCLUDE_PROPERTIES: return "uint8"; case MSG_PARAM_FORWARD: return "bool"; case MSG_PARAM_LARGEST_OBJECT: return "location"; + case MSG_PARAM_AUTHORIZATION_TOKEN: case MSG_PARAM_SUBSCRIPTION_FILTER: case MSG_PARAM_FILL_PARAMETERS: - case MSG_PARAM_INCLUDE_PROPERTIES: case MSG_PARAM_HOP_PATH: return "bytes"; default: + if (MSG_PARAM_REPEATABLE.includes(id)) return "bytes"; throw new Error(`unknown message parameter id: ${id.toString()}`); } } @@ -277,11 +299,33 @@ export class Parameters { vars: Map; bytes: Map; #locations: Map; + /** Every instance of a parameter that may repeat, decoded only; we never send one. */ + #repeated: Map; constructor() { this.vars = new Map(); this.bytes = new Map(); this.#locations = new Map(); + this.#repeated = new Map(); + } + + #repeat(id: bigint, value: Uint8Array) { + const values = this.#repeated.get(id) ?? []; + values.push(value); + this.#repeated.set(id, values); + } + + /** + * Whether the message carried a Range Filter. We advertise no MAX_FILTER_RANGES, so a + * request with one is refused rather than served unfiltered. + */ + get rangeFilters(): boolean { + return MSG_PARAM_RANGE_FILTERS.some((id) => this.#repeated.has(id)); + } + + /** Whether the message carried TRACK_PROPERTY_FILTER, which only a SUBSCRIBE_TRACKS may. */ + get trackPropertyFilter(): boolean { + return this.#repeated.has(MSG_PARAM_TRACK_PROPERTY_FILTER); } // --- Numeric accessors --- @@ -395,17 +439,18 @@ export class Parameters { /** INCLUDE_PROPERTIES: whether the peer wants Track Properties on the response. */ get includeProperties(): boolean | undefined { - const data = this.bytes.get(MSG_PARAM_INCLUDE_PROPERTIES); - if (!data) return undefined; + // Draft-16 and earlier frame every parameter as a Key-Value-Pair, so an odd id lands + // in `bytes`. It still has to read as present, so the draft-20 gate can refuse it. + const legacy = this.bytes.get(MSG_PARAM_INCLUDE_PROPERTIES); + const v = legacy ? (legacy.length === 1 ? BigInt(legacy[0]) : 2n) : this.vars.get(MSG_PARAM_INCLUDE_PROPERTIES); + if (v === undefined) return undefined; // The draft allows exactly 0 or 1; anything else is a protocol violation. - if (data.length !== 1 || data[0] > 1) { - throw new Error(`invalid INCLUDE_PROPERTIES value: ${data.join(",")}`); - } - return data[0] === 1; + if (v > 1n) throw new Error(`invalid INCLUDE_PROPERTIES value: ${v}`); + return v === 1n; } set includeProperties(v: boolean) { - this.bytes.set(MSG_PARAM_INCLUDE_PROPERTIES, new Uint8Array([v ? 1 : 0])); + this.vars.set(MSG_PARAM_INCLUDE_PROPERTIES, v ? 1n : 0n); } /** HOP_PATH: the hop chain an advertisement traversed, as its raw parameter value. */ @@ -552,7 +597,9 @@ export class Parameters { } else { const size = await r.u53(); const bytes = await r.read(size); - if (id === MSG_PARAM_LARGEST_OBJECT) { + if (MSG_PARAM_REPEATABLE.includes(id)) { + params.#repeat(id, bytes); + } else if (id === MSG_PARAM_LARGEST_OBJECT) { if (params.#locations.has(id)) { throw new Error(`duplicate message parameter id: ${id.toString()}`); } @@ -567,6 +614,12 @@ export class Parameters { continue; } + if (MSG_PARAM_REPEATABLE.includes(id)) { + const size = await r.u53(); + params.#repeat(id, await r.read(size)); + continue; + } + if (params.vars.has(id) || params.bytes.has(id) || params.#locations.has(id)) { throw new Error(`duplicate message parameter id: ${id.toString()}`); } diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index aeee04da778..e77257228e6 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -13,7 +13,7 @@ import type { Producer as TrackProducer } from "../track.ts"; import { wireOf } from "../wire.ts"; import { NativeSession, type Session } from "./adapter.ts"; import type * as Cluster from "./cluster.ts"; -import { FetchHeader } from "./fetch.ts"; +import { Fetch, FetchHeader } from "./fetch.ts"; import { Frame, Group as GroupMessage } from "./object.ts"; import { PublishDone } from "./publish.ts"; import { PublishNamespace, PublishNamespaceUpdate } from "./publish_namespace.ts"; @@ -174,6 +174,51 @@ test("TRACK_STATUS gets exact NOT_SUPPORTED refusal bytes on every draft", async } }); +// Legal requests we don't serve are refused NOT_SUPPORTED one at a time. +test("FETCH and a non-forwarding SUBSCRIBE get NOT_SUPPORTED", async () => { + const refusal = async (version: IetfVersion, run: (pub: Publisher, stream: Stream) => Promise) => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, version, true); + const { pub, origin } = publisher(pair.server, { session }); + const written: Uint8Array[] = []; + const stream = new Stream({ + readable: new ReadableStream(), + writable: new WritableStream({ + write: (chunk) => { + written.push(new Uint8Array(chunk)); + }, + }), + version, + }); + await run(pub, stream); + await stream.writer.closed; + origin.close(); + return written.flatMap((chunk) => Array.from(chunk)); + }; + + for (const version of [Version.DRAFT_14, Version.DRAFT_16, Version.DRAFT_20] as const) { + const fetch = await refusal(version, (pub, stream) => pub.runFetch(new Fetch({ requestId: 7n }), stream)); + // FETCH_ERROR on draft-14, REQUEST_ERROR after; the code follows the Length and any Request ID. + expect(fetch[0]).toBe(version === Version.DRAFT_14 ? 0x19 : 0x05); + expect(fetch[version <= Version.DRAFT_16 ? 4 : 3]).toBe(0x3); + + const paused = await refusal(version, (pub, stream) => + pub.runSubscribe( + new Subscribe({ + requestId: 7n, + trackNamespace: Path.from("test"), + trackName: "video", + subscriberPriority: 0, + forward: false, + }), + stream, + ), + ); + expect(paused[0]).toBe(0x05); + expect(paused[version <= Version.DRAFT_16 ? 4 : 3]).toBe(0x3); + } +}); + // The header is part of the group's lifetime too. If it blocks on flow control, advancing // the live edge must reset the stream without waiting for that write to finish. test("a blocked group header is reset when the group expires", async () => { diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index 8e04fbbb06d..acc3bbc3c0a 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -15,8 +15,8 @@ import * as Varint from "../varint.ts"; import { type Advertised, type Advertisements, wireOf } from "../wire.ts"; import type { Session } from "./adapter.ts"; import * as Cluster from "./cluster.ts"; -import { requestReason, toRequestCode } from "./error.ts"; -import { FetchHeader } from "./fetch.ts"; +import { type RequestKind, requestReason, toRequestCode } from "./error.ts"; +import { type Fetch, FetchError, FetchHeader } from "./fetch.ts"; import * as Filter from "./filter.ts"; import { FetchFrame, Frame, Group as GroupMessage } from "./object.ts"; import { fromWire, toWire } from "./priority.ts"; @@ -291,20 +291,34 @@ export class Publisher { const name = msg.trackNamespace; let broadcast: broadcast.Consumer | undefined; let refusal: { errorCode: number; reasonPhrase: string } | undefined; - try { - broadcast = - this.#publish && (wireOf(this.#publish).local(name) ?? (await wireOf(this.#publish).demand(name))); - if (!broadcast) { - refusal = { - errorCode: toRequestCode("does_not_exist", "subscribe", version), - reasonPhrase: "broadcast not found", - }; + + // Legal requests we can't honor are refused one at a time. A subscription that + // forwards nothing is only useful to a subscriber that later turns forwarding on, + // and serving a Range Filter unfiltered would deliver objects it excluded. + const unsupported = !msg.forward + ? "FORWARD=0 not supported" + : msg.rangeFilters + ? "range filters not supported" + : undefined; + + if (unsupported) { + refusal = { errorCode: toRequestCode("not_supported", "subscribe", version), reasonPhrase: unsupported }; + } else { + try { + broadcast = + this.#publish && (wireOf(this.#publish).local(name) ?? (await wireOf(this.#publish).demand(name))); + if (!broadcast) { + refusal = { + errorCode: toRequestCode("does_not_exist", "subscribe", version), + reasonPhrase: "broadcast not found", + }; + } + } catch (err: unknown) { + const e = error(err); + const condition = + e instanceof StreamError && e.code === StreamCode.NotFound ? "does_not_exist" : "internal"; + refusal = { errorCode: toRequestCode(condition, "subscribe", version), reasonPhrase: reason(e) }; } - } catch (err: unknown) { - const e = error(err); - const condition = - e instanceof StreamError && e.code === StreamCode.NotFound ? "does_not_exist" : "internal"; - refusal = { errorCode: toRequestCode(condition, "subscribe", version), reasonPhrase: reason(e) }; } if (refusal) { @@ -1291,22 +1305,57 @@ export class Publisher { * @internal */ async runTrackStatusRequest(msg: TrackStatusRequest, stream: Stream) { + // TRACK_STATUS_ERROR is 0x0f on draft-14. + await this.#refuseUnsupported(stream, msg.requestId, "track_status", 0x0f, "TRACK_STATUS is not supported"); + } + + /** + * Refuses an incoming SUBSCRIBE_TRACKS (draft-18+): we never send the PUBLISH it asks for. + * The reply carries no Request ID, so the message body is left unread. + * + * @internal + */ + async runSubscribeTracks(stream: Stream) { const version = this.#session.version; - const errorCode = toRequestCode("not_supported", "track_status", version); + await stream.writer.u53(RequestError.id); + await new RequestError({ + errorCode: toRequestCode("not_supported", "subscribe_namespace", version), + reasonPhrase: "SUBSCRIBE_TRACKS is not supported", + }).encode(stream.writer, version); + stream.close(); + } + + /** + * Handles an incoming FETCH on a bidi stream. We serve none. + * + * @internal + */ + async runFetch(msg: Fetch, stream: Stream) { + await this.#refuseUnsupported(stream, msg.requestId, "fetch", FetchError.id, "FETCH is not supported"); + } + + /** + * Refuse a request NOT_SUPPORTED. Draft-14 gives each request its own error message, + * `errorId14`, with the SUBSCRIBE_ERROR body; later drafts use REQUEST_ERROR. + */ + async #refuseUnsupported( + stream: Stream, + requestId: bigint, + kind: RequestKind, + errorId14: number, + reasonPhrase: string, + ) { + const version = this.#session.version; + const errorCode = toRequestCode("not_supported", kind, version); if (version === Version.DRAFT_14) { - // TRACK_STATUS_ERROR shares the SUBSCRIBE_ERROR body on draft-14. - await stream.writer.u53(0x0f); - await new SubscribeError({ - requestId: msg.requestId, - errorCode, - reasonPhrase: "TRACK_STATUS is not supported", - }).encode(stream.writer, version); + await stream.writer.u53(errorId14); + await new SubscribeError({ requestId, errorCode, reasonPhrase }).encode(stream.writer, version); } else { await stream.writer.u53(RequestError.id); await new RequestError({ - requestId: version === Version.DRAFT_15 || version === Version.DRAFT_16 ? msg.requestId : undefined, + requestId: version === Version.DRAFT_15 || version === Version.DRAFT_16 ? requestId : undefined, errorCode, - reasonPhrase: "TRACK_STATUS is not supported", + reasonPhrase, }).encode(stream.writer, version); } stream.close(); diff --git a/js/net/src/ietf/subscribe.ts b/js/net/src/ietf/subscribe.ts index 4bfc31125a7..fcd48f42607 100644 --- a/js/net/src/ietf/subscribe.ts +++ b/js/net/src/ietf/subscribe.ts @@ -38,6 +38,12 @@ export class Subscribe { /** Whether the subscriber wants Track Properties on the response (INCLUDE_PROPERTIES). */ propertiesWanted: boolean; + /** Whether Objects are forwarded (FORWARD). We only serve forwarding subscriptions. */ + forward: boolean; + + /** Whether the request carried a Range Filter, which we refuse. Never encoded. */ + rangeFilters: boolean; + constructor({ requestId, trackNamespace, @@ -46,6 +52,8 @@ export class Subscribe { filter, fill, propertiesWanted, + forward, + rangeFilters, }: { requestId: bigint; trackNamespace: Path.Valid; @@ -54,6 +62,8 @@ export class Subscribe { filter?: Filter.Filter; fill?: Filter.Fill; propertiesWanted?: boolean; + forward?: boolean; + rangeFilters?: boolean; }) { this.requestId = requestId; this.trackNamespace = trackNamespace; @@ -62,6 +72,8 @@ export class Subscribe { this.filter = filter ?? { kind: "unfiltered" }; this.fill = fill; this.propertiesWanted = propertiesWanted ?? true; + this.forward = forward ?? true; + this.rangeFilters = rangeFilters ?? false; } async #encode(w: Writer, version: IetfVersion): Promise { @@ -75,7 +87,7 @@ export class Subscribe { if (version === Version.DRAFT_14) { await w.u8(this.subscriberPriority); await w.u8(GROUP_ORDER); - await w.bool(true); // forward = true + await w.bool(this.forward); await w.write(Filter.encode(this.filter, version)); await w.u53(0); // no parameters } else { @@ -83,7 +95,7 @@ export class Subscribe { const params = new Parameters(); params.subscriberPriority = this.subscriberPriority; params.groupOrder = GROUP_ORDER; - params.forward = true; + params.forward = this.forward; params.subscriptionFilter = Filter.encode(this.filter, version); // FILL_PARAMETERS and INCLUDE_PROPERTIES arrived in draft-20. An older peer reads @@ -131,15 +143,11 @@ export class Subscribe { } const forward = await r.bool(); - if (!forward) { - throw new Error(`unsupported forward value: ${forward}`); - } - const filter = await Filter.decodeInline(r); await Parameters.decode(r, version); // ignore parameters - return new Subscribe({ requestId, trackNamespace, trackName, subscriberPriority, filter }); + return new Subscribe({ requestId, trackNamespace, trackName, subscriberPriority, filter, forward }); } // v15+: fields are in parameters const params = await Parameters.decode(r, version); @@ -152,18 +160,20 @@ export class Subscribe { groupOrder = GROUP_ORDER; // default to descending } - const forward = params.forward ?? true; - if (!forward) { - throw new Error(`unsupported forward value: ${forward}`); - } - - // FILL_PARAMETERS and INCLUDE_PROPERTIES are draft-20 additions. An unknown message - // parameter is a protocol violation, so they stay rejected on the drafts that predate - // them rather than being quietly tolerated. + // FILL_PARAMETERS and INCLUDE_PROPERTIES are draft-20 additions, and the Range + // Filters draft-19 ones. An unknown message parameter is a protocol violation, so + // they stay rejected on the drafts that predate them rather than being quietly + // tolerated. const draft20 = Filter.isDraft20(version); if ((params.fillParameters !== undefined || params.includeProperties !== undefined) && !draft20) { throw new Error("FILL_PARAMETERS and INCLUDE_PROPERTIES need draft-20"); } + if (params.rangeFilters && !Filter.hasRangeFilters(version)) { + throw new Error("Range Filters need draft-19"); + } + if (params.trackPropertyFilter) { + throw new Error("TRACK_PROPERTY_FILTER is not allowed on SUBSCRIBE"); + } // An absent LOCATION_FILTER means the subscription is unfiltered. const raw = params.subscriptionFilter; @@ -180,6 +190,8 @@ export class Subscribe { fill, // Defaults to 1, so an absent parameter means the subscriber wants them. propertiesWanted: params.includeProperties ?? true, + forward: params.forward ?? true, + rangeFilters: params.rangeFilters, }); } } diff --git a/js/net/src/ietf/subscribe_namespace.ts b/js/net/src/ietf/subscribe_namespace.ts index 25f5054a71f..a36090c910e 100644 --- a/js/net/src/ietf/subscribe_namespace.ts +++ b/js/net/src/ietf/subscribe_namespace.ts @@ -153,9 +153,8 @@ export class SubscribeNamespaceLegacy { /// SUBSCRIBE_TRACKS message ID (0x51) introduced in draft-18 (#1542). /// /// moq-lite does not implement PUBLISH replication through a CDN, which is the -/// only thing SUBSCRIBE_TRACKS enables. We never send it and reject it loudly -/// on receipt rather than silently ignoring, since the peer would otherwise -/// wait forever for a REQUEST_OK. +/// only thing SUBSCRIBE_TRACKS enables. We never send it and refuse each one +/// with REQUEST_ERROR NOT_SUPPORTED. export const SUBSCRIBE_TRACKS_ID = 0x51; export class SubscribeNamespaceOk { diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index a748d1aa4f3..03dd535166e 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -98,6 +98,7 @@ function sees(filter: Filter, path: Path.Valid): boolean { */ export class Subscriber { #session: Session; + #localClose = false; // The transport, so a request cut off by the session's close ends with the session's // error. Optional for tests that drive a bare session. @@ -143,6 +144,11 @@ export class Subscriber { // Whether the peer understands the HIDDEN parameter (MoQ Hidden). #hidden: boolean; + /** Marks this subscriber's deliberate local session close. @internal */ + close() { + this.#localClose = true; + } + /** * Creates a new Subscriber instance. * @@ -666,7 +672,7 @@ export class Subscriber { console.debug(`subscribe close: id=${requestId} broadcast=${broadcast} track=${request.name}`); } catch (err) { const e = await sessionCause(this.#quic, err); - producer.close(e); + producer.close(this.#localClose ? undefined : e); stream.abort(e); console.warn( `subscribe error: id=${requestId} broadcast=${broadcast} track=${request.name} error=${reason(e)}`, @@ -1020,7 +1026,7 @@ export class Subscriber { // The control message establishing this alias can arrive after the data stream. subscription = await this.#aliases.get(group.trackAlias); } catch (err: unknown) { - const e = error(err); + const e = await sessionCause(this.#quic, err); // Ours: we cancelled the subscription and the publisher has not stopped yet. // Anything else on this alias is the publisher sending data for a track it never // acknowledged, which is worth seeing. @@ -1107,7 +1113,7 @@ export class Subscriber { // A group with no objects still exists. open().close(); } catch (err: unknown) { - const e = error(err); + const e = await sessionCause(this.#quic, err); if (e instanceof ProtocolViolation) { // The publisher broke the track's end, which no later group can repair. producer?.close(e); diff --git a/js/net/src/ietf/track.ts b/js/net/src/ietf/track.ts index 668d3ad86d7..3dbeb118ac9 100644 --- a/js/net/src/ietf/track.ts +++ b/js/net/src/ietf/track.ts @@ -3,6 +3,7 @@ import type { Reader, Writer } from "../stream.ts"; import * as Message from "./message.ts"; import * as Namespace from "./namespace.ts"; import { Parameters } from "./parameters.ts"; +import { Subscribe } from "./subscribe.ts"; import { type IetfVersion, Version } from "./version.ts"; // we only support Group Order descending @@ -50,29 +51,12 @@ export class TrackStatusRequest { return Message.encode(w, (mw) => this.#encode(mw, version)); } + /** + * Every draft defines TRACK_STATUS as identical to SUBSCRIBE, so it decodes as one and + * keeps only what names the track. We refuse the request, so the rest goes unread. + */ static async decode(r: Reader, version: IetfVersion): Promise { - return Message.decode(r, (mr) => TrackStatusRequest.#decode(mr, version)); - } - - static async #decode(r: Reader, version: IetfVersion): Promise { - const requestId = await r.u62(); - if (version === Version.DRAFT_17) { - await r.u62(); // required_request_id_delta - } - const trackNamespace = await Namespace.decode(r); - const trackName = await r.string(); - - if (version === Version.DRAFT_14) { - await r.u8(); // subscriber_priority - await r.u8(); // group_order - await r.bool(); // forward - await r.u53(); // filter_type - await Parameters.decode(r, version); // parameters - } else { - // v15+: just parameters - await Parameters.decode(r, version); - } - + const { requestId, trackNamespace, trackName } = await Subscribe.decode(r, version); return new TrackStatusRequest({ requestId, trackNamespace, trackName }); } } diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index fcde98ca10d..07dc242d0c7 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -1151,6 +1151,63 @@ test("integration: ietf fetch group is unsupported", async () => { server.abort(); }); +// Draft-18 defines SUBSCRIBE_TRACKS, so a limited endpoint answers NOT_SUPPORTED and the +// subscription already on the session keeps going. +test("integration: ietf SUBSCRIBE_TRACKS is refused per request", async () => { + // SUBSCRIBE_TRACKS, Length, Request ID, Track Namespace Prefix ("room"), no parameters. + const subscribeTracks = new Uint8Array([0x51, 0x00, 0x08, 0x01, 0x01, 0x04, 0x72, 0x6f, 0x6f, 0x6d, 0x00]); + + for (const protocol of [ + Ietf.ALPN.DRAFT_18, + Ietf.ALPN.DRAFT_19, + Ietf.ALPN.DRAFT_20, + Ietf.ALPN.DRAFT_21, + Ietf.ALPN.DRAFT_22, + ]) { + const pair = createMockTransportPair(protocol); + const origin = new OriginProducer(); + + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client }), + accept(pair.server, url, { publish: origin.consume() }), + ]); + + const broadcast = publish(origin, Path.from("test")); + const served: TrackProducer[] = []; + const serving = (async () => { + for (;;) { + const req = await wireOf(broadcast).requested(); + if (!req) break; + const track = req.accept(); + track.writeString("before"); + served.push(track); + } + })(); + + const remote = wireOf(client).consume(Path.from("test")); + const track = remote.track("video").subscribe().ordered(); + expect(await track.readString()).toBe("before"); + + const bidi = await pair.client.createBidirectionalStream(); + const writer = bidi.writable.getWriter(); + await writer.write(subscribeTracks); + const reply: number[] = []; + for await (const chunk of bidi.readable as ReadableStream) reply.push(...chunk); + // REQUEST_ERROR (0x05), a two-byte length, then the error code: NOT_SUPPORTED (0x3). + expect(reply[0]).toBe(0x05); + expect(reply[3]).toBe(0x03); + + for (const producer of served) producer.writeString("after"); + expect(await track.readString()).toBe("after"); + + broadcast.close(); + await serving; + remote.close(); + client.close(); + server.close(); + } +}); + test("integration: ietf draft-14", async () => { await runPublishSubscribeFlow("", Ietf.Version.DRAFT_14); }); @@ -2373,7 +2430,11 @@ const DEATH = SessionCode(71); * Serve one track with a group left open, kill the publisher's session once the subscriber has * read into it, and return how the subscriber's track ended. */ -async function runSessionDeath(protocol: string, version?: number): Promise { +async function runSessionDeath( + protocol: string, + version?: number, + local = false, +): Promise<[Error | null, Error | null]> { const pair = createMockTransportPair(protocol); const origin = new OriginProducer(); @@ -2394,9 +2455,12 @@ async function runSessionDeath(protocol: string, version?: number): Promise { - const closed = await runSessionDeath(protocol, version); + const [closed, groupEnd] = await runSessionDeath(protocol, version); expect(closed).toBeInstanceOf(SessionError); expect((closed as SessionError).code).toBe(DEATH); + expect(groupEnd).toBeInstanceOf(SessionError); + expect((groupEnd as SessionError).code).toBe(DEATH); + }); + test(`integration: ${name} ends a locally closed track cleanly`, async () => { + const [closed] = await runSessionDeath(protocol, version, true); + expect(closed).toBeNull(); }); } diff --git a/js/net/src/lite/connection.test.ts b/js/net/src/lite/connection.test.ts index bf7717fefba..501f461c27c 100644 --- a/js/net/src/lite/connection.test.ts +++ b/js/net/src/lite/connection.test.ts @@ -1,7 +1,12 @@ import { expect, test } from "bun:test"; -import { probeLevel } from "./connection.ts"; +import { createMockTransportPair } from "../mock.ts"; +import { Stream } from "../stream.ts"; +import { wireOf } from "../wire.ts"; +import { Connection, probeLevel } from "./connection.ts"; +import { Goaway } from "./goaway.ts"; import { ProbeLevel } from "./setup.ts"; -import { Version } from "./version.ts"; +import { StreamId } from "./stream.ts"; +import { ALPN_04, Version } from "./version.ts"; /** A transport whose `getStats` behaves as described, or is absent entirely. */ function transport(getStats?: () => Promise): WebTransport { @@ -45,3 +50,39 @@ test("a throwing getStats advertises None rather than propagating", async () => }); expect(await probeLevel(quic, Version.DRAFT_05)).toBe(ProbeLevel.None); }); + +async function sendGoaway(server: WebTransport, uri: string): Promise { + const stream = await Stream.open(server, { version: Version.DRAFT_04 }); + await stream.writer.u53(StreamId.Goaway); + await new Goaway(uri).encode(stream.writer, Version.DRAFT_04); + stream.writer.close(); +} + +test("a lite GOAWAY keeps the session open, and a second one closes it", async () => { + const pair = createMockTransportPair(ALPN_04); + const connection = new Connection({ + url: new URL("https://relay.example/"), + quic: pair.client, + version: Version.DRAFT_04, + }); + + let closed = false; + void connection.closed.then(() => { + closed = true; + }); + + try { + await sendGoaway(pair.server, ""); + const drain = await wireOf(connection).goaway; + expect(drain.uri).toBe(""); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(closed).toBe(false); + + await sendGoaway(pair.server, "https://other.example/"); + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(closed).toBe(true); + } finally { + connection.abort(); + } +}); diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index 427c1e51a7b..a6136904871 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -1,9 +1,10 @@ -import { type Getter, Signal } from "@moq/signals"; +import { type Getter, Once, Signal } from "@moq/signals"; import type * as announce from "../announced.ts"; import type { Established } from "../connection/established.ts"; +import type { Drain } from "../connection/goaway.ts"; import { type Probe, type Stats, transportStats } from "../connection/stats.ts"; import { type Transport, transportOf } from "../connection/transport.ts"; -import { closeError, error, fromClose, StreamCode, StreamError, sessionCause } from "../error.ts"; +import { closeError, error, fromClose, ProtocolViolation, StreamCode, StreamError, sessionCause } from "../error.ts"; import { type Hop, randomHop } from "../hop.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import type * as Path from "../path.ts"; @@ -96,6 +97,9 @@ export class Connection implements Established { // Written by the Subscriber as PROBE messages arrive. #probe = new Signal({}); + // The peer's GOAWAY. Lite carries no deadline, so only the URI is set. + #goaway = new Once(); + /** * The {@link Role} the peer advertised in its SETUP, for a server deciding whether the * peer's authorization grants the direction it intends to use. @@ -133,7 +137,7 @@ export class Connection implements Established { this.hop = randomHop(); this.#publisher = new Publisher(this.#quic, this.#version, this.hop, publish); this.#subscriber = new Subscriber(this.#quic, this.#version, this.hop, this.#probe, this.#peerSetup); - registerWire(this, { consume: (path) => this.#subscriber.consume(path) }); + registerWire(this, { consume: (path) => this.#subscriber.consume(path), goaway: this.#goaway }); void this.#run(); } @@ -237,6 +241,10 @@ export class Connection implements Established { this.#runBidi(stream) .catch((err: unknown) => { stream.writer.reset(err); + // A protocol violation on one stream is the peer breaking the session. + // Resetting that stream leaves it free to repeat the violation; a duplicate + // GOAWAY is the one this dispatcher raises. + if (err instanceof ProtocolViolation) this.abort(); }) .finally(() => { stream.writer.close(); @@ -265,7 +273,9 @@ export class Connection implements Established { await this.#publisher.runProbe(stream); } else if (typ === StreamId.Goaway) { const msg = await Goaway.decode(stream.reader, this.#version); - console.info("received goaway:", msg.uri); + // A peer sends at most one; a second is a protocol violation. + if (this.#goaway.peek() !== undefined) throw new ProtocolViolation("duplicate GOAWAY"); + this.#goaway.set({ uri: msg.uri }); } else { throw new Error(`unknown stream type: ${typ.toString()}`); } diff --git a/js/net/src/lite/subscriber.test.ts b/js/net/src/lite/subscriber.test.ts index f203d8d295b..79565c0c7e1 100644 --- a/js/net/src/lite/subscriber.test.ts +++ b/js/net/src/lite/subscriber.test.ts @@ -1008,3 +1008,27 @@ test.each([ subscriber.close(); }); + +// A reader leaving partway through a fetched group cancels the FETCH: the truncated group +// must not end clean, as a FIN would make it read whole. +test("the last reader leaving mid-response cancels the fetch", async () => { + const { quic, streams } = fakeSession(); + const subscriber = new Subscriber(quic, Version.DRAFT_05, HopSchema.parse(1n)); + + const fetch = subscriber.fetchGroup(Path.from("room"), "video", 0); + await drainUntil(() => streams.length === 1); + await answerTrackInfo(streams[0]); + await drainUntil(() => streams.length === 2); + await streams[1].reading; + + // One frame of the group: a zero timestamp delta, then the sized payload. + streams[1].inbound.enqueue(new Uint8Array([0, 4, ...new TextEncoder().encode("head")])); + const group = await fetch; + expect(new TextDecoder().decode((await group.readFrame())?.payload)).toBe("head"); + group.close(); + + const err = fromTransport(await streams[1].aborted) as StreamError; + expect(err.code).toBe(StreamCode.Cancel); + + subscriber.close(); +}); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 228ec19d495..5f7e0fb5dd3 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -879,11 +879,13 @@ export class Subscriber { pending ??= stream.reader.decodeMaybe(decode); const next = await race([pending, group.closed, unused]); if (next === idle) { - if (!group.isClosed && group.used.peek()) { + if (group.isClosed) break; + if (group.used.peek()) { unused = group.unused().then((): typeof idle => idle); continue; } - break; + // Abandoned mid-group: the truncated group must never end clean. + throw new StreamError(StreamCode.Cancel, { message: "cancel" }); } pending = undefined; if (!next || next instanceof Error) break; @@ -1097,7 +1099,7 @@ export class Subscriber { producer.close(); stream.stop(new StreamError(StreamCode.Cancel, { message: "cancel" })); } catch (err: unknown) { - const e = error(err); + const e = await sessionCause(this.#quic, err); producer.close(e); stream.stop(e); } finally { diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index 6b184909b30..7f29e489693 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -1425,6 +1425,52 @@ test("a rejected request is not asked of the same route again", async () => { origin.close(); }); +// A relay migration lands the replacement session's route next to the draining one's. The +// request must hand over to it, not drop to nothing while the new session answers. +test("an outranked route keeps serving until its replacement answers", async () => { + const origin = new Producer(); + const consumer = origin.consume(); + const path = Path.from("migrating"); + + const older = new BroadcastProducer(); + const keepOlder = older.consume(); + const disposeOlder = serve(origin, path, () => keepOlder.clone()); + + const request = consumer.request(path); + await settle(); + const first = request.active.peek(); + expect(first).toBeDefined(); + + let gaps = 0; + const stop = request.active.subscribe((active) => { + if (active === undefined) gaps++; + }); + + // The newer route wins the tie but has not answered yet. + const newer = wireOf(origin).receive(path); + const asked = newer.requested().next(); + await settle(); + expect(request.active.peek()).toBe(first); + + // Once it answers, the request swaps straight across. + const replacement = new BroadcastProducer(); + const { value: req } = await asked; + req?.accept(replacement.consume()); + await settle(); + expect(request.active.peek()).toBeDefined(); + expect(request.active.peek()).not.toBe(first); + expect(gaps).toBe(0); + + stop(); + request.close(); + newer.close(); + disposeOlder(); + keepOlder.close(); + older.close(); + replacement.close(); + origin.close(); +}); + test("a refusal is terminal, never falling through to a broader route", async () => { const origin = new Producer(); const consumer = origin.consume(); diff --git a/js/net/src/stream.ts b/js/net/src/stream.ts index c2847f395a1..44b0aaea61c 100644 --- a/js/net/src/stream.ts +++ b/js/net/src/stream.ts @@ -304,7 +304,7 @@ export class Reader { while (this.#buffer.byteLength + this.#chunked < size) { if (!(await this.#fill())) { - throw new Error("unexpected end of stream"); + throw new UnexpectedEnd(); } } @@ -364,13 +364,22 @@ export class Reader { } } + // Like decode, but leaves the bytes buffered for the next read. + async #peek(decode: (c: Cursor) => T): Promise { + for (;;) { + const result = this.#try(decode, false); + if (!(result instanceof Short)) return result; + await this.#fillTo(result.need); + } + } + /** Like {@link decode}, but returns undefined if the stream ends cleanly first. */ async decodeMaybe(decode: (c: Cursor) => T): Promise { if (await this.done()) return undefined; return this.decode(decode); } - #try(decode: (c: Cursor) => T): T | Short { + #try(decode: (c: Cursor) => T, consume = true): T | Short { // A retry of the decode that last ran short, before the bytes it needs have arrived, // would only throw again. Every decode reads at least a byte, so none can succeed on // an empty buffer either. @@ -382,7 +391,7 @@ export class Reader { const cursor = new Cursor(this.#buffer, this.version); try { const result = decode(cursor); - this.#slice(cursor.offset); + if (consume) this.#slice(cursor.offset); this.#short = undefined; return result; } catch (err: unknown) { @@ -407,6 +416,15 @@ export class Reader { return this.#slice(this.#buffer.byteLength); } + // Reads to the end of the stream, dropping every byte instead of buffering it. + async discard(): Promise { + this.#buffer = new Uint8Array(); + do { + this.#chunks = []; + this.#chunked = 0; + } while (await this.#fill()); + } + async string(): Promise { return this.decode(STRING); } @@ -433,6 +451,11 @@ export class Reader { return this.decode(U62); } + /** Like {@link u62}, but leaves the varint buffered, so a stream's type can be read twice. */ + async peekU62(): Promise { + return this.#peek(U62); + } + async varint(): Promise { return this.decode(VARINT); } @@ -457,6 +480,13 @@ export class Reader { } } +/** The stream ended cleanly partway through a read. */ +export class UnexpectedEnd extends Error { + constructor() { + super("unexpected end of stream"); + } +} + // Thrown by a Cursor read that runs past the buffered bytes, carrying how many bytes from the // start of the buffer the decode needs. Not an Error: it ends every chunk, so it must not // capture a stack. diff --git a/js/net/src/wire.ts b/js/net/src/wire.ts index 5ac73aad851..75f356b2435 100644 --- a/js/net/src/wire.ts +++ b/js/net/src/wire.ts @@ -9,6 +9,7 @@ */ import type { Dispose, GetPromise, Getter } from "@moq/signals"; import type * as broadcast from "./broadcast.ts"; +import type { Drain } from "./connection/goaway.ts"; import type { Consumer as GroupConsumer } from "./group.ts"; import type { Route } from "./hop.ts"; import type * as origin from "./origin.ts"; @@ -70,9 +71,11 @@ export interface Advertised { */ export type Advertisements = ReadonlyMap; -/** The protocol-facing operation behind an established session. */ +/** The protocol-facing operations behind an established session. */ export interface Established { consume(path: Path.Valid): broadcast.Consumer; + /** Settles with the peer's GOAWAY; the session keeps serving until it closes. */ + readonly goaway: GetPromise; } type View = Broadcast | OriginProducer | OriginConsumer | Established; diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts new file mode 100644 index 00000000000..3abcc742080 --- /dev/null +++ b/js/watch/src/audio/replay.test.ts @@ -0,0 +1,100 @@ +import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test"; +import type * as Catalog from "@moq/hang/catalog"; +import { Time } from "@moq/net"; +import { type Arrival, type Options, QUANTUM, replay, target } from "./replay"; + +const RATE = 48000; + +/** Twenty ms frames for `seconds`, one per group, each arriving `lateness(i)` ms after it was captured. */ +function paced(seconds: number, lateness: (i: number) => number): Arrival[] { + return Array.from({ length: seconds * 50 }, (_, i) => ({ at: i * 20 + lateness(i), timestamp: i * 20, group: i })); +} + +/** How long {@link paced} observes `seconds` of frames: the frames, plus one more. */ +const observed = (seconds: number) => seconds * 1000 + 20; + +/** Quanta that came back short or empty while playing, once the ring had first played. */ +async function underruns(trace: Arrival[], options: Options): Promise { + let started = false; + let short = 0; + for await (const { output, stalled } of replay(trace, options)) { + const filled = output.findLastIndex((v) => v !== 0) + 1; + if (started && !stalled && filled < QUANTUM) short++; + if (filled > 0) started = true; + } + return short; +} + +describe.each(["shared", "post"] as const)("%s ring", (ring) => { + // The consumer warns on every group it skips. + let warn: Mock; + beforeEach(() => { + warn = spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => warn.mockRestore()); + + it("plays an evenly paced sender without a gap", async () => { + expect( + await underruns( + paced(10, () => 30), + { ring, rate: RATE, delay: 100, duration: observed(10) }, + ), + ).toBe(0); + }); + + it("underruns when a flush span outlasts the target, and not when it is covered", async () => { + // Five frames held and flushed at once: 80 ms of arrival spread. + const trace = paced(10, (i) => 30 + (4 - (i % 5)) * 20); + expect(await underruns(trace, { ring, rate: RATE, delay: 40, duration: observed(10) })).toBeGreaterThan(50); + expect(await underruns(trace, { ring, rate: RATE, delay: 150, duration: observed(10) })).toBe(0); + }); + + it("holds newer groups behind a missing one until the max age gives up on it", async () => { + const trace = paced(10, () => 30).filter((arrival) => arrival.group !== 250); + // The consumer delivers in group order, so the ring runs dry for longer than the missing frame + // while the groups behind it wait. Writing arrivals straight into the ring plays straight through. + const frame = Math.ceil((20 / 1000) * (RATE / QUANTUM)); + expect(await underruns(trace, { ring, rate: RATE, delay: 100, duration: observed(10) })).toBeGreaterThan(frame); + expect(warn).toHaveBeenCalled(); + }); + + it("leaves a frame missing inside a group as missing audio", async () => { + // Five frames a group, delivered on time, with one from the middle of a group never sent. + const trace = paced(10, () => 30) + .map((arrival, i) => ({ ...arrival, group: Math.floor(i / 5) })) + .filter((_, i) => i !== 252); + expect(await underruns(trace, { ring, rate: RATE, delay: 100, duration: observed(10) })).toBeGreaterThan(0); + }); + + it("renders through the end of the observation, past the last arrival", async () => { + let last = 0; + let heard = 0; + for await (const { at, output } of replay( + paced(10, () => 30), + { + ring, + rate: RATE, + delay: 100, + duration: 15_000, + }, + )) { + last = at; + if (at > 11_000 && output.some((v) => v !== 0)) heard++; + } + expect(last).toBeGreaterThanOrEqual(15_000); + expect(heard).toBe(0); + }); +}); + +describe("target", () => { + const config = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; + + it("sizes auto from the RTT and the rendition's jitter", async () => { + // 1.25 x 40 ms of RTT, plus the Opus frame and a render quantum. + expect(await target({ delay: "auto", config, rtt: 40 })).toBe(Time.Milli(50 + 20 + 3)); + }); + + it("adds the rendition's jitter to a fixed delay", async () => { + expect(await target({ delay: Time.Milli(250), config })).toBe(Time.Milli(250 + 20 + 3)); + }); +}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts new file mode 100644 index 00000000000..6102d61c100 --- /dev/null +++ b/js/watch/src/audio/replay.ts @@ -0,0 +1,269 @@ +/** + * Replay a recorded arrival trace through the player's consumer and rings on a simulated clock. + * + * Every frame is written into its group at the instant the trace says it arrived, and the player's + * real `Container.Consumer` decides what to deliver, wait for, and skip, at the max age the delay + * sets. Delivered frames go into the real ring, and a discontinuity resets it, the way the decoder + * does. One render quantum is read every quantum's worth of that same clock, which is what the + * AudioWorklet does. {@link target} resolves the delay with a real {@link Sync} fed the recorded + * catalog config and connection RTT, and the ring is sized from it the way the decoder sizes it, so + * a change to any of them moves what a replay hears. Decoding is taken as instant and sample exact. + * + * Deterministic, so the audio quality harness grades its output with no headroom, and a unit test + * can assert exact counts. + * + * @internal Test support, not part of the player. + */ +import type * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; +import { Group, Time, Track } from "@moq/net"; +import { Signal } from "@moq/signals"; +import { type Delay, Sync } from "../sync"; +import { playbackJitter } from "./config"; +import { ringSamples } from "./latency"; +import { AudioRingBuffer } from "./ring-buffer"; +import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; +import { Terminal } from "./terminal"; + +/** An AudioWorklet render quantum, in frames. */ +export const QUANTUM = 128; + +/** + * The replay's own subscription passes every recorded group through: the recorder's relay already + * applied a wire max age, so only the consumer's decisions are made again. + */ +const WIRE_MAX_AGE = Time.Milli(Number.MAX_SAFE_INTEGER); + +/** Any non-empty payload: an empty one is the legacy container's end marker. */ +const PAYLOAD = new Uint8Array(1); + +/** One frame reaching the container consumer. */ +export interface Arrival { + /** When it arrived, on the viewer's monotonic clock, in ms. */ + at: number; + /** Its media timestamp, in ms. */ + timestamp: number; + /** The group that carried it. */ + group: number; +} + +/** What the player resolves its delay from. */ +export interface Target { + /** The configured delay, as the element takes it. "instant" plays no audio, so it has no target. */ + delay: Exclude; + /** The rendition as the catalog described it: its advertised jitter. */ + config: Catalog.AudioConfig; + /** The connection's round trip, in ms, which "auto" sizes from. Unset is no PROBE. */ + rtt?: number; +} + +/** The delay a player resolves for `input`, in ms, from a real {@link Sync}. */ +export async function target(input: Target): Promise { + const sync = new Sync({ + delay: input.delay, + probe: input.rtt === undefined ? undefined : { rtt: Time.Milli(input.rtt) }, + }); + const jitter = playbackJitter(input.config); + const unregister = sync.register(new Signal(jitter)); + try { + // Sync's effects run on microtasks. The delay has settled once it holds this rendition's jitter + // on top of the resolved jitter, the only rendition registered. + while (sync.out.delay.peek() !== Time.Milli.add(sync.out.jitter.peek(), jitter)) { + await sync.out.delay.changed(); + } + return sync.out.delay.peek(); + } finally { + unregister(); + sync.close(); + } +} + +/** What a replay plays through. */ +export interface Options { + /** `shared` is the SharedArrayBuffer ring a cross-origin isolated page runs; `post` is the rest. */ + ring: "shared" | "post"; + /** The sample rate, in Hz. */ + rate: number; + /** + * The resolved delay, in ms. See {@link target}. It sizes the ring and is the consumer's max age, + * which `Sync` sets to the delay plus a lookahead a live viewer does not configure. + */ + delay: number; + /** + * How long the trace observed, from its first arrival, in ms. Rendering runs to here, so an + * outage after the last arrival is heard rather than cut off. + */ + duration: number; +} + +/** One render quantum, as the worklet produced it. */ +export interface Quantum { + /** When it finished rendering, on the trace's clock, in ms. */ + at: number; + /** The quantum: what the ring supplied at the front, zeros after it. Reused between yields. */ + output: Float32Array; + /** Whether the ring was stalled after this quantum. */ + stalled: boolean; + /** The playhead after this quantum, on the media clock, in ms. */ + timestamp: number; +} + +/** The two rings behind the calls the decoder and the worklet make on them. */ +interface Ring { + insert(timestamp: Time.Micro, data: Float32Array[]): void; + read(output: Float32Array[]): number; + reset(): void; + readonly stalled: boolean; + readonly timestamp: Time.Micro; +} + +/** The shared ring, sized the way `SharedAudioBuffer` sizes it. */ +function shared(rate: number, latency: number): Ring { + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(rate, latency * 2), rate)); + ring.setLatency(latency); + return ring; +} + +/** The postMessage ring, sized the way `PostAudioBuffer` asks the worklet to size it. */ +function post(rate: number, latency: number): Ring { + const ms = Time.Milli.fromSecond((latency / rate) as Time.Second); + const ring = new AudioRingBuffer({ rate, channels: 1, latency: ms }); + return { + insert: (timestamp, data) => ring.write(timestamp, data), + read: (output) => ring.read(output), + reset: () => ring.reset(), + get stalled() { + return ring.stalled; + }, + get timestamp() { + return ring.timestamp; + }, + }; +} + +/** A trace's media timestamp as the container carries it. */ +const micros = (ms: number) => Math.round(ms * 1000) as Time.Micro; + +/** + * Samples each frame carries, by its timestamp: the trace's typical frame, the median spacing. + * + * A frame within half a frame of that runs up to where the next one starts instead, so consecutive + * frames tile the timeline exactly despite rounding. Any wider spacing is a frame that never + * arrived, which stays missing audio rather than stretching the frame before it. + */ +function frameSamples(trace: Arrival[], rate: number): Map { + const starts = [...new Set(trace.map((a) => a.timestamp))].sort((a, b) => a - b); + const sample = (ms: number) => Math.round((ms * rate) / 1000); + const gaps = starts.slice(1).map((next, i) => sample(next) - sample(starts[i])); + const typical = [...gaps].sort((a, b) => a - b)[Math.floor(gaps.length / 2)]; + if (typical === undefined) throw new Error("a trace needs two frames"); + + const spans = new Map(); + starts.forEach((start, i) => { + const gap = gaps[i]; + const tiles = gap !== undefined && Math.abs(gap - typical) * 2 <= typical; + spans.set(micros(start), tiles ? gap : typical); + }); + return spans; +} + +/** + * Run every microtask already queued, and every one those queue, before returning. + * + * A macrotask starts only once the microtask queue is empty, and nothing between a group write and + * the ring arms a timer, so this orders the replay behind the consumer rather than waiting on time. + */ +const settle = () => new Promise((resolve) => setImmediate(resolve)); + +/** + * Play `trace` in simulated real time, yielding every render quantum from the first arrival to the + * end of the observation. + * + * A group's stream finishes with its last recorded frame. The audio is a constant, never zero, so + * what the ring did not supply is exactly the zeros at the end of a quantum, which is how the + * harness's output tap reads a real page. + */ +export async function* replay(trace: Arrival[], options: Options): AsyncGenerator { + const first = trace[0]; + const last = trace.at(-1); + if (!first || !last) return; + const end = first.at + options.duration; + if (end < last.at) throw new Error(`an arrival at ${last.at} ms is past the ${options.duration} ms observed`); + + const { rate } = options; + const samples = frameSamples(trace, rate); + const ring = (options.ring === "shared" ? shared : post)(rate, ringSamples(rate, Time.Milli(options.delay))); + + // Where each group's stream finishes. + const ends = new Map(); + trace.forEach((arrival, i) => { + ends.set(arrival.group, i); + }); + + const track = new Track.Producer("audio"); + const consumer = new Container.Consumer(track.subscribe({ maxAge: WIRE_MAX_AGE }), { + format: new Container.Legacy.Format("audio"), + maxAge: Time.Milli(options.delay), + }); + + // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. + const terminal = new Terminal(); + let failure: { error: unknown } | undefined; + const decode = (async () => { + for (;;) { + const next = await consumer.next(); + if (!next) return; + if (terminal.update(next)) ring.reset(); + if (next.end !== undefined || !next.frame) continue; + + const { timestamp } = next.frame; + const length = samples.get(timestamp); + if (length === undefined) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); + ring.insert(timestamp, [new Float32Array(length).fill(0.5)]); + } + })().catch((error: unknown) => { + failure = { error }; + }); + + const groups = new Map(); + const step = (QUANTUM / rate) * 1000; + const output = new Float32Array(QUANTUM); + let next = 0; + + try { + for (let n = 1; first.at + (n - 1) * step <= end; n++) { + // The quantum ending at `now` renders once everything that arrived by then is delivered. + const now = first.at + n * step; + const written = next; + while (next < trace.length && trace[next].at <= now) { + const arrival = trace[next]; + let group = groups.get(arrival.group); + if (!group) { + group = new Group.Producer(arrival.group); + groups.set(arrival.group, group); + track.writeGroup(group); + } + const timestamp = micros(arrival.timestamp); + group.writeFrame({ + payload: Container.Legacy.encodeFrame(PAYLOAD, timestamp), + timestamp: Time.Timestamp.fromMicros(timestamp), + }); + if (ends.get(arrival.group) === next) { + group.close(); + groups.delete(arrival.group); + } + next++; + } + if (next > written) await settle(); + if (failure) throw failure.error; + + output.fill(0); + ring.read([output]); + yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp) }; + } + } finally { + consumer.close(); + track.close(); + await decode; + } +} diff --git a/package.json b/package.json index 49e3a47edec..6f2f81a4db4 100644 --- a/package.json +++ b/package.json @@ -45,8 +45,10 @@ "demo/boy", "demo/web", "js/moq-boy", + "test/audio-quality/clients/js", "test/interop/clients/js", "test/interop/clients/js-native", + "test/drain", "test/wasm" ] } diff --git a/quest/README.md b/quest/README.md index 38d4b4b9bc4..51892ea6308 100644 --- a/quest/README.md +++ b/quest/README.md @@ -16,10 +16,10 @@ offer. m4 waits on an upstream release. Priority is separate from branch targeting: published API and wire breaks still land on dev under the repository rules. -A quest waiting on the outside world, in any milestone, states that condition -as a plain-text `Required` bullet, so `quest ready` reports it blocked. -`/quest-audit` re-checks those gates; when one clears, remove the bullet and -move the quest to the milestone its priority belongs in. +A quest waiting on the outside world, in any milestone, requires a small quest +beside it that names the condition. That condition quest stays ready, so +every `/quest-spawn` resurfaces it; when the condition clears, delete it and +move the blocked quest to the milestone its priority belongs in. ## Required diff --git a/quest/m0/README.md b/quest/m0/README.md index 5ec1c79c845..222dcc44499 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -17,8 +17,9 @@ done. moq.pro tracks this repository as a submodule rather than a release, so no release quest gates this milestone. The Pronto GPU integration lives in moq.pro. -Relay hardening: IETF interop leads the ranking, since only those quests -block Seattle. IETF stream types came from m1 in the 2026-09-30 audit +The branch flip ranks first: it is a short cutover that every later PR +targets. Relay hardening: IETF interop leads the rest, since only those +quests block Seattle. IETF stream types came from m1 in the 2026-09-30 audit because a session ended by legal input is exactly what Seattle would hit. The DoS hardening from an external review on 2026-09-29, verified against `main`, stays in m0 as security work. Its quests describe fixes, not exploits. @@ -28,11 +29,10 @@ work is done on the line branch and waits to land. Serving the relay's ingested-only view (`origin::Consumer::local()`) to localhost workers belongs to moq.pro's edge, which embeds moq-relay; it moved there on 2026-09-28. -Audio playout: the jitter target replaces the round-trip guess. The harness's -browser lane grades it nightly and records the traces it replays; the native -lane is a standalone m1 quest, since nothing here waits on it. The harness -line lands before the jitter line, since both add -`js/watch/src/audio/replay.test.ts`. The [A/V clock](/quest/m1/av-clock.md) +Audio playout: the jitter target replaces the round-trip guess. The browser +audio quality harness in `test/audio-quality/` has landed; it grades playout +nightly and records the traces it replays. Its native lane is a standalone m1 +quest, since nothing here waits on it. The [A/V clock](/quest/m1/av-clock.md) moved to m1 in the 2026-09-30 audit: it waits on the whole jitter line and is a published `@moq/watch` break on dev. @@ -40,20 +40,15 @@ Published API or wire breaks still land on dev; each quest's Plan says so. ## Required -- [Legal IETF input](/quest/m0/ietf-legal-input.md) - draft-20+ FETCH, allowed parameters, INCLUDE_PROPERTIES and FORWARD=0 decode and are refused per request, not session-fatal +- [Branch flip](/quest/m0/branch-flip.md) - `dev` becomes the default `main` trunk and today's `main` becomes `release`, where publishing runs - [IETF FIN semantics](/quest/m0/ietf-fin-not-cancel.md) - a request stream FIN stops updates without cancelling, and REQUEST_UPDATE on a subscribe is parsed -- [IETF stream types](/quest/m0/ietf-uni-stream-types.md) - padding streams are discarded stream-only and an unknown uni type closes the session, per draft-21 +- [JS unknown bidi](/quest/m0/ietf-unknown-bidi.md) - an unknown moq-transport bidi stream type closes the session in JS, as in Rust +- [moq-lite early streams](/quest/m0/lite-early-streams.md) - a lite uni stream that arrives before SETUP is held until SETUP lands, and a second SETUP is fatal - [Request caps](/quest/m0/request-caps.md) - lite message sizes, IETF request IDs, and per-session announces and subscriptions are bounded -- [quest check everywhere](/quest/m0/quest-check-everywhere.md) - `quest check` guards `main`, `dev`, and the line branches on push and PR, not only PRs into `main` - [noq reassembly cap](/quest/m0/noq-reassembly-cap.md) - noq carries quinn's stream reassembly cap and the connection receive window is finite by default -- [qmux reset race](/quest/m0/qmux-reset-race.md) - qmux handles RESET_STREAM under one lock instead of panicking -- [Path hunting](/quest/m0/path-hunting.md) - a withdrawn path is retracted about once per relay instead of hunting stale alternatives, with a hold-down on route updates and no wire change +- [qmux credit](/quest/m0/qmux-credit.md) - qmux returns connection credit for dropped and stopped streams and delivers its close frame, on both lines - [Shared fronts](/quest/m0/shared-fronts.md) - viewer sessions share a front, so fronts scale with peers, not viewers -- [Frame alloc budget](/quest/m0/frame-alloc-budget.md) - frame buffers pre-allocate within a per-session budget and otherwise grow with bytes received -- [Handshake deadline](/quest/m0/handshake-deadline.md) - an unfinished handshake or slow HTTP header times out -- [Revalidate overflow](/quest/m0/revalidate-overflow.md) - no auth duration can overflow a deadline and abort the relay - [Wildcard](/quest/m0/wildcard/README.md) - a relay resolves subscriptions against advertised prefixes, a service claims the prefix it could serve and refuses the rest instead of enumerating broadcasts, and the browser player treats a covering claim as availability -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - a browser playout latency regression fails a nightly run instead of arriving as a bug report, and its recorder supplies the jitter target's replay traces - [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the audio playout target is a measured estimate of arrival timing in both languages, not a round-trip guess ## Related diff --git a/quest/m0/audio-jitter-target/README.md b/quest/m0/audio-jitter-target/README.md index 9b5c88ad7de..1056392c713 100644 --- a/quest/m0/audio-jitter-target/README.md +++ b/quest/m0/audio-jitter-target/README.md @@ -35,9 +35,8 @@ was wrong, and both compile unchanged for existing callers. The line branch is about 200 commits behind `main` with conflicts in `js/watch/src/sync.ts` and `rs/moq-cli`; merge `main` in (never rebase the shared branch) before finishing the watch quest. The raw #3477 traces are gone, so record fresh -traces with the [audio quality -harness](/quest/m0/audio-quality-harness/README.md) instead of asking the -reporter; they replace the #3477 traces wherever the quests name them. +traces with the audio quality harness in `test/audio-quality/` instead of +asking the reporter; they replace the #3477 traces wherever the quests name them. The algorithm is written down at `doc/concept/audio-jitter.md`, with a conformance corpus beside it that both implementations will read. @@ -76,7 +75,6 @@ buffer against uneven arrivals. ## Required -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - lands first, since both lines add `js/watch/src/audio/replay.test.ts`; it also records the traces the watch quest replays - [Watch](/quest/m0/audio-jitter-target/watch.md) - js/watch and js/hang bring the #3954 estimator into conformance - [Native](/quest/m0/audio-jitter-target/native.md) - rs/moq-audio grows a measured jitter buffer from the same algorithm diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index 0151ecdd387..ea2f8c97c54 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -74,13 +74,23 @@ against it is likely cheaper than patching the branch's: came up on the 100 ms chip, so something is restoring or overriding it. Pin that down: a stored preference silently winning over the default is its own bug, and it also means auto gets far less real exposure than it looks like. -- Replay recorded traces rather than synthetic ones of the same shape. The - #3477 traces are gone, so record fresh ones with the [browser - harness](/quest/m0/audio-quality-harness/browser.md) (decided with the - maintainer instead of asking the reporter). Trim a copy into the repository - and replay it through both rings in `replay.test.ts`. +- Replay recorded traces rather than synthetic ones of the same shape. + `test/audio-quality/traces/` holds fresh ones (the #3477 traces are gone), + and `js/watch/src/audio/replay.ts` already plays a trace through + `Container.Consumer` and both rings on a simulated clock at the delay `Sync` + resolves. Replay them in `replay.test.ts`, move the replay's target onto the + estimator (its 100 ms fallback covers the public relay's missing PROBE RTT + until then), and tighten the harness's exact replay budgets with it. The + checked-in `test/audio-quality/budgets.json` was recorded locally (#4426); + re-record it from the nightly runner's first runs on `main` before + tightening, since nightly only runs `main`'s code. The replay sizes every + frame as the trace's median spacing, which a trace missing most of its + frames defeats; once the container knows the codec's frame duration, record + it in the trace and replay that instead. - Manual run against the public relay on Chrome and Safari, the two rows the - issue measured. Measure the publisher's audio encoder input-to-output lag in + issue measured. Re-record the `relay-mic` trace in the same run with a real + microphone (`just test audio-quality-record`); the checked-in one used + Chromium's fake capture device. Measure the publisher's audio encoder input-to-output lag in the same run using the reporter's instrumented harness; #3518 fixed the known cause, so the 7.35 s lag and the 88 to 275 ms/s drift the issue reported stand unconfirmed. If drift survives, the suspects are `writeFrame` opening a @@ -95,6 +105,3 @@ reading `probe`; removing it is part of the `SyncInput` reshape in [Plan: A/V clock](/quest/m1/av-clock.md). Land the estimator so that quest can adopt it without a second estimator change. -## Required - -- [Browser harness](/quest/m0/audio-quality-harness/browser.md) - records the arrival traces this quest replays diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md deleted file mode 100644 index 0262b1bc116..00000000000 --- a/quest/m0/audio-quality-harness/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Audio quality harness - -## Goal - -A regression in audio playout latency fails a run instead of arriving as a bug -report. The harness plays a broadcast over an impaired path, counts what the -listener would actually have heard (underruns, short quanta, discarded samples, -skip-aheads) and what each stage of the pipeline contributed to the delay, and -grades the result against a checked-in budget. This line is the browser; -the native lane is [Native audio -quality](/quest/m1/audio-quality-native.md), on the same jitter profiles and -reporting the same numbers. - -Boundaries: audio only. The stage breakdown is defined generically so video can -adopt it later, but no video assertion ships here. No perceptual scoring: the -grade is glitches and latency, not an opinion about how it sounds. - -## Plan - -Browser only, because that is where the traces and the reported bug both -are. The native lane moved to m1 as a standalone quest (decided in the -2026-09-28 quest audit): nothing in m0 waits on it, and it follows against the -same budgets and metric schema so the two implementations can be compared -rather than merely both passing. - -The starting point is not a blank page. The reporter on #3477 already built a -working browser lane on their fork (`fperex/moq`, branch -`debug-findings-solution`, under `test/audio-quality/`): a Playwright-driven -matrix over `moq-shaper`, a budget file graded under `--enforce`, a nightly -job, and a replay runtime. Upstream that rather than reinventing it. The raw traces it shipped with are gone, so -this harness records fresh ones, and the [jitter target's watch -quest](/quest/m0/audio-jitter-target/watch.md) replays them (decided with the -maintainer during the merged-PR audit). - -Jitter comes from the seeded userspace UDP shaper the transport drills run -under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake -arrival clock, so the transport's own contribution is measured rather than -assumed. Its `moq-shaper` binary is what a JS harness process puts in front of -a relay. - -Loss, reorder and rate limiting stay switched off in the profiles here. The -buffer's job is absorbing arrival spread, and mixing congestion response into an -audio quality number makes a failure hard to attribute. - -Nightly, not per-PR: the matrix is jitter profiles by runtime by codec and -sample rate, which is more than a merge gate should carry, and `nightly.yml` -already exists for exactly this trade. Budgets are keyed by the full row, since -each of those dimensions moves the expected floor. - -The budgets checked in with the browser lane (#4426) were recorded locally. -Once the line lands, re-record `test/audio-quality/budgets.json` from the -nightly runner's first runs, since nightly only runs `main`'s code. Tightening -the auto rows belongs to the [jitter target's watch -quest](/quest/m0/audio-jitter-target/watch.md). - -## Required - -- [Browser](/quest/m0/audio-quality-harness/browser.md) - upstream the fork's harness, grade it against a budget, run it nightly - -## Related - -- [Native audio quality](/quest/m1/audio-quality-native.md) - the same profiles and budgets through `moq play` on a dummy device -- [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the estimator this exists to keep honest -- [Latency ledger](/quest/m2/latency-ledger.md) - promotes this harness's probes into a public API -- [Time stretch](/quest/m1/watch-audio-time-stretch.md) - graded by this harness once it lands diff --git a/quest/m0/audio-quality-harness/browser.md b/quest/m0/audio-quality-harness/browser.md deleted file mode 100644 index a47aed79690..00000000000 --- a/quest/m0/audio-quality-harness/browser.md +++ /dev/null @@ -1,74 +0,0 @@ -# [L] The browser audio lane is graded against a budget, nightly - -## Goal - -`just test audio-quality` plays a broadcast in a headless browser over a seeded, -impaired path and prints a graded table: underruns, short quanta, discarded -samples, skip-aheads, and where the end-to-end delay went. It fails when a -profile exceeds its budget. A nightly job runs the full matrix and keeps the -run directory of a failure. - -The same run works on both ring paths, isolated and postMessage, since the -production path is the one without cross-origin isolation. - -## Plan - -Upstream the reporter's lane from `fperex/moq` branch -`debug-findings-solution` rather than rebuilding it, keeping the attribution. -It already has a built `test/audio-quality/` lane: `just test audio-quality` -over `moq-shaper`, a Playwright-driven Chromium matrix, a `budgets.json` -graded by `grade.ts` under `--enforce`, a nightly job that keeps a failure's -run directory, and `chromium`, `safari`, and `replay` runtimes. It also -carries player, estimator, and shaper changes (checked 2026-09-28: 222 commits -ahead of and 148 behind `main`), so land the lane on its own and hold its -schema to the contract below rather than taking the branch wholesale. - -- Land the lane under `test/`, alongside the existing `interop` and `drill` - lanes, wired into the `justfile` the way they are. -- Keep the instrumentation ad-hoc for now. The probes stay a debug surface, not - public API; promoting them is [Latency - ledger](/quest/m2/latency-ledger.md), which nothing here waits on. -- The metric schema is the deliverable that outlives this quest, because the - native lane and the ledger both have to emit the same thing. It is a - contract, so write it as one: - - Every counter defined, not just the two that are obvious. An underrun is a - quantum the ring could only partly fill; a skip-ahead is a re-anchor that - discarded buffered audio; short quanta and discarded samples need the same - treatment rather than being left to the reader. - - Units and clock domain stated once. Durations in one unit, and every - timestamp on a named clock, so a browser value and a native value are the - same measurement. Naming the clock is not enough on its own: the stages span - the publisher, the relay and the viewer, which are separate processes and - often separate machines, so either reduce every timestamp to one common - monotonic reference or define the calibration and drift correction that maps - between them. The ledger's sum-to-end-to-end identity is arithmetic across - these values, and it is meaningless if they sit on unaligned clocks. - - Aggregation stated per metric: a count over the run, a max, or a - percentile. "Underruns: 3" and "underruns: 3/s" grade differently. - - Stages as exclusive, non-overlapping spans (capture, encode, publish flush, - network, jitter buffer, decode, render). The ledger's sum-to-end-to-end - identity is unimplementable if two stages can claim the same milliseconds, - and an unaccounted remainder is the finding, so give it a name too. -- Run the `moq-shaper` binary (`rs/moq-shaper`, see `test/drill/README.md`) - in front of the relay, so this harness and the drills share one impairment - implementation. It exits nonzero when the profile never acted, which the - harness must treat as a failed run: a profile that silently did nothing - turns an impaired run into an unimpaired pass. -- Profiles: near-zero, mild, bursty (the flush-span shape from #3477), and a - step change that forces the target to move mid-run. Fixed seeds, recorded - with the results. -- One profile runs a fixed delay preset rather than auto, as the control. Every - other profile exercises adaptation, so without this the whole matrix can pass - while a fixed preset regresses, and fixed presets are what a viewer lands on - today. -- Budgets in one checked-in file, keyed by the whole matrix row and not by - profile alone: runtime, codec, sample rate, profile, and the ring path, since - each of those moves the expected floor. A profile-only key silently grades - one row against another's threshold. Tightening a budget is then a visible - diff and loosening one needs a reason in review. -- Record real arrival traces (the #3477 release traces are gone), trim them - into a fixture, and replay them too, so a real recorded arrival pattern is - graded next to the synthetic profiles. The jitter target's watch quest - replays the same fixture. -- Add the lane to `nightly.yml`, and extend its header comment with why this - one is not a PR gate. diff --git a/quest/m0/branch-flip.md b/quest/m0/branch-flip.md new file mode 100644 index 00000000000..63c4df5900d --- /dev/null +++ b/quest/m0/branch-flip.md @@ -0,0 +1,88 @@ +# [M] Trunk is main, releases ship from release + +## Goal + +`dev` is renamed `main` and is the default branch: the trunk, where breaking +changes are allowed and every quest and PR lands. Today's `main` is renamed +`release`, a permanent branch where release-plz and all publishing run. Code +reaches `release` when a release is cut, by merging `main` in; urgent fixes in +between are cherry-picked. AGENTS, CONTRIBUTING, and the quest tree describe +that model, with no `dev` left. + +## Plan + +Facts (2026-10-02): + +- Publishing on push: `release-rs` (release-plz), `release-js`, `release-py`, + and `release-go` trigger on `main`; `release-kt-lib` and `release-swift-lib` + on `main` and `dev`. Everything else publishes from tags and is unaffected. +- `workflow_run` chains run the default branch's workflow file, and a + checkout without a ref takes its head. `release-py`, `release-kt-lib`, and + the `release-swift-lib` build pin the triggering `head_sha`; `release-go`, + `release-brew`, and the other `release-swift-lib` checkouts do not, so after + the flip they would build trunk. +- `cache.yml` and `swift.yml` warm caches on `main` on both branches. PRs + read caches only from the default branch, so writers stay on trunk and + need no change. `obs`, `platform`, and `quest` push filters name `main` and + `dev`. Nightly and interop crons run on the default branch. +- Rulesets: "main" targets `~DEFAULT_BRANCH` (linear history, Check and Test + required), so it moves to trunk and forbids a merge-commit back-merge. + "dev" targets `refs/heads/dev` (PRs only, squash or merge commit) and does + not follow a rename. +- Renaming `main` makes GitHub retarget its ~35 open PRs to `release`; `dev`'s 10 + follow it to the new `main`. +- `sh/changed.sh` falls back to `origin/main`, which stays right for trunk. + `quest` hardcodes `main` as the milestone base, which becomes right too. +- About 50 quest files on `main` (40 on `dev`) say "on `dev`"; AGENTS.md (API, + Development, quest sections), CONTRIBUTING.md, `py/AGENTS.md`, and + `quest/README.md` encode the split. +- `nix run github:moq-dev/moq`, `npx skills add moq-dev/moq`, and docs "edit" + links follow the default branch. + +Decided (2026-10-02): + +- ✅ Flip, over keeping `dev` with continuous sync or dropping a stable branch. +- ✅ A release is cut by merging `main` into `release` (merge commit), by + hand. Rejected: resetting `release` to `main`, and backport-only flow. +- ✅ Backports are manual cherry-pick PRs into `release`; automate later if + it hurts. +- ✅ After each publish, a workflow merges `release` back into `main`, so trunk + carries the published versions and CHANGELOGs. Rejected: running the + release-plz PR on `main` and publishing from `release`. +- ✅ Trunk is the default branch. User-facing installs in docs pin `release` + (`github:moq-dev/moq/release`); all branch-triggered publishing, including + kt and swift libs, runs only from `release`. Nightly and edit links stay on + trunk. +- ✅ A new `release` ruleset (no deletion or force push, Check and Test + required, PRs only, merge commits allowed); the `dev` ruleset is deleted. +- ✅ Open PRs that land on `release` through the rename are retargeted to the + new `main`, except any explicitly meant as backports. +- ✅ One mechanical wording sweep: "on `dev`" goes from quests (trunk allows + breaks), the rules describe trunk, `release`, and backports, and + `dev`-only process quests are deleted. +- ✅ One quest, m0, right after dev-sync lands. +- ✅ Trunk's ruleset drops linear history, so the back-merge lands as a merge + commit and advances the merge base; ordinary PRs still squash. Rejected: + squash-merging the back-merge, which leaves the base at the last cut, so a + second publish before the next cut conflicts; and a ruleset bypass for the + workflow token. + +Sequence: + +1. dev-sync ([#4720](https://github.com/moq-dev/moq/pull/4720)) lands. +2. A PR on `dev`: the wording sweep, trunk stops publishing, `obs`, + `platform`, and `quest` push filters name `main` and `release`, + unpinned `workflow_run` checkouts pin the triggering `head_sha`, the + back-merge workflow, docs pin `release`. +3. A small PR on `main`: publish triggers move to `release`; cache writers + keep `main`, so they stop running there. +4. Admin, back to back, run by the agent only after the maintainer's + go-ahead in chat: rename `main` to `release`, rename `dev` to `main`, set + the default branch, create the `release` ruleset, delete the `dev` + ruleset, retarget open PRs. Release-plz PRs (such as #4596) stay on + `release`. Dry-run first and show the PR list with each PR's class. +5. Verify: a no-op push to `main` publishes nothing; release-plz runs on + `release`; a chained publish builds `release` while trunk differs; the + back-merges of two publishes without a cut between both land on `main`. + +Public API: none. Wire: none. Contributors see the new branch model. diff --git a/quest/m0/frame-alloc-budget.md b/quest/m0/frame-alloc-budget.md deleted file mode 100644 index 25c28fba887..00000000000 --- a/quest/m0/frame-alloc-budget.md +++ /dev/null @@ -1,32 +0,0 @@ -# [M] Frame buffers are backed by bytes received - -## Goal - -The memory a relay commits to an incoming frame tracks what the peer has -sent, not only the size it declared. Today the first payload byte allocates -the whole declared size (up to `MAX_CACHE_BYTES`, 32 MiB) with -`vec![0u8; size]` and charges it to the cache, with no aggregate bound. - -## Plan - -- Pre-allocating the declared size is a deliberate optimization, so keep it - within a per-session budget: a frame whose declared size fits the - session's remaining budget is allocated up front; past the budget the - buffer grows as bytes arrive. Bytes are returned to the budget when the - frame completes or aborts. Decided 2026-09-29. -- Charge the cache by bytes written rather than declared size - (`model/group.rs` `state.cache += frame.size`). -- Benchmark first (`rs/moq-net/benches`): frame receive throughput swept over - frame size and concurrent streams, for today, full growth, and the budget, - so the budget's default comes from the measurement. -- Paths: `model/frame.rs` `MutableFrameBuf::new`, `coding/reader.rs` - `poll_read_frame`, called from both `lite/subscriber.rs` and - `ietf/subscriber.rs`. - -Public API: none expected; if the budget needs a knob, it goes on the relay -config, not moq-net. Wire: none. - -## Related - -- [Frame slot charge](/quest/m1/frame-slot-charge.md) - lands first; both change the cache charge in `model/group.rs` -- [Peer limits](/quest/m1/quic/peer-limits.md) - stream counts and windows per peer diff --git a/quest/m0/handshake-deadline.md b/quest/m0/handshake-deadline.md deleted file mode 100644 index e8f998804b3..00000000000 --- a/quest/m0/handshake-deadline.md +++ /dev/null @@ -1,24 +0,0 @@ -# [S] Handshakes and HTTP headers have deadlines - -## Goal - -A connection that never completes its QUIC, WebTransport, WebSocket or MoQ -SETUP handshake is closed after a deadline, and the relay's HTTP listener -times out slow request headers. Today `moq-tokio/src/server.rs` pushes -`accept_request` futures with no deadline, and the 5 s keep-alive keeps an -idle pre-SETUP connection alive indefinitely. axum-server 0.8 builds hyper -without a timer, so hyper's `header_read_timeout` never fires -(`moq-relay/src/web.rs`). - -## Plan - -- Wrap the accept future in a timeout covering everything up to a completed - SETUP, then close with the transport's timeout code. The timeout is a - config value with a default of around 10 s. -- Configure `http1().timer(TokioTimer).header_read_timeout(..)` on the - relay's HTTP builder. -- Tests on a paused clock: a stalled SETUP is closed at the deadline, and a - completed one is not. -- Document both in `doc/bin/relay/config.md`. - -Public API: a config field on the server. Wire: none. diff --git a/quest/m0/ietf-fin-not-cancel.md b/quest/m0/ietf-fin-not-cancel.md index ccae4e3d282..bf48edc2202 100644 --- a/quest/m0/ietf-fin-not-cancel.md +++ b/quest/m0/ietf-fin-not-cancel.md @@ -26,10 +26,5 @@ silently ends the subscription. Public API: none. Wire: conformance fix; no draft change. -## Required - -- [Legal IETF input](/quest/m0/ietf-legal-input.md) - lands first; both change `ietf/fetch.rs` and the request close path - -## Related - -- [Lite request streams](/quest/m1/request-stream-serve.md) - lite deliberately treats a FIN as ending the request; don't unify the two +Lite deliberately treats a FIN as ending the request (`RequestServe` in +`rs/moq-net/src/lite/publisher.rs`); don't unify the two. diff --git a/quest/m0/ietf-legal-input.md b/quest/m0/ietf-legal-input.md deleted file mode 100644 index 32ebf87922b..00000000000 --- a/quest/m0/ietf-legal-input.md +++ /dev/null @@ -1,49 +0,0 @@ -# [L] Legal IETF input never fails the session - -## Goal - -Every message a conformant draft-14 to draft-22 peer may send decodes, and -anything we don't support is refused per request, not by closing the session. -moxygen and libquicr send several of these today, and Seattle interop is -2026-10-12. - -## Plan - -Each of these closes the session today with `PROTOCOL_VIOLATION`: - -- FETCH on drafts 20+ still decodes the removed Fetch Type field - (`ietf/fetch.rs`). Decode the draft-20 layout: namespace, name and params, - with the range in LOCATION_FILTER. Refuse it `NOT_SUPPORTED` until - [Draft-20 FETCH](/quest/m1/ietf-fetch-location.md) serves it. Fix the encoder and - the pinned test in `version.rs`. -- Request parameters the draft allows on a message fail `decode_params!`: - AUTHORIZATION TOKEN (0x03) anywhere, NEW_GROUP_REQUEST (0x32) and the - object and track filters (0x25 to 0x29) on SUBSCRIBE and REQUEST_UPDATE, - FILL_TIMEOUT (0x0A) and 0x21 and 0x35 on FETCH, and any parameter on TRACK_STATUS. Accept each - one where the draft allows it. The token is decoded and ignored, since the - session grant still applies; [Request token](/quest/m1/auth/request-token.md) - gives it meaning (decided 2026-09-29). An unknown key stays fatal, per the - draft. -- INCLUDE_PROPERTIES (0x35) is read as length-prefixed; the draft defines a - uint8 (`ietf/subscribe.rs`). Encode and decode it as `u8`, and delete the - doc comment arguing for key-parity framing: that rule covers Setup and - Properties KVPs, not message parameters. -- FORWARD=0 returns `DecodeError::Unsupported`. Decode it, then refuse the - request `NOT_SUPPORTED`. - -Not in scope: pre-draft-20 subscribe filters other than Largest Object stay -ignored. `older_drafts_are_ignored` in `ietf/publisher.rs` records that -choice, since honoring them would change what existing peers receive. - -- Tests: one decode test per message and draft, built from the draft's - message figures and bytes captured from moxygen and libquicr, plus a - session test showing each refusal leaves the session open. -- Mirror the decode in `js/net`. - -Public API: none. Wire: fixes conformance; no draft change. - -## Related - -- [Request token](/quest/m1/auth/request-token.md) - also edits `decode_params!`, turning the ignored token into a per-request grant -- [IETF FIN semantics](/quest/m0/ietf-fin-not-cancel.md) - the other interop blocker -- [IETF stream types](/quest/m0/ietf-uni-stream-types.md) - same stream-scoped-before-fatal rule for uni streams diff --git a/quest/m0/ietf-uni-stream-types.md b/quest/m0/ietf-uni-stream-types.md deleted file mode 100644 index 1dfd4e01040..00000000000 --- a/quest/m0/ietf-uni-stream-types.md +++ /dev/null @@ -1,45 +0,0 @@ -# [S] Validate IETF unidirectional stream types - -## Goal - -Accept valid padding streams and close the session for genuinely unknown -stream types according to the negotiated moq-transport draft. Behavior only, -so it ships on main. - -## Plan - -Moved from m1 to m0 in the 2026-09-30 audit: a padding stream answered with -INTERNAL_ERROR is legal input mishandled, which m0 fixes before Seattle -interop on 2026-10-12. - -`run_unis` in `rs/moq-net/src/ietf/session.rs` routes every non-SETUP uni -stream to `run_uni_group`, which rejects padding and unknown types alike while -leaving the session alive. That stream-only rejection reaches the wire as -INTERNAL_ERROR, because nothing registers a code for it: the handler maps a -session-scoped error to `StreamError::Internal` and aborts the reader. - -draft-21 settles what each stream type means: a stream whose type the -endpoint does not recognize MUST close the session, and a padding stream -(type 0x132B3E28) MUST be discarded, which an endpoint may do by cancelling -it. The tree negotiates drafts 14 through 22 (`ietf::Version` in -`rs/moq-net/src/ietf/version.rs`), so apply that split to every supported draft and check the earlier ones for -the padding type value and whether draining is required. - -- Classify stream types before spawning a group handler. Handle PADDING per - draft, draining it where required and otherwise cancelling it with a - stream-only code, and propagate a genuinely unknown type to the session - driver as a protocol violation. Keep ordinary group failures scoped to their - streams. -- The test `unknown_uni_type_does_not_claim_the_session_closed` in - `session.rs` asserts the current behavior, an INTERNAL_ERROR stop and - no session close, and flips: an unknown type now closes the session and - stops nothing on its own. -- Add a padding test asserting a stream-only cancel with no session close, and - keep `a_group_for_a_retired_alias_is_stopped_with_cancelled`, which - pins that a dropped group never closes the session. - -Consult [draft-21 section 11.5](https://www.ietf.org/archive/id/draft-ietf-moq-transport-21.html) -for the wording, and [draft-19 section 3.4 and section 11.5.1](https://www.ietf.org/archive/id/draft-ietf-moq-transport-19.html) -plus [draft-20 section 11.5.1](https://www.ietf.org/archive/id/draft-ietf-moq-transport-20.html) -and [draft-22](https://www.ietf.org/archive/id/draft-ietf-moq-transport-22.html) -for the other drafts the tree negotiates. diff --git a/quest/m0/ietf-unknown-bidi.md b/quest/m0/ietf-unknown-bidi.md new file mode 100644 index 00000000000..e075b860533 --- /dev/null +++ b/quest/m0/ietf-unknown-bidi.md @@ -0,0 +1,36 @@ +# [XS] JS closes an IETF session on an unknown bidi stream type + +## Goal + +In `@moq/net`, a moq-transport bidi stream whose message type is unknown closes +the session with PROTOCOL_VIOLATION, as Rust and the JS uni path already do, +instead of aborting only that stream. A message a draft defines but we don't +support keeps the refuse-per-request rule +(#4685, #4610). + +## Plan + +Facts (`origin/main`, 2026-10-01): + +- JS: the `default:` arm of the bidi dispatch in + `js/net/src/ietf/connection.ts` logs and calls `stream.abort(...)`, so the + `#runBidis` catch never reaches `#violated` and the session stays open. +- JS uni already throws `ProtocolViolation` for an unknown type. +- Rust: `run_dispatch`'s `_` arm in `rs/moq-net/src/ietf/session.rs` returns + `UnexpectedStream`, which maps to PROTOCOL_VIOLATION. Rust has a uni test + (`an_unknown_uni_type_closes_the_session`) but no bidi one. + +Decided (2026-10-01): unknown is fatal; defined but unsupported is refused per +request. Rejected: making Rust lenient to match JS. + +- JS reads the bidi type with `u53()`, which throws `RangeError` for a valid + varint above 2^53 before the switch runs, so `#runBidis` never sees a + `ProtocolViolation` for it either. The uni path already reads `u62()`. + +Work: read the bidi type at full wire width (`u62()`), throw +`ProtocolViolation` from the JS default arm, and add a bidi test in both +languages that asserts the PROTOCOL_VIOLATION close code for a small unknown +type and one at or above 2^53. moq-lite is out of scope: its draft says an unknown stream type +MUST be reset and MUST NOT be fatal, and both implementations comply. + +Public API: none. Wire: none; fixes conformance. diff --git a/quest/m0/lite-early-streams.md b/quest/m0/lite-early-streams.md new file mode 100644 index 00000000000..b9d005c2990 --- /dev/null +++ b/quest/m0/lite-early-streams.md @@ -0,0 +1,52 @@ +# [S] moq-lite streams that arrive before SETUP are held + +## Goal + +On lite-05 and later (the versions with a Setup Stream), a uni stream that +arrives before the peer's SETUP is held and handled once SETUP lands, never +aborted, in Rust and JS. A second Setup Stream closes the session with +PROTOCOL_VIOLATION in both languages, as the lite draft requires. + +## Plan + +Facts (`origin/main`, 2026-10-01): + +- Rust `lite::accept_setup` (`rs/moq-net/src/lite/session.rs`) aborts every + non-SETUP uni stream with `UnexpectedStream`. + `accept_request_skips_uni_stream_before_setup` in `rs/moq-net/src/server.rs` + pins that. An unknown `DataType` fails the handshake. +- JS lite has no pre-SETUP gate: `#runUni` in `js/net/src/lite/connection.ts` + handles SETUP like any other uni stream, so early groups already work. It + also accepts a second SETUP silently, and the `#runUnis` catch only stops the + stream, so throwing `ProtocolViolation` there would not close the session. + The `#runBidis` catch closes on `ProtocolViolation`, but with a bare + `close()` rather than the PROTOCOL_VIOLATION code. +- `drafts/draft-lcurley-moq-lite.md`: neither endpoint waits for the peer's + SETUP. A receiver MUST buffer only streams whose encoding depends on a + negotiated extension, and a second Setup Stream MUST close with + PROTOCOL_VIOLATION. Rust's abort is stricter than the draft. +- Lite-01 to lite-04 have no Setup Stream and are out of scope. + +Decided (2026-10-01): + +- ✅ Rust holds early uni streams until SETUP, then hands them to the normal + classifier, the same shape as + IETF early streams (#4686). Rust can't + route before SETUP anyway, since path and role come from it. Rejected: + processing immediately unless extension-dependent, which needs a session + before SETUP. +- ✅ The duplicate-SETUP check rides along, in both languages. In JS the uni + catch must close the session with the PROTOCOL_VIOLATION code, not copy the + bidi path's bare `close()`. + +The remaining JS gap is the duplicate SETUP and its close path; the hold is +Rust work unless JS turns out to need an extension-dependent hold. + +Crib from #4686: its `died_before_header` helper and peek-then-hold pattern. +QUIC stream credit bounds the held queue. Check whether JS lite must buffer any +extension-dependent stream type today, and hold those until SETUP. + +Tests: flip `accept_request_skips_uni_stream_before_setup` to expect the group +to be delivered. In both languages, a duplicate SETUP closes the session. + +Public API: none. Wire: none; matches the lite draft. diff --git a/quest/m0/path-hunting.md b/quest/m0/path-hunting.md deleted file mode 100644 index 7ca416b2467..00000000000 --- a/quest/m0/path-hunting.md +++ /dev/null @@ -1,49 +0,0 @@ -# [M] A withdrawn path stops hunting - -## Goal - -When a broadcast ends, each relay in a lite-06 mesh retracts it about once, -instead of re-announcing stale alternatives. Today one withdrawal on live's -34-relay graph costs hundreds to over a thousand extra announce starts, and -on Kick dev an ended name drew 575 to 57,805 mesh announce starts (median -about 13,000) while live names drew one flood (moq-dev/moq.pro#2116). New -and removed routes still propagate at once, and no wire changes. - -## Plan - -Promoted from the cluster routing line to m0 on 2026-09-30: it is today's -partial-mesh hunting that #4399 left open, and it is hurting production. - -Decided (2026-09-30), lite-06, no wire change: - -- A route *update*, where a relay's advertised route for a path switches to a - different route, waits a short hold-down before it is re-advertised. A new - path (nothing advertised yet) and a removal (nothing left) go out at once. -- So when the best route is withdrawn and an alternative remains, the relay - retracts at once and re-announces the alternative only if it survives the - hold-down. Stale alternatives are withdrawn inside that window and never - spread. A genuine failover, where a link dies but the origin still - publishes, is delayed by the hold-down; pick the value from the repro, and - report the failover cost it adds. -- Check the interaction with #4399's withdrawal hiding and with - subscriptions already pinned to a route (fronts keep serving from a live - source; only advertisements are held). - -Test first: land moq.pro's in-process repro upstream (real lite-06 sessions -over the mock transport on live's 34-relay graph, mocked time), asserting -about one retraction per relay for a single withdrawal from a non-hub -publisher, plus a failover test where a link dies and the path comes back -through another route after the hold-down. It must fail on `main` today. - -Then moq.pro moves its pin and confirms about one retraction wave per ended -name on Kick dev after a deploy the user approves (moq.pro's own quest). - -If no hold-down closes it, the fallback is a per-origin seqno on cluster -links in the wip lite version; report before building it. - -Public API: none expected. Wire: none. - -## Related - -- [Cluster routing](/quest/m1/cluster-routing/README.md) - tiers shrink the core graph this hunts on -- [Cross-relay delivery under bursts](/quest/m1/cross-relay-bursts.md) - closed broadcasts announced for minutes, the same symptom diff --git a/quest/m0/qmux-credit.md b/quest/m0/qmux-credit.md new file mode 100644 index 00000000000..eaf72e9e079 --- /dev/null +++ b/quest/m0/qmux-credit.md @@ -0,0 +1,53 @@ +# [M] qmux returns every byte's credit and sends its close frame + +## Goal + +A qmux session returns connection-level credit for every byte it receives, +whether the app reads it, drops the stream unread, or it arrives after +STOP_SENDING, so a long-lived session never stalls on MAX_DATA. `close()` +delivers its APPLICATION_CLOSE frame before the transport drops, so a TCP or +WebSocket peer sees the close code whenever the transport stays writable +within the close bound. Both hold on the 0.5 line that +`main` pins and the 0.6 line `dev` uses. + +## Plan + +Facts from moq-dev/web-transport `rs/qmux/src/session.rs` (0.5.1 and main): + +- Credit returns only through `RecvStream::report_consumed` on reads. + `Drop for RecvStream` sends STOP_SENDING and returns stream-count credit, + but never consumes bytes still buffered or queued in `inbound_data`. +- On STOP_SENDING the writer removes the recv entry, so later STREAM data is + dropped before the connection-credit check, and a later RESET_STREAM + returns early, so its final-size gap is never consumed. The peer counts all + of it against MAX_DATA. +- A new stream's entry is inserted only after `accept_uni`/`accept_bi` hands + it over, so a stream dropped in that window leaves a stale entry that keeps + charging the connection window. +- `close()` queues APPLICATION_CLOSE and marks the session closed at once. A + writer that is mid-write treats that as interrupted and drops the frame. + 0.6 made `close()` idempotent and keeps the first reason, but not the order. + +Work: + +- Consume credit for unread bytes on drop, for STREAM data and the RESET + final-size gap after STOP_SENDING (keep enough of a retired stream's state + to account for its final size), and close the accept gap. +- Mark the session closed only once the writer has sent the close frame, with + a bound so a stalled transport still drops. +- Tests in `rs/qmux`: a session that drops many unread streams keeps + delivering past its initial window, and a peer reads the close code after a + close issued mid-write. +- Release on both lines (0.5.x after 0.5.2, and 0.6.x), then bump + `main`'s pin to that 0.5.x. `main` stays on 0.5: 0.6 needs + web-transport-trait 0.5, a breaking change that belongs on `dev`, which + picks up 0.6.x. + +Why m0: the WebSocket fallback and the planned edge-to-core `tls://` links +both run on qmux, and MoQ drops streams constantly. + +Public API: none. Wire: none. + +## Related + +- [qmux on noq-proto](/quest/m2/quic-qmux.md) - replaces these stream maps later diff --git a/quest/m0/qmux-reset-race.md b/quest/m0/qmux-reset-race.md deleted file mode 100644 index 0aa6adcdd39..00000000000 --- a/quest/m0/qmux-reset-race.md +++ /dev/null @@ -1,27 +0,0 @@ -# [XS] qmux handles RESET_STREAM under one lock - -## Goal - -qmux's RESET_STREAM handling never panics when the writer task removes the -receive stream concurrently. Today `session.rs` checks -`streams.recv.contains_key`, drops the lock, and later re-locks with -`.expect("live recv stream")`; a `StopSending` queued by a dropped -`RecvStream` can remove the entry in between. A missing entry must be a -no-op, not a panic. - -## Plan - -- In moq-dev/web-transport `rs/qmux/src/session.rs`, look the entry up once - under a single lock (`if let Some(recv) = get_mut`); a missing entry means - the stream already closed and the reset is a no-op. -- Regression test that removes the entry between the two points. -- Audit the other `expect`s on the stream maps (`sched.rs`) for the same - pattern. -- Release a patch on the pinned 0.5 line and on 0.6, then bump the 0.5 pin - here. Moving to 0.6 is not part of this quest. - -Public API: none. Wire: none. - -## Related - -- [qmux on noq-proto](/quest/m2/quic-qmux.md) - replaces these stream maps entirely, later diff --git a/quest/m0/quest-check-everywhere.md b/quest/m0/quest-check-everywhere.md deleted file mode 100644 index 447eadfca61..00000000000 --- a/quest/m0/quest-check-everywhere.md +++ /dev/null @@ -1,19 +0,0 @@ -# [XS] quest check runs on every branch that carries quests - -## Goal - -`quest check` fails any push or PR that breaks quest structure on `main`, -`dev`, and the questline branches, not only PRs into `main`. - -## Plan - -- `check.yml` runs `quest check` on pull requests only. Also run it on push to - `main`, `dev`, and `quest/**`, so a direct merge commit (such as `main` - merged into a line) can't land a broken tree. Use a dedicated job that runs - `quest check` unconditionally: `check.yml`'s scope steps diff against - `origin/$GITHUB_BASE_REF`, which is empty on a push. -- `dev` already pins main's `quest` (8590d2a) and passes since #4428. Only the - wildcard line still pins 46d7fe8: merge `main` into it to bump the pin, and - fix what the new check reports. - -Public API: none. Wire: none. diff --git a/quest/m0/revalidate-overflow.md b/quest/m0/revalidate-overflow.md deleted file mode 100644 index f42c8ba0ad1..00000000000 --- a/quest/m0/revalidate-overflow.md +++ /dev/null @@ -1,18 +0,0 @@ -# [XS] Auth durations cannot overflow a deadline - -## Goal - -No auth-server or token duration can panic the relay. Today -`moq-auth/src/client.rs` computes `Instant::now() + cadence` for the -`revalidate` value, and `Grant::deadline` adds a token's `exp` the same way. -A value near `u64::MAX` seconds overflows and panics. - -## Plan - -- Use `checked_add`. `Grant::validate` refuses a `revalidate` longer than the - grant's expiry, and treats an `exp` beyond `Instant`'s range as no expiry. -- Unit tests with `u64::MAX` revalidate and exp. -- Grep the workspace for other `Instant + Duration` on peer- or - server-supplied values. - -Public API: none. Wire: none. diff --git a/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md b/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md index 886144d2d0e..21eff93ccd7 100644 --- a/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md +++ b/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md @@ -89,5 +89,5 @@ the closed producer's cache or terminal state. ## Related -- [Resume latest](/quest/m1/resume-latest.md) - edits `resume.rs` and its takeover tests (#4491) +- [#4491](https://github.com/moq-dev/moq/pull/4491) - edits `resume.rs` and its takeover tests - [Parked reads wake](/quest/m1/parked-read-wakes.md) - edits the same `resume.rs` wakeups diff --git a/quest/m1/README.md b/quest/m1/README.md index a6930d12aa7..7bda75e93da 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -22,24 +22,29 @@ QUIC studies there on that rule. ## Required -- [Cluster idle timeout](/quest/m1/cluster-idle-timeout.md) - a relay notices a silent peer relay within seconds, not after the shared 30 s QUIC idle timeout - [moq-transport peers are plain clients](/quest/m1/ietf-cluster-off.md) - no moq-transport session negotiates the cluster extension, so none can splice a pool's members on a Hop ID label -- [Cluster routing](/quest/m1/cluster-routing/README.md) - edge and core tiers carry each broadcast into a region once over path vector, with the backbone hidden from end users +- [Epoch primitive](/quest/m1/epoch.md) - one `Epoch` type in moq-net and @moq/net, carried as a trailing `@` path segment, shared by e2ee and broadcast epochs +- [Broadcast epochs](/quest/m1/broadcast-epoch/README.md) - each publish of a name gets a fresh `@` epoch, viewers follow the newest live one at once, and bare names still resolve on every version +- [Cluster routing](/quest/m1/cluster-routing/README.md) - any node routes toward a broadcast's origin over CDN and P2P links alike, with per-origin routes and path-less announces - [Remove `--hop`](/quest/m1/hop-removal.md) - on `dev`, redundant publishers share an explicit `@`, and `--hop` and the publisher's Hop ID are gone - [One route cost](/quest/m1/route-cost.md) - on `dev`, Warm and Cold collapse to one static route cost +- [Delete removed cluster flags](/quest/m1/cluster-shims.md) - on `dev`, once a release has carried their refusals, `mesh` and `linger` leave `cluster::Config` - [Track tail interop](/quest/m1/track-tail-interop.md) - a Rust publisher ending a track with a group in flight is read to its end by the JS subscriber, and the reverse, in `just test interop` +- [moq-mux truncated group](/quest/m1/mux-truncated-group.md) - a test pins that moq-mux skips a truncated spliced group as aborted +- [Demand everywhere](/quest/m1/demand-everywhere.md) - on `dev`, requests and producers at every level watch subscribers through `demand()` alone, ahead of the bindings exposing it - [Merge queue](/quest/m1/merge-queue.md) - the required checks run on `merge_group`, so a stale green check can no longer break main - [Binding audio delay](/quest/m1/binding-surface.md) - moq-ffi and every wrapper configure and observe audio playout delay - [FFI shape](/quest/m1/ffi-shape/README.md) - the bindings mirror Rust's layers: net at the root, then media, json, flate, audio, and video namespaces built from the handle below - [Raw stream codes](/quest/m1/raw-stream-codes.md) - raw QUIC stream resets and stops carry the application's code, not an HTTP/3-mapped one +- [moq-net owns its transport seam](/quest/m1/transport-seam.md) - moq-net names only its own transport traits, so a web-transport-trait or qmux major bump is a patch for it - [IETF drain before close](/quest/m1/ietf-drain-before-close.md) - moq-transport sessions deliver finished tracks before a graceful close, as moq-lite does - [Close waits for the tail](/quest/m1/close-tail.md) - on lite-07, `close()` returns `Ok` only after each subscriber FINs its Subscribe Stream, having read the track to its end - [moq play drain tail](/quest/m1/play-drain-tail.md) - retired renditions and finite tracks play their last 10 ms of audio +- [A moq-relay release refuses --cluster-mesh](/quest/m1/relay-mesh-refusal-release.md) - the release that lets the cluster flag shims go - [WebTransport close upstream](/quest/m1/wt-close-upstream.md) - web-transport-moq delivers the close capsule itself, and moq-tokio's `CLOSE_LINGER` is deleted - [moq announced](/quest/m1/cli-announced.md) - `moq ls` becomes a follow-only live view named `moq announced`, with local-only shell completion - [Delete the live marker](/quest/m1/announce-live-removal.md) - announce streams yield only route events; the `Live` marker and its hold machinery are gone - [Browser close code](/quest/m1/browser-close-code.md) - a playwright case proves the page reads a relay's close code and reason, on every web-transport backend -- [Resumed groups](/quest/m1/resume-latest.md) - a half-delivered group ends once the new copy is past it, so a group-only reader never parks after a mid-group failover - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it - [Subscribe ranges](/quest/m1/subscribe-ranges/README.md) - on dev, a lite-07 SUBSCRIBE asks for past and live ranges in either order and replaces FETCH; relays fill misses by range, including over moq-transport - [Cross-relay bursts](/quest/m1/cross-relay-bursts.md) - bursty small-group tracks cross two relays without lost groups, unanswered FETCHes, or stalls @@ -47,10 +52,12 @@ QUIC studies there on that rule. - [Parked reads wake](/quest/m1/parked-read-wakes.md) - a read parked on an evicted or aborted group wakes and re-judges, for resume successors and plain tracks alike - [Reader end parity](/quest/m1/reader-end-parity.md) - JS readers see a track's end once the newest group reaches the declared end, as Rust readers do - [JS cache window](/quest/m1/js-prune-cache-window.md) - js/net ages idle groups on a cache window and keeps max_age as media-time staleness, as Rust does -- [Request stream cancel](/quest/m1/request-stream-serve.md) - a lite publisher stops resolving a SUBSCRIBE or FETCH once the requester FINs or resets, through one wrapper that owns every request stream's reader -- [Session death parity](/quest/m1/session-death.md) - a local close ends tracks cleanly in both languages, and JS group readers see the session's error on session death +- [IETF FETCH abandonment](/quest/m1/ietf-fetch-abandonment.md) - the Rust moq-transport subscriber cancels a group FETCH once its last reader leaves - [Watch and publish under CSP](/quest/m1/csp-assets.md) - blob workers stay the default; strict-CSP apps host the files and set a base URL - [More tests under load](/quest/m1/test-flakes-2/README.md) - the second round of load-only failures, one quest per flake, fixed at the cause +- [Shaper step on mock time](/quest/m1/shaper-step-clock.md) - the `moq-shaper` step test asserts on a paused clock, so a loaded runner cannot fail it +- [A week of nightly interop after #4529](/quest/m1/interop-week.md) - the nightly traces the stall diagnosis reads +- [Nightly tests on macOS and Windows](/quest/m1/nightly-platform-tests.md) - `moq-auth`, `hang`, `moq-tokio`, and `moq-native` tests run nightly where `Instant` and sockets differ, not just compile - [CI runner stalls](/quest/m1/ci-runner-stalls.md) - the 0.4 to 0.8 s freezes of both interop tracks on CI are attributed from a week of nightlies and fixed or told apart from playback bugs - [Catalog estimate rate](/quest/m1/catalog-estimate-rate.md) - a rising `jitter`/`delay` estimate republishes the catalog at most once a second, in js/publish and moq-mux - [Rewind detail](/quest/m1/timestamp-rewind-detail.md) - a refused timestamp rewind names the frame's timestamp and the live edge @@ -58,7 +65,6 @@ QUIC studies there on that rule. - [Wire compatibility](/quest/m1/wire-compat.md) - a nightly run tests this checkout against the last published release for tokens, session wire, and catalog/container - [Same-hop importers](/quest/m1/hop-aligned-import.md) - importers fed one stream publish identical groups and timestamps, so failover between a redundant pair survives - [Auth client CA](/quest/m1/relay-auth-client-ca.md) - on dev, `auth::Config::validate` and `init` take the client-CA flag, so no caller can skip the check -- [TS PSI reassembly](/quest/m1/ts-psi-reassembly.md) - `import ts` reads a PAT or PMT that spans packets or follows a nonzero pointer_field instead of aborting, and one corrupted section costs a repetition and a counted `CRC_error`, not the import - [TS damaged units](/quest/m1/ts-damaged-units.md) - one malformed PES or access unit is dropped, counted as `damaged`, and resynced at the next keyframe instead of ending the import - [RTMP interleaving](/quest/m1/rtmp-interleaving.md) - isolate partial messages before optimizing assembly copies - [TS stats module](/quest/m1/ts-stats-module.md) - on dev, the TS stats types move under `ts::stats` as `Snapshot` and `Stream`, with an owned `track` @@ -66,15 +72,15 @@ QUIC studies there on that rule. - [Capture by default](/quest/m1/capture-default.md) - moq-video and moq-audio build `capture` by default, so pre-merge checks test it and the capture gate goes away - [Ship capture and playback](/quest/m1/cli-packaging.md) - a released moq binary can capture and play, which no distribution currently enables - [Go and Dart doc samples](/quest/m1/doc-samples-go-dart.md) - Go and Dart doc samples compile against their wrappers -- [Publishing requires a timestamp](/quest/m1/publish-timestamp.md) - every Rust and binding publish API takes a timestamp; nothing fills in now for the caller -- [JS publishing requires a timestamp](/quest/m1/js-publish-timestamp.md) - the same in @moq/json, @moq/flate, and @moq/net -- [Plan: untimed peer objects](/quest/m1/plan-untimed-objects.md) - decide what a frame's timestamp is when a moq-transport or pre-lite-05 peer sends none -- [Data consumer timestamps](/quest/m1/data-consumer-timestamps.md) - json and binary consumers return each value's timestamp, in Rust and every binding +- [Publishing never invents a timestamp](/quest/m1/publish-timestamp.md) - no Rust or binding publish API fills in a timestamp; an untimed payload goes out untimed, and moq-mux data producers take broadcast-clock timestamps +- [JS publishing never invents a timestamp](/quest/m1/js-publish-timestamp.md) - the same in @moq/json, @moq/binary, and @moq/net +- [Plan: untimed objects](/quest/m1/plan-untimed-objects.md) - map faithful timestamp absence, publisher to consumer, into implementation quests +- [Data consumer timestamps](/quest/m1/data-consumer-timestamps.md) - json and binary consumers return each value's timestamp, in Rust and every binding; snapshots add `latest()` beside an in-order `next()` +- [JS data consumer timestamps](/quest/m1/js-data-consumer-timestamps.md) - @moq/json and @moq/binary consumers return each value's timestamp, with snapshot `next()` and `latest()` - [Nested data configs](/quest/m1/data-config-nesting.md) - docs nest `BinaryConfig`/`JsonConfig` in an application section instead of flattening it - [Omit empty catalog sections](/quest/m1/catalog-omit-empty.md) - a Rust catalog with no video or audio leaves those keys out, as JS does -- [Public clock anchor](/quest/m1/clock-anchor-public.md) - an application demuxer shares one source offset across its tracks through `Anchor` and `Lane` -- [Delete SourceMap](/quest/m1/source-map-removal.md) - on dev, `Anchor` with its `Lane`s is the one way onto the broadcast clock - [Draft-20 FETCH](/quest/m1/ietf-fetch-location.md) - a draft-20+ FETCH within one group is served from its LOCATION_FILTER, as older drafts are +- [FETCH_OK properties](/quest/m1/fetch-ok-properties.md) - our FETCH_OK carries the track properties SUBSCRIBE_OK does, as draft 16+ requires - [Fetch without SUBSCRIBE](/quest/m1/ietf-fetch-only.md) - a relay fetches from an IETF upstream without subscribing, finished tracks included, with End of Track always reported - [IETF timestamp units](/quest/m1/ietf-timestamp-units.md) - drafts 14-16 stop sending Timestamp properties they can't give units for - [JS IETF datagrams](/quest/m1/js-ietf-datagram.md) - `@moq/net` sends and receives datagram groups over moq-transport, like Rust @@ -86,7 +92,9 @@ QUIC studies there on that rule. - [Dropped sources](/quest/m1/dropped-sources.md) - track consumers see the producer's real error on every end path, never `Dropped` - [C++ through moq-ffi](/quest/m1/cpp/README.md) - generated C++ over moq-ffi with futures and expected-style errors, shipped as a tarball, vcpkg, and Conan, and adopted by the OBS plugin - [Generated C bindings](/quest/m1/c/README.md) - C generated from moq-ffi ships as `moq-c` 0.8.0 and replaces the hand-written libmoq +- [The final libmoq release is the stub](/quest/m1/libmoq-final-release.md) - the release that lets `rs/libmoq` go - [Retire the libmoq stub](/quest/m1/libmoq-retire.md) - on dev, the published `libmoq` crate stops after its final release points users at `moq-c` +- [dev's decoded-frame surface reaches main](/quest/m1/obs-decoded-surface.md) - the surface the OBS source decodes into is on main - [OBS native codecs](/quest/m1/obs-moq-video/README.md) - replace FFmpeg video and audio decoding with moq-video and moq-audio, deliver GPU frames, and use native audio/video encoders - [AudioToolbox decode](/quest/m1/audio-decode-audiotoolbox.md) - macOS and iOS decode HE-AAC, multichannel AAC, and what else the framework offers - [HE-AAC catalog output](/quest/m1/he-aac-catalog-output.md) - HE-AAC catalog entries name the output rate and layout, not the LC core @@ -101,9 +109,11 @@ QUIC studies there on that rule. - [Opus mapping family](/quest/m1/opus-mapping-family.md) - on dev, the Opus head config keeps its mapping family only in `mapping` - [mp4-atom dOps mapping](/quest/m1/mp4-atom-dops-mapping.md) - a released mp4-atom reads and writes any `dOps` channel mapping family and table - [CMAF surround Opus](/quest/m1/cmaf-opus-surround.md) - fMP4 import and export carry an Opus channel mapping table +- [A self-hosted NVIDIA runner is registered](/quest/m1/gpu-runner.md) - the maintainer registers the host that runs the NVIDIA tests - [GPU CI](/quest/m1/gpu-ci.md) - NVIDIA tests run nightly on a self-hosted GPU runner, and `just rs nvidia` runs them locally instead of skipping - [Capture cut test](/quest/m1/capture-cut-e2e.md) - the capture loop's keyframe throttle is tested end to end against an encoder with its own GOP - [NVENC keyframe flag](/quest/m1/nvenc-keyframe-flag.md) - NVENC flags keyframes from its reported picture type instead of scanning the bitstream +- [Rendition preference](/quest/m1/rendition-preference.md) - automatic selection by ``, `Video::ranked`, and WHEP keeps the highest `preference` that decodes, so a compatibility transcode is only picked when nothing preferred decodes - [JS rendition ranking](/quest/m1/js-ranked.md) - `@moq/hang` ranks video renditions like Rust, and `@moq/watch`'s fallback uses it - [Audio rendition pick](/quest/m1/audio-ranked.md) - single-track FLV/RTMP and WHEP serve the best audio rendition, not the first by name - [FLV catalog stream](/quest/m1/flv-catalog-stream.md) - on dev, `flv::Export` takes a catalog stream like fmp4, replacing `with_select` @@ -112,7 +122,9 @@ QUIC studies there on that rule. - [QoS](/quest/m1/qos/README.md) - broadcast health: relay starvation and timeliness histograms, on dev - [Catalog track alias](/quest/m1/catalog-track-alias.md) - catalog rendition keys become aliases with an optional `track` name, so one catalog lists renditions from several broadcasts - [Media stats](/quest/m1/stats/README.md) - publishers announce a stats track in the catalog, viewers answer a soliciting catalog through a per-catalog `.echo` broadcast, and a Rust encoder adapts to them -- [Drain](/quest/m1/drain/README.md) - relay restarts drain sessions over GOAWAY instead of hard-dropping them +- [JS group-boundary handover](/quest/m1/js-group-handover.md) - a JS track subscription carries across a route swap at a group boundary, so `test/drain` passes at zero latency budget +- [JS GOAWAY requests](/quest/m1/js-goaway-requests.md) - after GOAWAY the JS client opens no new request on the old session, like Rust +- [Drain handshakes](/quest/m1/drain-handshakes.md) - a drain GOAWAYs and waits for sessions still in their handshake instead of exiting under them - [Strict Redirect::resolve](/quest/m1/redirect-resolve.md) - on dev, `Redirect::resolve` can no longer quietly turn a refused redirect into a redial - [Transport upgrade](/quest/m1/transport-upgrade/README.md) - a session that came up over WebSocket moves to QUIC once the QUIC dial lands, handing over at a group boundary - [Scope track priority](/quest/m1/track-priority-scope.md) - priority orders one owner's streams, and a shared cluster session is fair across tenants @@ -133,12 +145,14 @@ QUIC studies there on that rule. - [Plan: cache age-out](/quest/m1/cache-wall-eviction.md) - a swept benchmark decides whether the track cache ages groups out on wall time without a write - [Frame slot charge](/quest/m1/frame-slot-charge.md) - a group's frame slots past the first four count against the cache pool, including capacity a released group keeps - [Front deadlines](/quest/m1/front-deadline-index.md) - a front's per-event cost stops growing with its track count: an expiry index and per-track wakes, proven by a churn benchmark +- [Listener deadlines](/quest/m1/listener-deadlines.md) - io_uring, HTTP/2, and the internal listener bound slow handshakes and headers, and iroh honors `quic.keep_alive` +- [Papercuts](/quest/m1/papercuts.md) - JS refuses to serve a broadcast it did not produce, `just` skips `.scratch/`, and a uring test stops sleeping - [Front parking](/quest/m1/origin-front-parks.md) - an unroutable request waits on a front instead of re-asking on every route-table move +- [Route wakes](/quest/m1/route-wakes.md) - a route change wakes only the fronts it can move, so pool churn stops scaling with served paths +- [Admission bench](/quest/m1/admission-bench.md) - the driver's per-event admission walk is benchmarked over tracks and copies - [Publish channel count](/quest/m1/publish-audio-channel-count.md) - forcing a channel count on an Audio.Capture stops costing the subscriber gaps of silence - [JS abandonment](/quest/m1/js-subscribe-abandonment.md) - a viewer returning during IETF subscribe setup keeps its track across microtasks -- [Epoch primitive](/quest/m1/epoch.md) - one `Epoch` type in moq-net and @moq/net, carried as a trailing `@` path segment, shared by e2ee and broadcast epochs - [E2EE](/quest/m1/e2ee/README.md) - TypeScript and Rust peers interoperate over encrypted broadcasts no relay can decrypt -- [Broadcast epochs](/quest/m1/broadcast-epoch/README.md) - each publish of a name gets a fresh `@` epoch, viewers follow the newest live one at once, and bare names still resolve on every version - [#3056](/quest/m1/3056-watch-video-decoder-captures-the-rewind-generation-at.md) - watch: the video decoder resets on a declared discontinuity - [#933](/quest/m1/933-video-rotation-metadata-not-propagated-from-mobile-camera.md) - the catalog rotation follows the live camera's orientation - [#2848](/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md) - the Opus producer follows its bandwidth grant through the settled `moq_mux::rate::Control` @@ -153,6 +167,7 @@ QUIC studies there on that rule. - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay` +- [TS passthrough](/quest/m1/ts-passthrough.md) - `--passthrough` carries the multiplex as whole packets, listed in an `m2ts` catalog section, and writes it back byte-identical (less late drops) on a fixed delay - [FLV export delay](/quest/m1/flv-export-delay.md) - FLV interleaves through the shared fixed-delay release stage - [MKV export delay](/quest/m1/mkv-export-delay.md) - MKV interleaves through the shared fixed-delay release stage - [Release profile](/quest/m1/release-profile.md) - every release build gets fat LTO, one codegen unit, and stripping from the workspace profile instead of three script exports diff --git a/quest/m1/admission-bench.md b/quest/m1/admission-bench.md new file mode 100644 index 00000000000..288dc13e685 --- /dev/null +++ b/quest/m1/admission-bench.md @@ -0,0 +1,23 @@ +# [S] Admission walk benchmark + +## Goal + +A benchmark sweeps tracks per front against copies per track for the +driver's admission walk, so a per-event cost that grows with the front +shows up as a slope. + +## Plan + +On the wildcard line, `run_front` (`rs/moq-net/src/model/origin.rs`) walks +every track and every copy (`io.copies()`) and calls `Provenance::admit` after +each event whenever `front.admit()` returns an origin, even when nothing +changed; Abort and End walk the same way. #4279 left this unbenchmarked +(moq-dev/moq#4607 covered pool resolution). The walk only runs once a +lite-07 reply names an origin, so extend `rs/moq-net/benches/session.rs` over +lite-07 mock sessions. If the slope matters, say so rather than optimizing +here; [Front deadlines](/quest/m1/front-deadline-index.md) owns per-track +wakes. + +## Required + +- [Wildcard](/quest/m0/wildcard/README.md) - the line the admission walk lives on diff --git a/quest/m1/audio-quality-native.md b/quest/m1/audio-quality-native.md index ff4d16ab813..76b845c16e8 100644 --- a/quest/m1/audio-quality-native.md +++ b/quest/m1/audio-quality-native.md @@ -25,7 +25,8 @@ from whichever platform a user happened to be on. - `moq play` needs to emit the run's counters and stage timings as JSON for the analyzer. Add that output, and keep it useful outside the test: a user debugging their own latency wants the same dump. -- Reuse the browser lane's analyzer and budget file, with the JSON field names +- Reuse the browser lane's analyzer and budget file (`test/audio-quality/`, the + schema is `clients/js/src/schema.ts`), with the JSON field names and meanings matching its schema exactly. Native rows get their own budget values in the same file, keyed the same way, since the device floor differs; they do not get their own schema. @@ -40,11 +41,6 @@ budget. Only then compare end-to-end totals, with the backend-dependent stages isolated: this lane deliberately accepts real device callback noise, so a difference in totals alone proves nothing about the estimator. -Standalone in m1 rather than a child of the m0 [Audio quality -harness](/quest/m0/audio-quality-harness/README.md) line (decided in the -2026-09-28 quest audit): nothing in m0 waits on it. The native jitter target +Standalone in m1 rather than a child of the m0 audio quality harness line +(decided in the 2026-09-28 quest audit): nothing in m0 waits on it. The native jitter target it grades is done on the jitter target line. - -## Required - -- [Browser](/quest/m0/audio-quality-harness/browser.md) - defines the metric schema, the budget file, and the extracted shaper diff --git a/quest/m1/auth/peer-grant.md b/quest/m1/auth/peer-grant.md index bdf539b257d..0641129a7c1 100644 --- a/quest/m1/auth/peer-grant.md +++ b/quest/m1/auth/peer-grant.md @@ -67,3 +67,4 @@ Additive. ## Related - [Signaling and policy](/quest/m2/p2p/signal.md) - the first consumer +- [Route trust](/quest/m1/cluster-routing/route-trust.md) - extends the grant with the node ids its holder may advertise routes for, a second purpose beside serving paths diff --git a/quest/m1/auth/request-token.md b/quest/m1/auth/request-token.md index f7336308975..40676e2eecf 100644 --- a/quest/m1/auth/request-token.md +++ b/quest/m1/auth/request-token.md @@ -11,9 +11,8 @@ does not cover it, by the token on the request; with neither it is refused `UNAUTHORIZED`. The token's grant covers only the request it rode on and lives exactly as long as that request, and a REQUEST_UPDATE carrying a new token replaces it, which is how a peer refreshes. It scopes by path, never -by method. [Legal IETF input](/quest/m0/ietf-legal-input.md) decodes and -ignores the key first, so a token no longer fails the session; this quest -gives it meaning. +by method. Every request already decodes the key and ignores it, so a token +no longer fails the session; this quest gives it meaning. ## Plan @@ -21,9 +20,9 @@ gives it meaning. (`rs/moq-net/src/ietf/token.rs`, `js/net/src/ietf/token.ts`): `USE_VALUE` yields the token, `REGISTER` is a value since we advertise no `MAX_AUTH_TOKEN_CACHE_SIZE`, and `DELETE` or `USE_ALIAS` closes with `PROTOCOL_VIOLATION`. Both decoder families change: the strict - `decode_params!` path, which ignores the key after - [Legal IETF input](/quest/m0/ietf-legal-input.md), and the generic KVP - path the legacy drafts use, which also ignores it. + `decode_params!` path, where each request reads the repeatable key into an + ignored `Vec`, and draft-14's `Parameters::skip`, which consumes it + unread. `js/net` keeps every instance in `Parameters` and reads none. - Fallback only: a request the session grant already covers is served without verifying its token. Otherwise its token becomes an `auth::Request` on the session's `auth::Handle`, the seam an AUTH stream's @@ -49,9 +48,8 @@ gives it meaning. connection's id, and the auth server treats that as one more grant on the session that never POSTs `end`, so a request token would widen the whole connection for its life. The per-request call carries the token in - `moq_auth::Request.token` with its kind (a CAT reaches the CAT verifier, - not the JWT one) and the request's path, is never counted as a session - grant, and ends when the request ends. Two requests carrying the same bytes + `moq_auth::Request.token` with its kind and the request's path, is never + counted as a session grant, and ends when the request ends. Two requests carrying the same bytes get two leases. The request is resolved against the origin with the path checked against that lease's grant, not through the session's scoped origin handle. Name the call while implementing. @@ -75,7 +73,3 @@ to) and on `moq_auth::Client` (the per-request lease). Wire: none new; the param - [Relay tokens](/quest/m1/auth/relay-refresh.md) - supplies the lease revalidation the per-request lease reuses - -## Related - -- [Legal IETF input](/quest/m0/ietf-legal-input.md) - also edits `decode_params!`; refresh this Plan when it lands, since the strict decoder then decodes and ignores the token diff --git a/quest/m1/ci-runner-stalls.md b/quest/m1/ci-runner-stalls.md index ae944c35bfe..722c4a8570e 100644 --- a/quest/m1/ci-runner-stalls.md +++ b/quest/m1/ci-runner-stalls.md @@ -25,4 +25,4 @@ Public API: none. Wire: none. ## Required -- A week of nightly interop runs on `main` after #4529, which adds the `delay` column these traces need +- [A week of nightly interop after #4529](/quest/m1/interop-week.md) - the traces carry the `delay` column diff --git a/quest/m1/cli-packaging.md b/quest/m1/cli-packaging.md index 33a48b9b306..4cb9136bb02 100644 --- a/quest/m1/cli-packaging.md +++ b/quest/m1/cli-packaging.md @@ -33,7 +33,10 @@ then fails to open a device is the same gap one layer down. Also enable `v4l2` in the Linux ARM release build, so a released binary on a Raspberry Pi 4 publishes from `moq import capture` through the V4L2 M2M hardware encoder (`rs/moq-video/src/v4l2.rs`, already run on a Pi 4's -`bcm2835-codec`) with no GStreamer detour. Verify that once on a Pi 4. +`bcm2835-codec`) with no GStreamer detour. That Pi 4 run covered only +640x360 once, so the Pi 4 check also covers `set_bitrate` on a running +encoder (congestion control retunes through it) and 1080p, which codes as +1088 rows and relies on the compose rectangle to crop back. Add a board hardware note to `doc/bin/cli.md` next to the capture build instructions: Raspberry Pi 5 has no video encoder and Jetson Orin Nano ships diff --git a/quest/m1/clock-anchor-public.md b/quest/m1/clock-anchor-public.md deleted file mode 100644 index d491aa60981..00000000000 --- a/quest/m1/clock-anchor-public.md +++ /dev/null @@ -1,29 +0,0 @@ -# [S] moq-mux exports the per-source clock anchor - -## Goal - -An application running its own demuxer puts several tracks of one source on -the broadcast `Clock` with one shared offset, as the built-in TS, fMP4, and -FLV importers do, without going through a built-in importer. - -## Plan - -Requested by an external consumer (OneTooMany, Discord): their MPEG-TS -ingest publishes video, KLV, and sometimes audio from one program. One -`SourceMap` per track lets tracks drift apart by their first-PTS difference, -and one shared `SourceMap` reads interleave (beyond `MAX_REORDER`) as a reset. - -Decided: make `clock::Anchor` and `clock::Lane` public with the API the -importers already use: `Anchor::new(clock)`, `translate(&mut lane, pts)`, -`translate_at`, `extend`, and `Lane::restart`, with `Lane: Default`. Additive, -so it lands on `main` for a 0.10 patch. - -Open for the PR: the export path. Proposal: `pub mod clock` with -`moq_mux::clock::{Anchor, Lane}`, keeping the root `Clock` re-export. Ask the -maintainer with alternatives. - -Document both in `doc/lib/rs/moq-mux.md`, with a two-track example. - -Their other `ts::Import` gaps (MPEG-2 video, KLV reassembly) stay unplanned -until requested directly. The H.265 per-packet error is -[TS damaged units](/quest/m1/ts-damaged-units.md). diff --git a/quest/m1/close-tail.md b/quest/m1/close-tail.md index 1c1e75dfb9d..0afcfef2769 100644 --- a/quest/m1/close-tail.md +++ b/quest/m1/close-tail.md @@ -46,7 +46,7 @@ subscriber FIN rule, with a changelog entry) and `doc/concept/moq-lite.md` (a subscription ends with both sides' FIN, and a graceful close waits for it). Check that a lite-07 publisher already treats a subscriber FIN after SUBSCRIBE_END as the end of a finished subscription, not a cancel of one still in flight -([Request stream cancel](/quest/m1/request-stream-serve.md)). +(`RequestServe` in `rs/moq-net/src/lite/publisher.rs`). Tests: the reporter's `close_tail` case (one session, paused clock, a mock switch that acks a FIN as soon as it is sent, as a real transport does) fails diff --git a/quest/m1/cluster-idle-timeout.md b/quest/m1/cluster-idle-timeout.md deleted file mode 100644 index 5bbf7d739b8..00000000000 --- a/quest/m1/cluster-idle-timeout.md +++ /dev/null @@ -1,29 +0,0 @@ -# [S] Cluster idle timeout - -## Goal - -A relay notices a silent peer relay within seconds, not after the 30 s QUIC -idle timeout it shares with every viewer. moq.pro's routing simulator found -that failure detection, not routing, sets every outage window: a silent link -or relay loss goes unnoticed for the idle timeout in every routing design, -and subscribes through it go nowhere meanwhile. - -## Plan - -- Cluster sessions get their own idle timeout and keep-alive, shorter than - the `--quic-*` defaults (`rs/moq-tokio/src/quic.rs`, 30 s idle and 5 s - keep-alive). A keep-alive alone only keeps a quiet session open; the - timeout is what detects loss. -- QUIC's effective idle timeout is the smaller of the two endpoints' - (RFC 9000 section 10.1), so the dialing relay's setting bounds both sides of - a cluster link. Say so where the setting is documented. -- Pick the default in the PR with its reasoning: loss detection against - spurious drops on a lossy intercontinental link. -- Docs: `doc/bin/relay/cluster.md` and `doc/bin/relay/config.md`. - -Public API: adds a relay config field; lands on `main`. - -## Related - -- [Cluster routing](/quest/m1/cluster-routing/README.md) - its liveness and failover depend on this detection -- [Broadcast epochs](/quest/m1/broadcast-epoch/README.md) - viewers wait out the same timeout for an old route diff --git a/quest/m1/cluster-routing/README.md b/quest/m1/cluster-routing/README.md index 386f8655e93..63a54c88c09 100644 --- a/quest/m1/cluster-routing/README.md +++ b/quest/m1/cluster-routing/README.md @@ -2,130 +2,148 @@ ## Goal -A cluster carries each broadcast into a region once and never exposes its -backbone to end users. Relays take an explicit edge or core role: edges -serve end users and spread paths over their region's cores, cores link to -the cores of other regions over configured links with static costs, and -routing between them stays path vector. A broadcast under overlapping -prefixes routes to one origin deterministically. Redundancy an operator -configures may deliver more than one copy. - -Non-goals: edges acting as each other's intermediates (it makes edges a -bigger DDoS target), cache-aware route switching, and a permanently -mixed-version cluster. +Any MoQ node routes a subscription toward a broadcast's origin over whatever +links it holds, and a change in the network costs messages in proportion to +the origins it touches, not the broadcasts behind them. One protocol serves +every layout: moq.pro's edge and core tiers, a customer's own cluster, and a +drone swarm where some drones have an uplink, the rest reach it over a lossy +radio mesh, and every drone proxies for its neighbours. A node holding both a +direct link and a CDN link knows which one reaches an origin, so it uses the +CDN only when no direct route exists, and an endpoint linked to several CDNs +prefers its primary and fails over to the next. The CDN never exposes its backbone to +end users, and a broadcast under overlapping prefixes routes to one origin +deterministically. + +Non-goals: link-state topology, cache-aware route switching, TURN (the CDN +route is the fallback), and a permanently mixed-version cluster. ## Plan ### Decisions -Settled in the 2026-09-30 wildcard audit (cache tiers), replacing the -link-state and existence-split design planned earlier that day: - -- Two roles, edge and core. Where a region has one edge, that edge is also - its core. A region may have several cores, which do not link to each other. - An edge dials every core in its region over qmux on TLS (intra-region links - are not congested) and picks one per path by rendezvous hashing, so a - broadcast crosses into a region once. An edge never re-advertises one - core's routes to another, so it is never transit. Cores dial the cores of - the regions they link to, only those with lower names, so each pair has one - connection. Cores are hidden from end users, which is the DDoS shield. See - [Edge and core](/quest/m1/cluster-routing/roles.md). -- Core links are configured and static, and may skip PoPs when the RTT is - short: Seattle pulls from San Jose directly rather than via Oregon. Nothing - switches routes on cache state, so Warm and Cold collapse to one route cost - ([One route cost](/quest/m1/route-cost.md)), and the wildcard line's route - upgrade quest is deleted. -- No link-state topology. With split-horizon edges and a sparse core graph, - path vector with hop lists stays loop-free, path hunting is confined to the - cores, liveness flooding (the simulator's dominant link-state cost) is never - paid, and failure detection sets the outage window either way. The parked - implementation is #4631. If stale paths remain on the tiered layout, the - fix is [Path hunting](/quest/m0/path-hunting.md), promoted to m0: a hold-down - on route updates, with no wire change. -- The line lands on `main`: its children are additive. The two breaking - changes left it for `dev` on their own: - [Remove `--hop`](/quest/m1/hop-removal.md) and - [One route cost](/quest/m1/route-cost.md). -- Cluster links are moq-lite only; moq-transport peers are plain clients - ([moq-transport peers are plain clients](/quest/m1/ietf-cluster-off.md)). -- An epoch-qualified concrete path (`foo/@`) is a source's identity: - origins that announce the same one are interchangeable, which is how a - redundant pair is expressed. A path a claim produces keeps Wildcard's - per-origin identity. See [Selection](/quest/m1/cluster-routing/selection.md). -- Between clusters, announcements stay path vector with cluster ids as hops, - in the last child. -- Anything specific to moq.pro's deployment (generating peer lists and roles - from its inventory, its simulator) is planned in moq.pro, not here. +Settled in the 2026-10-01 audit (#4694), replacing the per-broadcast path +vector the 2026-09-30 cache-tiers audit kept: + +- **Two layers on one stream.** A ROUTE advertises reachability of one origin + node (node id, seqno, metric); an ANNOUNCE says a prefix lives at a route's + node and carries no path. Both ride the session's announce stream, so a + ROUTE always precedes the ANNOUNCEs naming it. A link flap updates one ROUTE + per affected origin instead of re-announcing every broadcast through it, + and ending a broadcast is one fact with nothing to path-hunt. The hop list + and the per-broadcast seqno plan (#4650) are gone. See + [Routes and announces](/quest/m1/cluster-routing/routes.md). +- **Loop freedom per node, not per prefix.** Routes use Babel's feasibility + condition (RFC 8966) keyed by node id. Babel was set aside earlier because + it is not loop-free for a prefix with several origins (RFC 8966 s2.7); + routing to a node removes that case, and a redundant pair is two ANNOUNCEs + of one epoch path. +- **Next-hop authority.** A node stores every neighbour's announces but treats + as live only those from its current next hop toward the origin. Babel's next + hops toward one origin form a tree, so an END is final and no cycle of + neighbours can keep a dead broadcast alive, while losing a neighbour just + moves the next hop and adopts the stored copies. +- **Every hop re-selects.** No origin pinning in SUBSCRIBE: a pin breaks + subscription aggregation and has no good answer when stale. Nearest-origin + selection over consistent metrics is still a shortest-path tree, so it is + loop-free once converged, and a transient cycle collapses into one + aggregated subscription until re-selection moves it. The reply names the + serving node for the splice rule. +- **Policy per link, not roles.** A link marked upstream never receives + routes learned on another upstream link, and a route learned upstream keeps + a down-only mark across other links so a mesh with two uplinks cannot leak + CDN routes back into the CDN. An edge marks its core links upstream; a + Starlink drone marks its CDN link upstream. See + [Upstream links](/quest/m1/cluster-routing/transit.md). +- **One node id space, no cluster ids.** A customer cluster is nodes behind + upstream or trusted links. Loop detection by cluster id would drop a + partitioned swarm's traffic to itself through the CDN (BGP's partitioned-AS + problem). Folding a large customer cluster into one id at its boundary is + [Routing cost domains](/quest/m3/routing-cost-domains.md)'s. +- **Every session speaks it**, relays, apps, and browsers, in the wip lite + version; a plain client with one link advertises only itself. Node ids are + opaque randoms, so routes reveal no backbone addresses. +- **Trust.** Cluster-peer links may advertise any node; a client link's node + ids stay scoped to its session. See + [Route trust](/quest/m1/cluster-routing/route-trust.md). +- **Multi-CDN by link preference.** A node may rank its links (moq.pro + primary, Cloudflare secondary) ahead of the metric, so metrics from + different operators are never compared; a CDN that speaks no ROUTE is just + a link. See [Multi-CDN endpoints](/quest/m1/cluster-routing/multi-cdn.md). +- **Metric on the wire, cost policy local.** One additive metric on ROUTE, + where each hop adds its link cost plus one so it strictly increases; the + origin's per-prefix cost rides ANNOUNCE untouched (`transcode/**` at 10 and + `transcode/foobar` at 1 from one node). Static + configured costs stay the default (the CDN); a radio link may measure its + own ([Link quality](/quest/m1/cluster-routing/link-quality.md)). +- **Topology now, wire later.** moq.pro's edge and core migration runs on + today's lite-06 path vector with upstream links; the route layer lands in + the wip version afterwards with no re-layout, once + [the simulator](/quest/m1/cluster-routing/sim.md) has compared it. + +Kept from 2026-09-30: core links are configured and may skip PoPs; Warm and +Cold collapse to one cost ([One route cost](/quest/m1/route-cost.md)); +epoch-qualified paths are a source's identity +([Selection](/quest/m1/cluster-routing/selection.md)); cluster links are +moq-lite only ([moq-transport peers are plain clients](/quest/m1/ietf-cluster-off.md)); +`--hop` removal and One route cost land on `dev` on their own. Children +changing the wip lite version count as additive and land on `main`. +Anything specific to moq.pro's deployment is planned in moq.pro. ### Simulator findings -Measured on the earlier flat mesh; they motivated the link-state design the -tiers decision replaced, and still bound what path vector costs. +Measured on the earlier flat mesh with moq.pro's `just rs sim` (live's graph) +and `just rs sim sweep` (synthetic regional graphs of 34, 340, and 1020 +relays). -The report is moq.pro's `just rs sim` (every scenario on live's graph) and -`just rs sim sweep` (synthetic regional graphs of 34, 340, and 1020 relays). -It carries messages and bytes by kind, per relay and cross-region, -convergence, loop, stall, and failover windows, and state per relay and per -registry. - -- Existence costs about one message per relay per event flooded, and one per - interested relay plus one per remote registry with registries. On live, a +- Existence costs about one message per relay per event flooded. On live, a wide customer prefix with `.stats` from every relay cost 2.5M announces - today, 4k flooded, and 3.5k with registries, whose relays held 63 records - on average against 94. -- Liveness is the split's dominant cost at scale: flooding every link repeats - each report once per neighbour. At 1020 relays (mean degree 35), a link - loss and restore plus a relay loss and restart cost about 290 MiB of - liveness flooding and 54 MiB of session-up digests against under 1 MiB of - existence for twenty publishes, while a publish costs exactly one message - per relay. -- Unbatched, one relay restart at 340 relays sent half a million liveness - messages; batched for 50 ms, 27k. + under path vector and 4k flooded. +- Link-state liveness flooding dominates at scale: at 1020 relays (mean + degree 35), a link loss and restore plus a relay loss and restart cost + about 290 MiB of liveness against under 1 MiB of existence for twenty + publishes. Per-node distance vector sends a route change only where the + best route changes, which is what the split candidate must show. - Failure detection, not routing, sets every outage window: a silent link or - relay loss is noticed after the 30 s QUIC idle timeout in every candidate, - and subscribes through it go nowhere until then. -- The simulator saw no loop while views agreed, and HRW split an equal-cost - pool 63/49 where a hash of the announced prefix sent all of it to one - sibling (Wildcard has since keyed the tie on the requested path). On live's graph no disagreement looped across 20 seeds of link, - cost, and relay churn. -- One registry per region and two cost about the same; two halves the - busiest registry's load. A registration sent on a dead registry session - that nobody has noticed yet waits for the failover. + relay loss is noticed only after the QUIC idle timeout. +- HRW split an equal-cost pool 63/49 where a hash of the announced prefix + sent all of it to one sibling. - [#4399](https://github.com/moq-dev/moq/pull/4399) trades path hunting for - flapping. In its own full mesh of 8 it retracts each of 100 withdrawn - broadcasts exactly once per relay, as its tests show. On live's graph, with - each writer coalescing for 1 ms, ending three paths loops subscribes for - 9 s instead of 41 s and edges report the ended paths for 38 s summed - instead of 93, but it costs about the same 60k messages, and clients see - 17k updates instead of 12k, with ended paths re-announced 4185 times - instead of 163: each re-announce by a peer revives every route through it, - stale ones included. Written without coalescing, the revivals cascade, and - the same scenario costs 18.6M messages instead of 189k. Per-origin seqnos, - above, end both. + flapping: each re-announce by a peer revives every route through it, stale + ones included, and without coalescing the revivals cascade (18.6M messages + instead of 189k). ANNOUNCEs that carry no path and follow the next hop end + both. ### Remaining work Once every child has landed: -- An end-to-end test: a two-region cluster in `rs/moq-relay/tests` where a - publish, an end, a core loss, and a redundant-pair failover each reach a - subscriber on a far edge, with time mocked. -- Rewrite `doc/bin/relay/cluster.md` into the operator's view (edge and core - layout, dial rules, TLS edge links, link costs, idle timeout, redundant - pairs), and add a routing page under `doc/concept`. +- End-to-end tests in `rs/moq-relay/tests` with time mocked: a two-region + tiered cluster (publish, end, core loss, redundant-pair failover reaching a + far edge), and a drone mesh with two uplinks that partitions and heals + (a broadcast from a drone without an uplink reaches a CDN subscriber, and + each island reaches the other through the CDN). +- Rewrite `doc/bin/relay/cluster.md` into the operator's view (upstream + links, tiers, dial rules, TLS edge links, link costs, redundant pairs), and + add a routing page under `doc/concept` that walks both the CDN tiers and the + drone mesh through the same ROUTE and ANNOUNCE rules. ## Required -- [Edge and core](/quest/m1/cluster-routing/roles.md) - relays take an explicit edge or core role; edges spread paths over their region's cores and are never transit +- [Upstream links](/quest/m1/cluster-routing/transit.md) - a link marked upstream never receives routes learned on another upstream link, which builds edge tiers and drone uplinks from one rule - [Selection](/quest/m1/cluster-routing/selection.md) - a broadcast under overlapping prefixes routes to one origin deterministically, and same-epoch origins are one source -- [Between clusters](/quest/m1/cluster-routing/inter-cluster.md) - announcements crossing a cluster boundary stay path vector with cluster ids as hops +- [Multi-CDN endpoints](/quest/m1/cluster-routing/multi-cdn.md) - an endpoint holds sessions to several CDNs, uses its preferred one, and fails over to the next +- [Simulate the split](/quest/m1/cluster-routing/sim.md) - moq.pro's simulator compares the route layer with path vector before the wire is written +- [Routes and announces](/quest/m1/cluster-routing/routes.md) - ROUTE per origin node and path-less ANNOUNCE on one stream, loop-free by Babel feasibility +- [Route trust](/quest/m1/cluster-routing/route-trust.md) - a peer grant lets a client link advertise the nodes behind it as one identity +- [Link quality](/quest/m1/cluster-routing/link-quality.md) - a radio link's cost follows its measured quality without flapping routes ## Related - [Wildcard](/quest/m0/wildcard/README.md) - the longest-prefix rule, pool spread, and reply identity Selection builds on +- [P2P](/quest/m2/p2p/README.md) - browser and native peers that become routing nodes over this layer - [Remove `--hop`](/quest/m1/hop-removal.md) - on `dev`: redundant publishers share an explicit `@` - [One route cost](/quest/m1/route-cost.md) - on `dev`: Warm and Cold collapse to one static cost - [Same-hop importers](/quest/m1/hop-aligned-import.md) - the importer half of a redundant pair; `--hop` removal re-keys it to a shared epoch - [Broadcast epochs](/quest/m1/broadcast-epoch/README.md) - a redundant pair shares one epoch -- [Cross-relay delivery under bursts](/quest/m1/cross-relay-bursts.md) - its #4349 report shows closed broadcasts announced for up to 229 s, evidence for per-origin seqnos -- [Routing cost domains](/quest/m3/routing-cost-domains.md) - cost across the cluster boundaries this keeps path vector +- [Cross-relay delivery under bursts](/quest/m1/cross-relay-bursts.md) - its #4349 report shows closed broadcasts announced for up to 229 s +- [Routing cost domains](/quest/m3/routing-cost-domains.md) - policy and aggregation at boundaries between operators diff --git a/quest/m1/cluster-routing/inter-cluster.md b/quest/m1/cluster-routing/inter-cluster.md deleted file mode 100644 index 7bc07c072fa..00000000000 --- a/quest/m1/cluster-routing/inter-cluster.md +++ /dev/null @@ -1,36 +0,0 @@ -# [M] Routing between clusters - -## Goal - -Announcements crossing a cluster boundary stay path vector with cluster ids -as the hops, like BGP between autonomous systems. A customer's on-prem -cluster is one hop, and an announcement naming the receiving cluster is -dropped. Inside a cluster, hop lists name relays; across a boundary they name -clusters. - -## Plan - -- A boundary is a configured link to a relay of another cluster; decide how - a relay knows its own cluster id and its peer's. -- What crosses a boundary is a cluster's announcement, plus its cluster-id - list. An imported record keeps that list as provenance, so - whichever boundary relay exports it appends its own cluster id to the full - list; stripping it on import would let A → B → C → A re-enter A. -- The remote origin is outside the receiving cluster, so the - importing boundary relay announces the record inside its cluster as the - origin, with the boundary link's cost added. The serving origin's identity - still rides the reply, so re-originating does not merge two sources. -- Cluster-id lists are short, so they need no `Hop Base`/`Hop Keep` - compression. -- Cost across the boundary is plain configured link cost; business policy - and incomparable costs are - [Routing cost domains](/quest/m3/routing-cost-domains.md)'s. - -Wire: the boundary announcement in the current wip lite version, with the -draft updated in the same PR. Tests: two clusters with two boundary links -and an origin that is not a boundary relay, and three clusters in a cycle with different ingress and egress relays, where -an announcement neither loops nor re-enters its origin cluster. - -## Related - -- [Routing cost domains](/quest/m3/routing-cost-domains.md) - designs policy on this path vector diff --git a/quest/m1/cluster-routing/link-quality.md b/quest/m1/cluster-routing/link-quality.md new file mode 100644 index 00000000000..4fc03107745 --- /dev/null +++ b/quest/m1/cluster-routing/link-quality.md @@ -0,0 +1,34 @@ +# [M] Link quality + +## Goal + +A link can take its cost from its measured quality instead of configuration, +so a drone mesh steers around a fading radio link without manual tuning, +while a small wobble never flips a route. Static configured cost stays the +default and is what the CDN runs; measured cost is opt-in per link. This is +local policy feeding the one metric on the wire, with no wire change. + +## Plan + +Decided 2026-10-01: the wire carries one additive metric and each node +computes its own link costs. + +- Derive the cost from what QUIC already measures (smoothed RTT, loss); an + ETX-style estimate (Babel RFC 8966 Appendix A, B.A.T.M.A.N.'s TQ) is the + reference. Pick the formula from the harness, not from intuition. +- Hysteresis: only advertise a cost change past a threshold or after it + holds, so the route layer sees a few changes per real shift. Report how + many ROUTE changes a fading link causes. +- Never mix it into business pricing: moq.pro's + [priced topology](https://github.com/moq-dev/moq.pro/blob/main/quest/m3/priced-topology.md) + keeps RTT out of monetary costs, and that stays true because measured cost + is opt-in on links the operator chooses. +- A lossy-link fixture over the mock transport with mocked time, reusable by + the line's drone-mesh end-to-end test. + +Public API: a per-link way to ask for measured cost in the peer entry. +Wire: none. + +## Required + +- [Routes and announces](/quest/m1/cluster-routing/routes.md) - a cost change costs one ROUTE per origin there, instead of a re-announce per broadcast diff --git a/quest/m1/cluster-routing/multi-cdn.md b/quest/m1/cluster-routing/multi-cdn.md new file mode 100644 index 00000000000..f073b061ec2 --- /dev/null +++ b/quest/m1/cluster-routing/multi-cdn.md @@ -0,0 +1,49 @@ +# [M] Multi-CDN endpoints + +## Goal + +A client or publisher holds sessions to several CDNs at once, ranked by its +own preference: moq.pro as primary, Cloudflare as secondary. A subscription +goes over the most preferred link that has a route and moves to the next one +when that route goes away; a publisher pushes the same epoch to every CDN so +the move splices instead of restarting. The secondary needs nothing from us: +a moq-transport relay that speaks no ROUTE is just a link. + +## Plan + +Decided 2026-10-01 (moq-dev/moq#4694): + +- Ranking: longest prefix, then the link's preference, then the metric, then + the rendezvous hash. Preference is a per-link rank the node sets (lower + wins, equal by default), separate from the additive link cost, so metrics + from different operators are never added together or compared; within one + preference the metric decides as before. Babel permits any choice among + feasible routes, so loop freedom is unchanged. The ranking lives in + [Selection](/quest/m1/cluster-routing/selection.md); this quest adds the + preference input and the endpoint side. +- A session that speaks no ROUTE (moq-transport, older lite) is a link whose + announces resolve to a session-scoped origin at that link's cost. +- Identity: two CDNs never share node ids, so failover across them splices + only for an epoch-qualified path; a bare path is a discontinuity. The + publisher side is the redundant pair of + [Selection](/quest/m1/cluster-routing/selection.md), from one process. +- API: the Rust client config and `js/net` accept several upstream URLs, each + with its preference (and cost), feeding one shared origin. Mirror the shape + in the bindings that expose connect config. Pick the smallest shape that + covers both publish and subscribe. + +Test with mocked time: a subscriber linked to a lite relay (preferred) and a +moq-transport relay (secondary), with a publisher pushing one epoch to both; +losing the preferred relay moves the subscription at a group boundary with +no timestamp rewind, and its return moves it back. A bare path ends and +resubscribes instead. + +Docs: a multi-CDN section in the routing concept page and the client +connect docs. + +Public API: per-upstream preference in client config, in Rust, JS, and the +bindings. Wire: none. + +## Required + +- [Selection](/quest/m1/cluster-routing/selection.md) - the ranking this adds the preference input to diff --git a/quest/m1/cluster-routing/roles.md b/quest/m1/cluster-routing/roles.md deleted file mode 100644 index c6e2f9628e9..00000000000 --- a/quest/m1/cluster-routing/roles.md +++ /dev/null @@ -1,54 +0,0 @@ -# [M] Edge and core relays - -## Goal - -A relay runs as an edge or a core, and a cluster built from them carries -each broadcast into a region once. An edge serves end users, dials every core -in its region, spreads paths over those cores by rendezvous hashing, and is -never transit between cores. A core serves only cluster peers, dials the -cores of the regions it links to, and never dials an edge. - -## Plan - -Decided in the 2026-09-30 wildcard audit (cache tiers): - -- The role is the node's own `cluster.role`, explicit, from config or - alongside the peer list the connect API returns. A relay whose links - contradict its role (an edge linked to another edge, a core dialing an - edge) fails at startup. Where a region has one edge, that edge is its core. -- A relay with no role is a legacy peer and links any peer, as before roles. - A core accepts and dials a legacy peer as a core-grade link, and an edge - never links one. This is what lets an operator migrate one region at a - time: a migrated region's cores bridge to the unmigrated mesh. A - permanently mixed cluster stays a non-goal. -- A core refuses a session that is not a cluster peer, loudly, so a - misrouted client fails instead of silently exposing the shield. -- Cores dial only cores with lower names, so each pair has one connection; - cores in one region do not link to each other. This is configuration: the - relay follows its peer list, and whatever generates the list applies the - rule. -- Edge links run qmux over TLS on TCP, not WebSocket and not QUIC: - intra-region links are not congested. qmux over TCP exists today only in - plaintext (`tcp://`); add `tls://`, wired to qmux's existing `tls` module. - (Scheme name decided 2026-09-30.) -- An edge never re-advertises a route learned from one core to another core: - its cluster dials publish only what entered locally - (`origin::Consumer::local()`), so no core routes through an edge. -- An edge picks a core per path with the path-keyed rendezvous hash the - wildcard line gave `route_order`, so every edge in a region sends a path to - the same core, and failover moves only the paths the lost core served. The - per-path spread across equally ranked cores is - [Selection](/quest/m1/cluster-routing/selection.md)'s. - -Test: a two-region cluster (two edges and two cores in one region, one core -in the other) in `rs/moq-relay/tests`, where subscribers on both edges of a -region cause one cross-region subscription per broadcast, a route from one -core never reaches another through an edge, a client dialing a core is -refused, and losing a core moves only its paths. - -Public API: a relay config field for the role and a TLS qmux URL scheme. -Wire: none expected. - -## Related - -- [moq-transport peers are plain clients](/quest/m1/ietf-cluster-off.md) - cluster links are lite-only diff --git a/quest/m1/cluster-routing/route-trust.md b/quest/m1/cluster-routing/route-trust.md new file mode 100644 index 00000000000..592e6bd8fe6 --- /dev/null +++ b/quest/m1/cluster-routing/route-trust.md @@ -0,0 +1,39 @@ +# [M] Route trust + +## Goal + +A session may advertise routes only to nodes it is trusted for, so an +authorized publisher cannot claim a cheap route to a CDN relay's node and +attract its subscriptions. Cluster-peer links may advertise any node. A +client link's node ids stay scoped to that session, unless its peer grant +names the nodes it may speak for: then a Starlink drone carries its fleet's +nodes as their real identities, and two uplinks of one swarm reach one origin +instead of two unrelated ones. + +## Plan + +Decided 2026-10-01: + +- Session scoping is the default and ships with + [Routes and announces](/quest/m1/cluster-routing/routes.md): a scoped id + resolves only that session's own authorized announces and never shadows a + route learned on a trusted link. +- Promotion rides [Peer grants](/quest/m1/auth/peer-grant.md), the + hop-bound credential a direct session already presents; extend it to name + the node ids (or an id range or prefix of them) the holder may advertise. + Decide the shape with the auth line. +- A promoted link is still subject to its upstream attribute and the + down-only bit. + +Test: a client advertising a cheap route to a cluster node does not move any +other session's subscription; two uplinks of one mesh holding grants for the +same drone node fail over between each other at a group boundary instead of +ending the subscription. + +Public API: the grant's node scope (`moq-token`, `js/token`, and the token +CLI docs). Wire: none beyond the grant. + +## Required + +- [Routes and announces](/quest/m1/cluster-routing/routes.md) - the node ids this scopes +- [Peer grants](/quest/m1/auth/peer-grant.md) - the credential this extends diff --git a/quest/m1/cluster-routing/routes.md b/quest/m1/cluster-routing/routes.md new file mode 100644 index 00000000000..511beded238 --- /dev/null +++ b/quest/m1/cluster-routing/routes.md @@ -0,0 +1,117 @@ +# [XL] Routes and announces + +## Goal + +Routing info splits into two layers on the session's announce stream, in the +wip lite version, for every session. A ROUTE advertises reachability of one +origin node; an ANNOUNCE says a prefix lives at a route's node and carries no +path. A link flap or relay loss sends one ROUTE change per origin whose best +route changed, never a re-announce per broadcast; ending a broadcast reaches +each node about once and nothing hunts; failover to another neighbour is +seamless; and no state of disagreeing neighbours keeps a dead broadcast +alive. + +## Plan + +Decided 2026-10-01 (see the [line's decisions](/quest/m1/cluster-routing/README.md)): + +- Wire sketch, flexible in detail. ROUTE_START carries the node id, a seqno, + the metric, and a down-only bit, and implicitly takes the next stream-local + Route ID, as ANNOUNCE_START takes the next Announce ID today. ROUTE_UPDATE + (Route ID, seqno, metric, down-only bit) and ROUTE_END (Route ID) reference + it; the bit rides UPDATE so a route moving between a mesh path and a CDN + fallback changes it without retracting its announces. + ANNOUNCE_START carries the prefix (keeping today's Path Base/Keep + compression), a Route ID, and the origin's cost for that prefix. + ANNOUNCE_UPDATE re-prices it and ANNOUNCE_END ends one broadcast while the + route stays up. An announce never changes its Route ID: another origin + serving the same path is another ANNOUNCE. An ANNOUNCE naming an unknown + Route ID is a protocol violation, and ROUTE_END ends that stream's + ANNOUNCEs on the route. The hop list leaves this version. + + ```text + ROUTE_START node=0x7a3f seqno=41 metric=12 -> route 0 + ANNOUNCE_START prefix=transcode/ route=0 cost=10 -> announce 0 + ANNOUNCE_START prefix=transcode/foobar route=0 cost=1 -> announce 1 + ROUTE_UPDATE route=0 seqno=42 metric=15 + ANNOUNCE_END announce=1 # one broadcast ends + ROUTE_END route=0 # ends announce 0 too + ``` +- A stream carries a ROUTE only while some ANNOUNCE on it references it: sent + just before the first, ended after the last. That bounds a client session + to the origins of what it asked for (announce interest already scopes it), + not the CDN's whole node table. +- The announce cost is the origin's per-prefix seed (`transcode/**` at 10 and + `transcode/foobar` at 1 from one node, decided 2026-10-01) and nobody + changes it in transit; a path's cost through a route is that seed plus the + route metric. Link changes therefore touch only ROUTE, and feasibility runs + on the route metric alone. This is [One route cost](/quest/m1/route-cost.md)'s + single cost, which stops accumulating per hop once this lands. +- Each hop adds its link cost plus one to the metric (decided 2026-10-01), so + the metric strictly increases as Babel requires while operators keep + configuring cost 0 for a free link. +- The node id is global and opaque: a relay's `cluster.id` or a random id, + and an app's handshake id. It is needed so routes from two neighbours to + one origin are recognized as one, which carries loop freedom, + deduplication, the reply's serving node, and P2P dialing (an app maps an + announce's node to a roster peer). +- Loop freedom is Babel's feasibility condition (RFC 8966) keyed by node: + accept a route if its seqno is newer, or equal with a metric below the + feasibility distance. Only the origin advances its seqno. Retraction is an + infinite metric or ROUTE_END; there is no count to infinity. +- Next-hop authority: store every neighbour's ANNOUNCEs, but a prefix is live + at a node only as its current next hop toward the origin announces it, the + neighbour a subscribe for that path would go to (metric, then the path-keyed + rendezvous hash among equal next hops). When the next hop changes, adopt + the new neighbour's stored set and send downstream only the difference. + No per-announce seqno. +- Down-only bit: set on a route learned on an upstream link, kept across + other links, and a route carrying it is never sent on an upstream link + ([Upstream links](/quest/m1/cluster-routing/transit.md)). +- A plain client with one link advertises a ROUTE for itself and the + ANNOUNCEs it publishes; its node id is scoped to its session (shared + identity across sessions is [Route trust](/quest/m1/cluster-routing/route-trust.md)). + A relay advertising routes to a client sends them as usual; node ids reveal + nothing about the backbone. +- Every hop re-selects; SUBSCRIBE names no origin. The reply names the + serving node, which is the identity Selection's splice rule uses. +- Mixed versions: a lite-06 peer keeps today's path vector, translated at the + relay that speaks both, for the rollout window only. + +Why not a hold-down: #4644 retracted at once and re-announced a replacement +after 1 s, which ended a downstream front whose only path ran through the +relay, dropping subscribers for about a second while the relay kept serving; +holding without retracting brought hunting back (moq-dev/moq.pro#2116 has the +measurements, and #4642's 300 ms cursor hold is the production mitigation). +Separating "the path changed" (ROUTE) from "the broadcast ended" (ANNOUNCE) +removes that trade, and the failover tests must show no front loses its +route while a working one exists. Decide whether #4642's hold can then go. + +Open, for the implementer to settle and record: + +- Seqno lifetime across restarts of a node with a configured stable id + (persist it, fold an incarnation into the id, or Babel's seqno request). +- Whether starvation recovery needs a request message, or session restart + covers it. +- The window where an origin's ANNOUNCE_END and a next-hop change cross in + flight, and whether it needs more than the next END to arrive. +- How much of today's route trie, fronts, and `route_order` survives intact. + +Tests, time mocked: #4644's live-graph withdrawal and failover tests +(`quest/m0/path-hunting` branch) as acceptance, run on the wip version; a +link flap with many broadcasts behind one origin costs one ROUTE change per +link; a ring of three where a stale neighbour cannot revive an ended +broadcast; a two-uplink mesh that never carries CDN routes back to the CDN; +an island reached through the CDN after a partition; an unknown Route ID +refused. + +Wire: `drafts/draft-lcurley-moq-lite.md` in the same PR, and `js/net` +encodes, decodes, and resolves it (JS transit stays in +[P2P](/quest/m2/p2p/README.md)). Public API: the route-change surface on +`broadcast::Route` and its bindings will likely change; report it. This may +split at start (Rust and draft, then JS), as long as both land in one +release. + +## Required + +- [Simulate the split](/quest/m1/cluster-routing/sim.md) - the numbers that confirm the design before the wire is written diff --git a/quest/m1/cluster-routing/selection.md b/quest/m1/cluster-routing/selection.md index 8cb3b0cbd50..8091855e8cd 100644 --- a/quest/m1/cluster-routing/selection.md +++ b/quest/m1/cluster-routing/selection.md @@ -3,8 +3,9 @@ ## Goal A broadcast under overlapping announcements routes to one origin chosen -deterministically, and an edge spreads paths over its region's cores so -every edge sends a given path the same way. +deterministically, and a relay spreads paths over equally ranked next hops +(an edge over its region's cores) so every relay sends a given path the same +way. Origins that announce the same epoch-qualified concrete path are one source, so a subscriber moves between them at a group boundary when the incumbent ends or becomes unreachable. @@ -27,14 +28,22 @@ Decided: announce the same epoch-qualified path and must not pool. This extends Wildcard's resume rule to concrete same-epoch origins. -Candidate mechanics (re-scoped to path vector by the 2026-09-30 tiers -decision; there is no topology to forward by): +Candidate mechanics: - Route selection stays `route_order` over the routes a relay holds: longest prefix, then cost, then a rendezvous hash (HRW) of the requested path and the route's origin. At an edge that spreads paths over the region's cores and makes every edge pick the same core for a path; failover moves only the paths the lost core won. +- Every hop re-selects (decided 2026-10-01): SUBSCRIBE names no origin, since + a pin breaks subscription aggregation. Once + [Routes and announces](/quest/m1/cluster-routing/routes.md) lands, the + candidates are the origin nodes announcing the path, ranked by longest + prefix, then the link's preference + ([Multi-CDN endpoints](/quest/m1/cluster-routing/multi-cdn.md)), then route + metric to the node, then HRW, and the reply's Origin is + the serving node id. Write the ranking so that change swaps its inputs, not + its shape. - A refusal follows Wildcard's refusal rule: every refusal is terminal, and an origin sheds load by withdrawing or re-pricing its route instead. - The serving origin's identity rides the Origin field of the reply, per @@ -54,9 +63,6 @@ subscribers end and resubscribe rather than splice. ## Required -- [Edge and core](/quest/m1/cluster-routing/roles.md) - the edges and cores this spreads over +- [Upstream links](/quest/m1/cluster-routing/transit.md) - the edges and cores this spreads over - [Wildcard](/quest/m0/wildcard/README.md) - the longest-prefix rule, pool spread, and reply identity this builds on - -## Related - - [Epoch primitive](/quest/m1/epoch.md) - parses the `@` segment that makes a path a source identity diff --git a/quest/m1/cluster-routing/sim.md b/quest/m1/cluster-routing/sim.md new file mode 100644 index 00000000000..d474d82282f --- /dev/null +++ b/quest/m1/cluster-routing/sim.md @@ -0,0 +1,21 @@ +# [XS] Simulate the split + +## Goal + +Waits on moq.pro's routing simulator. The condition clears when moq.pro's +[Simulate the edge and core layout](https://github.com/moq-dev/moq.pro/blob/main/quest/m1/routing-sim-tiers.md) +reports a ROUTE and ANNOUNCE split candidate (per-node distance vector with +Babel feasibility, path-less announces following the next hop) beside path +vector, on the tiered live layout, the tiered synthetic sweeps, and a +synthetic drone mesh (lossy links, two uplinks, a partition that heals). + +Per scenario (publish, end, link flap, relay loss and restart, uplink loss): +messages and bytes by kind, convergence time, stale-path seconds, transient +subscribe loops, and whether a two-uplink mesh ever carries CDN traffic. + +moq-dev/moq.pro#2136 rewrites that quest for the split; until it merges the +moq.pro quest still asks the older path-vector question. To check: look for +the report in a merged moq.pro PR. To advance: land #2136, then ask the +maintainer to start the moq.pro quest. Delete this quest once the report is +in and [Routes and announces](/quest/m1/cluster-routing/routes.md) is +confirmed or revised by it. diff --git a/quest/m1/cluster-routing/transit.md b/quest/m1/cluster-routing/transit.md new file mode 100644 index 00000000000..a938e75adb5 --- /dev/null +++ b/quest/m1/cluster-routing/transit.md @@ -0,0 +1,66 @@ +# [M] Upstream links + +## Goal + +A cluster link can be marked upstream, and a relay never re-advertises a +route learned on one upstream link to another upstream link. Everything else +still transits. That one rule builds moq.pro's edge tier (an edge marks its +region's cores upstream, so it is never transit between them) and a drone +uplink (a Starlink drone marks its CDN link upstream: the mesh reaches the +CDN and the CDN reaches the mesh, but the drone never carries CDN traffic +between CDN relays). Edge, core, and one-node region are an operator's names +for configurations, not relay modes. Clients still see every route a relay +knows. + +## Plan + +Decided 2026-10-01, replacing the per-node `cluster.transit` setting (#4683) +with a per-link attribute, because a drone must transit for its mesh but never +for its CDN uplink, which no per-node switch can express (BGP's +customer/provider export rule, RFC 9234, reduced to one bit): + +- `upstream` is an attribute of a peer entry, from config or the connect + API's peer list, off by default. It applies to the link whichever side + dialed: an edge that is dialed by a core still treats that core as + upstream, so accepted peer sessions need a way to be marked (the peer's + grant classification or identity is the likely source; pick the simplest + shape that covers the reverse-dial test). +- An upstream link is sent only routes that did not arrive on another + upstream link. On today's path vector that is the routes that entered + locally or over non-upstream links. +- The down-only mark that stops a two-uplink mesh from leaking CDN routes + back into the CDN needs a wire bit, so it lands with + [Routes and announces](/quest/m1/cluster-routing/routes.md), not here. +- No roles and no topology check. Who dials whom is the peer list, and + whatever generates it applies the operator's rules (an edge dials its + region's cores, cores dial only lower-named cores of linked regions); the + generator is where that is tested. +- Refusing clients on a hidden core is admission, not routing: the auth + server or embedder refuses a session that is not a cluster peer. +- Relays from before this attribute treat every link as transit, so a cluster + migrates one region at a time with no legacy mode. +- Spreading paths over equally ranked upstream links by the path-keyed + rendezvous hash is [Selection](/quest/m1/cluster-routing/selection.md)'s; + it is what makes every edge of a region send a given path to the same core. +- Edge links run qmux over TLS on TCP, not WebSocket and not QUIC: + intra-region links are not congested. qmux over TCP exists today only in + plaintext (`tcp://`, moq-tokio's `tcp.rs`); add `tls://` beside it on + moq-tokio's `tls` helpers. (Scheme name decided 2026-09-30.) + +Test: a two-region cluster in `rs/moq-relay/tests`. One region has two edges +whose links to two cores are upstream; the other is one relay that also +serves clients. Subscribers on both edges cause one cross-region subscription +per broadcast, a route from one core never reaches the other through an edge, +a client on the far relay gets a broadcast published on an edge, a core that +dials an edge still gets no routes through it, and losing a core moves only +its paths. Add a three-node line (mesh node, uplink node, CDN relay) where the +mesh node's broadcast reaches the CDN and a CDN broadcast reaches the mesh +node. + +Public API: an `upstream` peer attribute and a TLS qmux URL scheme. +`cluster.tier` is unrelated: it stays the billing and stats label. +Wire: none. + +## Related + +- [moq-transport peers are plain clients](/quest/m1/ietf-cluster-off.md) - cluster links are lite-only diff --git a/quest/m1/cluster-shims.md b/quest/m1/cluster-shims.md new file mode 100644 index 00000000000..5713f838f0d --- /dev/null +++ b/quest/m1/cluster-shims.md @@ -0,0 +1,21 @@ +# [XS] Delete the removed cluster flags + +## Goal + +`cluster::Config` no longer carries the hidden `mesh` and `linger` fields, +and `--cluster-mesh` and `--cluster-linger` are unknown flags like any other. + +## Plan + +Both fields exist only to refuse their removed settings at startup with a +pointer to the replacement (`deprecated()` in `rs/moq-relay/src/cluster.rs`, +checked there and in `config.rs`). The linger refusal shipped in +moq-relay 0.15.0; the mesh refusal (#4601) ships in the next release. Once a +release has carried both, delete the fields, `deprecated()`, and its checks, +per the no-shim rule. Lands on `dev`: removing public fields is a break. + +Public API: removes two hidden `cluster::Config` fields. Wire: none. + +## Required + +- [A moq-relay release refuses --cluster-mesh](/quest/m1/relay-mesh-refusal-release.md) - the release that carries the last refusal diff --git a/quest/m1/cpp/README.md b/quest/m1/cpp/README.md index 92bdb368985..9a8655c78bc 100644 --- a/quest/m1/cpp/README.md +++ b/quest/m1/cpp/README.md @@ -77,5 +77,4 @@ Confirmed in [#4100](https://github.com/moq-dev/moq/pull/4100): - [vcpkg registry](/quest/m2/cpp-vcpkg.md) - a registry we own serves the prebuilt package to `vcpkg` manifests - [Conan remote](/quest/m3/cpp-conan.md) - a remote we own serves the prebuilt package to Conan -- [C# through moq-ffi](/quest/m3/cs/README.md) - the same recipe with NordSecurity's C# generator - [Unreal prototype](/quest/m3/unreal.md) - a UE5 module consumes the package with exceptions disabled diff --git a/quest/m1/cpp/generator.md b/quest/m1/cpp/generator.md index 66dc7d8be83..dfca3a0dab4 100644 --- a/quest/m1/cpp/generator.md +++ b/quest/m1/cpp/generator.md @@ -41,7 +41,3 @@ stop here and write down why. group before dropping its write future). - Offer both the 0.32 port and the expected flag upstream to LiveKit and NordSecurity; the fork exists only until they tag. - -## Related - -- [C# generator](/quest/m3/cs/generator.md) - the same 0.32 port against NordSecurity's C# generator diff --git a/quest/m1/cross-relay-bursts.md b/quest/m1/cross-relay-bursts.md index a7a31241d4a..3dc14f10303 100644 --- a/quest/m1/cross-relay-bursts.md +++ b/quest/m1/cross-relay-bursts.md @@ -27,4 +27,4 @@ cluster-routing evidence, not this quest's scope. ## Related -- [Cluster routing](/quest/m1/cluster-routing/README.md) - owns the stale and flapping announcements from the same report +- [Routes and announces](/quest/m1/cluster-routing/routes.md) - owns the stale and flapping announcements from the same report diff --git a/quest/m1/data-consumer-timestamps.md b/quest/m1/data-consumer-timestamps.md index 3ac4aebdbe3..ec755c545d5 100644 --- a/quest/m1/data-consumer-timestamps.md +++ b/quest/m1/data-consumer-timestamps.md @@ -15,24 +15,37 @@ sync with video. Every `moq_net::Frame` has a timestamp, but the consumers decode only `frame.payload` and drop it: `moq_mux::{json,binary}::Consumer::next` and the -moq-json and moq-flate snapshot and stream consumers they wrap. A snapshot -consumer returns the timestamp of the frame it decoded last. +moq-json and moq-flate snapshot and stream consumers they wrap. Each snapshot +state carries the timestamp of the frame that produced it. Decided: `next()` and `poll_next()` return `Timed`, the type the producers -take, whose `at` becomes required in -[Publishing requires a timestamp](/quest/m1/publish-timestamp.md). The -timestamp is the frame's media timestamp on the track's timescale. -[Plan: untimed peer objects](/quest/m1/plan-untimed-objects.md) decides -whether a peer can deliver a frame without one; if it can, `at` becomes an -`Option` here, so this waits for it and breaks once. +take in [Publishing never invents a timestamp](/quest/m1/publish-timestamp.md). +`at` is the frame's media timestamp on the track's timescale, and `None` for +an untimed frame: [Plan: untimed objects](/quest/m1/plan-untimed-objects.md) +settled (2026-10-01) that absence survives the wire rather than becoming +arrival time. A republisher passes `at` straight to a moq-mux data producer. + +Decided (2026-10-01): snapshot consumers get two reads, both returning +`Timed`. Today the moq-json snapshot consumer applies every buffered delta +but yields only the newest state, and moq-flate jumps to the newest group, so +a 9s state is lost when 11s is already buffered. A caller syncing to a +playhead needs the newest state at or before it. + +- `next()` (and `poll_next()`) yields every state in order, like a stream + consumer. A reader that falls behind the drift budget (`Lagged`) still + jumps to the newest group, so a slow reader stays bounded. +- `latest()` (and `poll_latest()`) skips to the newest state, today's + behavior, kept as the optimization for callers that only want the current + value. +- Same names in the moq-mux wrappers, moq-ffi, and every binding. moq-ffi's json/flate consumers return the timestamp too, and the py, swift, -kt, go, and dart wrappers and `doc/lib/*` follow. JS readers stay with -[Data sync in watch](/quest/m3/watch-data-sync.md). +kt, go, and dart wrappers and `doc/lib/*` follow. JS is +[JS data consumer timestamps](/quest/m1/js-data-consumer-timestamps.md). Public API: breaking, on `dev`. Wire: none. ## Required -- [Publishing requires a timestamp](/quest/m1/publish-timestamp.md) - makes `Timed.at` required, the type this returns -- [Plan: untimed peer objects](/quest/m1/plan-untimed-objects.md) - decides whether the returned timestamp can be absent +- [Publishing never invents a timestamp](/quest/m1/publish-timestamp.md) - gives `Timed.at` its untimed meaning, the type this returns +- [Plan: untimed objects](/quest/m1/plan-untimed-objects.md) - the model must carry an absent timestamp to consumers diff --git a/quest/m1/demand-everywhere.md b/quest/m1/demand-everywhere.md new file mode 100644 index 00000000000..9db4a1b3aaa --- /dev/null +++ b/quest/m1/demand-everywhere.md @@ -0,0 +1,46 @@ +# [M] Demand is the one way to watch subscribers + +## Goal + +On `dev`, every model handle that serves subscribers watches them through a +`Demand` handle returned by `demand()`: `track::Request`, `group::Request`, +`group::Producer`, and `broadcast::Producer`. Their own +`used`/`unused`/`is_used`/`poll_unused` copies are gone, in Rust and `js/net`. +moq-ffi then exposes the same handle on its request and producer types through +the [FFI shape](/quest/m1/ffi-shape/README.md) line, so a binding's fetch +handler can see a group request abandoned. + +## Plan + +Facts (2026-10-01): `track::Demand` and `broadcast::Demand` already exist. +[#4528](https://github.com/moq-dev/moq/pull/4528) (the track-demand quest) moved +`track::Producer` to `demand()` on `dev`, and deliberately kept group and +broadcast `used`/`unused`, since group demand drives fetch coalescing. +`broadcast::Producer` is already `demand()`-only, with `used`/`unused` on +`broadcast::Demand`; there is no `group::Demand` yet. +`track::Request` and `track::Dynamic` have `poll_unused`, and #4691 added +`group::Request::poll_unused`. moq-ffi wraps only `track::Demand` +(`MoqTrackDemand`); `MoqGroupRequest` and `MoqTrackRequest` have no demand. + +Decided (2026-10-01): + +- ✅ Everything under `Demand`, reversing track-demand's call to keep group and + broadcast `used`/`unused`. Rejected: Requests only. +- ✅ The binding exposure follows this quest in the FFI shape line instead of + wrapping `poll_unused` now. Rejected: a standalone moq-ffi quest for + `poll_unused`, and waiting in m2 for a consumer. +- ✅ m1, ahead of FFI shape, so the bindings expose `Demand` once. + +- ✅ One type per level (`track::Demand`, `group::Demand`, + `broadcast::Demand`) with the same method set, like the two that exist. + Rejected: one generic `Demand`, which erases level-specific state such as + track priority. + +Keep fetch coalescing's group demand semantics, and keep `abort_unused` where +its race still needs an owner. + +Public API: breaking on `dev` in moq-net and `@moq/net`. Wire: none. + +## Related + +- [IETF FETCH abandonment](/quest/m1/ietf-fetch-abandonment.md) - first new consumer of group request demand; can ship on `poll_unused` without waiting for this diff --git a/quest/m1/drain-handshakes.md b/quest/m1/drain-handshakes.md new file mode 100644 index 00000000000..037d72f9f7a --- /dev/null +++ b/quest/m1/drain-handshakes.md @@ -0,0 +1,29 @@ +# [S] Drain in-flight handshakes + +## Goal + +A relay drain counts a session from admission, not only once it is +established, so a client still in its handshake when the drain starts is sent +a GOAWAY and waited for instead of being cut off by the relay exiting. The +drain still ends at its deadline whatever a handshake does. + +## Plan + +`shutdown::Observer::serve` is taken in `supervise`, after the handshake. A +drain that starts before any session reaches it sees nothing to wait for and +`Relay::run` returns at once. #4186 chose this on purpose, reasoning that with +DNS withdrawn first, arrivals are rare. The io_uring drain test hit it anyway: a +moq-lite-06 client sees its session before the relay has admitted it, so the +test now holds a straggler to keep the drain open. + +Take the count where the relay first commits to a session (the accept or +registration point on each of the QUIC, io_uring, WebSocket, and iroh paths), +and hand it to the session so the guard spans auth and the handshake. A +session admitted during a drain already gets a GOAWAY carrying the time left, +so the change should be the count alone. Revisit whether a handshake that fails +auth, or never completes, should hold a drain until the deadline. + +Add a regression test that triggers the drain between a client's connect and +the relay's admission and expects a GOAWAY, then drop the straggler workaround +in `rs/moq-relay/tests/runtime_uring.rs`. Update `doc/bin/relay/config.md`, +which documents the current behavior. diff --git a/quest/m1/drain/README.md b/quest/m1/drain/README.md deleted file mode 100644 index f1195be9dfa..00000000000 --- a/quest/m1/drain/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Graceful relay drains (GOAWAY) - -## Goal - -Relay restarts drain sessions instead of hard-dropping them. The end state: a -draining node is first withdrawn from DNS (marked unhealthy so resolvers stop -handing it out), waits out the DNS TTL plus a margin for monitor detection, -THEN sends GOAWAY on every MoQ session. Clients reconnect through a fresh DNS -resolve and land on a different relay, and a straggler that dials the draining -node anyway (a cached resolve, or a pool alias) just gets another GOAWAY. Only -after sessions drain or the stop deadline expires does the process exit and -the new software boot. - -GOAWAY only reaches MoQ sessions, and only the Rust client acts on it today: -`moq_tokio::Connection` migrates, while `js/net` decodes and logs the message -and at most closes the session afterwards (the IETF path does, the lite path -does not), leaving any reconnect to the ordinary close-triggered backoff -rather than migrating. The wire message is lite04+/IETF only besides. So the stop deadline is the -real backstop - for pre-lite04 versions, for client SDKs deployed before -client-goaway ships, and for in-process ingest gateways (RTMP/SRT/WHIP/WHEP), -which have no GOAWAY equivalent at all: their grace is the DNS-drain window -stopping new arrivals plus the encoder's own reconnect. The DNS-drain-first -ordering is what keeps that hard-close window small. - -## Plan - -This questline holds the two relay/client halves. The orchestration around -them (a planned-drain health state, the SIGTERM sequencing and stop timeouts, -per-PoP serial deploys, a two-node PoP floor, and the gateway drain contract) -is moq.pro's (downstream) fleet drain work, which consumes these quests. - -Drain stays at the MoQ layer. WebTransport's `WT_DRAIN_SESSION` capsule and -the browser `draining` promise are advisory and carry no redirect URI or -timeout, and qmux and WebSocket have no equivalent, so neither the relay nor -the clients send or act on them (decided 2026-09-26). - -**relay-drain-api.** A drain hook that GOAWAYs every established session and -immediately GOAWAYs any new arrival, so an embedding process can enter drain -on SIGTERM after the DNS window and still bound the total stop time. Expose -enough phase/session state to prove which bound ended a drain. - -**client-goaway.** The JS reconnector migrates like the Rust one, preserving -the app-visible session while resolving DNS again before dialing, and the Rust -path gains the regression test it lacks. This is a -scale-down prerequisite, not merely a deploy improvement. RTMP/SRT/WHIP/WHEP -cannot receive MoQ GOAWAY, so their contract remains DNS withdrawal followed -by the stop deadline and encoder reconnect. - -## Required - -- [Relay drain api](/quest/m1/drain/relay-drain-api.md) - a drain hook that - GOAWAYs every session, including new arrivals, triggered by the embedding - process on SIGTERM -- [Client goaway](/quest/m1/drain/client-goaway.md) - the JavaScript client - migrates on GOAWAY with a handover and the guarded redirect the Rust client - already has, and the Rust drain path gets its regression test -- [JS GOAWAY requests](/quest/m1/drain/js-goaway-requests.md) - after GOAWAY the JS client opens no new request on the old session, like Rust - -## Related - -- [Cluster routing](/quest/m1/cluster-routing/README.md) - the configured topology and link costs a second relay per PoP joins diff --git a/quest/m1/drain/client-goaway.md b/quest/m1/drain/client-goaway.md deleted file mode 100644 index 5d611d75dca..00000000000 --- a/quest/m1/drain/client-goaway.md +++ /dev/null @@ -1,87 +0,0 @@ -# [L] Client goaway - -## Goal - -The JavaScript client migrates on GOAWAY the way the Rust client already -does: it dials the replacement while the old session keeps serving, follows a -redirect URI under the same guard, and keeps the app-visible handle and its -origins across the swap. The Rust client's fleet-drain path (an empty-URI -GOAWAY redialed through a fresh DNS resolve) gains the regression test it -lacks today. - -## Plan - -moq.pro's (downstream) fleet drain orchestration relies on this behavior: a -drained node is already out of DNS when GOAWAY fires, so a re-resolve is what -lands clients on a healthy relay. - -Everything below describes `dev`, which is where this quest lands: `main` -still has `moq-native`'s close-only `Reconnect`. The drain loop is written -once into `Connection` and its pool. - -### Rust policy and proof - -`moq_tokio::Connection` already handles GOAWAY: the session loop returns the -message, `Redirect::resolve` guards the URI (scheme tier never drops, the host -is pinned to the configured one by default, and `follow` is the opt-in that -lets a peer name another), the loop redials while a -`Draining` handle keeps the old session serving until it closes or overstays -the handover cap, and `Status::Migrating` is visible to callers. Every dial -resolves DNS again, since `Addrs` holds URLs and each backend resolves at dial -time, so nothing is pinned. The relay always sends an empty URI, and the only -end-to-end coverage is the cluster sibling test with a redirect. - -Add the fleet-drain regression test: a relay GOAWAYs with an empty URI and a -timeout, the client redials the configured URL through a fresh resolve (a -resolver the test can repoint), live tracks hand over at a group boundary, -and the old session closes within the handover cap. - -### JavaScript is greenfield - -`js/net` logs the lite GOAWAY URI and keeps that session open, -logs the IETF draft-17+ URI and then closes the session when its control -loop ends, and on the draft-14 to -16 shared control stream reads the message -body but returns without decoding it. Nothing migrates: `Connection` -reconnects only after `closed` fires, through its backoff, tears the old -connection down in its effect cleanup, and the pool -(`js/net/src/connection/pool.ts`) keys by URL href. - -- Surface the peer's GOAWAY on the live session as a drain signal carrying the - resolved URI and the timeout, decoded on every wire the client speaks, - including the draft-14 to -16 adapter route that currently returns without - decoding the body it has already read. -- `Connection` mirrors `Draining`: on GOAWAY it dials the target immediately, - swaps the origin wiring (`forwardAnnounced`, `publish`, `subscribe`) once - the replacement is established, and leaves the old session to close on its - own or at a handover cap: the configured cap, lowered to the peer's timeout - when the wire carried a positive one. Lite and IETF drafts 14 to 16 carry no - timeout, and the IETF decoder reads an absent one as zero, so absence means - the cap and never a zero-length handover. Groups in flight finish. A GOAWAY does not go through the backoff delay; a failed - replacement dial does. -- Port the guard: same-host by default, refuse a scheme-tier drop or a - widening to a local host, and offer the follow mode. An empty URI preserves - the current address list, including caller-selected fallbacks, and starts - normal migration. A malformed or policy-refused explicit redirect ends the - connection with a typed terminal error; it must not trigger a retry against - the original address or another configured fallback. Only an accepted - redirect replaces the list. Apply and test this policy in Rust as well as JS; - do not assume the existing Rust behavior already satisfies it. A redirect with a certificate pin (`serverCertificateHashes`) - is refused unless the host is unchanged, since the pin cannot verify another - relay; the pool already refuses to share pinned connections. -- The pool re-keys its entry to the redirect target, so a later - caller configured with that URL shares the migrated connection. The app's - handle and shared origin are unchanged; only the pool key moves, and a - caller still asking for the original URL gets a fresh entry. When the - target key already holds a live entry, that entry wins: the migrating entry - is removed from the pool without replacing the target entry. Existing - handles keep its migrated connection, but a new lookup for the original URL - dials fresh and a target lookup joins the target entry. The migrated entry - retires when its last existing handle releases it. Two connections to one - relay for that overlap is the honest cost; entry removal is identity-guarded - so neither cleanup can delete another entry. -- Tests against the in-tree relay: an empty-URI drain migrates without a - dropped group, a GOAWAY without a timeout hands over at the configured cap, - a redirect moves the pool key and origins, a redirect onto an already - pooled key keeps existing handles on both entries but makes a new caller for - the original URL dial fresh, each guard refusal closes rather than - reconnects, and the draft-14 to -16 route decodes the URI. diff --git a/quest/m1/drain/relay-drain-api.md b/quest/m1/drain/relay-drain-api.md deleted file mode 100644 index 7ff107aa3f2..00000000000 --- a/quest/m1/drain/relay-drain-api.md +++ /dev/null @@ -1,22 +0,0 @@ -# [M] Relay drain api - -## Goal - -Expose a drain hook that sends GOAWAY on every relay session. Include new -arrivals while draining, so an embedding process can trigger it on SIGTERM. - -Identical behavior on both runtimes: a session served from an io_uring worker -drains the same as one served from a tokio worker. - -## Plan - -moq.pro's (downstream) fleet drain orchestration consumes this hook: its edge -process sheds the node from GeoDNS, waits out the TTL, then fires the drain. - -The signal path already reaches io_uring sessions: `drain_on_signal` fires the -broadcast, per-session `supervise` sends the GOAWAY from the shared runtime, -and `uring::Workers::shutdown` joins the threads only afterwards. What is -missing on both runtimes is the arrival half, and on the io_uring side nothing -proves any of it. Cover the new-arrival GOAWAY on the io_uring accept path -too, and add the drain case to `rs/moq-relay/tests/runtime_uring.rs`: a -session on a worker receives GOAWAY, closes, and the threads join cleanly. diff --git a/quest/m1/fetch-ok-properties.md b/quest/m1/fetch-ok-properties.md new file mode 100644 index 00000000000..69271af05bf --- /dev/null +++ b/quest/m1/fetch-ok-properties.md @@ -0,0 +1,38 @@ +# [S] FETCH_OK carries the track's properties + +## Goal + +Our IETF publisher's FETCH_OK carries the same track properties its +SUBSCRIBE_OK does: max cache duration, timescale, priority, and group order. +A fetch-only reader then learns the track as a subscriber would, and a relay +meets draft 16+'s rule to include all of a track's properties. + +## Plan + +Found (2026-10-01) while merging main into dev (#4647): a group FETCH with no +prior SUBSCRIBE_OK now reads its max age from FETCH_OK, but our publisher +sends an empty block. + +- `rs/moq-net/src/ietf/publisher.rs` sends FETCH_OK with + `properties: Default::default()`. SUBSCRIBE_OK fills them in the block near + `publisher.rs:655`. Share that code so the two can't diverge. +- Our group-fetch accept path reads only the max age from FETCH_OK and + hardcodes a microsecond timescale. Apply the same properties SUBSCRIBE_OK + does (timescale, priority, group order) there too. +- Drafts 14-15 allow the omission: MAX_CACHE_DURATION is a MAY there, and + FETCH_OK has no properties block. From draft 16, a relay MUST include all + Extension Headers / Properties associated with a track in FETCH_OK (d16 + §8.6, d22 §7.7). An empty block also misleads the reader: it infers + Ascending order and priority 128 where SUBSCRIBE_OK says Descending. +- Honour INCLUDE_PROPERTIES (0x35) on FETCH from draft 20. It defaults to + sending the properties; at 0 the block is present but empty. `fetch.rs` + decodes it on draft 20+ and drops it, so it is not honoured today. +- Test per draft range: FETCH_OK round-trips the properties SUBSCRIBE_OK would + carry, and INCLUDE_PROPERTIES = 0 empties the block. + +Public API: none. Wire: FETCH_OK gains its properties on drafts that define +the block. Interop: run `just test interop --all`. + +## Related + +- [Fetch without SUBSCRIBE](/quest/m1/ietf-fetch-only.md) - asks TRACK_STATUS for the same properties when a publisher omits them diff --git a/quest/m1/frame-slot-charge.md b/quest/m1/frame-slot-charge.md index 4b61d31b08a..8ac314fc252 100644 --- a/quest/m1/frame-slot-charge.md +++ b/quest/m1/frame-slot-charge.md @@ -28,7 +28,7 @@ up to. Two gaps follow: The fix is not a bigger constant. The charge has to follow the deque's capacity: charge the growth at each `charge.add` site (`write_frame`, the -`write_frames` loop, `create_frame`, and `create_frame_owned`), keeping +`write_frames` loop, and `GroupState::charge_partial` for a streamed frame), keeping `FRAME_SLOTS` as the part `CACHE_OVERHEAD` already paid. Decide what `MAX_CACHE_BYTES` compares against before touching any of it. @@ -47,7 +47,3 @@ Found by CodeRabbit on #3523, which fixed the one-frame-per-group undercount that was OOM-killing relays serving chat, and deliberately left out of it. Public API: none, unless `MAX_CACHE_BYTES` is restated. Wire: none. - -## Required - -- [Frame alloc budget](/quest/m0/frame-alloc-budget.md) - edits the same group cache charge; land it first diff --git a/quest/m1/gpu-ci.md b/quest/m1/gpu-ci.md index b3b0d6175df..6dbcbc6622c 100644 --- a/quest/m1/gpu-ci.md +++ b/quest/m1/gpu-ci.md @@ -42,16 +42,12 @@ skipping inside the Nix shell. job gated to `refs/heads/main`, never `pull_request`; a dedicated label only this job selects; read-only `permissions`. Read GitHub's self-hosted runner hardening guidance before wiring it. -- Share the runner with the io_uring one that #4132 plans - (`quest/m1/uring-runner.md` on the drain line, which wants a 6.12+ kernel on - the same host): one registration and one security posture, a label per - capability. Whichever quest lands second reuses the first's job shape. Public API: none. Wire: none. ## Required -- A self-hosted runner is registered for moq-dev/moq on the maintainer's host, with the NVIDIA driver +- [A self-hosted NVIDIA runner is registered](/quest/m1/gpu-runner.md) - the host the nightly job runs on ## Related diff --git a/quest/m1/gpu-runner.md b/quest/m1/gpu-runner.md new file mode 100644 index 00000000000..591b3b7bb8d --- /dev/null +++ b/quest/m1/gpu-runner.md @@ -0,0 +1,18 @@ +# [XS] A self-hosted NVIDIA runner is registered + +## Goal + +A self-hosted GitHub Actions runner is registered for moq-dev/moq on the +maintainer's Linux host (RTX 3070 Ti), with the NVIDIA driver installed. This +is a maintainer action: only someone with admin access to the repository and +the host can do it. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +As of 2026-09-30 `gh api repos/moq-dev/moq/actions/runners` lists none. +Register it with the dedicated label and hardening that +[GPU CI](/quest/m1/gpu-ci.md) describes, and share the registration with the +io_uring runner the drain line plans. diff --git a/quest/m1/hop-removal.md b/quest/m1/hop-removal.md index 7000837b894..f6cf8740156 100644 --- a/quest/m1/hop-removal.md +++ b/quest/m1/hop-removal.md @@ -19,15 +19,17 @@ connection cannot publish several broadcasts with different identities. an epoch a caller supplies ([Broadcast epochs](/quest/m1/broadcast-epoch/README.md)). Give `moq` a flag for it if Broadcast epochs has not. - Relays keep `cluster.id` as their identity; `--hop` fills it today - (`rs/moq-cli/src/args.rs`), so split that coupling. Decide whether relays - still need the Hop setup parameter or learn identity from topology. + (`rs/moq-cli/src/args.rs`), so split that coupling. Relays and P2P peers + keep the Hop setup parameter: it becomes the node id of + [Routes and announces](/quest/m1/cluster-routing/routes.md). Only the + publisher-facing use goes. - Re-key [Same-hop importers](/quest/m1/hop-aligned-import.md)'s docs and 1+1 tests to a shared epoch. - Docs: rewrite "Redundant publishers" in `doc/bin/cli.md` and `doc/concept/use-case/contribution.md`, then search the repo for `--hop`, `MOQ_HOP`, and `--cluster-id` examples, including demo recipes. -Open: whether anything declares an incumbent dead faster than the cluster +Open: whether anything declares an incumbent dead faster than the QUIC idle timeout (a failover service that retracts it, or active-active delivery to the relay). Neither is required to land this. @@ -39,7 +41,3 @@ parameter; lands on `dev`, outside the cluster routing line (which lands on `mai - [Selection](/quest/m1/cluster-routing/selection.md) - owns the same-epoch pool this relies on - [Broadcast epochs](/quest/m1/broadcast-epoch/README.md) - a restart is a new path, not a splice - [Same-hop importers](/quest/m1/hop-aligned-import.md) - the importer fixes this re-keys - -## Related - -- [Cluster idle timeout](/quest/m1/cluster-idle-timeout.md) - bounds how long a dead incumbent holds its pair diff --git a/quest/m1/ietf-cluster-off.md b/quest/m1/ietf-cluster-off.md index 4c8bd4be470..3aeda66a886 100644 --- a/quest/m1/ietf-cluster-off.md +++ b/quest/m1/ietf-cluster-off.md @@ -36,5 +36,5 @@ negotiating an optional extension. ## Related -- [Edge and core](/quest/m1/cluster-routing/roles.md) - cluster links are lite-only +- [Upstream links](/quest/m1/cluster-routing/transit.md) - cluster links are lite-only - [Wildcard](/quest/m0/wildcard/README.md) - the pool spread that makes a Hop ID label unsafe to splice on diff --git a/quest/m1/ietf-fetch-abandonment.md b/quest/m1/ietf-fetch-abandonment.md new file mode 100644 index 00000000000..6f3977b781c --- /dev/null +++ b/quest/m1/ietf-fetch-abandonment.md @@ -0,0 +1,32 @@ +# [S] An abandoned IETF group FETCH is cancelled + +## Goal + +When the last reader leaves a group that the Rust moq-transport subscriber is +fetching upstream, it cancels that FETCH (STOP_SENDING plus a reset of the +request stream, or FETCH_CANCEL on drafts that define it), before and after +FETCH_OK, the same as an idle SUBSCRIBE and the lite FETCH after #4691. The +partial group aborts with `Error::Cancel` and never ends clean. + +## Plan + +Facts (`origin/main`, 2026-10-01): `run_group_fetch` in +`rs/moq-net/src/ietf/subscriber.rs` sends the FETCH and waits only for the +request stream to close or the group slot to finish. It never watches demand. +It is cancelled only when its whole subscription ends. The subscribe path +already polls `poll_unused` and calls `cancel_subscribe`. Rust already encodes +and decodes `ietf::FetchCancel` (0x17, `rs/moq-net/src/ietf/fetch.rs`) and the +adapter routes it, but `run_group_fetch` never sends it. JS never issues an IETF FETCH (`fetchGroup` refuses on +moq-transport), so it is out of scope. + +Decided (2026-10-01): cancel after the answer too, matching SUBSCRIBE; the +fetched group is not filled into the cache for absent readers. Reuse +`group::Request::poll_unused` from #4691, or `demand()` if +[Demand everywhere](/quest/m1/demand-everywhere.md) lands first. Send the +existing `FetchCancel` on the drafts that define it (14 to 16) and reset the +request stream on later drafts, which removed FETCH_CANCEL. + +Tests: a mock-time test abandons a group FETCH before and after FETCH_OK and +asserts the upstream stream is cancelled. + +Public API: none. Wire: none. diff --git a/quest/m1/ietf-fetch-location.md b/quest/m1/ietf-fetch-location.md index 54daf01a6a2..daf1f8bf4e7 100644 --- a/quest/m1/ietf-fetch-location.md +++ b/quest/m1/ietf-fetch-location.md @@ -10,14 +10,10 @@ touching several groups is refused `NOT_SUPPORTED`, as on older drafts. ## Plan Today both sides refuse FETCH on draft-20+ `NOT_SUPPORTED`: the publisher in -`run_fetch_stream`, and the relay's upstream group fill in `run_group_fetch`, -because the codec still carries the removed Fetch Type field. -[Legal IETF input](/quest/m0/ietf-legal-input.md) fixes the codec. Then drop -both refusals, and flip `fetch_moq_transport_20` in `rs/moq-tokio/tests` back -to a served fetch. +`run_fetch_stream`, and the relay's upstream group fill in `run_group_fetch`. +The codec already decodes and encodes the draft-20 layout as +`FetchType::Filtered`, so drop both refusals, and flip +`fetch_moq_transport_20` in `rs/moq-tokio/tests` back to a served fetch. A +FETCH carrying Range Filters stays refused. Update the draft-20 note in `doc/concept/standard.md`. - -## Required - -- [Legal IETF input](/quest/m0/ietf-legal-input.md) - decodes the draft-20 FETCH this serves diff --git a/quest/m1/ietf-fetch-only.md b/quest/m1/ietf-fetch-only.md index a7bdc908e66..c904d30e600 100644 --- a/quest/m1/ietf-fetch-only.md +++ b/quest/m1/ietf-fetch-only.md @@ -24,6 +24,26 @@ TRACK_STATUS is the likely source of the info. The publisher refuses it today, so both sides are in scope. Keep the SUBSCRIBE path for real subscription demand. +Decided (2026-10-01): + +- Our subscriber learns a track's properties from TRACK_STATUS, sent in + parallel with every FETCH or SUBSCRIBE. From draft 20 it sets + INCLUDE_PROPERTIES to 0 on those requests, so the properties aren't sent + twice. Before draft 20 there's no opt-out, so it ignores the duplicate. +- First decouple object timestamps from INCLUDE_PROPERTIES: the publisher + derives its serving timescale from `properties_wanted` + (`publisher.rs:628`), so opting out today strips the Timestamp property + from every object and the subscriber falls back to arrival time. Test + TRACK_STATUS plus the opt-out with source timestamps that differ from + arrival. +- The publisher serves TRACK_STATUS. Its reply carries what it would set in a + SUBSCRIBE_OK (d22 §9.13), from the same code that fills SUBSCRIBE_OK and + FETCH_OK. +- The publisher still follows the spec for subscribers that don't opt out: + [FETCH_OK properties](/quest/m1/fetch-ok-properties.md). + ## Related +- [FETCH_OK properties](/quest/m1/fetch-ok-properties.md) - the publisher side of the same properties + - [JavaScript FETCH](/quest/m1/js-fetch.md) - the browser publisher answers these fetches diff --git a/quest/m1/interop-week.md b/quest/m1/interop-week.md new file mode 100644 index 00000000000..73dba3ddbef --- /dev/null +++ b/quest/m1/interop-week.md @@ -0,0 +1,16 @@ +# [XS] A week of nightly interop after #4529 + +## Goal + +Seven nightly interop runs on `main` have completed since +[#4529](https://github.com/moq-dev/moq/pull/4529) added the player's `delay` +column to the interop trace. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +#4529 merged on 2026-09-29 (`8b6abc0a1`). As of 2026-09-30 one scheduled +`interop.yml` run on `main` includes it, so the seventh lands around +2026-10-06. Check with `gh run list --workflow interop.yml --branch main`. diff --git a/quest/m1/js-data-consumer-timestamps.md b/quest/m1/js-data-consumer-timestamps.md new file mode 100644 index 00000000000..1deaa229e5a --- /dev/null +++ b/quest/m1/js-data-consumer-timestamps.md @@ -0,0 +1,34 @@ +# [M] JS data consumers return each value's timestamp + +## Goal + +`@moq/json` and `@moq/flate` snapshot and stream consumers return each value +with its frame's timestamp, so a browser can hold telemetry back until the +video playhead reaches it. Mirrors +[Data consumer timestamps](/quest/m1/data-consumer-timestamps.md) in Rust. + +## Plan + +Requested by OneTooMany, whose web frontend shows KLV and MAVLink telemetry +beside video and today reads raw frames to recover the timestamps. + +Decided (2026-10-01): `next()` and the async iterator yield `@moq/net`'s +`Timed` (`{ value, at? }`), the type the producers take after +[JS publishing never invents a timestamp](/quest/m1/js-publish-timestamp.md). +`at` is the frame's timestamp, absent for an untimed frame. Snapshot +consumers get the same two reads as Rust: `next()` yields every state in +order (`@moq/json` stops draining to the latest, `@moq/flate` reads groups in +order), and `latest()` skips to the newest state, today's behavior. A reader +that falls behind still jumps to the newest group. Update +`doc/lib/js/{json,flate}.md`. + +Public API: breaking, on `dev`. Wire: none. + +## Required + +- [JS publishing never invents a timestamp](/quest/m1/js-publish-timestamp.md) - adds `Timed` to `@moq/net` +- [Plan: untimed objects](/quest/m1/plan-untimed-objects.md) - js/net stops filling arrival time, so `at` can be absent + +## Related + +- [Synced data playback](/quest/m2/watch-data-sync.md) - the js/watch reader built on these timestamps diff --git a/quest/m1/drain/js-goaway-requests.md b/quest/m1/js-goaway-requests.md similarity index 100% rename from quest/m1/drain/js-goaway-requests.md rename to quest/m1/js-goaway-requests.md diff --git a/quest/m1/js-group-handover.md b/quest/m1/js-group-handover.md new file mode 100644 index 00000000000..42cc600ba15 --- /dev/null +++ b/quest/m1/js-group-handover.md @@ -0,0 +1,47 @@ +# [L] JS group-boundary handover + +## Goal + +A `js/net` track subscription survives its broadcast's route swapping to +another provider, such as a relay migration after a GOAWAY, and resumes from +the new provider at the first group it has not delivered. A viewer at the live +edge with no latency budget never loses a group across the swap, and never +has to notice the swap to keep reading. + +Done when `test/drain` passes with the viewer's `MAX_AGE` at zero, the +viewer subscribes once instead of following `request.active`, and the run is +stable enough for the nightly. + +## Plan + +Today the origin (`js/net/src/origin.ts`) holds the outranked route until the +new one answers, then closes the old front. Every track read through that +front ends, and the app has to subscribe again on the new broadcast. Once the +old relay drops its upstream pull, the new relay subscribes upstream from +scratch at the live edge. A group boundary that lands inside that window loses +the group in flight: about 1 run in 5 at 100 ms groups. + +Rust already solves this. `moq-net`'s origin fronts are spliced broadcasts, and +`model/resume.rs` resumes each track from the replacement at the first missing +group, capping the old segment so it ends at the boundary on its own. Mirror +that shape and naming in JS rather than inventing a second model. Things to +settle along the way: + +- Whether the request's `active` broadcast stays the same object across a swap, + with its tracks re-sourced underneath. That is the Rust behavior and the + simplest for players. It is also a behavior change for code that watches + `active` to resubscribe. +- The resumed subscription names where it left off (the `groups` floor) so the + new provider serves the missing group from its cache or upstream instead of + starting at its own live edge. +- Failover compatibility: Rust refuses to splice a source whose track + properties differ (timescale, retention, priority, order). Match it. +- `js/watch` and `js/hang` consumers that re-subscribe on `active` changes. + Check whether they still need to. + +Add unit coverage at the origin level against stand-in sessions, then flip +`test/drain` to zero budget (drop the resubscribe loop in `drain.ts` and the +budget note in its README). + +Public API: likely a behavior change to `Origin.Requesting.active` and track +subscriptions across a swap. Report it in the PR. diff --git a/quest/m1/js-publish-timestamp.md b/quest/m1/js-publish-timestamp.md index 40fbc836a1b..4a485b6c91a 100644 --- a/quest/m1/js-publish-timestamp.md +++ b/quest/m1/js-publish-timestamp.md @@ -1,18 +1,28 @@ -# [M] JS publishing requires a timestamp +# [M] JS publishing never invents a timestamp ## Goal -`@moq/json`, `@moq/flate`, and `@moq/net` publish APIs take a timestamp and -never fill in `Timestamp.now()` for the caller, mirroring -[Publishing requires a timestamp](/quest/m1/publish-timestamp.md). +`@moq/json`, `@moq/flate`, and `@moq/net` publish APIs never fill in +`Timestamp.now()` for the caller, mirroring +[Publishing never invents a timestamp](/quest/m1/publish-timestamp.md). A +payload published without a timestamp goes out untimed. ## Plan Today the json and flate snapshot and stream producers default `at = Timestamp.now()`, the json window producer always uses now, and js/net's -`writeString`, `writeJson`, and `writeBool` stamp now. Make the timestamp a -required argument on each, with the same shape as Rust's `Timed`. Callers in -the repository (js/hang catalog, js/publish, js/room) pass their clock's now -explicitly. Update `doc/lib/js/{json,binary,net}.md`. +`writeString`, `writeJson`, and `writeBool` stamp now. + +Decided (2026-10-01): `@moq/net` exports `Timed` +(`{ value: T; at?: Time.Timestamp }`), the Rust name and shape. Each producer +takes a `Timed`; an absent `at` is untimed, never now. The consumers return +the same type in +[JS data consumer timestamps](/quest/m1/js-data-consumer-timestamps.md). +Callers in the repository (js/hang catalog, js/publish, js/room) pass their +clock's now explicitly. Update `doc/lib/js/{json,flate,net}.md`. Public API: breaking, on `dev`. Wire: none. + +## Required + +- [Plan: untimed objects](/quest/m1/plan-untimed-objects.md) - an untimed payload must travel as untimed before producers stop filling in now diff --git a/quest/m1/js-ranked.md b/quest/m1/js-ranked.md index c7f2131e89f..2611d757066 100644 --- a/quest/m1/js-ranked.md +++ b/quest/m1/js-ranked.md @@ -4,11 +4,14 @@ `@moq/hang` ranks video renditions the same way as Rust's `hang::catalog::Video::ranked` (largest picture, then highest bitrate, ties in -name order), and `@moq/watch`'s fallback pick uses it, so the browser and the +name order), and `@moq/watch`'s no-target pick uses it, so the browser and the native egresses can't drift apart. ## Plan -Mirror the Rust name and semantics. `bestRendition` in `js/watch` already -matches today; replace it rather than keep two copies. Check whether the +Mirror the Rust name and semantics, including sorting by `preference` first +once [Rendition preference](/quest/m1/rendition-preference.md) lands. +`bestRendition` in `js/watch` already matches except exact ties, which keep +catalog order instead of name order; replace it rather than keep two copies. +Check whether the `byDimensions`/`byBitrate` filters can reuse the same ordering. diff --git a/quest/m1/libmoq-final-release.md b/quest/m1/libmoq-final-release.md new file mode 100644 index 00000000000..9ddcca14672 --- /dev/null +++ b/quest/m1/libmoq-final-release.md @@ -0,0 +1,15 @@ +# [XS] The final libmoq release is the stub + +## Goal + +The newest `libmoq` on crates.io is the code-free stub built from `dev`'s +`rs/libmoq`, whose README points at `moq-c`. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +As of 2026-10-02 the newest release is libmoq 0.6.9 (2026-09-30), cut from +`main`'s full crate, not the stub. Check with +`curl -s https://crates.io/api/v1/crates/libmoq` for a newer `max_version`. diff --git a/quest/m1/libmoq-retire.md b/quest/m1/libmoq-retire.md index 90717f5c66b..9534e1f3853 100644 --- a/quest/m1/libmoq-retire.md +++ b/quest/m1/libmoq-retire.md @@ -15,4 +15,4 @@ for anyone still resolving the old name. ## Required -- The final `libmoq` release, built from the `rs/libmoq` stub, is on crates.io +- [The final libmoq release is the stub](/quest/m1/libmoq-final-release.md) - the published crate points at `moq-c` before the source goes diff --git a/quest/m1/listener-deadlines.md b/quest/m1/listener-deadlines.md new file mode 100644 index 00000000000..bb1a0a1db26 --- /dev/null +++ b/quest/m1/listener-deadlines.md @@ -0,0 +1,40 @@ +# [L] Every listener bounds its handshake and headers + +## Goal + +Every relay listener bounds slow peers the way the default runtime does +after the handshake deadline (moq-dev/moq#4612): the io_uring +workers apply `listen.timeout`, the HTTPS listener's HTTP/2 path and the +`[internal]` listener drop a connection with no complete request in flight +for `listen.timeout` (covering slow headers and an idle keep-alive, even one +that ACKs every PING), and the iroh backend honors `quic.keep_alive`. + +## Plan + +- io_uring (`rs/moq-relay/src/uring.rs` `serve_connection`): bound the + WebTransport `Request::accept`, `respond`, and `accept_request_lite` by one + deadline from `listen::Config::resolved_timeout()`, on the worker's own + timer (`rs/moq-uring/src/timer.rs`), closing with the timeout code. Delete + the "There is no timeout here" note in `rs/moq-uring/src/quic/web.rs`. +- HTTP: `axum_server` builds hyper with no timer, and #4612's + `header_read_timeout` is HTTP/1 only. hyper's HTTP/2 PING keep-alive + (`http2().timer(..)`, `keep_alive_interval`, `keep_alive_timeout`) only + detects a dead peer: a client that ACKs every PING and never finishes a + request stays up. So the bound is a per-connection idle deadline: the + connection closes once it has had no complete request in flight for + `listen.timeout`, which covers both a trickled HEADERS block and an idle + keep-alive. hyper has no such knob, so it lives in the relay's serve path + (`rs/moq-relay/src/listener.rs` and `web.rs`, e.g. an in-flight counter in + the per-connection service driving that connection's graceful shutdown). + Set the PING keep-alive too, for dead peers. Apply both to the HTTPS + listener and the `[internal]` listener, which also gets the HTTP/1 header + timer. +- iroh (`rs/moq-tokio/src/iroh.rs`): set iroh 1.3's + `keep_alive_interval` from `quic.keep_alive`, and fix the docs that say iroh + has no knob (`rs/moq-tokio/src/quic.rs`, `doc/bin/relay/config.md`). +- Tests on a paused clock where the runtime allows: a stalled io_uring + handshake closes at the deadline, and an HTTP/2 client that ACKs PINGs but + never sends a request, or trickles one request's headers, is dropped at the + deadline while one with a request in flight is not. + +Public API: none beyond existing settings. Wire: none. diff --git a/quest/m1/mobile-ownership.md b/quest/m1/mobile-ownership.md index abfee2ab29d..d16beae6f12 100644 --- a/quest/m1/mobile-ownership.md +++ b/quest/m1/mobile-ownership.md @@ -17,6 +17,6 @@ carry the Rust codecs, and #4094 added the `CVPixelBuffer` bridge stack beside one that exists. What remains is recording the verdict in `rs/moq-ffi/AGENTS.md` (a maintainer -edit) and in [Android capture](/quest/m2/mobile-capture-android.md) and -[iOS capture](/quest/m2/mobile-capture-ios.md), which target `moq-video` as -written and no longer wait on this. +edit). [Android capture](/quest/m2/mobile-capture-android.md), +[iOS capture](/quest/m2/mobile-capture-ios.md), and the MediaCodec audio +quests already record it and no longer wait on this. diff --git a/quest/m1/mux-truncated-group.md b/quest/m1/mux-truncated-group.md new file mode 100644 index 00000000000..fb44cad34f9 --- /dev/null +++ b/quest/m1/mux-truncated-group.md @@ -0,0 +1,23 @@ +# [XS] moq-mux skips a truncated spliced group + +## Goal + +A moq-mux test feeds its consumer a spliced group whose continuation can never +arrive and asserts the group is skipped as aborted, not read as a short clean +group. + +## Plan + +Start after #4689 lands: it makes `group::Consumer` reads and `finished()` fail +on a truncated spliced group (`Error::Dropped` for a pruned seam, else the dead +route's error), where they used to end clean. moq-mux's consumer already drops a group +whose read fails, but only code reading verified that; no test covers a spliced +group. Build one with the moq-net resume test fixtures, or the smallest public +path to a pruned seam. + +Public API: none. Wire: none. + +## Related + +- [#4689](https://github.com/moq-dev/moq/pull/4689) - the truncated-group behavior this test pins +- [#4491](https://github.com/moq-dev/moq/pull/4491) - the `resume.rs` splice path diff --git a/quest/m1/nightly-platform-tests.md b/quest/m1/nightly-platform-tests.md new file mode 100644 index 00000000000..ea3c75c7c1c --- /dev/null +++ b/quest/m1/nightly-platform-tests.md @@ -0,0 +1,25 @@ +# [S] Nightly tests on macOS and Windows + +## Goal + +A nightly job runs `cargo test` for `moq-auth`, `hang`, `moq-tokio`, and +`moq-native` on macOS and Windows runners and alerts like the other nightly +jobs, so platform-only behavior is tested and not just compiled. + +## Plan + +Today `.github/workflows/platform.yml` runs `just rs windows|macos`, which only +`cargo check`s (`sh/rs/select.sh`), and every `nightly.yml` job runs on +`ubuntu-24.04-arm`. [#4600](https://github.com/moq-dev/moq/pull/4600)'s +regressions exercise different clocks: `Instant` overflow in `moq-auth` fails on +macOS and `SystemTime` overflow in `hang` fails on Windows. On Linux they only +prove no panic, so CI never runs them where they matter. + +Decision (2026-10-01): ✅ the platform-sensitive crates above. Rejected: the +whole workspace (slower, more flake surface) and only `moq-auth` + `hang`. + +Add a `just` recipe per platform beside the existing check, wire it into +`nightly.yml`'s matrix, and make sure `alert.yml` covers it. Fix or quarantine +anything that only fails there at its cause. + +Public API: none. Wire: none. diff --git a/quest/m1/obs-decoded-surface.md b/quest/m1/obs-decoded-surface.md new file mode 100644 index 00000000000..0585da10b1a --- /dev/null +++ b/quest/m1/obs-decoded-surface.md @@ -0,0 +1,14 @@ +# [XS] dev's decoded-frame surface reaches main + +## Goal + +`main` contains the moq-ffi decoded-frame surface from `dev` +([#4094](https://github.com/moq-dev/moq/pull/4094), renamed in `97575f002`). + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +As of 2026-09-30 both are on `dev` only. Check with +`git merge-base --is-ancestor 97575f002 origin/main`. diff --git a/quest/m1/obs-moq-video/source.md b/quest/m1/obs-moq-video/source.md index e7bf0fd977e..36161bb16d5 100644 --- a/quest/m1/obs-moq-video/source.md +++ b/quest/m1/obs-moq-video/source.md @@ -17,7 +17,7 @@ The MoQ source loads and plays supported video without FFmpeg's video libraries ## Required - [OBS migration](/quest/m1/cpp/obs.md) - the plugin is on the generated C++ before decode changes -- dev's decoded-frame surface (#4094, 97575f002) reaches main +- [dev's decoded-frame surface reaches main](/quest/m1/obs-decoded-surface.md) - the surface this decodes into ## Related diff --git a/quest/m1/open-gop-leading-pictures.md b/quest/m1/open-gop-leading-pictures.md index 0dc7c0af266..fe83d04f39b 100644 --- a/quest/m1/open-gop-leading-pictures.md +++ b/quest/m1/open-gop-leading-pictures.md @@ -50,6 +50,11 @@ decode, and the JS test should drive the software decoder. Every continuous group is passed through untouched. - The same rule in the Rust decode path (`moq-video` decode consumers), so native playback and the transcoder tune in the same way. +- The same rule in `moq export ts`. Decided (2026-10-01): the fixed-delay + export (#4645) still sends a join's orphaned leading pictures, so 3 of 500 + frames on the open-GOP fixture decode after they present + (`dts-before-pts`). Trim them at tune-in from the same signal, and make + `just test ts --open-gop` pass under `--strict`. - This quest owns the Rust non-continuous signal, which audio warmup and consumer warmup reuse rather than each adding one. Today `moq_mux::container::Consumer::poll_read` returns a bare frame, and diff --git a/quest/m1/papercuts.md b/quest/m1/papercuts.md new file mode 100644 index 00000000000..56a365d6e2f --- /dev/null +++ b/quest/m1/papercuts.md @@ -0,0 +1,26 @@ +# [S] Papercuts from the 2026-09-30 spawn + +## Goal + +Three small fixes found while landing the m0 quests: `@moq/net` refuses to +serve a broadcast it did not produce, `just fix` and `just check` leave the +gitignored `.scratch/` alone, and `remote_wake_unparks` passes under load. + +## Plan + +- JS republish: `@moq/net` publishes only what it produces (decided with + moq-dev/moq#4599), naming its own origin. Requests already skip received + routes (`#demand` resolves through `bestEntry(path, received)` in + `js/net/src/origin.ts`), so the gap is an app handing a consumed + `broadcast.Consumer` (one a session delivered) back to an origin for + serving, e.g. `Request.accept(consumer)`, which labels upstream content + with the local origin's hop. Refuse that loudly at the point it enters the + origin, not in the lite publisher's `local(..) ?? demand(..)` resolve. +- `.scratch/`: add it to `.taplo.toml`'s excludes and to `.remarkignore`, so + `taplo format` and `sh/markdown.sh` stop rewriting agents' scratch clones. + biome, nixfmt, just, and shfmt already skip it. +- `rs/moq-uring/src/worker.rs` `remote_wake_unparks`: observe that the worker + parked instead of sleeping 50 ms and asserting elapsed wall time, per the + rule that unit tests mock time. + +Public API: none. Wire: none. diff --git a/quest/m1/parked-read-wakes.md b/quest/m1/parked-read-wakes.md index 3e403f8b261..77b75479a24 100644 --- a/quest/m1/parked-read-wakes.md +++ b/quest/m1/parked-read-wakes.md @@ -22,12 +22,9 @@ Follow-ups from [#4484](https://github.com/moq-dev/moq/pull/4484), which made Settled: one quest, since both are the same wakeup invariant in `rs/moq-net/src/model/track.rs` and `resume.rs`. -Land after [resume latest](/quest/m1/resume-latest.md) (#4491), which edits -the same `resume.rs` wakeups, and rebase onto it. - Public API: none. Wire: none. ## Related -- [Resume latest](/quest/m1/resume-latest.md) - edits the same `resume.rs` wakeups; lands first (#4491) +- [#4491](https://github.com/moq-dev/moq/pull/4491) - a resumed group holds demand on its replacement copy in the same `resume.rs` wakeups - [#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) - extends the `resume.rs` takeover tests diff --git a/quest/m1/perf/group-cost.md b/quest/m1/perf/group-cost.md index 9dea750f034..1bf3bdeccfd 100644 --- a/quest/m1/perf/group-cost.md +++ b/quest/m1/perf/group-cost.md @@ -26,14 +26,16 @@ the next largest source from there. A measured no-win abandons the quest, per this line's rules. Decided in the 2026-09-30 audit: two suspected per-group costs merged here -as candidates to measure with `SESSION_ALLOCS`, not separate quests. +as candidates to measure, not separate quests. - Egress cache refresh: `group::Consumer::keep_alive` takes a state read guard and calls `Charge::refresh` between completed frame writes on both - the lite and IETF publishers. It protects a drain longer than - `latency_max`, so keep per-frame liveness and - `slow_prefetch_reader_survives_expiry` (rs/moq-net/src/model/track.rs); - fewer refresh calls alone are not a win. + the lite and IETF publishers. That is lock, clock, and atomic work that may + allocate nothing, so time it rather than count allocations: CPU per + viewer-group for fast fanout and flow-controlled readers, over SUBSCRIBE + and FETCH. It protects a drain longer than `latency_max`, so keep + per-frame liveness and `slow_prefetch_reader_survives_expiry` + (rs/moq-net/src/model/track.rs); fewer refresh calls alone are not a win. - Owned decode copies: `rs/moq-net/src/coding/decode.rs` decodes `Vec` through `Buf::copy_to_bytes` then `to_vec`, and `String` consumes that vector. Measure on the real reader input types (contiguous and chained) diff --git a/quest/m1/plan-untimed-objects.md b/quest/m1/plan-untimed-objects.md index 33344c05b2e..c495e94c703 100644 --- a/quest/m1/plan-untimed-objects.md +++ b/quest/m1/plan-untimed-objects.md @@ -1,12 +1,12 @@ -# [S] Plan: untimed peer objects +# [S] Plan: untimed objects ## Goal -Decide what a frame's timestamp is when a peer sends none, and rewrite this -quest into implementation quests. That covers IETF objects on a track without -TIMESCALE or without a Timestamp property, and lite-01 to lite-04, which have -no timestamp on the wire. The decision must not invent a timeline that -changes with the relay path. +Rewrite this quest into implementation quests that carry an absent timestamp +faithfully, from publisher to consumer, through every relay. That covers +untimed publishes, IETF objects on a track without TIMESCALE or without a +Timestamp property, and lite-01 to lite-04, which have no timestamp on the +wire. Nothing may invent a timeline that changes with the relay path. ## Plan @@ -17,30 +17,33 @@ relay then forwards those stamps as if they were real. A failover to another first hop changes them, so the timeline jumps, and they break the "one clock per broadcast" property. The maintainer dislikes this. -Settled elsewhere: publishing always requires a timestamp -([Publishing requires a timestamp](/quest/m1/publish-timestamp.md)), and -lite-05 and lite-06 (published) can't encode an absent one. - -Candidates: - -- Faithful absence (leaning): the model and consumers carry - `Option` so a relay forwards "none" as none. lite-07-wip encodes - it by shifting the FRAME Timestamp Delta and the DATAGRAM Timestamp by one - (0 = absent); an absent frame doesn't move the delta baseline. A lite-05/06 - encoder writes its send time. Untimed groups are never media-stale; the - pool's wall-clock expiry still reclaims them, and start resolution picks the - latest group instead of replaying the cache. -- First-hop arrival, documented: no change, but the failover jump stays. -- 0 as a sentinel: rejected. It collides with a real pts of 0. -- `max_age` on max(wall, pts): the maintainer asked. Rust keeps `max_age` as - media-time staleness so a congestion stall can't age content out, with the - pool's expiry as the wall-clock bound (`rs/moq-net/src/model/cache.rs`). +Decided (2026-10-01): faithful absence, on both the publish and the subscribe +side. The model and every consumer carry `Option`, a relay forwards +"none" as none, and no layer substitutes arrival or wall time. +[Publishing never invents a timestamp](/quest/m1/publish-timestamp.md) stops +the producers filling in now. Untimed groups are never media-stale; the +pool's wall-clock expiry still reclaims them, and start resolution picks the +latest group instead of replaying the cache. + +- lite-07-wip encodes absence by shifting the FRAME Timestamp Delta and the + DATAGRAM Timestamp by one (0 = absent); an absent frame doesn't move the + delta baseline. +- lite-05 and lite-06 (published) can't encode absence, so their encoder + writes its send time, documented in the lite draft as a downgrade for old + peers. +- Rejected: first-hop arrival (the failover jump stays), 0 as a sentinel + (collides with a real pts of 0), and `max_age` on max(wall, pts) (Rust keeps + `max_age` as media-time staleness so a congestion stall can't age content + out, with the pool's expiry as the wall-clock bound, + `rs/moq-net/src/model/cache.rs`). + +What remains is mapping the fallout into implementation quests. Facts to gather: which IETF implementations send TIMESCALE and Timestamp (moxygen, the interop matrix), and every model decision that reads a timestamp (`track.rs` staleness and `GroupExpiry`, `resume.rs` successors, `group.rs` `poll_timestamp`, which treats "has a timestamp" as "has a frame"). -Output: implementation quests, and updates to -[Data consumer timestamps](/quest/m1/data-consumer-timestamps.md), whose -return type depends on this, and to the timestamp and lite drafts. +Output: implementation quests (the model and wire carrying `Option`, lite-07 +encoding, and the timestamp and lite drafts), and their places in the +`Required` lists of the publish and consumer timestamp quests. diff --git a/quest/m1/publish-timestamp.md b/quest/m1/publish-timestamp.md index 78b11642cb1..0426aa75c5b 100644 --- a/quest/m1/publish-timestamp.md +++ b/quest/m1/publish-timestamp.md @@ -1,61 +1,74 @@ -# [L] Publishing requires a timestamp +# [L] Publishing never invents a timestamp ## Goal -Every publish API takes a timestamp, and nothing below it invents one. A -frame's timestamp is the publisher's statement of when the content happened. -A library never fills in "now" on the caller's behalf. Covers moq-net, +No publish API fills in a timestamp the caller didn't give. A frame's +timestamp is the publisher's statement of when the content happened; a payload +published without one goes out untimed and arrives untimed. Covers moq-net, moq-json, moq-flate, moq-mux, moq-ffi, and every binding wrapper (Python, Swift, Kotlin, Go, Dart). Not libmoq, which the [Generated C bindings](/quest/m1/c/README.md) replace. ## Plan -Decided (2026-09-30): the wire timestamp stays required, so the publish side -must be too. Today `Timed.at: None` means "stamp when written": moq-json and -moq-flate fill `Timestamp::now()` (moq-net's clock, not the broadcast's), and -moq-mux's `Clock::stamp` fills the broadcast clock's `now()`. The FFI raw +Decided (2026-10-01), reversing the 2026-09-30 "timestamp required" plan: the +timestamp is optional end to end, on the wire and in every API, and absent +means untimed. A library never substitutes now, wall clock, or arrival time. +How absence travels on the wire is +[Plan: untimed objects](/quest/m1/plan-untimed-objects.md); this quest +removes every place that fills one in. + +Today `Timed.at: None` means "stamp when written": moq-json and moq-flate +fill `Timestamp::now()` (moq-net's clock, not the broadcast's), and moq-mux's +`Clock::stamp` fills the broadcast clock's `now()`. The FFI raw `MoqFrame.timestamp_us` and `MoqDatagram.timestamp_us` default to 0. -- `moq_net::Timed` keeps its name and gets a required `at: T`, with no - `From`. The json/flate consumers return the same type (see - [Data consumer timestamps](/quest/m1/data-consumer-timestamps.md)). +- `moq_net::Timed` keeps its name and its `at: Option`; only the + meaning of `None` changes. The json/flate consumers return the same type + (see [Data consumer timestamps](/quest/m1/data-consumer-timestamps.md)). - moq-json (snapshot, stream, and window, whose `push` always stamps now) and - moq-flate producers take `Timed<_, Timestamp>`. The moq-mux data producers - take `Timed<_, Instant>` and map it onto the broadcast clock, refusing one - ahead of now (`Error::InvalidCapture`), as today; `Clock::stamp` stops - filling `None`. `catalog::data::Listing::record` then always has a capture - time for `delay` and `jitter`. + moq-flate producers take `Timed<_, Timestamp>` and write `None` as untimed. +- moq-mux data producers take `Timed<_, Timestamp>` on the broadcast clock, + the same contract as `container::Frame`, with no ahead-of-now check. A + caller already holding a broadcast-clock timestamp passes it as is: the + source's own timestamp on a catalog clock anchored to that source (video + + KLV from one MPEG-TS program), or the `at` of a consumer's `Timed` it derived the value + from (MAVLink telemetry republished). A source clock running slightly fast + must not be refused. +- `Clock::capture(Instant)` becomes public for callers holding a capture + `Instant`; it keeps refusing one ahead of now or before the epoch + (`Error::InvalidCapture`). `Clock::stamp` is deleted. + `catalog::data::Listing::record` samples `delay` and `jitter` only for timed + writes, and records zero delay for a timestamp ahead of now rather than + refusing it. +- Document on the moq-mux producers that a carried-over timestamp lines up + with media only on a broadcast sharing the source's clock mapping. - Publishers inside the repository (hang catalog snapshots, MSF, stats, room chat, examples) pass their clock's now explicitly. -- moq-ffi: the - capture time is a media timestamp on the broadcast's timeline. moq-ffi - exposes the broadcast clock's `now()` as a timestamp; callers stamp payloads - with values taken from it, and moq refuses one ahead of now. That keeps a - device or process clock out, as the Rust `Instant` mapping does. Either map - the timestamp back through the clock inside moq-ffi or give the clock a - typed timestamp moq-mux accepts; keep a raw `Timestamp` from compiling - there. The json/flate `update` and `append` take it as a required - argument, and the raw frame and datagram records lose their `default = 0`. +- moq-ffi: data producers take an optional timestamp in microseconds on the + broadcast clock, unchecked, as Rust does. moq-ffi exposes the broadcast + clock's `now()` so callers have a value to stamp with. The raw frame and + datagram records' `timestamp_us` becomes optional instead of `default = 0`. - Go (`go/wrapper/moq`) and Python (`py/moq-rs`) wrap only the JSON producers; [#4137](https://github.com/moq-dev/moq/pull/4137) added `publish_binary_snapshot` / `publish_binary_stream` (now - `publish_flate_*`) to moq-ffi without them. Add hand-written flate wrappers - there. The maintainer asked for this. - Each wrapper gets a test that a past capture time is accepted and a future - one refused. + `publish_flate_*`) to moq-ffi without them. Add hand-written flate wrappers there. The maintainer asked for this. + Each wrapper gets a test that a timestamp round-trips and an untimed payload + arrives untimed. - Docs: "stamped when written" in `doc/lib/rs/{moq-json,moq-flate}.md` and `rs/moq-net/src/model/timed.rs`, plus `doc/lib/{py,swift,kt,go,dart}`. -What a receiver fills in when a peer sends no timestamp is out of scope: -[Plan: untimed peer objects](/quest/m1/plan-untimed-objects.md). +The broadcast-clock input was requested by OneTooMany, who align KLV and +MAVLink-derived telemetry with video. -Public API: breaking, on `dev`. Wire: none. +Public API: breaking, on `dev`. Wire: none here; absence on the wire lands +with the untimed implementation quests. ## Required - [JSON and flate namespaces](/quest/m1/ffi-shape/json.md) - moves the data producers this changes, so the two breaks land in order rather than colliding +- [Plan: untimed objects](/quest/m1/plan-untimed-objects.md) - an untimed payload must travel as untimed before producers stop filling in now ## Related -- [JS publishing requires a timestamp](/quest/m1/js-publish-timestamp.md) - the same change in the JS packages +- [JS publishing never invents a timestamp](/quest/m1/js-publish-timestamp.md) - the same change in the JS packages diff --git a/quest/m1/quic/README.md b/quest/m1/quic/README.md index 856696ec7b1..2cab9c3e8c7 100644 --- a/quest/m1/quic/README.md +++ b/quest/m1/quic/README.md @@ -9,9 +9,10 @@ quinn and not noq). One core serves the tokio backend, the thread-per-core BBR correctness, reliable stream resets, hierarchical stream scheduling with per-broadcast fairness, wider limits for relay peers, and endpoint sharding. Per-stream acknowledgment progress, per-stream deadlines, qmux on the shared -stream state machine, and the experiments (GCC, receive timestamps, the egress -profile, media probing, L4S, careful resume, deadline keep-alive) live in -[m2](/quest/m2/README.md) and do not gate this line. +stream state machine, and the experiments (the egress profile, media probing, +L4S, careful resume, deadline keep-alive) live in [m2](/quest/m2/README.md); +GCC with receive timestamps lives in [m3](/quest/m3/README.md). None of them +gate this line. ## Plan @@ -78,7 +79,5 @@ consumes them. - [L4S on the backbone](/quest/m2/quic-ecn.md) - an ECT(1) option in the fork, an `ecn` config knob, and a dualpi2 measurement - [Careful resume on reconnect](/quest/m2/quic-careful-resume.md) - a redial starts at the previous connection's rate - [Keep-alive by deadline](/quest/m2/quic-keep-alive.md) - a PING only when the idle deadline nears, no fixed timer -- [Receive timestamps](/quest/m3/quic-receive-ts.md) - per-packet arrival - times for GCC and deadlines -- [GCC egress experiment](/quest/m3/quic-gcc.md) - a measured verdict on +- [GCC egress experiment](/quest/m3/quic-gcc.md) - receive timestamps and a measured verdict on WebRTC-style delay control diff --git a/quest/m1/quic/scheduler.md b/quest/m1/quic/scheduler.md index 9b2748bd7d0..e6f995919f1 100644 --- a/quest/m1/quic/scheduler.md +++ b/quest/m1/quic/scheduler.md @@ -38,8 +38,8 @@ levels. Browsers (js/net and web-transport-wasm) never create send groups: browser groups are flat and byte-fair, which would trade strict priority between subscriptions (audio over video) for fairness nobody on a browser session needs, so they keep the default group and pack priority and group -order into `sendOrder` (decided 2026-09-26 with -[Firefox 155](/quest/m2/firefox-155-webtransport.md)). +order into `sendOrder` (decided 2026-09-26 alongside Firefox 155's +send groups). Give every MoQ subscription one native send group. A SUBSCRIBE_UPDATE changes the group priority atomically. Group streams use their position within the diff --git a/quest/m1/redirect-resolve.md b/quest/m1/redirect-resolve.md index 7ee9ef70141..8e315909367 100644 --- a/quest/m1/redirect-resolve.md +++ b/quest/m1/redirect-resolve.md @@ -19,12 +19,7 @@ the connection it documents. Recommendation: make it private. Nothing outside `moq-tokio` calls it (only its own unit tests), and the repo keeps things private until a consumer needs them. If a consumer turns up, the alternative is returning the same -`Result>` as the internal `target` does once the drain line lands -(on `main` it is still `Option`, folding a refusal into "keep the -current addresses"), so empty and refused stay distinct. Removing or changing a published method is a break, so this +`Result>` as the internal `target` does, so empty and refused stay distinct. Removing or changing a published method is a break, so this targets `dev`; update `doc/lib/rs` if it mentions the method. -## Required - -- [Graceful relay drains](/quest/m1/drain/README.md) - the stricter `Connection` lands with the line -- `dev` has merged `main` after the drain line lands +Start by merging `main` into `dev` if `dev` does not have the drain line yet. diff --git a/quest/m1/relay-mesh-refusal-release.md b/quest/m1/relay-mesh-refusal-release.md new file mode 100644 index 00000000000..a4b71f3156d --- /dev/null +++ b/quest/m1/relay-mesh-refusal-release.md @@ -0,0 +1,15 @@ +# [XS] A moq-relay release refuses --cluster-mesh + +## Goal + +A `moq-relay` release on crates.io carries +[#4601](https://github.com/moq-dev/moq/pull/4601), which refuses +`--cluster-mesh` at startup with a pointer to its replacement. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +As of 2026-10-01 the newest release is moq-relay 0.15.8 (2026-09-28), cut +before #4601 merged. diff --git a/quest/m1/rendition-preference.md b/quest/m1/rendition-preference.md new file mode 100644 index 00000000000..d591afe19f8 --- /dev/null +++ b/quest/m1/rendition-preference.md @@ -0,0 +1,78 @@ +# [M] Rendition preference + +## Goal + +A hang video rendition carries an optional `preference` (a signed integer, +absent means 0, higher wins). Automatic selection keeps only the highest +preference among the renditions it can decode, then picks by target and +bitrate inside that tier as today. ``, +`hang::catalog::Video::ranked`, and WHEP's codec choice honor it, so a viewer +and every single-rendition egress (single-track RTMP play, WHEP, FLV export, +moq-transcode's source pick) only subscribe to a compatibility transcode +(such as H.265 republished as H.264, marked `-1`) when nothing preferred +decodes. A multitrack RTMP client still receives every rendition. + +## Plan + +Requested by an external consumer (OneTooMany), who publishes an H.265 source +with its own H.264 transcode beside it, so a viewer that can decode H.265 +should never pull the transcode. + +Decided in planning interviews on 2026-10-01: + +- A strict numeric tier, not `fallback: bool`. Two tiers cover the request, + but a ladder per codec (AV1 > H.265 > H.264) needs three, and today's + area-then-bitrate sort would pick H.264 over a same-size AV1 rung because + its bitrate is higher. The tier costs the same to implement as the bool. +- Not a total order (DASH `@qualityRanking`, HLS `SCORE`): the publisher + would have to rank every rung, and ABR still needs size and bitrate to + step down a ladder. +- Not a routing-style `cost`: an honest "produced on demand" cost would mark + moq-transcode's rungs, so the strict rule would stop capable viewers from + stepping down. The publisher sets `preference` explicitly. +- moq-transcode's rungs inherit their source's `preference`, so a ladder + adapts within one tier even when the source is not at 0. `rung_entry` + copies it like `optimize_for_latency`, and the per-snapshot refresh beside + `inherit_stalled` keeps it current when the source catalog changes. +- Named `preference`, higher wins, after DASH's `@selectionPriority` (strict + across per-codec Adaptation Sets, higher preferred). Not `priority`, which + already means track send priority. Signed, so a new preferred ladder is `1` + without renumbering existing renditions, and a fallback is `-1`. +- Video only. Optional on the wire, omitted when 0 (like `stalled`). + Additive, so older players ignore it. +- Selection order in `js/watch/src/video/source.ts`: decode support, then + keep the highest preference among supported renditions, then `stalled`, + then the existing target and bitrate pick within what is left. Preference + is about decodability only: a stalled source does not move capable viewers + onto a lower tier, which is usually derived from it and likely stalled too. +- A manual `target.name` still wins, as it does for `stalled`. The quality + picker keeps listing every tier. +- `Video::ranked` sorts by preference (highest first), then by picture and + bitrate as today. RTMP play, FLV export, and moq-transcode take the first + rendition they support, so they need no change. Update the + [JS rendition ranking](/quest/m1/js-ranked.md) Plan if it is still open. +- WHEP needs its own step: `Session::handle_media` (`rs/moq-rtc`) takes the + peer's first negotiated payload type, then `pick_video` filters `ranked()` + to that codec, so a peer offering the fallback's codec first would get the + fallback. Choose across every negotiated video codec so the highest + supported preference wins. +- Update `rs/hang` `VideoConfig`, `js/hang` `VideoConfigSchema`, + `drafts/draft-lcurley-moq-hang.md` (next to `stalled`, with a + source-plus-fallback example), and `doc/concept/hang.md`. No new docs page. +- Tests: a supported source wins over a lower-preference rendition with a + higher bitrate, over a bitrate budget that fits only the lower one, and + while the source is stalled and the lower one is not; an unsupported + source selects the lower preference; three tiers resolve to the highest + supported one; a manual `target.name` selects a lower preference; `ranked` + orders a larger lower-preference rendition after a smaller source; a WHEP + peer offering the fallback's codec first still gets the source; a + moq-transcode source at preference 1 under a budget that fits only a rung + still selects the rung. +- Out of scope: moq-ffi, libmoq, and the bindings until a native player needs + the field. moq-transcode producing same-size codec fallbacks; the consumer + publishes its own. + +## Related + +- [JS rendition ranking](/quest/m1/js-ranked.md) - mirrors `Video::ranked` in `@moq/hang`, which sorts by preference first once this quest lands +- [Audio rendition pick](/quest/m1/audio-ranked.md) - audio ranking, where `preference` could join later diff --git a/quest/m1/request-stream-serve.md b/quest/m1/request-stream-serve.md deleted file mode 100644 index 287deadb3db..00000000000 --- a/quest/m1/request-stream-serve.md +++ /dev/null @@ -1,43 +0,0 @@ -# [M] Lite request streams stop when the requester leaves - -## Goal - -A lite publisher stops resolving a SUBSCRIBE or FETCH as soon as the -requester closes its send direction, by FIN or reset, as TRACK already does -since #4494 (the draft ends a transaction when either side closes), so an -abandoned request releases its upstream lookup along the whole mesh. Serving -a request stream can't forget that watch: the handlers for TRACK, SUBSCRIBE, -and FETCH never hold the reader while they wait. - -## Plan - -`TrackInfoServe`, `SubscribeServe`, and `FetchServe` in -`rs/moq-net/src/lite/publisher.rs` each repeat the same decode, origin, and -broadcast steps, the same error classification and `abort`, and only -`TrackInfoServe` watches the reader for a FIN or reset. SUBSCRIBE waiting in -`Request` or `Confirm`, and FETCH waiting in `Request` or `Fetch`, pin the -upstream lookup for a requester that already left. - -Share one request-stream wrapper that owns the reader: it decodes the -message, then drives a per-kind handler that gets only the writer, watching -the reader for a FIN or reset for the whole wait. Pending data is not a close: -SUBSCRIBE forwards each SUBSCRIBE_UPDATE to its handler, and only a FIN or a -read error ends the handler. Fold the origin and broadcast steps into one shared -resolve step, so each state enum loses its `Hop` and `Request` variants. - -Rejected: a generic `kio::until(task, scope)`. The interest signal is owned -by the task itself (the reader, a `track::Request`), it covers one phase -rather than the whole task, and cancelling a track is a commit that can lose -to returning demand (`reject_unused`). Those are moq-net concerns, so the -guarantee comes from ownership in moq-net, not a kio combinator. The lite and -IETF subscribers keep their inline `poll_unused` checks. - -Tests: a SUBSCRIBE and a FETCH whose requester FINs or resets while the publisher -waits on the broadcast leave the upstream request unused, over real lite-05 -and lite-06 sessions like `tests/track_info_cancel.rs`. - -Public API: none. Wire: none. - -## Related - -- [Cross-relay bursts](/quest/m1/cross-relay-bursts.md) - unanswered FETCHes across relays, where an abandoned request pinning its lookup would show up diff --git a/quest/m1/resume-latest.md b/quest/m1/resume-latest.md deleted file mode 100644 index 3949bfec69e..00000000000 --- a/quest/m1/resume-latest.md +++ /dev/null @@ -1,75 +0,0 @@ -# [S] A resumed group ends once the new copy is past it - -## Goal - -A reader that awaits only the in-flight group's `read_frame` never parks forever -after its source fails mid-group. Once the track fails over to a new copy, the -half-delivered group either completes from that copy or ends with an error, on -every lite version and for local and remote sources alike. The one exception -is a copy that later drops the group it is serving; that waits on -[SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md). - -Two reports, one mechanism: - -- #4392: a publisher aborts a track mid-group over a session. On lite-03/04 the - far subscriber's in-flight `read_frame` stays pending forever; on lite-05/06 - it errors. -- #4365: a publisher aborts a track mid-group and re-creates it under the same - name in one origin. The origin reader parks in `read_frame` while a direct - reader gets the abort. - -## Plan - -Why it hangs: `track_ended` in `rs/moq-net/src/model/front.rs` treats a copy -that dies after delivering as failover and splices a new copy. The resumed -`resume::Group` then waits in `track.poll_peek_group` -(`rs/moq-net/src/model/resume.rs`) on that copy, but never asks it for -anything. Only `resume::Subscriber::apply`, driven by `recv_group`, subscribes -to a new copy. On lite-03/04 the re-query accepts the track without asking the -peer (no TRACK stream), so no SUBSCRIBE goes out and the refusal never comes -back. An in-process copy declares no start, so its peek answers "not yet" -forever. The doc comment on `resume::Group` promises that a group reader alone -picks up the continuation; that promise is what breaks. - -Decided: - -- While a resumed group waits on a copy, it holds a subscription on that copy - with `start = latest`, the same shape as a fresh subscriber. A refusal ends - the group with that error. -- If the copy's latest group is past the resumed group, the group ends with an - error at once. A same-name re-create is a continuation, so group 2 of the old - track ends and group 3 of the new one arrives. No special case for local - broadcasts. -- If the resumed group is the copy's latest, it reads the rest from that - subscription. A copy that later drops it says so with SUBSCRIBE_DROP once - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) lands; until then that case - waits as today. -- No FETCH, not even where the version supports it. Fetching older groups from - the new copy may come later as its own quest. -- Don't abort the old group eagerly on failover instead: long-lived groups - like a catalog must survive a failover. - -`poll_finished` needs the same subscription as `poll_current`. Drop the -subscription once the wait resolves. - -A prototype of the subscription half (about 20 lines in `resume.rs`) turned -all eight #4392 cases into errors with every moq-net test passing. - -Regression tests, on a paused clock (`#[tokio::test(start_paused = true)]`): - -- The #4392 repro as `rs/moq-net/tests/track_abort_over_session.rs`: abort - mid-group over one and two hops on lite-03 through lite-06; the in-flight - read errors. -- The #4365 repro: abort mid-group 2, re-create at group 3 through the origin; - group 2 ends with an error and group 3 arrives. - -## Closes - -- [#4392](https://github.com/moq-dev/moq/issues/4392) - close this issue when the quest finishes -- [#4365](https://github.com/moq-dev/moq/issues/4365) - close this issue when the quest finishes - -## Related - -- [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - tells a resumed latest group when the new copy dropped it -- [Parked reads wake](/quest/m1/parked-read-wakes.md) - edits the same `resume.rs` wakeups; lands after this quest's #4491 -- [#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) - extends the same `resume.rs` takeover tests diff --git a/quest/m1/route-cost.md b/quest/m1/route-cost.md index e058957d870..b5bddafe6ec 100644 --- a/quest/m1/route-cost.md +++ b/quest/m1/route-cost.md @@ -24,6 +24,9 @@ state, so Warm has no job. - Replace `Cost { warm, cold }` in `rs/moq-net/src/model/origin.rs` and its JS mirror, and delete what computes Warm from cache state. - The moq-transport cluster extension already carries one cost. +- Once [Routes and announces](/quest/m1/cluster-routing/routes.md) lands, this + single cost stays on ANNOUNCE as the origin's per-prefix seed and stops + accumulating per hop; link costs move to the ROUTE metric. Public API: `Cost` changes shape (a break, so it lands on `dev`). Wire: the wip version drops a field; lite-06 is unchanged. diff --git a/quest/m1/route-wakes.md b/quest/m1/route-wakes.md new file mode 100644 index 00000000000..e90a09936da --- /dev/null +++ b/quest/m1/route-wakes.md @@ -0,0 +1,92 @@ +# [M] A route change wakes only the fronts it affects + +## Goal + +A route change under a prefix (join, withdraw, re-price, restale) makes the +origin driver act only on the fronts that could change their selection: +fronts still waiting for a source, fronts served by the changed route, and +publisher-pinned fronts on its first hop. Today it wakes every front below +the prefix, so one equal-cost pool member joining or leaving a prefix that +serves 10k paths costs the driver 100 to 600 ms of single-threaded work +while no front switches (`origin/pool_churn`, #4607). A root claim, such as +an archive's catch-all, makes every front on the relay pay for every change +to it. Done when `origin/pool_churn` is flat in served paths. + +## Plan + +Facts from the wildcard line (`rs/moq-net/src/model/origin.rs` there): + +- Every route change calls `sync_route`, which calls + `routes.poke_below(prefix)`. That bumps a counter-only `Watch` on every path + below the prefix, so a woken front cannot tell what changed. Each re-runs + `select` under the table read lock (`retain_routes`, then `best_route`, + which hashes every qualifying pool member unless a Stay front's route + still serves), then the driver polls every track and rescans deadlines. +- `route_order` is rendezvous-style: FNV over the path and hop ids, lowest + wins. A join moves only paths the newcomer wins, a leave only paths the + leaver was winning. +- A front's `Pin` limits which routes it can take. `Stay` (an origin-named + front while serving) keeps its route while it serves; `Route` qualifies + only its own route; `Publisher` only routes whose first hop matches; + `Local` only local routes; `Any` (identity not yet known, or origin-named + and not serving) every route. Pool fronts in `origin/pool_churn` never + get an origin-naming reply, so they are `Publisher`, each on a different + member's hop. So on a join the fronts that can act are the `Any` ones, + `Publisher` ones on the new route's first hop, and `Local` ones for a + local route. On a leave, the fronts served by or requesting through that + route, plus those same pin sets: a deeper prefix appearing shadows + shallower routes for them, and one leaving can unshadow a broader route + for a waiter it never served. + +Decisions: + +- Covers every route change, not only equal-cost pools. The pool is where + the benchmark found it, but the wake is per prefix regardless of cost. + (2026-09-30) +- An index, not a payload filter on the watch: each route records the + fronts it serves, and each prefix node records only its non-Stay watches + (waiting fronts, publisher- and local-pinned fronts, `routed_broadcast` + waiters). A change wakes the ones it qualifies for (next decision) plus + the changed route's own fronts. A + payload filter would still walk every watch under the prefix, which stays + linear in paths. (2026-09-30) +- The prefix node keys its watches by what they qualify for: `Any` and + `routed_broadcast` waiters in one set, `Publisher` by hop, `Local` in its + own set. `Route` fronts hang off their route only. A front moves its + entries when it selects, and drops them when it closes. Any change to a + route wakes the `Any` set, its first hop's `Publisher` set, the `Local` + set if local, and the fronts it serves or is requesting for, so an + unrelated publisher joining or leaving wakes no serving pool front. The + pin sets wake on a loss too: withdrawing a deeper claim can unshadow a + broader route for a parked waiter it never served. A withdrawal takes + the served set before dropping the record. Fronts sit at paths below the changed prefix, so the lookup + walks descendants but skips subtrees whose matching set is empty, keeping + cost in woken fronts rather than served paths. (2026-09-30) +- A same-publisher join still wakes that hop's whole `Publisher` set, even + where the new route cannot win. Narrowing it to the paths the route wins + is left out: the `Publisher` pin goes with `--hop` removal in the cluster + routing line. (2026-09-30) +- Builds on shared-fronts' keying, so the index hangs off the final front + identity. `origin-front-parks.md` replaces the `routed_broadcast` retry + loop, one of the watch consumers here; whichever lands second adapts it. + (2026-09-30) +- Stay fronts never need a join or re-price: the edge/core tiers decision + (2026-09-30) dropped route upgrades, so a serving front moves only on + failover. + +Verification: `origin/pool_churn` flat in served paths at every pool width +it already sweeps, and a unit test that counts `select` calls per route +change and asserts only affected fronts re-select, with zero for an +unrelated publisher's join and leave, and a parked waiter retrying when a +deeper advertise-only claim over a served root is withdrawn. Keep `pool_resolve` unchanged. + +Public API: none. Wire: none. + +## Required + +- [Viewer sessions share a front](/quest/m0/shared-fronts.md) - re-keys fronts, which this index hangs off; it lands after the wildcard line, which is where the pool code lives + +## Related + +- [Front deadlines](/quest/m1/front-deadline-index.md) - each spurious wake also pays that per-track poll and deadline scan +- [Front parking](/quest/m1/origin-front-parks.md) - replaces the `routed_broadcast` watch loop this wakes diff --git a/quest/m1/session-death.md b/quest/m1/session-death.md deleted file mode 100644 index 2d848eebcf1..00000000000 --- a/quest/m1/session-death.md +++ /dev/null @@ -1,42 +0,0 @@ -# [S] Session death parity - -## Goal - -Rust and JS end a session's tracks and groups the same way: - -- A session closed locally, on purpose, ends the tracks it was receiving - cleanly in both languages. Groups still in flight end as they do today. -- A session that dies (peer close, transport failure) ends its tracks and - its group readers with the session's error, carrying the peer's code, - never the raw transport error, `Dropped`, or `Cancel`. - -## Plan - -https://github.com/moq-dev/moq/pull/4120 made a dying session end its tracks -with the session's error in both languages, and left two gaps. - -Since then #4351 and #4378 changed Rust abort semantics: an abort keeps the -finished groups and drops only the open ones, so readers get what finished -and then the abort, or a clean end once the declared end settled. #4385 (open) -mirrors that in JS. Build the clean end below on those semantics. - -Decided by the maintainer: - -- **A local close is a close, not an error.** JS already ends tracks cleanly - on `close()`. Rust has no clean end short of a finished track, so #4120 - ends them with the close error (previously `Cancel` or `Dropped`). Rust - needs a clean end at the current edge that is not a declared end: readers - get what was delivered and then `None`. Keep #4120's rule that an abort - before a declared end settles still wins, and #4116's clean end for a - dropped producer after `finish_at`; the new path must not mask either. - Whether this is a new `track::Producer` method or a driver-internal path - is an API call, so keep it private unless a consumer needs it. -- **JS group readers see the session's error.** Tracks go through - `sessionCause` in `js/net/src/error.ts`, but `runGroup` in the lite and - IETF subscribers closes a group with the raw error. Route it the same way. - -Extend the existing session-death tests (Rust -`a_session_death_ends_the_track_with_its_error`, the JS lite and IETF -integration cases) with a local close and with a group reader, on lite and -IETF. Behavior change, no signature change on either side unless the Rust -clean end needs a public method. diff --git a/quest/m1/shaper-step-clock.md b/quest/m1/shaper-step-clock.md new file mode 100644 index 00000000000..dca86b4b88d --- /dev/null +++ b/quest/m1/shaper-step-clock.md @@ -0,0 +1,16 @@ +# [XS] moq-shaper step test on mock time + +## Goal + +On `dev`, `moq-shaper`'s `a_step_gets_worse_part_way_through` asserts on a +paused or mock clock, not wall-clock latency, so a loaded runner cannot fail it. + +## Plan + +It failed once under a full-suite `just check --all` during the +[#4720](https://github.com/moq-dev/moq/pull/4720) merge (median latency +61.8ms before the 150ms step, 62.0ms after, against an expected +30ms), and +passed alone 3 times. Drive the shaper's delays from a paused Tokio clock or +the crate's own time source, then assert the step on that clock. + +Public API: none. Wire: none. diff --git a/quest/m1/source-map-removal.md b/quest/m1/source-map-removal.md deleted file mode 100644 index f4eddc12de4..00000000000 --- a/quest/m1/source-map-removal.md +++ /dev/null @@ -1,17 +0,0 @@ -# [XS] Delete SourceMap - -## Goal - -On `dev`, `moq_mux::SourceMap` and `Clock::source` are gone, so an `Anchor` -with its `Lane`s is the one way to map a source onto the broadcast clock. - -## Plan - -`SourceMap` is an `Anchor` plus one `Lane`, and nothing in the repository uses -it. A single-track source uses `Anchor` with one `Lane`. Migrate the -`SourceMap` tests in `rs/moq-mux/src/clock.rs` onto `Anchor`, and update -`doc/lib/rs/moq-mux.md`. - -## Required - -- [Public clock anchor](/quest/m1/clock-anchor-public.md) - `Anchor` and `Lane` must be public first diff --git a/quest/m1/subscribe-drop.md b/quest/m1/subscribe-drop.md index a74d3596df7..6b836be43c5 100644 --- a/quest/m1/subscribe-drop.md +++ b/quest/m1/subscribe-drop.md @@ -32,8 +32,8 @@ Decided: - Datagram groups stay best effort. A publisher counts a datagram as delivered, so a lost one leaves an uncovered hole that waits out the tail grace, as today. -- A resumed group ([Resumed groups](/quest/m1/resume-latest.md)) that is the - new copy's latest ends with the DROP's error when the copy drops it. +- A resumed group that is the new copy's latest ends with the DROP's error + when the copy drops it. - A dropped or aborted group is visible to readers, not silently skipped. #4533 found the Rust model releases an aborted group's sequence and skips it, so a truncated first object is indistinguishable from a group never sent. diff --git a/quest/m1/test-flakes-2/media-late-join.md b/quest/m1/test-flakes-2/media-late-join.md index c795da647c8..943532ef8bf 100644 --- a/quest/m1/test-flakes-2/media-late-join.md +++ b/quest/m1/test-flakes-2/media-late-join.md @@ -2,8 +2,8 @@ ## Goal -`just test media` late join holds its one-GOP budget under load, or the -regression behind it is fixed. +Late join holds its one-GOP budget under load in both `just test media` and +the interop browser lane, or the regression behind it is fixed. ## Plan @@ -13,3 +13,10 @@ It failed once after [#4181](https://github.com/moq-dev/moq/pull/4181): joining at the previous GOP's keyframe) or an off-by-one in how the fixture samples the live edge. Fix whichever it is; don't widen the budget without a reason. + +The interop browser check "late join starts live" +(`test/interop/clients/js/media.ts`, `MAX_LATE_JOIN_LAG_FRAMES`, one GOP +compared with `<=`) fails the same way on `main`: 16 frames behind with a +15-frame GOP on 09-24, 09-25, 09-28, and in +[#4577](https://github.com/moq-dev/moq/pull/4577). It samples `live` just +before the page opens. Treat it as the same flake (decided 2026-10-01). diff --git a/quest/m1/transport-seam.md b/quest/m1/transport-seam.md new file mode 100644 index 00000000000..2b2794778ff --- /dev/null +++ b/quest/m1/transport-seam.md @@ -0,0 +1,51 @@ +# [M] moq-net owns its transport seam + +## Goal + +moq-net's public API names only its own transport traits, and it has no +`web-transport-trait` dependency. A `web-transport-trait` or qmux major bump is +then a patch for moq-net: only the adapters outside moq-net change. + +## Plan + +Today `rs/moq-net/src/lib.rs` re-exports `web_transport_trait`, and +`Client::connect` / `Server::accept` (via `transport::poll::Boxable`) and the +`*_lite` entry points are bounded by `transport::poll::Session`, which on both +`main` and `dev` is a supertrait of `web_transport_trait::poll::Session` with a +blanket impl. So the upstream trait is public either way, and moving `main` from +qmux 0.5 to 0.6 (which needs `web-transport-trait` 0.5) would break moq-net. +That kept qmux 0.5.2 a hand-published backport on 2026-10-01. + +Decisions (2026-10-01): + +- ✅ Goal: trait bumps stop breaking moq-net. Rejected: only dropping the + `pub use`, since the trait still bounds `connect`/`accept`. +- ✅ The seam is the poll traits moq-net already has: `transport::poll::{Session, + SendStream, RecvStream}` become standalone traits with their own `poll_*` + methods and error type, not supertraits of `web_transport_trait::poll`. + Rejected: a new async trait. +- ✅ Adapters live outside moq-net: moq-tokio (native) and moq-wasm each wrap + web-transport sessions in a newtype implementing moq-net's traits. Callers of + moq-tokio's `Client`/`Server` see no change; direct moq-net users wrap + explicitly. Rejected: a blanket impl behind a feature in moq-net (versioned or + not) and a new adapter crate. +- ✅ Standalone m1 quest on `dev`, related to sans-io rather than a child of it. + +Work: + +- Remove `pub use web_transport_trait` and the dependency from moq-net; + `Error::from_transport` takes moq-net's own transport error. +- Port moq-net's tests and test transports to the owned traits. +- Add the adapters in moq-tokio and moq-wasm, plus every other crate that hands + a transport session straight to moq-net: moq-uring (`quic::Connection` / + `quic::web::Session` into `connect_lite` / `accept_lite`) and moq-ffi's wasm + branch. Add an adapter test each. +- Update `doc/` and the rustdoc on `transport`. + +Public API: breaking (moq-net's transport bounds and re-export), so `dev`. +Wire: none. + +## Related + +- [Sans-IO moq-net](/quest/m1/rs2ts/sans-io/README.md) - builds on the same seam +- [Raw stream codes](/quest/m1/raw-stream-codes.md) - the trait bump that exposed this diff --git a/quest/m1/transport-upgrade/README.md b/quest/m1/transport-upgrade/README.md index 25a197bed54..88ccc2afb10 100644 --- a/quest/m1/transport-upgrade/README.md +++ b/quest/m1/transport-upgrade/README.md @@ -33,9 +33,9 @@ every version (a client may send one with an empty URI; only a redirect URI is forbidden to a moq-transport client). The origin's multi-route front prefers the newest of two equal routes and `resume` splices each track at a group boundary, capping the old segment so the old session's subscription ends at the -boundary on its own. The JavaScript handover is the -[drain](/quest/m1/drain/README.md) line's (client goaway, done there), so the -JS half requires it. +boundary on its own. The JavaScript GOAWAY handover landed with the drain line, but it does not +splice tracks yet; the JS half requires +[JS group-boundary handover](/quest/m1/js-group-handover.md) for that. Shared decisions: @@ -59,7 +59,3 @@ Shared decisions: - [Rust](/quest/m1/transport-upgrade/rust.md) - moq-tokio keeps the QUIC dial after WebSocket wins and migrates through the existing Draining path - [JavaScript](/quest/m1/transport-upgrade/js.md) - js/net keeps the WebTransport dial after WebSocket wins and migrates through the client-goaway handover - -## Related - -- [Drain](/quest/m1/drain/README.md) - the peer-initiated half of the same handover diff --git a/quest/m1/transport-upgrade/js.md b/quest/m1/transport-upgrade/js.md index f5c85620356..eefefe9002d 100644 --- a/quest/m1/transport-upgrade/js.md +++ b/quest/m1/transport-upgrade/js.md @@ -11,10 +11,9 @@ nothing changes. ## Plan -Lands in `js/net`, after the [drain](/quest/m1/drain/README.md) line ships -the GOAWAY handover it reuses (client goaway is done on the line, JS -group-boundary handover is its remaining child): dial the replacement while the old session -keeps serving, swap the origin wiring once it is established, leave the old +Lands in `js/net`, reusing the GOAWAY handover the drain line shipped once +[JS group-boundary handover](/quest/m1/js-group-handover.md) splices tracks +across it: dial the replacement while the old session keeps serving, swap the origin wiring once it is established, leave the old session to close on its own or at the handover cap. See the [questline](/quest/m1/transport-upgrade/README.md) for the shared decisions. @@ -33,7 +32,7 @@ session to close on its own or at the handover cap. See the configured cap. A WebTransport attempt that fails after WebSocket won is logged at debug and the session stays on WebSocket. - A self-sent GOAWAY must gate new requests on the old session too, not only - a received one: [JS GOAWAY requests](/quest/m1/drain/js-goaway-requests.md) + a received one: [JS GOAWAY requests](/quest/m1/js-goaway-requests.md) covers the received case, so check it also covers this path. - On a successful upgrade delete the URL from `websocketWon`. - Tests in the browser harness against the in-tree relay: with the @@ -47,5 +46,5 @@ session to close on its own or at the handover cap. See the ## Required -- [Drain](/quest/m1/drain/README.md) - the GOAWAY handover this upgrade reuses -- [JS GOAWAY requests](/quest/m1/drain/js-goaway-requests.md) - no new request opens on a session that is going away +- [JS group-boundary handover](/quest/m1/js-group-handover.md) - tracks carry across the handover this upgrade reuses without a dropped group +- [JS GOAWAY requests](/quest/m1/js-goaway-requests.md) - no new request opens on a session that is going away diff --git a/quest/m1/ts-damaged-units.md b/quest/m1/ts-damaged-units.md index 8de2efbd990..e7124059e68 100644 --- a/quest/m1/ts-damaged-units.md +++ b/quest/m1/ts-damaged-units.md @@ -19,10 +19,10 @@ packet (clears `pending[pid]` and calls `stream.desync()`), and it is the model. Decided (maintainer, 2026-09-30): -- Fail-loud holds at unit granularity, as [TS PSI reassembly](/quest/m1/ts-psi-reassembly.md) - holds it at section granularity: a damaged unit is refused whole, never half +- Fail-loud holds at unit granularity, as the importer + holds it at section granularity for PSI (#4584): a damaged unit is refused whole, never half published, and counted, and nothing else about the feed changes. PSI damage - stays that quest's. + is already handled. - Drop the unit the way the TEI branch does: clear the PID's pending PES and desync its stream, so a codec that needs one waits for the next keyframe. Make sure the scratch buffer still advances past the packet on this path @@ -47,10 +47,6 @@ wherever the TS stats fields are described. Public API: additive, one stats field. Wire: none. -## Required - -- [TS PSI reassembly](/quest/m1/ts-psi-reassembly.md) - moq-mux owns the demux and PES header parsing this drops through - ## Closes - [#4581](https://github.com/moq-dev/moq/issues/4581) - one malformed packet ends the TS import diff --git a/quest/m1/ts-passthrough.md b/quest/m1/ts-passthrough.md new file mode 100644 index 00000000000..6a3aa3b5605 --- /dev/null +++ b/quest/m1/ts-passthrough.md @@ -0,0 +1,118 @@ +# [M] TS passthrough carries the multiplex verbatim on a fixed delay + +## Goal + +`moq import ts --passthrough` publishes a transport stream without +demultiplexing it, and `moq export ts --passthrough --delay ` writes the +same bytes back out, paced on the source's own PCR and released at a fixed +delay like an SRT receiver. From the first group it releases, the output is +byte-identical to the input, so it is exactly as T-STD-conformant as the +source. It carries what the demultiplexed lane cannot: TS-level scrambling, +private PIDs, and PSI/SI as authored. There is no JS player for it. + +## Plan + +Decided (2026-10-01), from a discussion with t0ms: + +- Why: the [T-STD line](/quest/m1/tstd/README.md) names a passthrough lane as + the only one that can carry primary distribution until the remux passes, and + none exists. Some feeds will want it permanently: the demultiplexed lane + cannot parse a scrambled elementary stream into frames, and an operator + who must hand on SI and private PIDs as authored has nothing to gain from + a remux. +- The track is listed in a new `m2ts` root section of the hang catalog, + not in `video`/`audio` (it has no codec to describe) and not in the + `mpegts` section, whose members (`tracks`, `program`, `si`, ...) record + what demultiplexing loses and are all in-band here. `m2ts` is one object, + `{ track, randomAccess, muxRate }`, naming the broadcast's one passthrough + track. Passthrough always carries the whole multiplex: splitting out a + program would rewrite the PAT, so there is no scope field, and a second + multiplex is a second broadcast. `muxRate` is present only while the + source holds a constant rate, as in `mpegts`. 188-byte packets only; a + 192-byte source is refused, so there is no packet-size field. This quest + specifies the section in `drafts/draft-lcurley-moq-mpegts.md` next to + `mpegts`, with its mapping onto MSFTS's track fields. A track is one shape + or the other. +- Objects are runs of whole 188-byte packets on one track. A group starts at + a packet with `random_access_indicator` set on the program's video PID + (its `stream_type` read from the PMT), or on the PCR PID when the program + has no video, so a joiner lands on a decodable picture whichever PID + carries the PCR. A group also starts at any PCR with + `discontinuity_indicator` set, since MSFTS forbids a PCR discontinuity + inside a group. A source that never sets the random-access indicator + falls back to a group every N PCRs (N to be fixed in the PR, with a group + spanning at most 1 s of PCR time). Pacing stays on the PCR PID throughout. +- `randomAccess` is true only while the PAT lists a single program with at + most one video PID, and every group has started at a + `random_access_indicator`. The first group that starts without one (a + fallback, or a `discontinuity_indicator` alone) republishes the catalog + with it false. Any other layout always publishes false, since programs or + video streams sharing a clock can stagger their GOPs. + Both indicators sit in the adaptation field, which TS scrambling leaves in + clear, so a scrambled feed still groups and paces. +- The publisher reads the PAT and PMT (with the existing `ts/psi.rs` + parsers) to find the PCR and video PIDs and count programs, and parses + nothing else past the adaptation field. PSI is never scrambled, and every + byte is still published as received. +- Each object's timestamp is the PCR time of its first byte, interpolated + between PCRs at the stream's own rate, carried in hang's `legacy` + container (a varint timestamp before the packets) as verbatim tracks are. + Object boundaries and timestamps then depend only on the bytes, so two + publishers of one feed publish identical objects, which 1+1 needs. +- A multiplex is paced on one PCR PID: the `PCR_PID` in the PMT of the PAT's + first program, or `--pcr-pid`. Every byte stays in order behind it. Programs + on independent clocks are out of scope. +- The export reuses the [fixed-delay release](/quest/m1/tstd/delay.md) + stage, keyed on each object's PCR time instead of a DTS, which also + spreads each object's bytes at the PCR-implied rate. It must pace on the + source's PCR, not on arrival: a pacer that re-clocks on arrival moves the + PCR-to-PTS offset over a long capture and fails the decoder buffers, even + though every byte is intact. +- An object that misses its deadline is dropped and counted, as the release + stage does. The output shows a continuity error there; passthrough never + rewrites continuity counters. +- Source clock drift is handled by the fixed-delay release's clock recovery + (#4645), shared with the demultiplexed export, not here. +- Passthrough gives 1+1 identity for free: two exporters fed the same + objects emit identical TS packets for every object both release, + continuity counters included, since passthrough never rewrites them; an + object one leg drops late is a gap in that leg only. Aligning the legs in + time is left to the `--sync` anchor planned for the demultiplexed export's + 2022-7 legs, which passthrough can adopt. + That is TS identity, not ST 2022-7 recovery, which also needs matching RTP + headers from a coordinated RTP egress. +- Rust only. `js/hang` does not parse `m2ts`, and a player sees no + rendition for the track. +- `--passthrough` publishes only the passthrough track. Publishing it + alongside the demultiplexed tracks is out of scope. + +Test: an export of a broadcast capture is byte-identical to the input from the +first released group, and the strict T-STD check gives the same verdict on +output and input. A scrambled fixture (`transport_scrambling_control` set on +its elementary PIDs) groups and paces the same as its clear twin. A +two-program fixture on one clock with staggered GOPs publishes `randomAccess` +false, as does one program with two staggered video PIDs. A single-program +fixture with the PCR on its audio PID starts groups at video random access +points only. PCR discovery finds a PCR PID that differs from the PMT PID, with +the PMT section split across packets. Two exporters fed the same objects with +different arrival skew emit identical bytes; when only one misses a deadline, +its output is the other's less that object's packets. A dropped object is +counted, and the rest still go out on time. Rerun the #4613 netem rig (10% +loss, 120 s) against it. + +Update `doc/bin/cli.md` for both flags, `doc/concept` for the section, and +the draft's comparison section to say this repository now publishes both +shapes. + +Public API: new import and export modes; nothing existing breaks. Wire: the +hang catalog gains an `m2ts` root section; additive. + +## Required + +- [Fixed-delay release](/quest/m1/tstd/delay.md) - the release stage this reuses, with its clock recovery (#4645) + +## Related + +- [TS hitless](/quest/m2/ts-hitless.md) - the demultiplexed lane's 2022-7 legs and the `--sync` anchor +- [MSFTS convergence](/quest/m2/msfts-convergence.md) - the ES-level side of the same mapping +- [TS byte schedule](/quest/m1/tstd/byte-schedule.md) - the remux's equivalent of pacing on the source PCR diff --git a/quest/m1/ts-psi-reassembly.md b/quest/m1/ts-psi-reassembly.md deleted file mode 100644 index d5fa4556ed3..00000000000 --- a/quest/m1/ts-psi-reassembly.md +++ /dev/null @@ -1,69 +0,0 @@ -# [L] TS PSI reassembly - -## Goal - -The MPEG-TS importer reads a PAT or PMT whose section spans several packets or -starts after a nonzero `pointer_field`, instead of aborting the import. Today -the `mpeg2ts` 0.6.1 reader parses PSI from a single packet and rejects a nonzero -`pointer_field`, and its error ends the whole import, so a valid long PMT (many -audio languages, long descriptors) or a PAT with more than about 40 programs -cannot be imported. `ts::Programs` finds such a PAT too. - -## Plan - -Settled decisions: - -- moq-mux owns the packet demux, and the import stops using - `mpeg2ts::ts::TsPacketReader`. It can't be fixed from outside: the reader's - PAT and PMT parsers are private to the crate, and it learns PES PIDs only - from PMTs it parsed itself. Feeding it a synthesized one-packet PMT to - register PIDs was rejected as fragile coupling, and fixing it upstream in - sile/mpeg2ts was rejected as slower and outside this repository. Switching to - `mpeg2ts-reader` was rejected too: it is a callback-driven demux framework - (last release January 2025), and moq-mux already owns the hard part in - `SectionReassembler`. Owning the demux also deletes the importer's - workarounds for the reader: the `Feed` mutex it reads through, and the - `pmt_pids`/`streams` gate that keeps unknown PIDs away from it. `mpeg2ts` - stays for export, tests, and plain types such as `StreamType`. -- PID 0 and every PMT PID go through the existing `SectionReassembler`, which - already handles `pointer_field`, split headers, and continuity gaps. The - PAT (`transport_stream_id`, version, program loop, possibly several sections) - and the PMT (program number, PCR PID, program descriptors, and the - elementary-stream loop with descriptors) are parsed in moq-mux. PES is - routed by the PIDs of the parsed PMT, and its header (PTS/DTS) is also parsed - in moq-mux. -- Each reassembled section's CRC-32/MPEG-2 is checked with the `crc` crate. - [Per-program SI](/quest/m2/ts-program-si.md) needs the same dependency, and - whichever change lands first adds it. A section with a bad CRC is dropped and - the last good table stays in force, so the import continues; it no longer - aborts as the `mpeg2ts` reader does. The section is refused whole, never - half-applied, so fail-loud holds at section granularity. A malformed - adaptation field on the PAT or a PMT PID, which ends the import today too, - costs that section the same way. A feed that never delivers a good PAT and - PMT still publishes nothing. Line noise on a live feed then costs one - repetition (0.5 s at most), not the broadcast; today one flipped CRC byte on - `bbb.ts` makes `decode` return `CRC32 mismatch` and `moq import ts` exit. -- Each section dropped for a CRC mismatch is counted in `Import::stats` as a - cumulative `crc_error`, so a corrupt feed is visible rather than silently - held on a stale table. It is stream-wide on `Stats`, since the PAT and PMT - PIDs have no elementary-stream row, and named for the TR 101 290 check that - [TS import health](/quest/m2/ts-import-health.md) adopts. A section dropped - for another reason (a malformed adaptation field, a parse failure) is not a - `crc_error`; that quest counts it under `PAT_error` or `PMT_error`. `Stats` - is `#[non_exhaustive]`, so the new field is additive; its docs (today per - elementary stream) and `is_empty` widen to cover it, and `ts::stats::Log` - reports it. -- `ts::Programs` reads through the same PAT path, so a PAT spanning packets - is found before any program publishes. -- One quest, because the demux refactor alone changes nothing observable. - -Keep every existing TS import test and fixture passing unchanged. Add tests for -a PMT spanning two packets, a PAT behind a nonzero `pointer_field`, a -multi-packet PAT with enough programs to need it (read by `ts::Programs` and -by `with_program`), and a PAT and a PMT with a corrupt CRC, each between good -repetitions, that is dropped and counted once while the import keeps its -previous layout and a later good PMT revision still applies. A positive -control: a feed whose only PAT is corrupt publishes nothing and counts it. - -Folded in from `ts-import-psi-crc` while landing the TR 101 290 plan (#4496): -that quest duplicated this one's bad-CRC drop and `CRC_error` count. diff --git a/quest/m1/ts-stats-module.md b/quest/m1/ts-stats-module.md index a68a4203fbd..93ce37348a5 100644 --- a/quest/m1/ts-stats-module.md +++ b/quest/m1/ts-stats-module.md @@ -30,16 +30,10 @@ Decided while planning the follow-ups of downcast. - Update moq-cli's `publish.rs`, the only consumer that names these types. moq-srt only uses `stats::Log`. -- Main quests still add fields under the old names: - [TS PSI reassembly](/quest/m1/ts-psi-reassembly.md), - [TS import health](/quest/m2/ts-import-health.md) and - [TS export liveness](/quest/m2/ts-export-liveness.md). The rename carries - those fields at merge time. Update the type names in the quests still open +- Main still adds fields under the old names: #4584 added `crc_error`, and + [TS import health](/quest/m2/ts-import-health.md) adds more. `Export::stats` + (#4577) also returns the old `ts::Stats`. The rename carries those at merge time. Update the type names in the quests still open when this lands, including [media stats schema](/quest/m1/stats/schema.md) and [Rust reporters](/quest/m1/stats/rust.md). Public API: breaking renames in moq-mux. Wire: none. - -## Required - -- `main` merged into `dev`, carrying #4506's `ts::stats` module (dev has none yet) diff --git a/quest/m1/tstd/README.md b/quest/m1/tstd/README.md index ec8f9dc2cff..9e502c0b057 100644 --- a/quest/m1/tstd/README.md +++ b/quest/m1/tstd/README.md @@ -25,3 +25,7 @@ nightly. - [Fixed-delay release](/quest/m1/tstd/delay.md) - frames go out at media time plus a fixed `--delay`, in one order under loss - [T-STD check](/quest/m1/tstd/check.md) - the harness grades the full buffer model instead of the transport buffer alone - [TS byte schedule](/quest/m1/tstd/byte-schedule.md) - PCRs sit on the byte grid the mux rate implies, paced against the fixed delay + +## Related + +- [TS passthrough](/quest/m1/ts-passthrough.md) - the passthrough lane named in the Goal diff --git a/quest/m1/wt-close-upstream.md b/quest/m1/wt-close-upstream.md index ac4b763988a..168d8748a73 100644 --- a/quest/m1/wt-close-upstream.md +++ b/quest/m1/wt-close-upstream.md @@ -47,10 +47,6 @@ header to `UnknownSession`, flooding relay logs with WARNs; the fork now keeps the read's cause and logs a reset at debug. After the bump, confirm on a moq.pro relay that the flood stops. -## Required - -- A `web-transport-moq` 1.3.x release that carries moq-dev/noq#24 - ## Related - [Browser close code](/quest/m1/browser-close-code.md) - the playwright case that proves Chromium reads the code this quest unblocks diff --git a/quest/m2/README.md b/quest/m2/README.md index 867c989db8e..b47fafec37a 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -19,7 +19,11 @@ waiting on an upstream release waits in [m4](/quest/m4/README.md). - [One port](/quest/m2/one-port/README.md) - a relay speaks QUIC and STUN on one UDP port and HTTP, RTMP, and RTMPS on one TCP port; WebRTC media is an embedder hook - [Ladder](/quest/m2/ladder/README.md) - a transcode ladder adapts to the uplink it publishes over, instead of encoding every live rung at its ceiling - [Processor](/quest/m2/processor/README.md) - a customer-run worker publishes an on-demand contribution under its own service prefix with scoped access +- [Synced data playback](/quest/m2/watch-data-sync.md) - js/watch releases JSON and binary payloads on the media playhead, and a slow data track holds media back - [Stream sessions](/quest/m2/uring-tcp/README.md) - serve WebSocket and HTTP from the io_uring workers, where io_uring pays off most +- [Hitless TS legs](/quest/m2/ts-hitless.md) - two `--sync` export legs emit packet-identical TS for ST 2022-7 +- [DVB E-AC-3](/quest/m2/ts-eac3.md) - E-AC-3 private data is split per sync frame and buffer-modelled in TS export +- [T-STD controls](/quest/m2/tstd-controls.md) - the harness gains an MB-overflow control and an AAC broadcast reference - [fMP4 emsg](/quest/m2/emsg.md) - settles the shared framing and missing-data semantics before this section adopts them - [ID3 catalog section](/quest/m2/id3.md) - timed ID3 as a first-class container-neutral catalog section - [FLV script tags](/quest/m2/flv-script.md) - onMetaData and AMF data messages survive RTMP and FLV import @@ -67,8 +71,8 @@ waiting on an upstream release waits in [m4](/quest/m4/README.md). - [Catalog colour model](/quest/m2/color-catalog.md) - the catalog describes a rendition's colour and HDR properties once a renderer consumes them - [Archive recovery listing](/quest/m2/archive-recovery-listing.md) - a resumed DVR lists what changed since its checkpoint, not every stored group - [Relay io_uring packages](/quest/m2/relay-io-uring-package.md) - Linux relay packages ship io_uring once the ring is on par with tokio -- [iOS capture](/quest/m2/mobile-capture-ios.md) - camera and screen capture if the mobile ownership decision selects Rust -- [Android capture](/quest/m2/mobile-capture-android.md) - NDK/JNI capture using the existing codecs if mobile ownership selects Rust +- [iOS capture](/quest/m2/mobile-capture-ios.md) - Rust captures the camera and screen on iOS +- [Android capture](/quest/m2/mobile-capture-android.md) - Rust captures through NDK/JNI on Android, reusing the existing codecs - [Mobile completion](/quest/m2/mobile-completion.md) - verify the selected native/mobile path before closing #700 - [Opus implementation](/quest/m2/audio-opus-backend.md) - compare Opus codec quality, CPU, build cost, and the loss recovery each backend offers - [Latency ledger](/quest/m2/latency-ledger.md) - a session reports where its end-to-end audio delay went, stage by stage @@ -96,17 +100,17 @@ waiting on an upstream release waits in [m4](/quest/m4/README.md). - [Keep-alive by deadline](/quest/m2/quic-keep-alive.md) - a PING only when the idle deadline nears, no fixed timer - [Socket close](/quest/m2/noq-socket-close.md) - `moq-quic` releases an endpoint's socket on close, so moq-tokio drops its wrapper - [GOP overhead](/quest/m2/gop-overhead.md) - price the I-frames a short GOP pays for, deciding whether a long GOP plus a keyframe request is worth designing -- [Per-program SI](/quest/m2/ts-program-si.md) - a selected TS program's broadcast carries only its own service's SDT and EIT - [TS import health](/quest/m2/ts-import-health.md) - `moq import ts` counts the TR 101 290 errors of the feed it receives, PCR and PTS graded on its own values -- [TS export liveness](/quest/m2/ts-export-liveness.md) - `moq export ts` reports each elementary stream's access units and quiet time, catching a per-track stall - [TS health stats](/quest/m2/ts-health-stats.md) - the TS counters ride the stats plumbing beside the media counters +- [TS stopped log: audio and video only](/quest/m2/ts-stopped-av-only.md) - sparse data PIDs like SCTE-35 stop logging "stopped delivering" every quiet second - [Teleoperation](/quest/m2/teleop/README.md) - MoQ carries robot video down and control up on one session as a library capability - [Media QA on other engines](/quest/m2/browser-media-qa-engines.md) - the media harness measures a Firefox or WebKit player over the fallback and names what each engine lacks -- [Firefox 155 WebTransport](/quest/m2/firefox-155-webtransport.md) - Firefox negotiates the version by subprotocol, and the other new WebTransport features stay unused on purpose -- [Windows.Graphics.Capture](/quest/m2/capture-wgc.md) - one WGC backend for display and window capture with the cursor, replacing Desktop Duplication and GDI +- [Windows.Graphics.Capture](/quest/m2/capture-wgc.md) - the WGC display and window backend verified on real Windows hardware - [Windows capture parity](/quest/m2/capture-windows.md) - system audio and a settled app-capture policy - [Linux capture parity](/quest/m2/capture-linux.md) - Wayland window/system-audio capture with explicit display-selection and app-capture limits - [Audio capture time](/quest/m2/audio-capture-time.md) - native audio stamps a buffer's capture instant, not when the driver reads it +- [CMAF frame timestamp](/quest/m2/cmaf-frame-timestamp.md) - CMAF decoders time samples from the moq-lite frame timestamp, using `tfdt` only within the fragment +- [Shared import clock](/quest/m2/shared-clock.md) - an importer joining a clock already in use offsets its PTS instead of moving it, so captures and imports share one timeline - [X11 capture transport](/quest/m2/x11-capture-shm.md) - move X11 capture to shared memory and RandR events instead of a per-frame socket copy - [Interop graceful close](/quest/m2/interop-graceful-close.md) - successful runner publications withdraw before disconnecting; external repository approval required - [Egress profile](/quest/m2/quic-egress-profile.md) - measure relay send-path syscalls, pacing bursts, and allocations before optimizing any of them diff --git a/quest/m2/audio-decode-mediacodec.md b/quest/m2/audio-decode-mediacodec.md index 5158823f91d..ed9e496e966 100644 --- a/quest/m2/audio-decode-mediacodec.md +++ b/quest/m2/audio-decode-mediacodec.md @@ -8,6 +8,10 @@ device's codec list opens. ## Plan +Rust owns capture and codecs on mobile, settled in the 2026-09-30 audit +([Mobile ownership](/quest/m1/mobile-ownership.md)), so this backend is +the Android audio codec path. + The audio counterpart of `rs/moq-video/src/decode/backend/mediacodec.rs`, behind a new optional audio `mediacodec` feature and the decode seam, on `target_os = "android"`. @@ -23,10 +27,6 @@ behind a new optional audio `mediacodec` feature and the decode seam, on `target - The binding ships in the moq-ffi Android slice, which is how Kotlin and Dart reach it. -## Required - -- [Mobile ownership](/quest/m1/mobile-ownership.md) - if Kotlin owns platform codecs, this backend is moot - ## Related - [Android capture](/quest/m2/mobile-capture-android.md) - the video MediaCodec family this sits beside diff --git a/quest/m2/audio-encode-mediacodec.md b/quest/m2/audio-encode-mediacodec.md index 52ead7973e8..c5d5d3a9e6c 100644 --- a/quest/m2/audio-encode-mediacodec.md +++ b/quest/m2/audio-encode-mediacodec.md @@ -7,6 +7,10 @@ where the device's encoder supports it. ## Plan +Rust owns capture and codecs on mobile, settled in the 2026-09-30 audit +([Mobile ownership](/quest/m1/mobile-ownership.md)), so this backend is +the Android audio codec path. + The audio counterpart of `rs/moq-video/src/encode/backend/mediacodec.rs`, behind the `mediacodec` feature and the encode seam. @@ -23,5 +27,4 @@ behind the `mediacodec` feature and the encode seam. ## Required -- [Mobile ownership](/quest/m1/mobile-ownership.md) - if Kotlin owns platform codecs, this backend is moot - [MediaCodec decode](/quest/m2/audio-decode-mediacodec.md) - the round-trip regression decodes through it diff --git a/quest/m2/audio-opus-backend.md b/quest/m2/audio-opus-backend.md index 0a5ccf0e6fc..ee9a2256804 100644 --- a/quest/m2/audio-opus-backend.md +++ b/quest/m2/audio-opus-backend.md @@ -40,7 +40,3 @@ tested only by reading the codec's control value. Retain fixtures and repeatable measurements in the existing CI/nightly audio harness. Public API and wire: no change for the study; validate compatibility before a separately scoped backend implementation. - -## Related - -- [Audio quality](/quest/m0/audio-quality-harness/README.md) - shared measurement infrastructure diff --git a/quest/m2/browser-media-qa-engines.md b/quest/m2/browser-media-qa-engines.md index 2cf1adce000..c20350bb349 100644 --- a/quest/m2/browser-media-qa-engines.md +++ b/quest/m2/browser-media-qa-engines.md @@ -33,7 +33,6 @@ fallback run covers the player, not Firefox's transport path (see open PR a measurement of that engine, not of the player. - Playwright WebKit is not Safari. Say so in the report; a Safari defect such as #2812 still needs a manual run. - -## Related - -- [Firefox 155 WebTransport](/quest/m2/firefox-155-webtransport.md) - the real Firefox transport path, verified by hand +- Real Firefox 155 still needs a manual run too: it reports a non-empty + `WebTransport.protocol` and lands on lite-06 against the in-tree relay, while + 153/154 take the draft-14 SETUP path. diff --git a/quest/m2/capture-wgc.md b/quest/m2/capture-wgc.md index 80302d8254a..5a25173cb09 100644 --- a/quest/m2/capture-wgc.md +++ b/quest/m2/capture-wgc.md @@ -1,97 +1,34 @@ -# [L] Windows.Graphics.Capture backend for display and window capture +# [S] Verify Windows.Graphics.Capture on real hardware ## Goal -One Windows.Graphics.Capture (WGC) backend serves `Source::Display` and -`Source::Window` on Windows and honors `config.cursor` through -`IsCursorCaptureEnabled`. It replaces both current backends: DXGI Desktop -Duplication (`capture/desktopduplication.rs`, which ignores `config.cursor`) -and GDI window capture (`capture/window.rs`, whose `PrintWindow` runs on the -target's UI thread). Frames stay on the GPU as NV12 `Surface::Texture`. +The WGC backend for `Source::Display` and `Source::Window` (`capture/wgc.rs`, +`capture/wgc/native.rs`) behaves correctly on real Windows desktops. Hosted +Windows CI has no desktop, so only the notification, pacing, startup, and +selector logic run there. Fix any bug the matrix turns up. Out of scope: application capture and system audio, which stay in -[Windows capture parity](/quest/m2/capture-windows.md). WGC captures one -monitor or one window per item and has no per-process source, so it does not -answer app capture. +[Windows capture parity](/quest/m2/capture-windows.md). ## Plan -Decided in planning (09-30): - -- **Replace, no fallback.** Delete `desktopduplication.rs` and the GDI capture - path. Keeping DD as a fallback means two backends for Windows builds that - are out of support anyway. -- **Mirror `capture/screencapture.rs`.** `CreateForMonitor` or - `CreateForWindow` (through `IGraphicsCaptureItemInterop`) builds the - `GraphicsCaptureItem`. Everything after that is shared: one session, a - free-threaded frame pool, a first frame to learn the size, and a guard that - closes the session on drop. `GraphicsCaptureItem.Closed` ends the stream. -- **GPU NV12 output.** Each BGRA pool frame goes through an - `ID3D11VideoProcessor` blit into an owned NV12 `d3d11::Texture` on the - capture device, so it can reach the Media Foundation hardware encoder - zero-copy, just as camera frames do. Pool frames are recycled, so copying out - is required. Generalize the existing NV12 `Scaler` in `frame.rs` to take BGRA - input instead of writing a second processor. openh264 already downloads a - `Texture` through `to_i420`. The existing scaler copies its input color - space to its output, which is right for resizing but wrong for converting. - Set the RGB range and the output YCbCr matrix explicitly so they match what - the encoder signals. Otherwise HD frames get converted with BT.601 while the - stream advertises BT.709. NV12 also needs even dimensions, and a window's - size is often odd. Keep the BGRA pool at its native size, and crop the NV12 - texture and the stream geometry down to even, as both current backends do. -- **The `windows` crate directly** (0.62, already a public re-export), not - `windows-capture` or another wrapper. We need our own D3D11 device, so it can - be shared with the encoder, and one `windows` version. This adds WinRT - features (`Graphics_Capture`, `Graphics_DirectX_Direct3D11`, - `Win32_System_WinRT_Graphics_Capture`, `Win32_System_WinRT_Direct3D11`, - `Win32_Graphics_Dwm`, and whatever else they pull in). Adding features is - additive. -- **Minimum Windows 10 2004 (build 19041).** That is where - `IsCursorCaptureEnabled` appears. Refuse loud below it with - `Error::Unsupported`, never silently drop the cursor. Set - `IsBorderRequired = false` where the property exists (build 20348+), but - request `GraphicsCaptureAccessKind::Borderless` first: without that access, - setting the property can fail with `E_ACCESSDENIED`. If access is denied, - keep the border rather than failing the capture. Older builds keep the - yellow capture border. Document the minimum in - `doc/lib/rs/moq-video.md`. -- **Resize ends the stream.** When `ContentSize` changes, debounce with - `Settle` and then return `Read::Done` so the caller reopens, the way - `window.rs` does today. The encoder's geometry is fixed. This also removes - DD's stale-size bug, where it keeps converting at the old width and height - after a mode change. -- **Displays enumerate across every adapter.** Use `EnumDisplayMonitors` - rather than adapter 0's DXGI outputs, which miss monitors attached to the - other GPU on hybrid laptops. Keep the user-visible `display:N` ids and names. -- **Windows skip DWM-cloaked entries.** Filter on `DWMWA_CLOAKED` (hidden UWP - windows and other virtual desktops), and size windows from - `DWMWA_EXTENDED_FRAME_BOUNDS` rather than `GetWindowRect`, which includes the - invisible borders. Enumeration never opens a capture: `ContentSize` belongs - to a frame, so it is only used for resize detection on the selected - capture. Keep the `window:{hwnd}` ids. - -Public API: no Rust signature changes. `config.cursor` starts working for -displays. Windows builds older than 19041 go from working to refused. - -Verification: Windows CI only runs `cargo check`, and hosted runners have no -real desktop. Add `#[ignore]` display and window tests, run them by hand on -real Windows hardware, and record this checklist in the PR: +Run the ignored `wgc_*` tests and `moq` capture by hand, and record each +result in the PR: - cursor on and off - an odd-sized window - color on an SD and an HD source, from BGRA through NV12 to the encoder, on both the Media Foundation path and the openh264 readback path -- window resize, close, and minimize -- multi-monitor +- window resize, close, and minimize, including minimizing between + `StartCapture` and the first frame +- a static screen keeps delivering paced frames without the rendition stalling +- multi-monitor, including a hybrid-GPU laptop - one Windows 10 2004+ machine, showing that the yellow border appears there -Fix the docs this makes stale in the same PR: `rs/moq-video/README.md`, -`doc/lib/rs/moq-video.md`, `doc/bin/cli.md`, the `capture/mod.rs` module doc, -the Cargo feature comment, and the `frame.rs` docs on which producers make a -`Texture`. +Full-display capture with the cursor on was verified on Windows 11 25H2 with +one monitor. ## Related - [Windows capture parity](/quest/m2/capture-windows.md) - app capture and system audio, which WGC does not answer -- [X11 capture transport](/quest/m2/x11-capture-shm.md) - owns the X11 frame-buffer reuse; the GDI half disappears with `window.rs` - [Direct3D11 render import](/quest/m2/render-d3d11.md) - the other end of keeping Windows frames on the GPU diff --git a/quest/m2/cmaf-frame-timestamp.md b/quest/m2/cmaf-frame-timestamp.md new file mode 100644 index 00000000000..42abb97467f --- /dev/null +++ b/quest/m2/cmaf-frame-timestamp.md @@ -0,0 +1,38 @@ +# [S] CMAF decoders time samples from the frame timestamp + +## Goal + +A CMAF fragment decodes at its moq-lite frame timestamp, even when its `tfdt` +says otherwise. Today both decoders take sample times from `tfdt` and ignore +the frame timestamp: `fmp4::decode` in `rs/moq-mux/src/container/fmp4/mod.rs` +and `Format.decode` in `js/hang/src/container/cmaf/format.ts`. A publisher +that moves a passthrough track onto another timeline (an importer joining a +shared clock, ad insertion splicing unrelated PTS bases) would play at the +source's PTS. + +## Plan + +Decided (2026-10-01): the frame timestamp is the broadcast timeline. A +payload's own timestamps are only relative within the frame. Rewriting `tfdt` +on the publisher was rejected: it can force a v0 to v1 box upgrade, which +shifts `trun` data offsets. + +- A sample's time is the frame timestamp plus its PTS offset from the + fragment's first sample. The encoder already stamps the frame with that + first sample's PTS (`fmp4::encode`), so today's output decodes unchanged. +- Rust's decode gets the frame timestamp from the caller. The fMP4 exporter + and moq-hls re-fragment decoded frames, so they follow without changes; + confirm with a test. +- Check the TS exporter's verbatim carriage, which keeps PES PTS inside the + payload. Rewrite those from the frame timestamp, or refuse a mismatch. +- `main`: a behavior fix with no API change. + +Test: in Rust and JS, a fragment whose `tfdt` disagrees with its frame +timestamp decodes at the frame timestamp, with B-frame offsets preserved. + +Public API: none. Wire: none; this states what the frame timestamp already +means. + +## Related + +- [Shared clock](/quest/m2/shared-clock.md) - the first publisher to rely on it diff --git a/quest/m2/firefox-155-webtransport.md b/quest/m2/firefox-155-webtransport.md deleted file mode 100644 index dc8af1eb6d5..00000000000 --- a/quest/m2/firefox-155-webtransport.md +++ /dev/null @@ -1,47 +0,0 @@ -# [XS] Firefox 155 negotiates the version by WebTransport subprotocol - -## Goal - -Firefox 155 reports the negotiated `protocol`, so a Firefox watcher lands on -the version the relay picks from `WT-Available-Protocols` (lite-06 today) -instead of the draft-14 SETUP fallback it used before, as Chrome 143+ does. -js/net's comments and types stop claiming native WebTransport lacks -`protocol`. - -## Plan - -- By hand against the in-tree relay, confirm Firefox 155 reports a non-empty - `protocol` and negotiates lite-06, and Firefox 153/154 still reach the - draft-14 SETUP path. Playwright Firefox has no WebTransport, so this cannot - run in CI (see [Media QA on other engines](/quest/m2/browser-media-qa-engines.md)). -- Delete the stale comment in `negotiate` (`js/net/src/connection/connect.ts`) - and the `@ts-expect-error` on `transport.protocol` in - `js/net/src/connection/accept.ts`, typing it the way `connect.ts` does if the - DOM lib still lacks the property. The empty-string and `undefined` fallback - stays for Firefox 153/154 and draft-14 peers. -- Fix any Firefox line in `doc/lib/js/index.md` or `doc/concept/transport.md` - this makes stale. - -Firefox 155 shipped five WebTransport features; decided 2026-09-26: - -- **Subprotocols** are the only one MoQ adopts in the browser, and js/net and - every Rust backend already speak them; this quest is the Firefox check. -- **Send groups** are native only. MoQ wants strict priority between - subscriptions (audio over video), and browser send groups are flat and - byte-fair, so js/net keeps the default group and its packed `sendOrder`. - moq-noq gains send groups in the - [scheduler](/quest/m1/quic/scheduler.md), and relays use one per broadcast on - cluster links in [Scope track priority](/quest/m1/track-priority-scope.md). - Pooling several publishers on one browser connection might want them later, - but sessions sharing the same content make that murky. -- **`datagrams.createWritable()`** is already feature-detected in js/net and - web-transport-wasm; nothing to do. -- **`draining`** is not used. Drain stays at the MoQ layer: GOAWAY carries a - redirect URI (and a timeout on moq-transport draft-17+; elsewhere the - deadline is sender-local) and works over qmux and WebSocket, while - `WT_DRAIN_SESSION` is advisory and carries neither (see - [drain](/quest/m1/drain/README.md)). -- **`exportKeyingMaterial()`** has no consumer: the exporter is per hop, so it - cannot key e2ee, which is end to end. Binding auth tokens to the TLS session - is the plausible future use; moq-noq already exposes the exporter, but - web-transport-moq does not. diff --git a/quest/m2/ietf-cluster-peers.md b/quest/m2/ietf-cluster-peers.md index 24c9d708499..a0803b2505a 100644 --- a/quest/m2/ietf-cluster-peers.md +++ b/quest/m2/ietf-cluster-peers.md @@ -18,8 +18,9 @@ The extension must carry what a lite cluster link carries by then: FETCH_OK Origin, so a downstream stitches failover on the member serving the path rather than the advertisement's first Hop ID. The draft's "Several Publishers of One Namespace" section changes with it. -- Whatever the [cluster routing line](/quest/m1/cluster-routing/README.md) - settles for topology and selection, since cluster links are lite-only there. +- The route layer of the [cluster routing line](/quest/m1/cluster-routing/README.md) + (per-node ROUTEs with seqno and metric, path-less announces, the down-only + bit) and its selection, since cluster links are lite-only there. Test: two workers behind a pool relay behind a moq-transport downstream relay; killing the serving worker ends the downstream subscription rather than diff --git a/quest/m2/latency-ledger.md b/quest/m2/latency-ledger.md index ad3cee94e8e..0ccc99977a7 100644 --- a/quest/m2/latency-ledger.md +++ b/quest/m2/latency-ledger.md @@ -9,8 +9,8 @@ come from the same place. ## Plan -The audio quality harness lands on ad-hoc debug probes, which is the right -trade to get it running. This quest promotes them. +The audio quality harness (`test/audio-quality/`) landed on ad-hoc debug +probes, which was the right trade to get it running. This quest promotes them. - Take the stage schema the harness already defines (capture, encode, publish flush, network, jitter buffer, decode, render) and expose it as fields of @@ -33,10 +33,8 @@ trade to get it running. This quest promotes them. ## Required -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - defines the stage schema and lands the probes this promotes - [Media stats schema](/quest/m1/stats/schema.md) - adds the stats and feedback snapshots this extends ## Related -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - defines the stages and is the first consumer - [QoS](/quest/m1/qos/README.md) - relay-side health, the same idea from the other end diff --git a/quest/m2/mobile-capture-android.md b/quest/m2/mobile-capture-android.md index 7482917a804..b408e7b8bd6 100644 --- a/quest/m2/mobile-capture-android.md +++ b/quest/m2/mobile-capture-android.md @@ -7,24 +7,19 @@ camera, MediaProjection for the screen, and MediaCodec for encode and decode. ## Plan -MediaCodec encode/decode already exist in moq-video. Reuse them rather than +Rust owns capture and codecs on mobile, settled in the 2026-09-30 audit ([Mobile ownership](/quest/m1/mobile-ownership.md)). MediaCodec encode/decode already exist in moq-video. Reuse them rather than planning a second backend family. The remaining capture and native Surface integration needs NDK/JNI lifecycle, synchronization, and actual device proof. -Weigh the cost honestly before starting. `moq-kit` already does this in Kotlin -over `moq-ffi`, and raw frames cannot cross the FFI boundary zero-copy, so a -Rust Android media path pays off for Rust-native consumers and not for the -mobile SDK. That is a real audience (this is the same gap that made -`iroh-live` reimplement the native layer) but it is worth naming, since it -decides whether XL is worth spending. +This is the Android SDK's media path, not a Rust-only extra. `moq-kit`'s +Kotlin capture is the parallel stack the verdict retires once this lands. +Frames cross `moq-ffi` as opaque `HardwareBuffer`/`Surface` handles rather than +copies. Rust-native consumers benefit too (the same gap that made `iroh-live` +reimplement the native layer). `moq-tokio` already reaches into Android through JNI for `tls::init_android`, so the mechanism exists. -## Required - -- [Ownership boundary](/quest/m1/mobile-ownership.md) - decides whether an NDK/JNI backend family is worth building - ## Related - [iOS capture](/quest/m2/mobile-capture-ios.md) - the other half of mobile, which diff --git a/quest/m2/mobile-capture-ios.md b/quest/m2/mobile-capture-ios.md index 9816b6a7d30..6d0fd59d6ad 100644 --- a/quest/m2/mobile-capture-ios.md +++ b/quest/m2/mobile-capture-ios.md @@ -7,6 +7,7 @@ through ReplayKit. ## Plan +Rust owns capture and codecs on mobile, settled in the 2026-09-30 audit ([Mobile ownership](/quest/m1/mobile-ownership.md)). Not a new codec backend. VideoToolbox encode, decode, and the native PixelBuffer surface compile on iOS (the `apple` cfg in moq-video). Verify the runtime path on a device rather than assuming desktop behavior. The new work is @@ -22,10 +23,6 @@ Reuse the `capture::Source` shape the other platforms use rather than growing an iOS-specific entry point, so device enumeration and selection behave the same everywhere. -## Required - -- [Ownership boundary](/quest/m1/mobile-ownership.md) - decides whether Rust owns capture on mobile at all - ## Related - [Android capture and encode](/quest/m2/mobile-capture-android.md) - the other half of diff --git a/quest/m2/mobile-completion.md b/quest/m2/mobile-completion.md index 8490656ed71..ed7f100b60c 100644 --- a/quest/m2/mobile-completion.md +++ b/quest/m2/mobile-completion.md @@ -9,15 +9,13 @@ work and deferred mobile phases are complete before #700 closes. ## Plan This quest owns the cross-phase completion proof, not another implementation. -Use the chosen Rust or platform-owned capture path and the existing binding -APIs. Record the supported path, limitations, and reproducible device results; -wire repeatable coverage into CI and identify the hardware evidence separately. -Update the native/mobile getting-started docs with the working path. - -The ownership decision may replace the Rust capture quests with platform-owned -work. In that case, update this quest's Required links to the replacement -implementation and proof quests before removing the abandoned blockers. -Abandoning a backend is not evidence that native/mobile support is complete. +Use the Rust capture path and the existing binding APIs: Rust owns capture +and codecs on mobile, settled in the 2026-09-30 audit +([Mobile ownership](/quest/m1/mobile-ownership.md)). Record the supported +path, limitations, and reproducible device results; wire repeatable coverage +into CI and identify the hardware evidence separately. Update the +native/mobile getting-started docs with the working path. + Do not close #700 merely because its next subset or a design decision finished. ## Required @@ -26,7 +24,6 @@ Do not close #700 merely because its next subset or a design decision finished. - [Dart codec parity](/quest/m1/dart-codecs.md) - codec-enabled artifacts and Dart video consumer integration - [iOS capture](/quest/m2/mobile-capture-ios.md) - deliver the selected iOS capture path - [Android capture](/quest/m2/mobile-capture-android.md) - deliver the selected Android capture and codec path -- [Mobile ownership](/quest/m1/mobile-ownership.md) - select and scope the mobile media architecture ## Closes diff --git a/quest/m2/msfts-convergence.md b/quest/m2/msfts-convergence.md index ac9a67dc4a3..010289aa5f4 100644 --- a/quest/m2/msfts-convergence.md +++ b/quest/m2/msfts-convergence.md @@ -4,7 +4,9 @@ A subscriber author can map this repository's demultiplexed TS lane (access units, Hang catalog `mpegts` section) onto MSFTS ES-level carriage -without guessing. Transporting TS verbatim is a non-goal. +without guessing. Transporting TS verbatim is out of scope here: +[TS passthrough](/quest/m1/ts-passthrough.md) carries it in an `m2ts` +catalog section that maps onto MSFTS's track fields. ## Plan @@ -20,7 +22,9 @@ egress moved to msfts#37 (closed). Two differences remain: matter to a subscriber, and converge or document the mapping. Update `drafts/draft-lcurley-moq-mpegts.md` and `doc/concept` with whatever -lands. Open TS PRs [#4577](https://github.com/moq-dev/moq/pull/4577) (per-ES -access units at export), [#4579](https://github.com/moq-dev/moq/pull/4579) -(export on the mux rate), and [#4580](https://github.com/moq-dev/moq/pull/4580) -(per-program SI) touch the same area; land or rebase on them first. +lands. [#4577](https://github.com/moq-dev/moq/pull/4577) (per-ES access +units at export) has merged, and [#4579](https://github.com/moq-dev/moq/pull/4579) +(export on the mux rate) closed in favour of the +[T-STD line](/quest/m1/tstd/README.md). +[#4580](https://github.com/moq-dev/moq/pull/4580) (per-program SI) is still +open and touches the same area; land or rebase on it first. diff --git a/quest/m2/one-port/README.md b/quest/m2/one-port/README.md index d7c055b883a..bbef222df9c 100644 --- a/quest/m2/one-port/README.md +++ b/quest/m2/one-port/README.md @@ -78,7 +78,7 @@ pre-accepted streams can stand behind. - [UDP demux](/quest/m2/one-port/udp-demux.md) - one socket carries QUIC and STUN answers, with greasing off and a WebRTC hook for embedders - [TCP acceptor](/quest/m2/one-port/tcp-demux.md) - one listener carries TLS-terminated HTTP, RTMP, and RTMPS -- [SRT on the shared socket](/quest/m2/one-port/srt-demux.md) - srt-tokio accepts a virtual socket and the flow table pins its 4-tuples +- [SRT on the shared socket](/quest/m2/one-port/srt-demux.md) - moq-srt drives `srt-protocol` on demuxed packets and the flow table pins its 4-tuples ## Related diff --git a/quest/m2/one-port/srt-demux.md b/quest/m2/one-port/srt-demux.md index d12a4920833..f540d2b5116 100644 --- a/quest/m2/one-port/srt-demux.md +++ b/quest/m2/one-port/srt-demux.md @@ -9,12 +9,12 @@ induction handshake. ## Plan `srt-tokio` 0.4 takes a `tokio::net::UdpSocket` in `bind_with_socket` and -nothing more abstract. Preferred: an upstream PR giving `SrtListener` a -socket trait (`poll_recv_from` and `poll_send_to`) that a real socket and our -virtual socket both implement, then `moq_srt::Server` takes the virtual -socket. Fallback if refused: `srt-protocol` is sans-io, so `moq-srt` drives -its `Listen` and `Connection` state machines directly on fed packets, which -is more code but removes the dependency on `srt-tokio`'s socket handling. +nothing more abstract. Decided 2026-09-30: `srt-protocol` is sans-io, so +`moq-srt` drives its `Listen` and `Connection` state machines directly on the +demuxed packets. That is more code, but it removes the dependency on +`srt-tokio`'s socket handling. An upstream socket trait was rejected: +upstream (now rosalyntg/srt-rs) has not been pushed since 2024-05, and its +maintainer said they do not actively support it. The demux side is the flow table from [UDP demux](/quest/m2/one-port/udp-demux.md): a 4-tuple already pinned to WebRTC (ICE succeeded) is never tested for SRT. @@ -36,4 +36,3 @@ with a QUIC client active on the same port; a QUIC short header from a new ## Required - [UDP demux](/quest/m2/one-port/udp-demux.md) -- srt-tokio accepts a caller-supplied socket abstraction upstream, or the sans-io fallback is chosen diff --git a/quest/m2/p2p/README.md b/quest/m2/p2p/README.md index f6e40ddb044..5067a3e8de1 100644 --- a/quest/m2/p2p/README.md +++ b/quest/m2/p2p/README.md @@ -5,9 +5,10 @@ Opted-in moq-lite clients serve each other directly, over WebRTC data channels between any two peers and over iroh between native ones, while the relay stays the rendezvous and the path of last resort. A client subscribes through the -relay at once, negotiates peers in the background, and each subscription -migrates to a peer when the origin ranks that route cheaper and back to the -relay when the peer goes away. Nothing waits on ICE and nothing breaks when it +relay at once, negotiates peers in the background, and a subscription +migrates to a direct peer when that peer is the broadcast's origin (or a +routing node such as `moq-cli --p2p`) and back to the relay when the peer +goes away. A browser never carries another peer's traffic. Nothing waits on ICE and nothing breaks when it fails: a symmetric NAT, a denied local-network prompt, or a filtered multicast segment leaves the client exactly where it started. @@ -93,23 +94,19 @@ the same independence without fighting the browser. ### Routes and migration -Rust already does the serving-side work: `best_route` re-runs on every table -change and a live subscription re-splices onto a cheaper route with the same -first hop at a group boundary, while an anonymous chain never wins. The JS -origin ranks the same way (`compareRoutes` in `js/net/src/origin.ts`: cost, -then fewest hops, then newest). The JS handshake -already declares a random hop id, so browser hops are identified; the roster -id is that hop id, held once per origin rather than once per session. +Routing is the [cluster routing line](/quest/m1/cluster-routing/README.md)'s: +every session speaks per-node ROUTEs and path-less ANNOUNCEs, so a broadcast's +announce names its origin node, the roster maps that node to a peer, and the +route metric over the node's own link costs picks the direct link or the +relay. The roster id is the node id, held once per origin rather than once +per session. -Watcher-to-watcher offload needs a tab to forward what it receives, which the -JS origin does not do today: [transit](/quest/m2/p2p/transit.md). - -How a P2P route outranks the relay's is deliberately open. Costs today live -in one scope, a relay mesh pricing its own links; a P2P link is neither free -nor the CDN's egress, and `warm` only accumulates, so a tab forwarding the -relay's route ties the relay and loses on chain length. -[Cost across scopes](/quest/m2/p2p/cost-scopes.md) writes the rule before -the watcher depends on it. +Decided 2026-10-01: a direct link wins only when the peer is the origin or a +routing node; a watching tab never re-serves what it receives. Offloading a +room of watchers is a customer relay's job (`moq-cli --p2p` is the small +one), which keeps warm and cold costs and cache-aware routing out of the +protocol. [Direct peers win](/quest/m2/p2p/cost-scopes.md) sets the cost +knobs and defaults. No Rust + WASM in-tab hop: [rs2ts](/quest/m1/rs2ts/remove-wasm.md) removes the WASM build, so the browser side stays TypeScript. @@ -132,8 +129,7 @@ WASM build, so the browser side stays TypeScript. - [Signaling and policy](/quest/m2/p2p/signal.md) - opted-in peers find each other under the prefix, the application picks who to dial, and the roster-size gate decides whether STUN is used - [Native data channel transport](/quest/m2/p2p/webrtc.md) - `moq-tokio` holds a moq-net session with a browser over str0m with a full ICE agent - [moq-cli joins](/quest/m2/p2p/cli.md) - `--p2p` publishes a roster entry with its iroh endpoint, dials iroh between native peers, and serves browsers as a transit hop -- [Transit in the JS origin](/quest/m2/p2p/transit.md) - a tab forwards the routes it receives to its peers with split horizon and its hop id appended -- [Cost across scopes](/quest/m2/p2p/cost-scopes.md) - the written rule for how relay egress, mesh links, and P2P links compare, so a peer that already carries a broadcast wins +- [Direct peers win](/quest/m2/p2p/cost-scopes.md) - a direct link wins only when the peer is the origin or a routing node, by link costs the node sets itself - [Watch opts in](/quest/m2/p2p/watch.md) - one attribute turns it on in the demo and the watcher migrates to the cheapest route - [Harness](/quest/m2/p2p/harness.md) - the Playwright harness and the numbers behind every mapping decision - [Unordered qmux](/quest/m2/p2p/unordered.md) - qmux tolerates reordering so the data channel runs unordered and a loss stalls one stream diff --git a/quest/m2/p2p/cost-scopes.md b/quest/m2/p2p/cost-scopes.md index fd0a2e5b2ce..c6223259a31 100644 --- a/quest/m2/p2p/cost-scopes.md +++ b/quest/m2/p2p/cost-scopes.md @@ -1,52 +1,45 @@ -# [S] Cost across scopes +# [S] Direct peers win ## Goal -A written rule, in `doc/concept/transport.md` and the drafts, for how a route -through a P2P peer compares with the relay's own route, such that a peer -already carrying a broadcast wins and an idle peer never has traffic pulled -through it. The rule is implementable by the Rust and JS origins with the -existing `Cost { warm, cold }` and the per-link price a publisher declares in -SETUP. +A subscriber uses a direct P2P link only when the peer at the other end is +the broadcast's origin or a routing node (a `moq-cli --p2p` hop or a relay +the customer hosts), and the CDN otherwise. A browser never carries another +peer's traffic. The rule is the route layer's ordinary metric over link +costs the node sets itself, with defaults that make a direct link beat the +relay, so a watcher next to the publishing tab pulls from it and everyone +else pulls from the CDN. ## Plan -This is a planning quest; it produces the rule and rewrites the -implementation quests that depend on it. - -What is settled today: a publisher declares an egress price in SETUP on -lite-06 and the receiver charges it per link, with local policy able to -override the peer's declaration; `warm` and `cold` accumulate per link and -compare in that order; a chain containing an anonymous hop loses to any -identified chain; Rust migrates a live subscription only to a route with the -same first hop. - -What is not: costs live in one scope, a mesh pricing its own links. A P2P -link is neither free (peer uplink, reliability) nor the CDN's egress, and -nothing flattens `warm` when an origin already carries the content, so a tab -forwarding the relay's route at the relay's price plus its own ties the relay -and loses on chain length. Candidate rules to weigh, with the numbers from -the [harness](/quest/m2/p2p/harness.md) where they exist: - -- a warm discount: an origin actively receiving a broadcast re-announces it - at `warm` 0 and `cold` unchanged, over every session, in both languages, - which is what the two magnitudes were designed for; -- explicit prices per scope: the relay's client-facing link priced as egress, - the P2P link priced by the application, and a documented comparison between - them; -- a scope tag on the route, so a P2P route is compared with a relay route by - a rule rather than by subtraction. - -Decide who sets the P2P link price (the application, per `Peers` knob), what -the relay's client-facing default is, whether the first-hop migration gate -holds when the first hop is a publisher tab, and whether the warm discount -belongs in the mesh too. Write the rule beside route selection in -`doc/concept/transport.md`, mirror it in `draft-lcurley-moq-lite.md` and -`draft-lcurley-moq-cluster.md` where the wire carries it, and pass -`just drafts check`. Then update [watch](/quest/m2/p2p/watch.md) and -[transit](/quest/m2/p2p/transit.md) with the chosen rule and open the -implementation quest it needs. - -## Related - -- [Cluster routing](/quest/m1/cluster-routing/README.md) - the mesh-side use of cost +Decided 2026-10-01 (moq-dev/moq#4694): + +- No warm or cold cost and no cache state. A peer that merely watches a + broadcast does not offer it to other peers; deduplicating a site's internet + traffic is a customer relay's job, which is a routing node like any other. + This deleted the JS transit quest. +- Link costs are the node's own policy: `Peers` sets the cost of P2P links + and of the relay session, mirrored by `moq-cli --p2p`. The relay declares + nothing per client. Pick defaults so one direct LAN hop beats the relay + route, and a relay route beats two P2P hops through a routing node only + when the application says so. +- The comparison needs the node ids and metrics of + [Routes and announces](/quest/m1/cluster-routing/routes.md): a broadcast's + ANNOUNCE names its origin node, and the roster maps that node to a peer the + application may dial, so an app learns whom to dial from the announce. +- Switching between the peer and the relay splices for an epoch-qualified + path (same source), and is a discontinuity otherwise, per + [Selection](/quest/m1/cluster-routing/selection.md). + +Deliverables: the `Peers` and `moq-cli` cost knobs with their defaults, the +rule written beside route selection in the routing concept page the cluster +routing line adds, and tests with the mock transport: a watcher with a direct +link to the publishing tab uses it and returns to the relay when it drops; a +watcher whose only direct peer is another watcher stays on the relay. + +Public API: cost knobs on `Peers` and `moq-cli`. Wire: none beyond the route +layer. + +## Required + +- [Routes and announces](/quest/m1/cluster-routing/routes.md) - the origin node ids and metrics this compares diff --git a/quest/m2/p2p/harness.md b/quest/m2/p2p/harness.md index 0863d93dd2d..383581bfffa 100644 --- a/quest/m2/p2p/harness.md +++ b/quest/m2/p2p/harness.md @@ -38,4 +38,3 @@ points. - [moq-cli joins](/quest/m2/p2p/cli.md) - [Signaling and policy](/quest/m2/p2p/signal.md) -- [Transit in the JS origin](/quest/m2/p2p/transit.md) - the watcher-to-watcher row diff --git a/quest/m2/p2p/signal.md b/quest/m2/p2p/signal.md index c0884266119..2857a6881c4 100644 --- a/quest/m2/p2p/signal.md +++ b/quest/m2/p2p/signal.md @@ -59,8 +59,7 @@ failure is terminal for the pair until either side re-announces; no retry loop. Each established connection goes through `connect` with the supplied transport on the dialing side and `accept` on the answering side, both with `publish: origin.consume()` and `consume: origin`, so the tab serves what it -publishes and, once [transit](/quest/m2/p2p/transit.md) lands, what it -receives. +publishes, never what it receives (a tab is not transit). Trust: publishing under the prefix proves only that the relay admitted the peer to the prefix, not that it may read everything this tab can. Tokens diff --git a/quest/m2/p2p/transit.md b/quest/m2/p2p/transit.md deleted file mode 100644 index cde7482763c..00000000000 --- a/quest/m2/p2p/transit.md +++ /dev/null @@ -1,35 +0,0 @@ -# [S] Transit in the JS origin - -## Goal - -A test proves a tab forwards the routes it receives on one session to the -sessions it serves, so a watcher can re-serve a broadcast to a peer and a room -of watchers can cost the relay one egress stream. Split horizon holds: a route -is never announced back to the session it arrived on, and every forwarded -chain carries the tab's hop id. - -## Plan - -Decided in the 2026-09-30 audit: shrink to a test. The Rust origin already -does transit (`with_publisher` and `with_subscriber` splice and share remote -routes), and [Generated lite](/quest/m1/rs2ts/lite.md) replaces the -hand-written JS origin with one generated from it, so there is no JS -forwarding step to write by hand. - -The hop id must be one per origin, not per session, so the roster id from -[signaling](/quest/m2/p2p/signal.md) and the id in forwarded chains agree. - -Tests with the mock transport pair against the generated origin: a route -received on A appears on B with the id appended and never on A; a chain -already containing the id or at `MAX_HOPS` is dropped; a retraction on A -retracts on B; two watchers of one tab share one upstream subscription. If a -case fails, the fix goes in the Rust origin. - -## Required - -- [Generated lite](/quest/m1/rs2ts/lite.md) - the JS origin generated from moq-net, which already does transit - -## Related - -- [Watch opts in](/quest/m2/p2p/watch.md) - the first topology that needs a forwarding tab -- [Cost across scopes](/quest/m2/p2p/cost-scopes.md) - adds the receiving link's cost to forwarded routes diff --git a/quest/m2/p2p/watch.md b/quest/m2/p2p/watch.md index 8307f6c0649..631b74ef2ac 100644 --- a/quest/m2/p2p/watch.md +++ b/quest/m2/p2p/watch.md @@ -3,27 +3,24 @@ ## Goal One attribute turns P2P on in the demo. A watcher tab keeps subscribing -through the shared origin, its subscriptions migrate to a peer when the -origin ranks that route cheaper, and they return to the relay when the peer -goes away, with no visible interruption. +through the shared origin, its subscriptions migrate to a direct peer that is +the broadcast's origin or a routing node, and they return to the relay when +the peer goes away, with no visible interruption. ## Plan `hang-watch` and `hang-publish` gain a `p2p` attribute that constructs `Peers` on the shared connection's origin with the demo's ICE servers and a `max` from the page; `demo/web` exposes the toggle. The route pick is the -origin's, from its cost ranking (`compareRoutes` in `js/net/src/origin.ts`) under the rule from -[cost across scopes](/quest/m2/p2p/cost-scopes.md): a peer already carrying -the broadcast wins, and its retraction falls back to the relay. +origin's, under the rule from [Direct peers win](/quest/m2/p2p/cost-scopes.md): +a direct link to the origin or a routing node wins, and losing it falls back +to the relay. -Three topologies and the demo shows all of them: a publishing tab serving -watcher tabs directly, watcher tabs pulling from a `moq-cli --p2p` hop, and a -watcher tab re-serving to another watcher through -[transit](/quest/m2/p2p/transit.md). +Two topologies and the demo shows both: a publishing tab serving watcher tabs +directly, and watcher tabs pulling from a `moq-cli --p2p` hop. ## Required - [Signaling and policy](/quest/m2/p2p/signal.md) - [moq-cli joins](/quest/m2/p2p/cli.md) - the native hop the second topology shows -- [Transit in the JS origin](/quest/m2/p2p/transit.md) -- [Cost across scopes](/quest/m2/p2p/cost-scopes.md) +- [Direct peers win](/quest/m2/p2p/cost-scopes.md) diff --git a/quest/m2/quic-careful-resume.md b/quest/m2/quic-careful-resume.md index d00d185ebd0..b009be4dcb8 100644 --- a/quest/m2/quic-careful-resume.md +++ b/quest/m2/quic-careful-resume.md @@ -38,5 +38,6 @@ only when the jump never makes the first second worse than slow start. - [Transport upgrade](/quest/m1/transport-upgrade/README.md) - one of the reconnects this speeds up -- [Drain](/quest/m1/drain/README.md) - GOAWAY redials are the other +- [JS group-boundary handover](/quest/m1/js-group-handover.md) - GOAWAY + redials are the other - [noq#815](https://github.com/n0-computer/noq/issues/815) - the careful-resume proposal to n0 diff --git a/quest/m2/quic-deadline.md b/quest/m2/quic-deadline.md index a644a87043b..cdfcd5b1b81 100644 --- a/quest/m2/quic-deadline.md +++ b/quest/m2/quic-deadline.md @@ -21,7 +21,7 @@ Implement in the fork. - On loss detection, before queueing a retransmission for a stream with a deadline, estimate the arrival instant as now plus the forward one-way delay. Start with `min_rtt / 2`, corrected by the peer's reported ACK delay; - the [receive-timestamps spike](/quest/m3/quic-receive-ts.md) replaces that + the [GCC experiment](/quest/m3/quic-gcc.md)'s receive timestamps replace that guess with a measured forward delay. If the estimate is past the deadline, reset the stream with a dedicated error code and drop its retransmit ranges, including bytes already lost, so flow control is returned in one step. @@ -52,8 +52,8 @@ Decided in the 2026-09-30 audit: moved to m2. No m1 quest consumes it. ## Related -- [Receive timestamps](/quest/m3/quic-receive-ts.md) - a measured forward - delay replaces the half-RTT estimate +- [QUIC GCC](/quest/m3/quic-gcc.md) - its receive timestamps give a measured + forward delay that replaces the half-RTT estimate - [Discover media headroom](/quest/m2/quic-probe.md) - can reuse retransmission machinery if redundant capacity probes prove worthwhile - [noq#813](https://github.com/n0-computer/noq/issues/813) - the per-stream deadline proposal to n0 diff --git a/quest/m2/quic-keep-alive.md b/quest/m2/quic-keep-alive.md index 609172872f4..063277f616d 100644 --- a/quest/m2/quic-keep-alive.md +++ b/quest/m2/quic-keep-alive.md @@ -3,8 +3,8 @@ ## Goal A connection sends a PING only when its idle deadline is near, never on a -fixed clock. An idle connection with a 30 s idle timeout costs one packet per -roughly 30 s minus a few PTOs; a busy connection costs none. The +fixed clock. An idle connection with a 10 s idle timeout costs one packet per +roughly 10 s minus a few PTOs; a busy connection costs none. The `keep_alive` setting on `quic::Client` and `quic::Server` becomes an optional maximum, default none. Redefining it is a config and CLI break, so this targets `dev`. diff --git a/quest/m2/render-d3d11.md b/quest/m2/render-d3d11.md index 5da44e0ebdc..ef902188fef 100644 --- a/quest/m2/render-d3d11.md +++ b/quest/m2/render-d3d11.md @@ -24,7 +24,7 @@ consumers land" comes due for `d3d11::Texture`: the renderer is that consumer, so whatever access it needs becomes public here rather than ahead of it. Worth doing on Windows specifically because 4K screen sharing is where the -download plus re-upload costs most, and Desktop Duplication makes that the +download plus re-upload costs most, and WGC screen capture makes that the common case. ## Related diff --git a/quest/m2/shared-clock.md b/quest/m2/shared-clock.md new file mode 100644 index 00000000000..1eb45c09494 --- /dev/null +++ b/quest/m2/shared-clock.md @@ -0,0 +1,68 @@ +# [M] One timeline for every source on a catalog + +## Goal + +Every source sharing a catalog (captures, and any number of container +importers) lands on one forward timeline, whichever starts first. Today the +first importer's first frame re-anchors the catalog clock +(`catalog::Producer::anchor`). A capture that started earlier has copied the +old clock, and a second importer's anchor is a no-op, so it publishes its own +PTS base on a clock it didn't place. Either way the sources drift apart. Following the new clock doesn't help: sources near PTS 0 step it +backwards, and `container::Producer::write` refuses a group below the last one +(`TimestampRewind`), even across a discontinuity. + +## Plan + +Follow-up of [#4668](https://github.com/moq-dev/moq/pull/4668), which moved +data tracks onto the anchored clock. The anchor exists only on `dev`, so this +targets `dev`. + +Decided (2026-10-01): + +- The clock never moves once something stamps with it. The public + `catalog::Producer::clock()` fixes the mapping (sets `anchored`). Crate-internal + readers, like #4668's `Listing`, read the state without fixing it. +- An importer anchors once, on its first frame, and gets back an offset: zero + if it placed the mapping (PTS stays verbatim), `clock.now() - first_pts` + otherwise. Every frame from that importer, on all of its tracks, is shifted + by that offset, so its audio and video stay in sync. Separate importers get + separate offsets. +- Captures read the clock once and never re-check it. Audio already stamps by + sample count from one `clock.now()` per epoch; there is no per-frame clock + read to add. +- Rejected: captures re-read the clock per frame and mark a break when it + changes. A backward anchor would hit `TimestampRewind`, and every audio + buffer would take the catalog lock to watch for an event that should never + happen while it is live. +- The offset rewrites only the moq-lite frame timestamp, never the payload. An + fMP4 passthrough fragment keeps its source `tfdt`, so the two disagree. The + frame timestamp is the broadcast timeline; a payload's timestamps are only + relative within the frame. Ad insertion (splicing sources with unrelated PTS + bases) needs the same rule. +- m2: nothing in-tree reaches it. `moq-cli publish` runs a capture or a stdin + import, never both, and moq-ffi does not expose capture. + +Guidance: + +- The offset is signed: a stream starting at 3600 s on a clock reading 10 s + shifts down. Refuse a frame that would land below zero rather than clamp it. +- A `with_clock` catalog is fixed from the start, so its importers offset too. + Today they publish verbatim PTS on a clock they didn't place. +- Document on `catalog::Producer::clock` that taking the clock fixes it. +- Test: start a synthetic capture, then import an fMP4 starting at PTS 0. Both + tracks advance from the capture's timeline with no rewind. Also cover the + reverse order: an importer first keeps its PTS verbatim. A passthrough + fragment whose `tfdt` disagrees with its frame timestamp decodes at the frame + timestamp, in Rust and JS. + +Public API: no new items. `catalog::Producer::clock()` now fixes the mapping, +and importers no longer publish verbatim PTS when the clock was already taken. +Wire: none. + +## Required + +- [CMAF frame timestamp](/quest/m2/cmaf-frame-timestamp.md) - decoders honour an offset frame timestamp on passthrough tracks + +## Related + +- [Audio capture time](/quest/m2/audio-capture-time.md) - maps audio's capture timeline onto the broadcast clock once per open diff --git a/quest/m2/ts-eac3.md b/quest/m2/ts-eac3.md new file mode 100644 index 00000000000..424fba20598 --- /dev/null +++ b/quest/m2/ts-eac3.md @@ -0,0 +1,33 @@ +# [S] DVB E-AC-3 gets a buffer model in TS export + +## Goal + +`moq export ts` splits DVB E-AC-3 private-data PES (`stream_type` 0x06 with +an `enhanced_AC3_descriptor`) one access unit per PES, schedules each against +the E-AC-3 buffer ETSI TS 101 154 gives DVB, and `test/ts/compliance.py` +grades it. That's the same treatment DVB AC-3 got in +the fixed-delay export (#4645). + +## Plan + +Found in #4645: E-AC-3 carried as DVB private data still goes out unsplit, +with no buffer model, so a multi-frame PES can overflow B. + +- Split it the way #4645 splits DVB AC-3, but per access unit: an + independent sync frame plus the dependent substreams that follow it share + one presentation interval (ETSI TS 102 366 Annex E), so they go out in one + PES on one decode time. Read sizes from each sync frame's `frmsiz`, and + drop a frame shorter than its header says. +- The buffer is the DVB one, not ATSC's: #4645's 12,896 B is A/52 Annex G's + for `stream_type` 0x87, and DVB AC-3 uses 5,696 B. Take the E-AC-3 value + from ETSI TS 101 154, cite the clause, and add it to the schedule's + per-PID admission and to the `tstd` model in `compliance.py`, which + refuses DVB E-AC-3 today. +- Test: a synthetic multi-frame E-AC-3 PES exports one access unit per PES; + an independent-plus-dependent fixture keeps both substreams on one + interval (32 ms for six 48 kHz blocks, not 64); and a control in + `tstd-controls.py` overflows B without the split. + +## Required + +- [T-STD TS export](/quest/m1/tstd/README.md) - the per-PID schedule this joins diff --git a/quest/m2/ts-export-liveness.md b/quest/m2/ts-export-liveness.md deleted file mode 100644 index f8c2a92b03f..00000000000 --- a/quest/m2/ts-export-liveness.md +++ /dev/null @@ -1,50 +0,0 @@ -# [S] moq export ts reports each elementary stream's access units and how long it has been quiet - -## Goal - -`Export::stats` returns `ts::Stats` with the import's per-PID row (#4502) for -every elementary stream the exporter writes: `units`, the access-unit count, -and `quiet`, the time since the last, on the output's own PCR. An operator can tell that one track stalled -at the exporter while PSI and the other PIDs kept flowing. The other fields of -`ts::Stats` stay zero at egress. `moq export ts` logs a line when a row stops -advancing. - -## Plan - -Decided while planning [#1838](https://github.com/moq-dev/moq/issues/1838): - -- **Liveness only, no ETSI counters at egress.** The media-aware lane - regenerates PAT, PMT, PCR and CC, so an egress check grades our muxer, never - the contribution feed. Sync, TEI, CRC, CC, PAT and PMT on bytes the muxer - constructs fail only on a code defect, which the `test/ts` hard checks catch - in CI, and `PAT_error` timed on the output's own PCR never fires on a whole - stall because that clock freezes too. `PCR_accuracy_error` would be - permanently in alarm: the exporter's PCR fails ±500 ns on every file sample - and the downstream groomer's output passes on every one - ([T33](https://github.com/tdrapier-wbd/moq-mpegts-paper/blob/d4c7573f9519a1c9c4882fab2021fbf82616fdcc/lab/test-33-gate2-preparation.md)). - PCR accuracy and wire timing belong to the groomer's monitoring of its own - output, outside this tree. -- **The runtime failure worth catching is a per-track stall.** #3533 stalled - video and primary audio at the exporter while PSI and the other PIDs - continued and the relay kept transmitting; only a per-PID liveness detector - on a subscriber found it - ([T27](https://github.com/tdrapier-wbd/moq-mpegts-paper/blob/d4c7573f9519a1c9c4882fab2021fbf82616fdcc/lab/test-27-liveness-detector.md)). -- **Same type as ingest.** `Export::stats` returns `ts::Stats`, so a - dashboard reads one schema at both edges. Additive on main. -- **No window in-tree.** The raw exporter delivers by group, with 102 - simultaneous media-time gaps over 0.3 s in 39 s on every PID, and thresholds - learned there differ from the groomed wire's for the same content (T27). A - window means something only with its monitoring point named, so the - consumer picks it. - -Implementation: - -- `rs/moq-mux/src/container/ts/export.rs`: count an access unit per PID when - its PES is written, and the quiet time on the output PCR. Fix the stale "TR 101 - 290 flags a gap over 40 ms" comment near `PCR_INTERVAL` (V1.4.1's limit is - 100 ms), and the matching 40 ms defaults and the "inserts no null packets, - and paces PCR once per media frame" sentence in `test/ts/README.md`. -- `rs/moq-cli/src/subscribe.rs` logs a stopped row the way `publish.rs` does. -- Tests in `export_test.rs`: a healthy export advances every row; a track that - stops mid-run grows its quiet time while PSI and the other PIDs continue, the - #3533 shape. diff --git a/quest/m2/ts-health-stats.md b/quest/m2/ts-health-stats.md index 28e8fc6b9d3..17a521065cb 100644 --- a/quest/m2/ts-health-stats.md +++ b/quest/m2/ts-health-stats.md @@ -43,4 +43,3 @@ Decided while planning [#1838](https://github.com/moq-dev/moq/issues/1838): - [Media stats schema](/quest/m1/stats/schema.md) - the snapshot the counters flatten into - [Rust reporters](/quest/m1/stats/rust.md) - `moq import --stats` and the stats interval - [TS import health](/quest/m2/ts-import-health.md) - the ingest counters -- [TS export liveness](/quest/m2/ts-export-liveness.md) - the egress rows diff --git a/quest/m2/ts-hitless.md b/quest/m2/ts-hitless.md new file mode 100644 index 00000000000..a055a89fc30 --- /dev/null +++ b/quest/m2/ts-hitless.md @@ -0,0 +1,42 @@ +# [L] Two TS export legs emit identical packets for ST 2022-7 + +## Goal + +Two `moq export ts --sync` legs of one broadcast, joined at different +moments, emit packet-identical TS from their first common packet, across +skips and source drift, continuity counters included. A ST 2022-7 selector +can then switch between them hitlessly. + +## Plan + +Decided (2026-10-01), from t0ms's review of the fixed-delay export (#4645). +They call 2022-7 a differentiator for primary distribution, not a +requirement, so this waits until the T-STD line settles. + +- Measured acquisition stays the default. `--sync` opts in to anchoring on + the catalog clock: release at `clock.wall + pts + delay`, so every leg + anchors the same way however it joined. This needs NTP-synced hosts. If the + measured age is implausible (negative, or beyond the delay), the export + fails loud. +- Continuity counters restart at each group. Each media PID's first packet in + a group sets `discontinuity_indicator`, and its counter is numbered from + there, so it comes from the media and not from what a leg has sent. The PCR + moves to its own PID, because a discontinuity flag on the PCR PID would + declare a time-base break. Loss is still detected within a group. +- Once anchored identically, both legs must make the same skip decision and + lay out the same slots. t0ms's `two_legs_render_a_skip_the_same_way` stays + ignored in #4645 until this lands. With the audio late, it also showed a + PES header byte that differs at the skip; find and fix that. +- Test with mocked time: + - two legs, joined a second apart, give byte-identical output including + the counters; + - the same across a mid-run skip, and under ±30 ppm drift; + - `--sync` with an implausible age fails loud. + +## Required + +- [T-STD TS export](/quest/m1/tstd/README.md) - the export, clock recovery, and schedule these legs share + +## Related + +- [TS passthrough](/quest/m1/ts-passthrough.md) - byte-identical legs for free in its own lane; adopts `--sync` for alignment diff --git a/quest/m2/ts-import-health.md b/quest/m2/ts-import-health.md index 77188dc12b8..4ef2a6f64b1 100644 --- a/quest/m2/ts-import-health.md +++ b/quest/m2/ts-import-health.md @@ -46,7 +46,7 @@ Decided while planning [#1838](https://github.com/moq-dev/moq/issues/1838): - **`CRC_error` covers PAT and PMT only.** TR 101 290 names CAT, PAT, PMT, NIT, EIT, BAT, SDT and TOT, and its table 5.1b cuts that to PAT and PMT for systems with reduced SI, which a contribution feed is. The importer parses - only PAT and PMT, and [TS PSI reassembly](/quest/m1/ts-psi-reassembly.md) + only PAT and PMT, and since #4584 it already drops a bad-CRC section and counts it as `crc_error`; this quest adopts that field. SI captured verbatim (NIT, SDT, EIT, BAT, TOT) is not CRC-checked, and CAT is out with the scrambled services below. @@ -106,10 +106,6 @@ Implementation: else. A TEI packet that also jumps its continuity counter counts one `Transport_error` and no `Continuity_count_error`. -## Required - -- [TS PSI reassembly](/quest/m1/ts-psi-reassembly.md) - the bad-CRC PAT or PMT the importer survives and counts as `crc_error` - ## Related - [TS health stats](/quest/m2/ts-health-stats.md) - where these counters are published diff --git a/quest/m2/ts-program-si.md b/quest/m2/ts-program-si.md deleted file mode 100644 index 168022f0a11..00000000000 --- a/quest/m2/ts-program-si.md +++ /dev/null @@ -1,52 +0,0 @@ -# [M] Per-program SI - -## Goal - -When an MPEG-TS import selects a program (`ts::Import::with_program`, so -`moq import ts --program ` and each broadcast of `--program all`), that -program's broadcast carries SI describing only its own service. Other services' -EIT sub-tables are dropped, and the SDT for this transport stream is rewritten -to list only the selected service. Network-wide tables (NIT, BAT, TDT/TOT, SDT -other, EIT other) pass through unchanged. An import without a selection does -no filtering by program and keeps today's capture behavior. - -## Plan - -Settled decisions: - -- Filter at import, in `si::Capture`, before a generation commits, so a - program's SI tracks never carry another service's bytes and exporters stay - unchanged. Filtering at export was rejected: every subscriber would still - receive the whole multiplex's schedule. -- Filter only under an explicit selection. A single-program input whose SDT - lists extra services is left alone, because default capture keeps the - sections it accepts verbatim. -- EIT actual (`table_id` 0x4E and 0x50..=0x5F): keep a sub-table only when its - `table_id_extension` (the service_id) is the selected program_number. Whole - sub-tables are dropped; nothing is rewritten. EIT other (0x4F and - 0x60..=0x6F) describes other transport streams and passes through. -- SDT actual (`table_id` 0x42 on PID 0x11): its one sub-table lists every - service, so once a generation is complete, rebuild it as a single section - (`section_number` and `last_section_number` 0) holding the selected service's - loop entry, wherever it sat in the source's sections. Keep the header fields - (transport_stream_id, version_number, original_network_id), recompute - `section_length`, and write a fresh CRC-32/MPEG-2 using the `crc` crate, which - is already in `Cargo.lock`, as a direct moq-mux dependency rather than a - hand-written table. [TS PSI reassembly](/quest/m1/ts-psi-reassembly.md) - needs it too, and whichever change lands first adds it. -- When the SDT has no entry for the selected service, carry no SDT actual rather - than fabricating a table the source never gave for this service. If a later - version drops the service, the next snapshot retires the SDT captured before - it, and its catalog entry goes too if nothing else is left on it. A later - version that lists the service again is captured normally. -- A `(PID, table_id)` whose every sub-table is filtered away gets no catalog - `mpegts.si` entry or track. -- The same change updates the program-selection paragraph in `doc/bin/cli.md` - and the `si.rs` module docs; no new page is needed. - -Test with a synthetic two-service multiplex whose SDT lists both services -across two sections, with EIT present/following for each. Each selected import -carries one SDT entry with a valid CRC and only its own EIT; an unselected -import keeps the sections verbatim; a selection missing from the SDT carries -none; an SDT version that drops the selected service retires the earlier one; -and the re-exported TS still parses. diff --git a/quest/m2/ts-stopped-av-only.md b/quest/m2/ts-stopped-av-only.md new file mode 100644 index 00000000000..e5d97c52dca --- /dev/null +++ b/quest/m2/ts-stopped-av-only.md @@ -0,0 +1,34 @@ +# [XS] TS stopped log grades only audio and video + +## Goal + +The TS import and export stats log ("elementary stream stopped delivering +access units") fires only for audio and video PIDs. Sparse data PIDs such as +SCTE-35 and ID3 are still counted but never logged as stopped. + +## Plan + +`ts::Log::sample` (`rs/moq-mux/src/container/ts/stats.rs`), added for SRT +ingest in [#4506](https://github.com/moq-dev/moq/pull/4506) and used for export +since [#4577](https://github.com/moq-dev/moq/pull/4577), logs any PID whose +access-unit count did not move in a 1 s interval. That assumes a continuous +stream; SCTE-35 sends a section now and then, so it logs nearly every quiet +second after each cue. + +Decision (2026-10-01): ✅ grade only audio/video. Rejected: learning each +PID's usual gap (more logic) and leaving the noise. + +`Log::sample` only sees `Stats`, and `StreamStats` carries no stream type; the +`track` suffix can't tell (`.ts` is any verbatim data, `""` is MPEG-1/2 video). +Carry the classification import already resolved into `StreamStats` for active, +retired, and section rows, not the raw PMT `stream_type`: `0x86` is DTS or, +with a CUEI descriptor, SCTE-35, and import routes the latter to sections. +Test a sparse CUEI-marked `0x86` PID beside a stalled video PID. + +Public API: an additive field on the `#[non_exhaustive]` `StreamStats`. Wire: +none. + +## Related + +- [TS stats module](/quest/m1/ts-stats-module.md) - renames these types on dev +- [TS health stats](/quest/m2/ts-health-stats.md) - publishes the same per-PID liveness diff --git a/quest/m2/tstd-controls.md b/quest/m2/tstd-controls.md new file mode 100644 index 00000000000..07ef58ed98d --- /dev/null +++ b/quest/m2/tstd-controls.md @@ -0,0 +1,28 @@ +# [S] The T-STD harness covers MB overflow and an AAC broadcast reference + +## Goal + +`just test ts-tstd` has a negative control that overflows the video multiplex +buffer (MB), and a positive reference that carries AAC the way a broadcast +encoder does. Every buffer and codec the export emits then has a control on +each side. + +## Plan + +Found in #4643. Grading against the declared HRD made MB about 3.6 MB on the +Kyrion capture, more than its 4 s holds, so no control reaches MB any more. +The only positive reference carries MPEG audio, but the export emits AAC. + +- MB: a longer clip with no declared HRD (level defaults), restamped to a + burst, which overflows MB and passes as captured. +- AAC: a shareable broadcast-style reference whose AAC passes B. + - FFmpeg's default 0.7 s audio lead overflows the 3,584 B AAC buffer, so try + its mux delay options first. + - If no generated clip passes, ask t0ms whether a short AAC capture can be + shared. +- Each control fails or passes for the stated reason, and `interop.yml` + runs them. + +## Required + +- [T-STD TS export](/quest/m1/tstd/README.md) - `just test ts-tstd` and `tstd-controls.py` land with it (#4645) diff --git a/quest/m2/watch-data-sync.md b/quest/m2/watch-data-sync.md new file mode 100644 index 00000000000..c63a06bf99c --- /dev/null +++ b/quest/m2/watch-data-sync.md @@ -0,0 +1,32 @@ +# [M] js/watch: play data tracks in sync with media + +## Goal + +A browser viewer receives JSON and binary track payloads on the same playhead +as the video and audio it plays, so telemetry shown beside a frame describes +that frame. A data track whose advertised `delay` and `jitter` exceed the +media's holds the media back by that much, and dropping the data track +releases it. + +## Plan + +- A `js/watch` data-track reader subscribes a catalog JSON or binary entry + (built-in section or an application section embedding the config), releases + each payload when the playhead reaches its frame timestamp, and registers its + `delay` and `jitter` with `Sync` like a media rendition. +- Snapshot tracks release the newest state at or before the playhead, picked + from the in-order states the consumer yields; stream tracks release every + record in order. +- An untimed payload adds no timestamp wait of its own. On a stream track it + still waits for the records before it, so a timed record at 11s followed by + an untimed one releases both at an 11s playhead, in order. An untimed + snapshot state applies once the states before it have. Test both + sequences. +- OneTooMany is the application this waited for (2026-10-01): their web + frontend holds KLV and MAVLink telemetry back to the video playhead with + its own sync code, which this replaces. In m2 rather than m1 because + they aren't blocked. + +## Required + +- [JS data consumer timestamps](/quest/m1/js-data-consumer-timestamps.md) - the reader releases each value by the timestamp its consumer returns diff --git a/quest/m3/3201-moq-uring-use-sendmsg-zc-for-large-udp-gso-trains.md b/quest/m3/3201-moq-uring-use-sendmsg-zc-for-large-udp-gso-trains.md index 9930a37408f..260460eddf6 100644 --- a/quest/m3/3201-moq-uring-use-sendmsg-zc-for-large-udp-gso-trains.md +++ b/quest/m3/3201-moq-uring-use-sendmsg-zc-for-large-udp-gso-trains.md @@ -41,7 +41,7 @@ ships in no package. ## Required -- A physical-NIC remote peer to measure zero-copy sends against +- [A physical-NIC peer for the zero-copy sweep](/quest/m3/zero-copy-peer.md) - loopback only measures the forced copy ## Closes diff --git a/quest/m3/README.md b/quest/m3/README.md index 57d33620742..02d73b4c1c6 100644 --- a/quest/m3/README.md +++ b/quest/m3/README.md @@ -9,31 +9,29 @@ Nothing here should start by opening an editor. ## Plan A quest lands here when its gate is the outside world, not its priority. Its gate -is a plain-text `Required` bullet, re-checked as the root -[questline](/quest/README.md) describes. A +is a condition quest beside it, as the root [questline](/quest/README.md) +describes. A speculative feature with no consumer parks here rather than in m2, and is deleted when it goes stale; git history keeps it. ## Required +- [Video validation hardware is on hand](/quest/m3/video-hardware-access.md) - the machines the validation runs on - [Video hardware validation](/quest/m3/video-hardware.md) - run the encode, capture, and zero-copy paths that were never run on real machines, including PipeWire on KDE, the camera portal, and a Pi +- [An Ada NVIDIA GPU is available](/quest/m3/ada-gpu.md) - the hardware AV1 NVENC is verified on - [NVENC AV1](/quest/m3/nvenc-av1.md) - AV1 encode through NVENC, once an Ada-generation GPU is available +- [An embedded video device is on hand](/quest/m3/embedded-device.md) - the device EGL import is validated on - [Embedded video](/quest/m3/video-embedded.md) - EGL import in the renderer, so moq-video presents on a Pi +- [A physical-NIC peer for the zero-copy sweep](/quest/m3/zero-copy-peer.md) - the rig the zero-copy sweep runs on - [#3201: moq-uring: use SENDMSG_ZC for large UDP GSO trains](/quest/m3/3201-moq-uring-use-sendmsg-zc-for-large-udp-gso-trains.md) - complete the prerequisite issue first - [#3204](/quest/m3/3204-moq-uring-register-tx-pool-buffers-for-zero-copy-sends.md) - moq-uring: register TX-pool buffers for zero-copy sends -- [Receive timestamps](/quest/m3/quic-receive-ts.md) - per-packet arrival times in ACKs, the feedback GCC and deadlines need -- [QUIC GCC](/quest/m3/quic-gcc.md) - a measured verdict on delay-based congestion control for media egress, shipping as `RealTime` +- [QUIC GCC](/quest/m3/quic-gcc.md) - receive timestamps in ACKs and a measured verdict on delay-based congestion control for media egress, shipping as `RealTime` - [AF_XDP UDP path](/quest/m3/af-xdp.md) - the kernel-bypass verdict on today's virtio hosts that gates DPDK -- [C# through moq-ffi](/quest/m3/cs/README.md) - generated C# over moq-ffi as a NuGet package with native runtimes -- [Unity prototype](/quest/m3/unity.md) - the C# package under IL2CPP, playing subscribed audio - [Unreal prototype](/quest/m3/unreal.md) - a UE5 module on the C++ package with exceptions disabled, rendering a subscribed broadcast to a texture -- [LiveKit client shim](/quest/m3/livekit-shim.md) - a media compatibility facade over the room SDK - [Conan remote](/quest/m3/cpp-conan.md) - a remote we own serves the same tarball to `conan install` -- [WHEP ABR](/quest/m3/whep-abr.md) - a WHEP viewer switches renditions from its own congestion feedback -- [Synced data playback](/quest/m3/watch-data-sync.md) - js/watch releases JSON and binary payloads on the media playhead, and a slow data track holds media back - [Linux OBS GPU input](/quest/m3/obs-linux-gpu.md) - publish OBS compositor frames without CPU readback on a validated Linux graphics/encoder combination - [Routing cost domains](/quest/m3/routing-cost-domains.md) - design operator boundaries and policy without adding incomparable costs -- [Common Access Tokens](/quest/m3/cat/README.md) - a moq-transport client presents a CAT in SETUP and `moq auth serve` admits it with the scope its `moqt` claim names +- [The moq.pro mesh runs lite-07](/quest/m3/lite07-mesh.md) - the deployment that makes the exemption dead code - [Drop the hidden cluster exemption](/quest/m3/hidden-exemption.md) - relays stop forcing hidden broadcasts on cluster peers once every peer opts in on the wire - [MAVLink bridge](/quest/m3/teleop-mavlink.md) - a `moq-mavlink` gateway replacing the VPN plus two unmanaged UDP flows, with QGroundControl and diff --git a/quest/m3/ada-gpu.md b/quest/m3/ada-gpu.md new file mode 100644 index 00000000000..ba14ea6991b --- /dev/null +++ b/quest/m3/ada-gpu.md @@ -0,0 +1,14 @@ +# [XS] An Ada NVIDIA GPU is available + +## Goal + +An Ada-generation or newer NVIDIA GPU is available to verify AV1 NVENC +encoding, on a CI host or a maintainer's machine. The current GPU host is an +RTX 3070 Ti, which has no AV1 NVENC. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +A consumer asking for AV1 encode is also a reason to get one. diff --git a/quest/m3/cat/README.md b/quest/m3/cat/README.md deleted file mode 100644 index 40871479531..00000000000 --- a/quest/m3/cat/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# Common Access Tokens - -## Goal - -A moq-transport client presents a Common Access Token -([draft-ietf-moq-c4m](https://datatracker.ietf.org/doc/draft-ietf-moq-c4m/)) -in the SETUP `AUTHORIZATION TOKEN` option and the relay admits it with the -scope the token's `moqt` claim names, through the same -`moq-auth` request contract every other credential uses. Our own clients can present one; `moq auth serve` verifies one; the -relay stays crypto-free and forwards the bytes. This is moq-transport only: -a CAT scopes itself by moq-transport message (`SUBSCRIBE`, `FETCH`, -`PUBLISH_NAMESPACE`, ...), which moq-lite has no equivalent for, so it rides -the SETUP option and nothing else. Nothing changes on the moq-lite wire or in -the URL, and the JWT stays the URL credential. - -Boundaries decided while planning: - -- SETUP only. Per-operation authorization is not planned: the relay - authorizes by path through origin scopes. A token on any other request is - [Request tokens](/quest/m1/auth/request-token.md)'s path-scoped fallback - for that request, and a CAT's per-message scope is never enforced. -- A token whose `moqt` scope restricts the track name is refused naming the - token. Grants are broadcast-path patterns and tracks are not scoped. -- Core CWT claims plus `moqt` and `moqt-reval` are enforced; any other CAT - claim present (`catu`, `catm`, `catnip`, `catreplay`, `catdpop`, `cnf`, - geo, composite `or`/`and`/`nor`) refuses the token naming the claim, so a - restriction we do not evaluate never widens silently. DPoP (c4m section 3) - is out of scope. -- Wire codes follow c4m-01: Token Type `0x01` is the value the draft - registers for CAT. The `moqt` and `moqt-reval` claim keys are still - `TBD_MOQT`, so they are hard-coded constants with a doc comment saying so. - We do not expect to interop on these until the draft registers them. -- Rust only. `@moq/auth` gains nothing; `js/net` only carries bytes it was - handed into SETUP. -- [draft-ietf-moq-privacy-pass-auth](https://datatracker.ietf.org/doc/draft-ietf-moq-privacy-pass-auth/) - was considered and dropped. A Privacy Pass token carries only the digest of - its `TokenChallenge`, the draft never says how a relay hands a client the - challenge whose `origin_info` scopes the token, VOPRF tokens need the - issuer's secret at the verifier, and nobody is asking. Do not re-plan it - until the draft has a challenge exchange and a registered token type. - -## Plan - -In m3 because nobody is asking for CAT yet and the claim keys are -unregistered; a consumer presenting CATs, or the draft registering the keys, -brings it back. - -Order: the SETUP option already reaches the auth server as -`moq_auth::Request.token`; verification comes first, then our clients present -one. Everything rides `moq_auth::Request` and -`moq auth serve`. The JWT types stay at the crate root, so the published -`moq-auth` names do not break; `cat` is an additive module beside them. - -## Required - -- draft-ietf-moq-c4m registers the moqt claim keys -- [Verify](/quest/m3/cat/verify.md) - `moq_auth::cat` turns a CAT into a - grant and `moq auth serve` admits one; `moq auth sign|verify` mint and - check the format -- [Present](/quest/m3/cat/present.md) - a CAT is one kind of configured - token, riding the SETUP option the in-band token quest already writes - -## Related - -- [In-band auth](/quest/m1/auth/README.md) - credentials presented after - SETUP, including on IETF requests; its [Token in - band](/quest/m1/auth/token-in-band.md) quest owns the client token - configuration a CAT joins diff --git a/quest/m3/cat/present.md b/quest/m3/cat/present.md deleted file mode 100644 index 53b2a8f877e..00000000000 --- a/quest/m3/cat/present.md +++ /dev/null @@ -1,46 +0,0 @@ -# [S] A CAT is one kind of configured token - -## Goal - -`moq_tokio::connect::Config.tokens` and `js/net`'s connect `tokens` accept a -Common Access Token beside the JWTs they already hold, and a moq-transport -session carries it in the SETUP `AUTHORIZATION TOKEN` option with Token Type -`0x01`. The library does not read the bytes. A CAT never rides the URL or an -AUTH stream, so a client that configures one while offering a version without -the option fails loud instead of dropping the credential. - -## Plan - -- [Token in band](/quest/m1/auth/token-in-band.md) defines the client token - configuration and writes the first configured token into the setup option - as type 0. `moq_net::setup::Token { kind, value }` already carries the - kind, with CAT = `0x1` (#4278); a JWT keeps kind `0x0` and its URL and - AUTH stream behavior. `--connect-token` - keeps taking a JWT; `--connect-cat ` (and `MOQ_CONNECT_CAT`) - adds a CAT. One CAT per connection: it is the connection credential, so a - configured CAT takes the setup option and the JWT that would have gone - there rides its AUTH stream instead, never the URL. `moq auth serve` - refuses a SETUP token beside a different `?jwt=` (see [Token in - band](/quest/m1/auth/token-in-band.md)), so a JWT on the URL would refuse the - CAT's connect; on a session without AUTH the JWT is simply absent, like - any extra token. -- A CAT with any offered version that lacks the setup option (every lite - version) fails `init` with `Unsupported` naming the token; `js/net` - rejects the same way before dialing. -- `moq` CLI publish and subscribe get the flag through the shared connect - config; `doc/bin/cli.md` and `doc/lib/rs/moq-net.md` gain it. -- Tests: a Rust server's `Handshake::token()` sees kind `0x01` and the - bytes on every draft that carries the setup option, from both a Rust and a JS client (a JS server is out - of scope: #4278 added no JS accept-side API, since nothing in `js/net` authorizes an IETF session); a JWT rides its - AUTH stream and not the URL when a CAT holds the option; a lite offer with - a CAT refuses at init; end to end against `moq auth serve` with a CAT from - `moq auth sign --format cat`. - -Public API: additive on `moq-tokio` and `js/net`. Wire: none. - -## Required - -- [Verify](/quest/m3/cat/verify.md) - the server that admits the token the - end-to-end test presents -- [Token in band](/quest/m1/auth/token-in-band.md) - the token - configuration and setup-option writer this adds a kind to diff --git a/quest/m3/cat/verify.md b/quest/m3/cat/verify.md deleted file mode 100644 index e606294f045..00000000000 --- a/quest/m3/cat/verify.md +++ /dev/null @@ -1,89 +0,0 @@ -# [M] moq auth serve admits a Common Access Token - -## Goal - -`moq_auth::cat` verifies a CAT (CTA-5007-B, a CWT under COSE_Mac0 or -COSE_Sign1) and turns its `moqt` and `moqt-reval` claims into a `Grant`, and -`moq auth serve` admits a connection whose SETUP token is one, with keys the -operator hands it. `moq auth sign --format cat` mints one and `moq auth -verify --format cat` checks one, so tests and operators have a way to make -tokens. Every claim we do not evaluate refuses the token naming the claim. - -## Plan - -The JWT types (`Claims`, `Key`, `Jwk`, `KeyId`, `Algorithm`, `Scope`, the -key set) sit at the root of the published `moq-auth` 0.1.x, and they stay -there: `cat` is an additive module beside them, so this lands on `main`. -Moving them under `moq_auth::jwt` would break every published caller; if it -is still wanted, it is its own `dev` change, not part of this quest. Decided -in the 2026-09-28 quest audit. `@moq/auth` stays flat. - -- Crates: `coset` for COSE and CWT claims, `ciborium` for CBOR; HMAC through - the `aws-lc-rs` the crate already links, ES256 through `p256`. Keys reuse - `moq_auth::Key` files: a JWK with `alg` maps to the COSE algorithm - (`HS256` to HMAC 256/256, `ES256` to -7, `EdDSA` to -8, `RS256` to -257); - a JWK whose algorithm has no COSE mapping is refused at load naming it. -- `cat::Claims { issuer, audience, subject, expires, not_before, issued, - id, scopes: Vec, revalidate: Option }` where - `cat::Scope { actions: Vec, namespace: Option>, - exact_depth: bool, track: Option }` follows the c4m-01 CDDL: - `cat::Match` is `Exact(bytes)`, `Prefix(bytes)`, or `Suffix(bytes)`, one - per namespace field in order, the trailing `nil` becomes `exact_depth`, - and `nil` anywhere else refuses the token. `cat::Action` is the c4m-01 - table 1 enum (`ClientSetup`, `ServerSetup`, `PublishNamespace`, - `SubscribeNamespace`, `Subscribe`, `RequestUpdate`, `Publish`, `Fetch`, - `TrackStatus`). Claim keys `moqt` and `moqt-reval` are provisional - constants in the CWT private-use range with a doc comment; they change - when the draft registers them. -- `Claims::verify(token, keys)` checks the signature or MAC against the key - set by `kid`, then `exp`, `nbf`, and `iat` against now, then walks the - claim set: anything outside `iss`, `sub`, `aud`, `exp`, `nbf`, `iat`, - `cti`, `moqt`, and `moqt-reval` refuses the token naming the label. -- `Claims::grant(&self, root) -> Result`: for every scope, - `Publish` adds the namespace pattern to `publish` and `Subscribe` adds it - to `subscribe`. The grant has no narrower verbs, so `PublishNamespace` is - accepted only beside `Publish`, and `SubscribeNamespace`, `RequestUpdate`, - `Fetch`, and `TrackStatus` only beside `Subscribe`; a scope whose actions - are narrower than either verb refuses the token naming the scope rather - than widening a fetch-only token into a live subscription. `ClientSetup` - and `ServerSetup` add nothing. The namespace fields become one `moq_pattern::Pattern` segment - each, the mapping `rs/moq-pattern` documents under "CAT / C4M", relative - to the dialed path exactly as the JWT `Claims::root` is: `Exact(f)` is the - literal segment, `Prefix(f)` is `f*`, `Suffix(f)` is `*f`, a named - namespace without `exact_depth` appends `/**`, and an absent namespace is - bare `**` with nothing appended. A field value containing `/` or `*` - cannot be a segment and refuses the token naming the scope. `f*` and `*f` - are patterns the relay admits through the pattern scopes landed in - [#3746](https://github.com/moq-dev/moq/pull/3746). A scope with a track - match is refused naming the scope. - `exp` is `expires`, `moqt-reval` is - `revalidate`, and `Grant::validate` keeps refusing a cadence without an - expiry, so the serve default `--expires` applies to a token without `exp` - the way it does for a JWT. A token that grants nothing is refused. -- `moq auth serve`: `--cat-key ` and `--cat-key-dir ` beside - `--key`, `--cat-issuer ` and `--cat-audience ` required to - match `iss` and `aud` when set. Policy order: a SETUP `token` is decided - first and never falls through; then the existing `jwt` query, mTLS, - public. -- `moq auth sign --format cat` and `verify --format cat` in `rs/moq-cli`, - sharing `--publish`/`--subscribe` patterns with the JWT form and emitting - one `moqt` scope per pattern through the inverse of the mapping above (a - literal segment is `Exact`, `f*` is `Prefix`, `*f` is `Suffix`, a trailing - `**` drops `nil`, a pattern with `**` elsewhere or `a*b` is refused naming - it) with the actions each verb implies; `--revalidate ` sets - `moqt-reval`. Output is the raw CWT bytes, or base64url with `--base64`. -- Docs: `doc/lib/rs/moq-auth.md` gains the module, `doc/bin/cli.md` the - flags, `doc/bin/relay/auth.md` a CAT section with the provisional codes - and the refused-claims list. -- Tests: sign and verify under each mapped algorithm; each refused claim; - each match kind mapped or refused, `nil` placement, and the moq-pattern - vectors round-tripped both ways; expiry and not-before; the grant - produced from every c4m-01 example; serve admitting a CAT over a real - moq-transport session on every supported draft and refusing an unknown - token kind, a bad MAC, and a CAT beside any `jwt` (a CAT is type `0x01` - and a `jwt` is a JWT, so equal bytes are still two credentials; only a - type-0 SETUP token equal to `jwt` is admitted once, per [Token in - band](/quest/m1/auth/token-in-band.md)); the CLI round trip. - -Public API: `moq_auth::cat` new, `moq auth serve` and `moq auth sign|verify` -gain flags. Wire: none. diff --git a/quest/m3/cs/README.md b/quest/m3/cs/README.md deleted file mode 100644 index dca620bd43a..00000000000 --- a/quest/m3/cs/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# C# through moq-ffi - -## Goal - -A .NET developer adds the NuGet package and holds the moq-ffi surface as -classes whose async operations are `Task`s, with native libraries for every -runtime identifier in the package. Same recipe as the C++ line: generate from -moq-ffi with a pinned generator, add a thin idiomatic wrapper, never hand-roll -a mirror. Unity is a separate prototype. - -## Plan - -In m3 until a .NET or Unity consumer asks; the C++ line covers native -embedders first. - -NordSecurity's `uniffi-bindgen-cs` (latest `v0.11.0+v0.31.0`) already emits -async methods as `Task` and async callback interfaces; it needs the same -uniffi 0.32 port the Go, Dart, and C++ generators got. Plain .NET first; the -IL2CPP constraints Unity adds (static `MonoPInvokeCallback` trampolines, no -dynamic loading) are measured in the next prototype rather than designed around -up front. - -## Required - -- [Generator](/quest/m3/cs/generator.md) - uniffi-bindgen-cs on uniffi 0.32, pinned and generating `cs/ffi` in CI -- [Package](/quest/m3/cs/package.md) - the `cs/moq` wrapper, NuGet package with native runtimes, interop client, and docs - -## Related - -- [C++ through moq-ffi](/quest/m1/cpp/README.md) - the sibling line this copies -- [Unity prototype](/quest/m3/unity.md) - the package under IL2CPP diff --git a/quest/m3/cs/generator.md b/quest/m3/cs/generator.md deleted file mode 100644 index 0966cbebb88..00000000000 --- a/quest/m3/cs/generator.md +++ /dev/null @@ -1,26 +0,0 @@ -# [M] uniffi-bindgen-cs on uniffi 0.32 - -## Goal - -`just cs check` regenerates `cs/ffi` from `rs/moq-ffi` with a pinned C# -generator and runs a small xunit program that connects, awaits a subscribed -frame as a `Task`, cancels a pending read with a `CancellationToken`, and -catches a typed `MoqException`, on the host runtime. - -## Plan - -- Fork `NordSecurity/uniffi-bindgen-cs` at `v0.11.0+v0.31.0` into - `kixelated/uniffi-bindgen-cs` and port it to uniffi 0.32.x, following the - Go port. Tag `vX.Y.Z+v0.32.0`; offer the port upstream and drop the fork - when NordSecurity tags. -- Pin it in `flake.nix` beside the other generators with the same list of - places that name the version. `just cs check` regenerates into `cs/ffi` - and runs the probe; wire it into the check workflow like `just go check`. -- Map moq-ffi's abort-on-drop futures onto `CancellationToken` in the - generator: a cancelled token drops the Rust future, and `IDisposable` on a - handle cancels its pending calls. If upstream lacks the mapping, it lands in - the fork here, never in the wrapper; the probe above is the acceptance test. - -## Related - -- [C++ generator](/quest/m1/cpp/generator.md) - the same port against the C++ generator diff --git a/quest/m3/cs/package.md b/quest/m3/cs/package.md deleted file mode 100644 index 66aecee5c5a..00000000000 --- a/quest/m3/cs/package.md +++ /dev/null @@ -1,27 +0,0 @@ -# [M] cs/moq: the wrapper, NuGet package, interop client, and docs - -## Goal - -`dotnet add package Moq.Net` (final name decided in the quest; `Moq` collides -with the mocking library on NuGet) gives a .NET 8 program the full moq-ffi -surface as `await`-able classes, with `runtimes//native` libraries for -the release matrix. A `test/interop/clients/cs` client joins `just test -interop --all` and `doc/lib/cs` documents the package. - -## Plan - -- `cs/ffi` is generated only, committed and regenerated by `just cs check`. - `cs/moq` is the thin wrapper: namespace `Moq`, dropped `Moq` type prefixes, - `TimeSpan` for timestamps, `ReadOnlyMemory` payloads, `IAsyncEnumerable` - over the consumer `next()` loops, `IAsyncDisposable` on handles. Nothing - re-implements a method. -- Package: one NuGet with managed code plus native `libmoq_ffi` per RID - (win-x64, osx-arm64, linux-x64, linux-arm64, matching `moq-c.yml`), built by - `release-cs.yml` on the nightly dry-run chain and published on a `cs-v*` - tag. -- Interop: `test/interop/clients/cs` publishes and subscribes like the Go client. -- Docs: `doc/lib/cs/index.md` and a row in `doc/lib/index.md`. - -## Required - -- [Generator](/quest/m3/cs/generator.md) - the pinned generator that emits `cs/ffi` diff --git a/quest/m3/embedded-device.md b/quest/m3/embedded-device.md new file mode 100644 index 00000000000..8e30e309992 --- /dev/null +++ b/quest/m3/embedded-device.md @@ -0,0 +1,13 @@ +# [XS] An embedded video device is on hand + +## Goal + +Someone has a Raspberry Pi or similar V4L2 M2M device without a usable +Vulkan driver, and can validate the renderer's EGL import on it. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +Record the device, OS image, and driver when it exists. diff --git a/quest/m3/hidden-exemption.md b/quest/m3/hidden-exemption.md index 97dc8acac6e..762a9de603b 100644 --- a/quest/m3/hidden-exemption.md +++ b/quest/m3/hidden-exemption.md @@ -20,4 +20,4 @@ the mesh runs lite-07. ## Required -- The moq.pro mesh deploys lite-07 +- [The moq.pro mesh runs lite-07](/quest/m3/lite07-mesh.md) - every peer opts in on the wire diff --git a/quest/m3/lite07-mesh.md b/quest/m3/lite07-mesh.md new file mode 100644 index 00000000000..8fb2d43a835 --- /dev/null +++ b/quest/m3/lite07-mesh.md @@ -0,0 +1,15 @@ +# [XS] The moq.pro mesh runs lite-07 + +## Goal + +Every relay in the moq.pro mesh dials its cluster peers with moq-lite-07, so +each one opts in to hidden broadcasts on the wire. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +As of 2026-09-30 lite-07 ships only as the off-by-default `moq-lite-07-wip` +ALPN, and moq.pro still has its lite-07 rollout quest open +(`quest/m1/lite07.md` in moq-dev/moq.pro). diff --git a/quest/m3/livekit-shim.md b/quest/m3/livekit-shim.md deleted file mode 100644 index 74e35d14d41..00000000000 --- a/quest/m3/livekit-shim.md +++ /dev/null @@ -1,40 +0,0 @@ -# [L] LiveKit client shim - -## Goal - -A drop-in `livekit-client`-compatible JS package (e.g. `@moq/livekit`) that -runs a multi-participant room entirely over MoQ. The v1 surface is media plus -core events: Room connect/disconnect, local and remote participants, -camera/mic/screenshare publish, auto-subscribe, and the TrackSubscribed event -family; data surfaces (publishData, streams, RPC) are stubbed. The token slot -takes an ordinary moq-auth token, no LiveKit JWT parsing. Done when an -off-the-shelf LiveKit JS sample runs against a MoQ relay with only the import -and the connect URL/token changed. - -## Plan - -In m3 until a LiveKit user asks to migrate; `@moq/room` already serves -new rooms directly. - -- The shim is a LiveKit-API facade over `@moq/room`, which carries hang.live's convention: the room is a path prefix in the connection URL and token root, - participants are discovered from the bare announce stream, identity is the - path before the broadcast name, and each participant publishes - `/camera.hang` (camera + mic, hd/sd renditions) and - `/screen.hang` (screenshare, - whose announce/unannounce is the screenshare lifecycle). The shim groups - the two paths per identity into one RemoteParticipant and maps catalog - entries to TrackPublications. -- Build on `@moq/publish` and `@moq/watch`. LiveKit quality hints - (setVideoQuality, adaptive settings) map to the receiver-driven pixel - target, or no-op gracefully. -- v1 is identity-only: `participant.identity` comes from the path and muted - state derives from catalog track presence. Names and coarse state are a - follow-up wired to the room SDK's `hang/*.json` metadata, not a rival - scheme. -- Mint tokens with `@moq/room`'s `claims(room, identity)`, which scopes the - `publish` claim to the identity's subtree so participants cannot publish at - each other's paths. - -## Required - -- A LiveKit user asks for a drop-in client diff --git a/quest/m3/nvenc-av1.md b/quest/m3/nvenc-av1.md index 1715d6ed32e..7af1ec9ac7b 100644 --- a/quest/m3/nvenc-av1.md +++ b/quest/m3/nvenc-av1.md @@ -23,7 +23,7 @@ Public API: additive. Wire: existing AV1 codec signaling. ## Required -- Ada or newer NVIDIA GPU available for verification +- [An Ada NVIDIA GPU is available](/quest/m3/ada-gpu.md) - the hardware to verify on ## Related diff --git a/quest/m3/quic-gcc.md b/quest/m3/quic-gcc.md index ff482bdd4d1..a1dfea3718c 100644 --- a/quest/m3/quic-gcc.md +++ b/quest/m3/quic-gcc.md @@ -1,18 +1,38 @@ -# [L] QUIC GCC egress experiment +# [XL] QUIC GCC egress experiment ## Goal Record a measured verdict on WebRTC-style delay-based congestion control for subscriber-facing media egress against noq's production -controller. Ship it only if it reduces queueing delay and rate variation +controller, driven by the QUIC receive-timestamps extension +(draft-smith-quic-receive-ts): the peer reports when each acknowledged packet +arrived, so the sender sees per-packet one-way delay variation the way +WebRTC's transport-wide congestion control feedback does. Ship it only if it reduces queueing delay and rate variation without collapsing throughput. A written abandonment is a successful outcome. ## Plan +### Receive timestamps + +- Implement the extension in the fork: the transport parameter, the ACK + frame variant with timestamp ranges, and an `on_ack` extension on + `Controller` that hands each acknowledged packet its receive instant. Both + ends are ours; browsers never see it. +- Compare the forward delay it measures against the half-RTT estimate the + [deadline quest](/quest/m2/quic-deadline.md) starts with, on the impaired + path profile with asymmetric delay. Report how often the half-RTT estimate + would have kept a hopeless retransmission or reset a deliverable one. +- Measure the ACK overhead the timestamps add on a fanout-shaped relay egress, + with and without the ACK-frequency extension reducing ACK rate. + +A delay-based controller without per-packet arrival times is a different, +weaker experiment. + +### GCC + Implement the candidate in the fork as a `congestion::Controller`, driven by -the per-packet receive timestamps the receive-timestamps spike delivers; the -sender-side inter-arrival filter is what makes it GCC rather than another -RTT-based controller. If it ships, it joins MoQ's backend-neutral congestion +those per-packet receive timestamps; the sender-side inter-arrival filter is +what makes it GCC rather than another RTT-based controller. If it ships, it joins MoQ's backend-neutral congestion family as `CongestionControl::RealTime`, beside `Loss` (Cubic) and `Delay` (BBR3); MoQ owns which algorithm each name means, and the config never exposes algorithm names. Egress only: relay ingest keeps BBR3. @@ -28,14 +48,12 @@ behavior against production cross traffic or real wifi and cellular loss. Receive timestamps are native-only: browsers never negotiate the extension, so GCC can only target native peers or relay-to-relay sessions, not browser -egress. Decided in the 2026-09-30 audit: parked in m3 until such a consumer -exists. +egress. Decided 2026-09-30: the receive-timestamps spike folds in here, and +neither waits on a native consumer. ## Required - [Hard fork](/quest/m1/quic/fork/README.md) - the change lands in `moq-quic`, not the frozen fork -- [Receive timestamps](/quest/m3/quic-receive-ts.md) - the per-packet - arrival times the delay filter runs on ## Related diff --git a/quest/m3/quic-receive-ts.md b/quest/m3/quic-receive-ts.md deleted file mode 100644 index 7cf17d9bd8f..00000000000 --- a/quest/m3/quic-receive-ts.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] Receive timestamps - -## Goal - -A measured verdict on the QUIC receive-timestamps extension -(draft-smith-quic-receive-ts): the peer reports when each acknowledged packet -arrived, so the sender sees per-packet one-way delay variation the way -WebRTC's transport-wide congestion control feedback does, instead of inferring -it from ACK arrival on the return path. The result is either a released fork -feature with the `Controller` trait carrying per-packet receive instants, or a -written reason it does not pay. - -## Plan - -- Implement the extension in the fork: the transport parameter, the ACK - frame variant with timestamp ranges, and an `on_ack` extension on - `Controller` that hands each acknowledged packet its receive instant. Both - ends are ours; browsers never see it. -- Compare the forward delay it measures against the half-RTT estimate the - [deadline quest](/quest/m2/quic-deadline.md) starts with, on the impaired - path profile with asymmetric delay. Report how often the half-RTT estimate - would have kept a hopeless retransmission or reset a deliverable one. -- Measure the ACK overhead the timestamps add on a fanout-shaped relay egress, - with and without the ACK-frequency extension reducing ACK rate. - -The GCC experiment requires this; a delay-based controller without per-packet -arrival times is a different, weaker experiment. - -Both ends must be ours, so this is native-only and serves native peers or -relay-to-relay sessions. Decided in the 2026-09-30 audit: parked in m3 with -GCC until such a consumer exists. - -## Required - -- [Hard fork](/quest/m1/quic/fork/README.md) - the change lands in `moq-quic`, not the frozen fork -- A native-to-native consumer of receive timestamps (browsers never see them) - -## Related - -- [QUIC GCC](/quest/m3/quic-gcc.md) - the controller that consumes it -- [Per-stream deadlines](/quest/m2/quic-deadline.md) - the other consumer diff --git a/quest/m3/routing-cost-domains.md b/quest/m3/routing-cost-domains.md index f32615b6aed..abb942b3e68 100644 --- a/quest/m3/routing-cost-domains.md +++ b/quest/m3/routing-cost-domains.md @@ -3,9 +3,11 @@ ## Goal Settle how independently operated MoQ networks exchange reachability without -adding incomparable costs, at the cluster boundaries where -[Cluster routing](/quest/m1/cluster-routing/README.md) keeps path vector with cluster -ids as hops. Cost inside one cluster is cluster-routing's. Produce a +adding incomparable costs, at the boundaries between operators on the one node id space +[Cluster routing](/quest/m1/cluster-routing/README.md) routes over. Cost inside +one operator's network is cluster-routing's. This includes whether a boundary +may fold a large customer cluster's nodes into one advertised node, and how +that keeps loop safety. Produce a reviewed design and scoped implementation quests, not a protocol implementation. Cloudflare, moq.pro, and self-hosted relays can retain their own business policy; no RTT/loss-driven repricing or automatic performance @@ -45,8 +47,7 @@ The design must work through examples of two operators using different scales, multiple entrances to one domain, asymmetric charges, mixed-provider nodes in one domain, unknown or untrusted peers, and a route leaving and re-entering a domain. Preserve publisher identity and loop safety across any metric rewrite. -Explain how warm-route marginal savings interact with border policy without -pretending those savings erase upstream delay. State tradeoffs, migration and +State tradeoffs, migration and mixed-version behavior, and the limits of any convergence claim. Reconcile with cluster-routing's configured link costs rather than creating @@ -56,7 +57,7 @@ quests. Open wire/API choices belong to this design exercise. ## Required -- [Routing between clusters](/quest/m1/cluster-routing/inter-cluster.md) - this designs on its inter-cluster path vector +- [Cluster routing](/quest/m1/cluster-routing/README.md) - the route layer this designs policy on ## Related diff --git a/quest/m3/unity.md b/quest/m3/unity.md deleted file mode 100644 index 3c3e42cd7c4..00000000000 --- a/quest/m3/unity.md +++ /dev/null @@ -1,29 +0,0 @@ -# [M] Unity prototype: the C# package under IL2CPP - -## Goal - -A written go/no-go verdict on a Unity package. The evidence is a sample -project that imports the NuGet package's managed assembly and native -libraries as a Unity plugin, builds with IL2CPP for desktop and one mobile -target, subscribes to a broadcast, and plays decoded audio through an -`AudioSource`, measured for startup and GC pressure. - -## Plan - -In m3 with the C# line; a Unity consumer asking brings both back. - -- IL2CPP forbids dynamic callback marshaling: every reverse P/Invoke needs a - static method with `[MonoPInvokeCallback]`. Audit what the generated `cs/ffi` - emits for callback interfaces and futures, and whether the generator needs - an IL2CPP option or Unity needs a shim. -- Native libraries: `Plugins/` layout from the NuGet `runtimes/` - tree, plus an Android `.so` and iOS static library the .NET package does not - ship, which the verdict prices. -- Threading: continuations must hop to the main thread for any `UnityEngine` - call; a `SynchronizationContext`-based executor is the likely answer. -- Verdict promotes a Unity package quest into m1 or abandons this with the - reasons. - -## Required - -- [Package](/quest/m3/cs/package.md) - the NuGet whose contents Unity imports diff --git a/quest/m3/video-embedded.md b/quest/m3/video-embedded.md index a844e2f7df9..c362078006a 100644 --- a/quest/m3/video-embedded.md +++ b/quest/m3/video-embedded.md @@ -27,5 +27,4 @@ libcamera source composes with what already exists, because ## Required -- Someone with a Raspberry Pi or similar V4L2 M2M device without a usable - Vulkan driver validates the EGL import on it +- [An embedded video device is on hand](/quest/m3/embedded-device.md) - the device to validate on diff --git a/quest/m3/video-hardware-access.md b/quest/m3/video-hardware-access.md new file mode 100644 index 00000000000..ee99144ac5e --- /dev/null +++ b/quest/m3/video-hardware-access.md @@ -0,0 +1,21 @@ +# [XS] Video validation hardware is on hand + +## Goal + +Someone with the hardware can run the video validation: an Intel GPU +exposing the VAAPI low-power entrypoint, a second render node, a Windows +machine with MJPEG and YUY2 cameras, a live camera per platform, a KDE/Wayland +desktop with an Intel or AMD GPU, a sandbox that can show the camera portal +dialog, and a Raspberry Pi whose CSI camera appears as a PipeWire node. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +As of 2026-09-30 none of it is on hand. Check by asking the maintainer which +machines are available. + +The validation can start with whatever subset is available; split the rest +out of [Video hardware validation](/quest/m3/video-hardware.md) when that +happens. diff --git a/quest/m3/video-hardware.md b/quest/m3/video-hardware.md index d2f420a456d..960318bcae9 100644 --- a/quest/m3/video-hardware.md +++ b/quest/m3/video-hardware.md @@ -66,11 +66,7 @@ is not a bug any amount of review finds. ## Required -- Someone with the hardware runs it: an Intel GPU exposing the VAAPI low-power - entrypoint, a second render node, a Windows machine with MJPEG and YUY2 - cameras, a live camera per platform, a KDE/Wayland desktop with an Intel or - AMD GPU, a sandbox that can show the camera portal dialog, and a Raspberry - Pi whose CSI camera appears as a PipeWire node +- [Video validation hardware is on hand](/quest/m3/video-hardware-access.md) - the machines to run it on ## Closes diff --git a/quest/m3/watch-data-sync.md b/quest/m3/watch-data-sync.md deleted file mode 100644 index c48272e8791..00000000000 --- a/quest/m3/watch-data-sync.md +++ /dev/null @@ -1,21 +0,0 @@ -# [M] js/watch: play data tracks in sync with media - -## Goal - -A browser viewer receives JSON and binary track payloads on the same playhead -as the video and audio it plays, so telemetry shown beside a frame describes -that frame. A data track whose advertised `delay` and `jitter` exceed the -media's holds the media back by that much, and dropping the data track -releases it. - -## Plan - -- A `js/watch` data-track reader subscribes a catalog JSON or binary entry - (built-in section or an application section embedding the config), releases - each payload when the playhead reaches its frame timestamp, and registers its - `delay` and `jitter` with `Sync` like a media rendition. -- Snapshot tracks release the newest value at or before the playhead; stream - tracks release every record in order. -- In m3 until an application needs synchronized data playback, such as - teleop telemetry beside video; until then, a raw consumer reads payloads as - they arrive. diff --git a/quest/m3/whep-abr.md b/quest/m3/whep-abr.md deleted file mode 100644 index b91c396e618..00000000000 --- a/quest/m3/whep-abr.md +++ /dev/null @@ -1,23 +0,0 @@ -# [L] WHEP adaptive rendition switching - -## Goal - -A WHEP viewer of a multi-rendition broadcast is switched between video -renditions from its own congestion feedback, the way `@moq/watch` switches -over MoQ, so a weak link gets SD instead of stalling on HD. - -## Plan - -In m3 until a WHEP viewer needs adaptive quality; MoQ viewers already switch -renditions, and a WHEP viewer gets the best rendition today. - -Open questions: whether to drive switching from str0m's bandwidth estimate -(TWCC) or from loss and REMB, how to switch without a keyframe gap (subscribe -the new rendition and splice at its next group, as the JS decoder does), and -whether offering the renditions as WebRTC simulcast layers (RIDs) buys -anything for a receive-only browser. Keep the best rendition -(`hang::catalog::Video::ranked`) as the starting rendition. - -## Required - -- A WHEP viewer that needs rendition switching diff --git a/quest/m3/zero-copy-peer.md b/quest/m3/zero-copy-peer.md new file mode 100644 index 00000000000..97422ae5e88 --- /dev/null +++ b/quest/m3/zero-copy-peer.md @@ -0,0 +1,13 @@ +# [XS] A physical-NIC peer for the zero-copy sweep + +## Goal + +A remote peer reachable through a physical NIC is available to sweep +`SENDMSG_ZC` against. Loopback only measures the kernel's forced copy. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +Record the host and link when it exists, so the sweep is reproducible. diff --git a/quest/m4/README.md b/quest/m4/README.md index 5ef8cbfb662..96567c2f3bc 100644 --- a/quest/m4/README.md +++ b/quest/m4/README.md @@ -7,9 +7,10 @@ so it is not forgotten. ## Plan -Each quest states its gate as a plain-text `Required` bullet, re-checked as the -root [questline](/quest/README.md) describes. +Each quest's gate is a condition quest beside it, as the root +[questline](/quest/README.md) describes. ## Required +- [Safari ships the WebKit 319818 fix](/quest/m4/webkit-319818.md) - the Safari release the gate admits - [Safari WebTransport](/quest/m4/safari-webtransport.md) - WebKit browsers return to WebTransport once WebKit 319818 ships fixed diff --git a/quest/m4/safari-webtransport.md b/quest/m4/safari-webtransport.md index 85b3a454a0f..4d0cc76fb94 100644 --- a/quest/m4/safari-webtransport.md +++ b/quest/m4/safari-webtransport.md @@ -23,7 +23,7 @@ this. ## Required -- WebKit bug 319818 (https://bugs.webkit.org/show_bug.cgi?id=319818) is fixed and shipping in a Safari release +- [Safari ships the WebKit 319818 fix](/quest/m4/webkit-319818.md) - the release to gate on ## Closes diff --git a/quest/m4/webkit-319818.md b/quest/m4/webkit-319818.md new file mode 100644 index 00000000000..de6fc2f323f --- /dev/null +++ b/quest/m4/webkit-319818.md @@ -0,0 +1,15 @@ +# [XS] Safari ships the WebKit 319818 fix + +## Goal + +A shipping Safari release carries the fix for +[WebKit bug 319818](https://bugs.webkit.org/show_bug.cgi?id=319818), so it +refills MAX_DATA and MAX_STREAMS credit. + +This quest tracks a condition outside the repository. When it holds, delete +this quest and every `Required` entry that links it. + +## Plan + +As of 2026-09-30 the bug is NEW with no fix. Check the bug and the Safari +release notes. diff --git a/rs/hang/src/catalog/clock.rs b/rs/hang/src/catalog/clock.rs index 83584e401da..00c6664896a 100644 --- a/rs/hang/src/catalog/clock.rs +++ b/rs/hang/src/catalog/clock.rs @@ -72,7 +72,10 @@ impl Clock { let unix_millis = MOQ_EPOCH_UNIX_MILLIS as u128 + total * 1000 / scale.as_u64() as u128; let unix_millis = u64::try_from(unix_millis).map_err(|_| crate::Error::TimestampOverflow(moq_net::TimeOverflow))?; - Ok(std::time::UNIX_EPOCH + std::time::Duration::from_millis(unix_millis)) + // Windows' `SystemTime` cannot reach the furthest wall a catalog carries. + std::time::UNIX_EPOCH + .checked_add(std::time::Duration::from_millis(unix_millis)) + .ok_or(crate::Error::TimestampOverflow(moq_net::TimeOverflow)) } } @@ -141,6 +144,18 @@ mod test { assert!(Clock::new(wall).is_err()); } + /// Regression: the furthest wall a catalog carries overflowed `SystemTime` on + /// Windows and panicked. Unix reaches it, so only Windows sees the error. + #[test] + fn wall_clock_past_the_system_clock_is_an_error() { + let wall = moq_net::Timestamp::new(MAX_SAFE_INTEGER, moq_net::Timescale::SECOND).unwrap(); + let clock = Clock::new(wall).unwrap(); + match clock.wall_clock(moq_net::Timestamp::ZERO) { + Ok(at) => assert!(at > std::time::UNIX_EPOCH + std::time::Duration::from_secs(MAX_SAFE_INTEGER)), + Err(err) => assert!(matches!(err, crate::Error::TimestampOverflow(_)), "{err}"), + } + } + #[test] fn wall_clock_converts_across_timescales() { let clock = Clock::new(moq_net::Timestamp::from_micros(1_000_000).unwrap()).unwrap(); diff --git a/rs/moq-auth/src/client.rs b/rs/moq-auth/src/client.rs index 21d1560c6a1..061be2e0cdc 100644 --- a/rs/moq-auth/src/client.rs +++ b/rs/moq-auth/src/client.rs @@ -507,6 +507,29 @@ mod tests { assert!(matches!(script.connect().await, Err(Error::GrantExpired))); } + /// Regression: a cadence past the clock's range overflowed `Instant` and panicked + /// the driver, at connect and on each reply. It never fires; the grant expires. + #[tokio::test] + async fn a_cadence_past_the_clock_never_rechecks() { + tokio::time::pause(); + let log = Log::default(); + let script = Script::new(log.clone(), |_| { + Some(Ok(grant( + Some(Duration::from_secs(3)), + Some(Duration::from_secs(u64::MAX)), + ))) + }); + + let consumer = script.connect().await.unwrap(); + // A nudge's reply schedules the next re-check the same way. + consumer.revalidate(); + let reason = tokio::time::timeout(Duration::from_secs(4), consumer.closed()) + .await + .expect("closed at expires"); + assert_eq!(reason, Reason::Expired); + assert_eq!(log.revalidates(), 1, "only the nudge re-checked"); + } + /// An outage is evidence of nothing: the grant stands through failed re-checks /// until `expires`, and closes then, not later. #[tokio::test] diff --git a/rs/moq-auth/src/error.rs b/rs/moq-auth/src/error.rs index c32d0d06b16..6825f063084 100644 --- a/rs/moq-auth/src/error.rs +++ b/rs/moq-auth/src/error.rs @@ -105,6 +105,9 @@ pub enum Error { #[error("session limits need a revalidate cadence, which ages out the slots of a relay that died")] LimitsWithoutRevalidate, + #[error("the grant bound reaches past the system clock's range")] + ExpiresOutOfRange, + #[error("grant asks to be revalidated at no interval")] ZeroRevalidate, diff --git a/rs/moq-auth/src/grant.rs b/rs/moq-auth/src/grant.rs index dd774fd52eb..bb414a56c06 100644 --- a/rs/moq-auth/src/grant.rs +++ b/rs/moq-auth/src/grant.rs @@ -61,11 +61,12 @@ impl Grant { } } - /// Snapshot the expiry on Tokio's clock; one already past is now. + /// Snapshot the expiry on Tokio's clock; one already past is now, and one beyond + /// the clock's range is never. #[cfg(feature = "tokio")] pub fn deadline(&self) -> Option { let remaining = self.expires?.duration_since(SystemTime::now()).unwrap_or_default(); - Some(tokio::time::Instant::now() + remaining) + tokio::time::Instant::now().checked_add(remaining) } /// Refuse a grant that admits nothing, asks to be revalidated without a bound or @@ -195,4 +196,17 @@ mod tests { "a past expiry is now, not a grace window" ); } + + /// Regression: the furthest `expires` the wire carries overflowed `Instant` on + /// clocks with a narrower range than `SystemTime` (macOS) and panicked. Windows' + /// `SystemTime` cannot represent it, so the grant does not parse there. + #[cfg(all(feature = "tokio", unix))] + #[tokio::test(start_paused = true)] + async fn deadline_past_the_clock_is_never() { + let start = tokio::time::Instant::now(); + let grant: Grant = serde_json::from_str(r#"{"publish":["**"],"expires":9223372036854775807}"#).unwrap(); + // Linux's clock reaches that far; macOS's does not, which reads as no expiry. + let deadline = grant.deadline(); + assert!(deadline.is_none_or(|at| at > start + Duration::from_secs(1 << 40))); + } } diff --git a/rs/moq-auth/src/serve.rs b/rs/moq-auth/src/serve.rs index b22425c561a..fb571a5539c 100644 --- a/rs/moq-auth/src/serve.rs +++ b/rs/moq-auth/src/serve.rs @@ -87,6 +87,8 @@ pub struct Policy { pub revalidate: Option, /// How long a grant with no bound of its own lasts: an anonymous session, a token /// without `exp`, a certificate without one. `None` leaves those unbounded. + /// A bound past the system clock's range is refused by [`Server::new`], and by + /// [`decide`](Self::decide) for each session it would bound. pub expires: Option, /// Live session caps. pub limits: Limits, @@ -122,6 +124,8 @@ pub enum Refusal { UnsupportedToken(u64), #[error("both a JWT and a client certificate were presented; present one")] TokenAndCertificate, + #[error("the policy's grant bound reaches past the system clock's range")] + ExpiresOutOfRange, } impl Policy { @@ -155,7 +159,11 @@ impl Policy { }; let mut grant = Grant::new(permissions.publish, permissions.subscribe); - grant.expires = expires.or_else(|| self.expires.map(|bound| SystemTime::now() + bound)); + grant.expires = match (expires, self.expires) { + (Some(expires), _) => Some(expires), + (None, Some(bound)) => Some(SystemTime::now().checked_add(bound).ok_or(Refusal::ExpiresOutOfRange)?), + (None, None) => None, + }; grant.revalidate = self.revalidate; grant.tier = self.tier.clone(); Ok(grant) @@ -256,7 +264,7 @@ impl Sessions { fn sweep(&mut self, cadence: Duration) { let now = Instant::now(); self.slots - .retain(|_, slot| now.saturating_duration_since(slot.seen) < 2 * cadence); + .retain(|_, slot| now.saturating_duration_since(slot.seen) < cadence.saturating_mul(2)); } /// Admit a `connect`, refusing over the cap. A known id refreshes instead. @@ -323,11 +331,18 @@ pub struct Server { impl Server { /// A server answering with `policy`, refusing one that asks for a re-check without - /// a bound, or caps sessions without the re-check that ages out a dead relay's. + /// a bound, bounds grants past the clock's range, or caps sessions without the + /// re-check that ages out a dead relay's. pub fn new(policy: Policy) -> crate::Result { if policy.revalidate.is_some() && policy.expires.is_none() { return Err(crate::Error::UnboundedRevalidate); } + if policy + .expires + .is_some_and(|bound| SystemTime::now().checked_add(bound).is_none()) + { + return Err(crate::Error::ExpiresOutOfRange); + } let limited = policy.limits.token.is_some() || policy.limits.remote.is_some(); if limited && policy.revalidate.is_none() { return Err(crate::Error::LimitsWithoutRevalidate); @@ -970,6 +985,41 @@ mod tests { assert!(matches!(Server::new(unbounded), Err(Error::UnboundedRevalidate))); } + /// Regression: `--expires` past the system clock's range panicked every answer. + /// The server refuses it up front, and the policy alone refuses the session + /// rather than grant it no bound. + #[tokio::test] + async fn a_bound_past_the_clock_is_refused() { + let endless = Policy { + public: rules(&["**"], &["**"]), + expires: Some(Duration::MAX), + ..Default::default() + }; + assert_eq!( + endless.decide(&request("/")).await.unwrap_err(), + Refusal::ExpiresOutOfRange + ); + assert!(matches!(Server::new(endless), Err(Error::ExpiresOutOfRange))); + } + + /// Regression: `--revalidate` near `Duration::MAX` overflowed the two-cadence + /// sweep and panicked every answer once a limit kept a session table. + #[tokio::test] + async fn a_cadence_past_the_clock_keeps_every_slot() { + let server = Server::new(Policy { + public: rules(&["**"], &["**"]), + limits: Limits { + token: None, + remote: Some(1), + }, + revalidate: Some(Duration::MAX), + ..limited() + }) + .unwrap(); + server.answer(&request("/")).await.unwrap(); + assert_eq!(server.answer(&request("/")).await.unwrap_err(), Refusal::RemoteLimit); + } + /// With no limit to count against, nothing is kept per session, so a relay that /// dies without sending `end` leaks nothing here. #[tokio::test] diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 6f2799028bc..00539f4ec1e 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -4291,7 +4291,7 @@ fn zero_with_a_flag_set_is_a_real_value() { // Without the flags the same zeroes mean nothing at all. let defaults = parsed(&client_config()); assert_ne!(defaults.connect.backoff.timeout, std::time::Duration::ZERO); - assert_eq!(defaults.quic.keep_alive, std::time::Duration::from_secs(5)); + assert_eq!(defaults.quic.keep_alive, std::time::Duration::from_secs(3)); } #[test] diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 2d1d8f67c58..96d3ba18bca 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -322,6 +322,10 @@ impl Subscribe { // hand-rolled poll instead of `ts.next()`. let mut delivery = Delivery::new(self.args.max_age); let linger = self.args.linger; + // Reports a track that stops reaching the output while the rest keeps flowing, + // the way `publish` reports one that stops arriving. + let mut log = moq_mux::container::ts::stats::Log::default(); + let mut sampled = tokio::time::Instant::now(); loop { let end = loop { let mut waited = false; @@ -341,6 +345,11 @@ impl Subscribe { }; delivery.update(&frame, ts.discontinuity()); delivery.deliver(&frame, waited, &mut stdout).await?; + + if sampled.elapsed() >= moq_mux::container::ts::stats::Log::INTERVAL { + sampled = tokio::time::Instant::now(); + log.sample(ts.stats()); + } }; // Any end waits out the linger, and on expiry the last one is the result: a diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index d1ca805abdf..7d45ede9cbc 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -1512,7 +1512,7 @@ mod tests { "{name} does not declare MUTABLE_READY" ); } - for (name, value) in [("quic-idle-timeout", 30_000), ("quic-keep-alive", 5_000)] { + for (name, value) in [("quic-idle-timeout", 10_000), ("quic-keep-alive", 3_000)] { assert_eq!(spec(&sink, name).value_type(), u64::static_type()); assert_eq!(sink.property::(name), value); } diff --git a/rs/moq-gst/tests/element.rs b/rs/moq-gst/tests/element.rs index 833590c89a3..b72f0e86828 100644 --- a/rs/moq-gst/tests/element.rs +++ b/rs/moq-gst/tests/element.rs @@ -552,11 +552,11 @@ fn a_pipeline_description_names_the_track() { fn a_pipeline_description_configures_quic_timeouts() { init(); let sink = gst::parse::launch( - "moqsink url=https://127.0.0.1:1 broadcast=test quic-idle-timeout=15000 quic-keep-alive=3000", + "moqsink url=https://127.0.0.1:1 broadcast=test quic-idle-timeout=15000 quic-keep-alive=2000", ) .expect("parse the description"); assert_eq!(sink.property::("quic-idle-timeout"), 15_000); - assert_eq!(sink.property::("quic-keep-alive"), 3_000); + assert_eq!(sink.property::("quic-keep-alive"), 2_000); } #[test] diff --git a/rs/moq-mux/Cargo.toml b/rs/moq-mux/Cargo.toml index 3c5a09e6be3..431cf98da05 100644 --- a/rs/moq-mux/Cargo.toml +++ b/rs/moq-mux/Cargo.toml @@ -16,6 +16,7 @@ categories = ["multimedia", "network-programming", "web-programming"] anyhow = { workspace = true } base64 = { workspace = true } bytes = { workspace = true } +crc = "3" h264-parser = { version = "0.4.2" } hang = { workspace = true } kio = { workspace = true } diff --git a/rs/moq-mux/src/catalog/select.rs b/rs/moq-mux/src/catalog/select.rs index febce02d2a5..4105e9f2446 100644 --- a/rs/moq-mux/src/catalog/select.rs +++ b/rs/moq-mux/src/catalog/select.rs @@ -85,4 +85,21 @@ mod tests { assert_eq!(out.video.renditions.keys().collect::>(), vec!["hi"]); assert!(matches!(stream.poll_next(&kio::Waiter::noop()), Poll::Ready(Ok(None)))); } + + #[test] + fn selects_boxed_stream() { + let mut catalog = Catalog::default(); + catalog.video.renditions = BTreeMap::from_iter(vec![h264("lo"), h264("hi")]); + + let boxed: Box> = Box::new(Once(Some(catalog))); + let selection = select::Broadcast::default().video(select::Video::default().name("lo")); + let mut stream = boxed.select(selection); + + let out = match stream.poll_next(&kio::Waiter::noop()) { + Poll::Ready(Ok(Some(c))) => c, + other => panic!("expected snapshot, got {other:?}"), + }; + assert_eq!(out.video.renditions.keys().collect::>(), vec!["lo"]); + assert!(matches!(stream.poll_next(&kio::Waiter::noop()), Poll::Ready(Ok(None)))); + } } diff --git a/rs/moq-mux/src/catalog/stream.rs b/rs/moq-mux/src/catalog/stream.rs index 444a2936980..8fced0225c6 100644 --- a/rs/moq-mux/src/catalog/stream.rs +++ b/rs/moq-mux/src/catalog/stream.rs @@ -47,3 +47,12 @@ pub trait Stream: kio::MaybeSend + 'static { Select::new(self, selection) } } + +/// A boxed stream forwards to the stream it holds, so `Box>` is a stream. +impl Stream for Box { + type Ext = S::Ext; + + fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>> { + (**self).poll_next(waiter) + } +} diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 35ebe48ca31..f88bc263aea 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -24,14 +24,13 @@ pub(crate) enum Event { /// /// Groups can arrive on the wire out of order. The consumer always reads frames *within* /// a group in arrival order, but across groups it advances by sequence number, skipping -/// stalled or missing groups when the difference between the oldest pending timestamp -/// and the newest available timestamp exceeds the configured max age. With the default -/// max age of zero, the consumer skips aggressively. Any group that has a newer -/// alternative is dropped. With a non-zero max age, slow groups are tolerated up to that -/// budget before being skipped. A missing sequence gets the same tolerance: there is no -/// way to tell a stream that lost the delivery race from one the cache evicted, so the -/// consumer waits for it until everything it could still present (bounded by where the -/// next group begins) falls a full budget behind the newest content. +/// a stalled or missing group once everything it could still present (bounded by where +/// the next group begins) falls a full budget behind the newest content, the same reach +/// [`moq_net::track::Subscription::max_age`] measures. With the default max age of zero, +/// the consumer skips aggressively: any group that has a newer alternative is dropped. +/// With a non-zero max age, slow groups are tolerated up to that budget before being +/// skipped. A missing sequence gets the same tolerance: there is no way to tell a stream +/// that lost the delivery race from one the cache evicted. /// /// Delivery starts at the [`Subscription::start`](moq_net::track::Subscription::start) /// floor when one is named, waiting for that group under the budget; without one it @@ -307,17 +306,16 @@ impl Consumer { } } - // Get the current group's min timestamp (the reference for age - // comparison) and its furthest presentation point (timestamp + duration). - let (oldest_timestamp, current_end) = if let Some(current) = self.pending.front_mut() + // The current group's furthest presentation point (timestamp + duration). + let current_end = if let Some(current) = self.pending.front_mut() && current.sequence <= self.current { match current.poll_min_timestamp(waiter, &self.format) { - Poll::Ready(Ok(ts)) => (Some(std::time::Duration::from(ts)), current.max_end), - _ => (None, None), + Poll::Ready(Ok(_)) => current.max_end, + _ => None, } } else { - (None, None) + None }; // Find the first newer group with data (our skip target) and where it starts. @@ -369,24 +367,18 @@ impl Consumer { continue; } - let should_skip = if let Some((_, next_start)) = next_group { - if let Some(oldest) = oldest_timestamp { - // Current group is blocking. Skip if newer groups have pulled past - // the max age budget, or if the current group has already presented - // up to where the next group begins (duration coverage) so there's - // nothing left worth waiting for. - let over_max_age = max_timestamp.saturating_sub(oldest) >= self.max_age; - let covered = current_end.is_some_and(|end| end >= next_start); - over_max_age || covered - } else { - // The current group has arrived but has no frame yet. Its content - // is bounded by where the next stamped group begins the same way a - // missing one's is, so give it the same budget before giving up. - max_timestamp.saturating_sub(next_start) >= self.max_age - } - } else { - false - }; + // The current group is blocking. Everything it could still present ends + // where the next group begins, so skip once that reach falls a full budget + // behind the newest content, the same measure as a missing group. Its own + // first frame is no bound: a group longer than the budget would be cut + // short whenever it blocks behind a newer one, as a join's backlog does + // while the newer groups race ahead on their own streams. Skip early too + // once it has presented up to where the next group begins (duration + // coverage), since nothing is left worth waiting for. + let should_skip = next_group.is_some_and(|(_, next_start)| { + max_timestamp.saturating_sub(next_start) >= self.max_age + || current_end.is_some_and(|end| end >= next_start) + }); if let Some((new_idx, next_start)) = next_group && should_skip @@ -1115,6 +1107,62 @@ mod tests { finisher.await.expect("finisher task panicked"); } + /// A group longer than the budget that blocks for a moment behind a newer group is + /// not cut short while what it could still present (up to where the newer group + /// begins) is within the budget. A join's backlog does exactly this: the newer group + /// races ahead on its own stream while the older one is still arriving. + #[test] + fn a_long_group_blocked_behind_a_newer_one_is_not_cut_short() { + let waiter = kio::Waiter::noop(); + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); + let mut consumer = container_max_age_only(track.subscribe(None), Duration::from_secs(2)); + let write = |group: &mut moq_net::group::Producer, timestamp: Timestamp| { + let frame = Frame { + timestamp, + payload: Bytes::from_static(&[0xDE, 0xAD]), + keyframe: false, + duration: None, + }; + Container::Legacy(crate::container::Kind::Data) + .write(group, &[frame]) + .unwrap(); + }; + let read = |consumer: &mut Consumer| match consumer.poll_read(&waiter) { + Poll::Ready(Ok(Some(frame))) => Some(frame.timestamp), + Poll::Pending => None, + other => panic!( + "unexpected read: {:?}", + other.map(|r| r.map(|f| f.map(|f| f.timestamp))) + ), + }; + + // A 2.5 s group, half arrived, and the next one already 0.5 s in. + let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + write(&mut group0, ts(0)); + write(&mut group0, ts(500_000)); + let mut group1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + write(&mut group1, ts(2_500_000)); + write(&mut group1, ts(3_000_000)); + + assert_eq!(read(&mut consumer), Some(ts(0))); + assert_eq!(read(&mut consumer), Some(ts(500_000))); + // Group 0's reach (2.5 s) is 0.5 s behind the newest frame: wait for it. + assert_eq!(read(&mut consumer), None, "cut group 0 short"); + write(&mut group0, ts(1_000_000)); + assert_eq!(read(&mut consumer), Some(ts(1_000_000))); + group0.finish().unwrap(); + assert_eq!(read(&mut consumer), Some(ts(2_500_000))); + + // Once the reach falls a full budget behind, the stalled group is skipped. + let mut group2 = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + write(&mut group2, ts(5_000_000)); + write(&mut group2, ts(7_000_000)); + assert_eq!(read(&mut consumer), Some(ts(3_000_000))); + assert_eq!(read(&mut consumer), Some(ts(5_000_000))); + group1.finish().unwrap(); + group2.finish().unwrap(); + } + // ---- Malformed rewind ---- #[tokio::test] diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index e493c9a3afc..99da67de8f8 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -58,7 +58,7 @@ const PSI_INTERVAL: Duration = Duration::from_millis(500); /// table asking for faster repetition than this still gets it. const SI_REVISION_INTERVAL: Duration = Duration::from_secs(1); /// Emit a PCR on every crossing of this media-time grid ([`Export::emit`]). -/// TR 101 290 flags a gap over 40 ms; broadcast muxes emit every 25-40 ms. +/// TR 101 290 V1.4.1 flags a gap over 100 ms; broadcast muxes emit every 25-40 ms. pub(super) const PCR_INTERVAL: Duration = Duration::from_millis(25); /// How many missed PCR slots to backfill at most: one second's worth. Frames /// coarser than the grid cross several slots at a time and every one is filled so @@ -158,8 +158,9 @@ pub struct Export { /// Offsets into [`pending`](Self::pending) where a keyframe's packets begin, so /// the output frame carrying one keeps the flag. keyframes: Vec, - /// Output frames ready to hand out, one per grid slot the last span covered. - queue: VecDeque, + /// Output frames ready to hand out, one per grid slot the last span covered, each with + /// what it tells [`Self::stats`] once returned. + queue: VecDeque<(Frame, Tally)>, /// Continuity counter of the last packet emitted on the PCR PID. A clock packet /// carries no payload, so it repeats whatever preceded it on the wire rather /// than advancing the counter ([`Export::pcr_at`]). @@ -197,6 +198,24 @@ pub struct Export { /// up before it ever configures video. `None` until the tables are built, and for /// programs with no video track (nothing to align to). video_start: Option, + /// Offsets into [`pending`](Self::pending) where each access unit's packets begin, with + /// its PID, so the output frame carrying it counts it. + units: Vec<(usize, u16)>, + /// Clock packets written into the output frame being laid out ([`Self::push`]). + tally: Tally, + /// Each elementary stream's access units and silence on the PCR returned ([`Self::stats`]). + liveness: super::import::Liveness, +} + +/// What a queued output frame tells [`Export::stats`], applied only once the frame is +/// returned: a rewind drops the queue unwritten. +#[derive(Default)] +struct Tally { + /// The frame's clock packets in order: the PCR in 27 MHz ticks, and whether it flags a + /// new time base. + pcrs: Vec<(u64, bool)>, + /// The PID of each access unit whose packets begin in the frame, after its clock packets. + units: Vec, } struct Pending { @@ -408,6 +427,23 @@ enum Kind { }, } +impl Kind { + /// The track suffix [`Import`](super::Import) gives a PID carrying this kind, so a stream's + /// row is named alike at both edges. + fn suffix(&self) -> &'static str { + match self { + Kind::Video(StreamType::H265) => ".hev1", + Kind::Video(_) => ".avc3", + Kind::Aac { .. } => ".aac", + Kind::Opus { .. } => ".opus", + Kind::Mp2 { .. } => ".mp2", + Kind::Ac3 => ".ac3", + Kind::Eac3 => ".eac3", + Kind::Verbatim { .. } => ".ts", + } + } +} + /// The null stuffing owed to the multiplex rate ([`Export::stuff`]). #[derive(Default)] struct Stuffing { @@ -698,6 +734,9 @@ impl Export { mux_rate: None, mux_rate_override: None, stuffing: Stuffing::default(), + units: Vec::new(), + tally: Tally::default(), + liveness: Default::default(), }) } @@ -819,8 +858,7 @@ impl Export { // own values imply and lets the caller's pacer release each at the instant // it asserts. See [`Self::advance`]. loop { - if let Some(out) = self.queue.pop_front() { - self.emitted_epoch = self.epoch; + if let Some(out) = self.pop() { return Poll::Ready(Ok(Some(out))); } let Some(name) = self.pick_next_track(waiter) else { @@ -877,8 +915,7 @@ impl Export { let drained = !self.tracks.is_empty() && self.tracks.values().all(|t| t.finished); if drained { self.emit(None)?; - if let Some(out) = self.queue.pop_front() { - self.emitted_epoch = self.epoch; + if let Some(out) = self.pop() { return Poll::Ready(Ok(Some(out))); } // SI emission rides media frames, so a snapshot that arrived behind the @@ -899,6 +936,22 @@ impl Export { Poll::Pending } + /// Return the next queued frame, committing what it carries to the output boundary. + fn pop(&mut self) -> Option { + let (out, tally) = self.queue.pop_front()?; + self.emitted_epoch = self.epoch; + for (pcr, discontinuity) in tally.pcrs { + if discontinuity { + self.liveness.discontinuity(); + } + self.liveness.written_pcr(pcr); + } + for pid in tally.units { + self.liveness.delivered(pid, 1); + } + Some(out) + } + /// The trailing SI frame for end of stream: every entry's current sections, /// re-emitted once. Duplicates of already-emitted sections are harmless /// repetitions; `None` when there are no tables or no program was ever built. @@ -1257,6 +1310,30 @@ impl Export { self.emitted_epoch } + /// Snapshot the access units each elementary stream has written, and how long each has + /// been quiet on the PCR the output carries. + /// + /// A track stalled upstream stops advancing its row while the PSI and the other PIDs + /// keep flowing, which nothing graded on the output bytes alone can see. Empty until the + /// program tables are built. Cheap enough to poll per frame. + pub fn stats(&self) -> super::Stats { + let mut stats = super::Stats::default(); + if self.psi.is_none() { + return stats; + } + for track in self.tracks.values() { + let (units, quiet) = self.liveness.stream(track.pid); + let row = super::StreamStats { + track: track.kind.suffix(), + units, + quiet, + ..Default::default() + }; + stats.streams.insert(track.pid, row); + } + stats + } + /// Carry on with the broadcast that replaced the one this export was reading. /// /// Call it once [`Source::returned`](crate::Source::returned) has resolved, after @@ -1301,6 +1378,8 @@ impl Export { } self.pending.clear(); self.keyframes.clear(); + self.units.clear(); + self.tally = Tally::default(); self.queue.clear(); self.watermark = None; // The stall carries across with its budget, and so does a held frame's @@ -1499,6 +1578,9 @@ impl Export { es_info, }; + for track in &tracks { + self.liveness.register(track.pid); + } self.psi = Some(Psi { pat, pmt, @@ -1664,6 +1746,8 @@ impl Export { } } + // A malformed section writes nothing, so the unit counts only if bytes went in. + let written = out.len(); match es_payload { // Section-framed verbatim (SCTE-35, ...) rides in private sections, not PES; // carry the bytes verbatim. @@ -1689,6 +1773,9 @@ impl Export { self.write_pes(&mut out, &unit, &es_payload)?; } } + if out.len() > written { + self.units.push((self.pending.len() + written, pid)); + } if keyframe { self.keyframes.push(self.pending.len()); } @@ -1752,6 +1839,7 @@ impl Export { self.span_counters = None; let bytes = std::mem::take(&mut self.pending); let keyframes = std::mem::take(&mut self.keyframes); + let units = std::mem::take(&mut self.units); let Some(to) = self.low.take() else { return Ok(()) }; let packets = bytes.len() / TsPacket::SIZE; @@ -1814,7 +1902,7 @@ impl Export { }; self.send(&mut payload, &bytes[cut * TsPacket::SIZE..next * TsPacket::SIZE]); self.stuff(index, &mut payload); - self.push(at, payload, &keyframes, cut, next); + self.push(at, payload, &keyframes, &units, cut, next); cut = next; at = boundary; let before = counter_before(&bytes, cut * TsPacket::SIZE, pcr_pid, self.pcr_cc); @@ -1824,7 +1912,7 @@ impl Export { } self.send(&mut payload, &bytes[cut * TsPacket::SIZE..]); - self.push(at, payload, &keyframes, cut, packets); + self.push(at, payload, &keyframes, &units, cut, packets); if let Some(cc) = counter_before(&bytes, bytes.len(), pcr_pid, None) { self.pcr_cc = Some(cc); } @@ -1891,19 +1979,34 @@ impl Export { } /// Queue one output frame, unless it would be empty. `from`..`to` are the packet - /// indices it carries, which decide whether a keyframe begins in it. - fn push(&mut self, timestamp: Timestamp, payload: Vec, keyframes: &[usize], from: usize, to: usize) { + /// indices it carries, which decide whether a keyframe, or which access units, begin in it. + fn push( + &mut self, + timestamp: Timestamp, + payload: Vec, + keyframes: &[usize], + units: &[(usize, u16)], + from: usize, + to: usize, + ) { if payload.is_empty() { return; } let (from, to) = (from * TsPacket::SIZE, to * TsPacket::SIZE); let keyframe = keyframes.iter().any(|&at| at >= from && at < to); - self.queue.push_back(Frame { + let mut tally = std::mem::take(&mut self.tally); + tally.units = units + .iter() + .filter(|&&(at, _)| at >= from && at < to) + .map(|&(_, pid)| pid) + .collect(); + let frame = Frame { timestamp, duration: None, payload: Bytes::from(payload), keyframe, - }); + }; + self.queue.push_back((frame, tally)); } /// The clock packet for grid slot `index`, and record that the slot is served. @@ -1938,9 +2041,11 @@ impl Export { }; self.last_pcr = Some(index); let mut packet = pcr_packet(pcr_pid, ticks, cc)?; - if std::mem::take(&mut self.pcr_discontinuity) { + let discontinuity = std::mem::take(&mut self.pcr_discontinuity); + if discontinuity { packet[5] |= 0x80; } + self.tally.pcrs.push(((ticks & TS_TIMESTAMP_MASK) * 300, discontinuity)); Ok(packet) } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 778c226325b..273ed39ecab 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -17,7 +17,7 @@ use mpeg2ts::ts::{ReadTsPacket, TsPacketReader, TsPayload}; use crate::catalog::hang::Container as HangContainer; use crate::container::ts::export::PCR_INTERVAL; -use crate::container::ts::{Export, catalog as tscat}; +use crate::container::ts::{Export, Stats, catalog as tscat}; use crate::container::{Frame, Kind, Producer}; use moq_net::Timestamp; @@ -1885,9 +1885,9 @@ async fn scte35_fixtures_survive_roundtrip() { } /// Build a well-formed long-form section: the full generic header (extension, -/// current version, `number` of `last`), then `body` and a CRC placeholder -/// (capture is verbatim, nothing checks it). The SI store buffers a sub-table -/// until its generation completes, so the header fields must be coherent. +/// current version, `number` of `last`), then `body` and a valid CRC-32/MPEG-2. +/// The SI store buffers a sub-table until its generation completes, so the header +/// fields must be coherent. fn make_long_section(table_id: u8, ext: u16, version: u8, number: u8, last: u8, body: &[u8]) -> Vec { let section_length = 5 + body.len() + 4; let mut s = vec![ @@ -1901,7 +1901,8 @@ fn make_long_section(table_id: u8, ext: u16, version: u8, number: u8, last: u8, last, ]; s.extend_from_slice(body); - s.extend_from_slice(&[0xde, 0xad, 0xbe, 0xef]); + let crc = crc::Crc::::new(&crc::CRC_32_MPEG_2).checksum(&s); + s.extend_from_slice(&crc.to_be_bytes()); s } @@ -3036,6 +3037,357 @@ async fn eit_now_next_and_schedule_are_captured() { ); } +/// [`si_rig`] importing only `program`. +fn si_rig_selecting(program: u16) -> SiRig { + let SiRig { + import, + catalog, + consumer, + } = si_rig(); + SiRig { + import: import.with_program(program), + catalog, + consumer, + } +} + +/// An SDT service loop entry for `service`, running, with a service_descriptor naming it. +fn sdt_service_entry(service: u16, name: &[u8]) -> Vec { + let mut descriptor = vec![0x48, 0, 0x01, 1, b'P', name.len() as u8]; + descriptor.extend_from_slice(name); + descriptor[1] = (descriptor.len() - 2) as u8; + let mut entry = service.to_be_bytes().to_vec(); + entry.push(0xfd); + entry.extend_from_slice(&[0x80, descriptor.len() as u8]); + entry.extend_from_slice(&descriptor); + entry +} + +/// SDT actual section `number` of `last` for TSID 1 on ONID 2, listing `services`. +fn sdt_actual(version: u8, number: u8, last: u8, services: &[(u16, &[u8])]) -> Vec { + let mut body = vec![0x00, 0x02, 0xff]; + for &(service, name) in services { + body.extend(sdt_service_entry(service, name)); + } + make_long_section(0x42, 1, version, number, last, &body) +} + +/// EIT present/following actual section `number` of 1 for `service` on TSID 1, ONID 2. +fn eit_pf(service: u16, number: u8) -> Bytes { + let body = [0x00, 0x01, 0x00, 0x02, 0x01, 0x4E, service as u8, number]; + Bytes::from(make_long_section(0x4E, service, 0, number, 1, &body)) +} + +fn nit() -> Bytes { + Bytes::from(make_long_section(0x40, 1, 0, 0, 0, &[0xbb; 4])) +} + +/// SI for services 1 and 2: an SDT listing service 1 in section 0 and service 2 in +/// section 1, EIT present/following for each, and a NIT. +fn two_service_si() -> Vec { + let mut out = si_packet(0x0011, &sdt_actual(0, 0, 1, &[(1, b"One")])); + out.extend(si_packet_cc(0x0011, &sdt_actual(0, 1, 1, &[(2, b"Two")]), 1)); + for (cc, section) in [eit_pf(1, 0), eit_pf(1, 1), eit_pf(2, 0), eit_pf(2, 1)] + .iter() + .enumerate() + { + out.extend(si_packet_cc(0x0012, section, cc as u8)); + } + out.extend(si_packet(0x0010, &nit())); + out +} + +/// `section` is one whole SDT actual for TSID 1 on ONID 2 at `version`, listing only +/// `service` named `name`, under a valid CRC. +fn assert_sdt_lists_only(section: &[u8], version: u8, service: u16, name: &[u8]) { + let crc = crc::Crc::::new(&crc::CRC_32_MPEG_2); + assert_eq!(crc.checksum(section), 0, "a valid CRC-32/MPEG-2"); + let section_length = (usize::from(section[1] & 0x0f) << 8) | usize::from(section[2]); + assert_eq!(section.len(), 3 + section_length, "section_length covers the section"); + assert_eq!(section[0], 0x42, "SDT actual"); + assert_eq!( + u16::from_be_bytes([section[3], section[4]]), + 1, + "transport_stream_id kept" + ); + assert_eq!((section[5] >> 1) & 0x1f, version, "version_number kept"); + assert_eq!((section[6], section[7]), (0, 0), "section 0 of 0"); + assert_eq!( + u16::from_be_bytes([section[8], section[9]]), + 2, + "original_network_id kept" + ); + assert_eq!( + §ion[11..section.len() - 4], + &sdt_service_entry(service, name)[..], + "the service loop holds the selected service alone" + ); +} + +/// A selected program's SI describes its own service alone: one SDT actual section +/// listing it, wherever it sat in the source, and only its own EIT. Network-wide +/// tables pass through. +#[tokio::test(start_paused = true)] +async fn a_selected_program_carries_only_its_own_si() { + for (program, name) in [(1u16, &b"One"[..]), (2, b"Two")] { + let mut input = crate::container::ts::import::test::two_programs(); + input.extend(two_service_si()); + let mut rig = si_rig_selecting(program); + rig.import.decode(&BytesMut::from(&input[..])).unwrap(); + rig.import.finish().unwrap(); + + let si = rig.catalog.snapshot().ext.mpegts.si.clone(); + let sdt = read_si_sections(&rig.consumer, &si[&0x0011][&0x42].track).await; + assert_eq!(sdt.len(), 1, "program {program}: one SDT actual section"); + assert_sdt_lists_only(&sdt[0], 0, program, name); + assert_eq!( + read_si_sections(&rig.consumer, &si[&0x0012][&0x4E].track).await, + vec![eit_pf(program, 0), eit_pf(program, 1)], + "program {program}: only its own EIT" + ); + assert_eq!( + read_si_sections(&rig.consumer, &si[&0x0010][&0x40].track).await, + vec![nit()], + "the NIT passes through" + ); + } +} + +/// Without a selection nothing is filtered by program: every section is kept verbatim. +#[tokio::test(start_paused = true)] +async fn an_unselected_import_keeps_every_service() { + let mut rig = si_rig(); + rig.import.decode(&BytesMut::from(&two_service_si()[..])).unwrap(); + rig.import.finish().unwrap(); + + let si = rig.catalog.snapshot().ext.mpegts.si.clone(); + assert_eq!( + read_si_sections(&rig.consumer, &si[&0x0011][&0x42].track).await, + vec![ + Bytes::from(sdt_actual(0, 0, 1, &[(1, b"One")])), + Bytes::from(sdt_actual(0, 1, 1, &[(2, b"Two")])), + ], + "the SDT verbatim" + ); + assert_eq!( + read_si_sections(&rig.consumer, &si[&0x0012][&0x4E].track).await, + vec![eit_pf(1, 0), eit_pf(1, 1), eit_pf(2, 0), eit_pf(2, 1)], + "every service's EIT" + ); +} + +/// A selected service the SDT does not list gets no SDT actual rather than a +/// fabricated one, and a table filtered to nothing gets no catalog entry. +#[tokio::test(start_paused = true)] +async fn a_selection_missing_from_the_sdt_carries_none() { + let mut rig = si_rig_selecting(3); + rig.import.decode(&BytesMut::from(&two_service_si()[..])).unwrap(); + rig.import.finish().unwrap(); + + let si = rig.catalog.snapshot().ext.mpegts.si.clone(); + assert!(si.contains_key(&0x0010), "the NIT was captured (control)"); + assert!(!si.contains_key(&0x0011), "no SDT entry: {si:?}"); + assert!(!si.contains_key(&0x0012), "no EIT entry: {si:?}"); +} + +/// An SDT revision that drops the selected service retires the SDT captured before +/// it, catalog entry and all; a later revision listing it again is captured normally. +/// Driven on the capture itself so each revision cuts without waiting out the debounce. +#[tokio::test(start_paused = true)] +async fn an_sdt_revision_dropping_the_service_retires_it() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone()); + capture.select(1); + let mut revise = |version: u8, services: &[(u16, &[u8])]| { + capture.section(0x0011, sdt_actual(version, 0, 0, services)).unwrap(); + capture.flush(Timestamp::ZERO, true).unwrap(); + catalog + .snapshot() + .ext + .mpegts + .si + .get(&0x0011) + .map(|tables| tables[&0x42].track.clone()) + }; + + let track = revise(0, &[(1, b"One"), (2, b"Two")]).expect("the SDT is advertised"); + assert_eq!(revise(1, &[(2, b"Two")]), None, "the revision retired the SDT entry"); + assert_eq!( + revise(2, &[(1, b"One"), (2, b"Two")]), + Some(track.clone()), + "listed again, on the same track" + ); + capture.finish(Timestamp::ZERO).unwrap(); + + let groups = read_si_groups(&consumer, &track).await; + assert_eq!(groups.len(), 3, "one snapshot per revision"); + assert_sdt_lists_only(&groups[0][0], 0, 1, b"One"); + assert!(groups[1].is_empty(), "the retiring snapshot carries no SDT"); + assert_sdt_lists_only(&groups[2][0], 2, 1, b"One"); +} + +/// A corrupted SDT actual is not rebuilt under a fresh, valid CRC: the section is +/// dropped and the last good snapshot stays in force, rather than the corruption +/// reading as a revision (or as the service leaving). An intact revision after it +/// is captured normally. +#[tokio::test(start_paused = true)] +async fn a_corrupt_sdt_keeps_the_last_good_snapshot() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone()); + capture.select(1); + let mut feed = |section: Vec| { + capture.section(0x0011, section).unwrap(); + capture.flush(Timestamp::ZERO, true).unwrap(); + catalog + .snapshot() + .ext + .mpegts + .si + .get(&0x0011) + .map(|tables| tables[&0x42].track.clone()) + }; + + let track = feed(sdt_actual(0, 0, 0, &[(1, b"One")])).expect("the SDT is advertised"); + // One unflagged bit flip turns the service name "One" into "Nne" under the source CRC. + let mut corrupt = sdt_actual(1, 0, 0, &[(1, b"One")]); + let name = corrupt.len() - 4 - 3; + corrupt[name] ^= 0x01; + assert_eq!(feed(corrupt), Some(track.clone()), "the corrupt revision kept the SDT"); + assert_eq!(feed(sdt_actual(2, 0, 0, &[(1, b"Uno")])), Some(track.clone())); + capture.finish(Timestamp::ZERO).unwrap(); + + let groups = read_si_groups(&consumer, &track).await; + assert_eq!(groups.len(), 2, "no snapshot for the corrupt revision"); + assert_sdt_lists_only(&groups[0][0], 0, 1, b"One"); + assert_sdt_lists_only(&groups[1][0], 2, 1, b"Uno"); +} + +/// A multi-section SDT revision with the selected service's section corrupt does not +/// commit as complete-as-observed when the other section repeats: that would read as +/// the service leaving. The revision commits once the section arrives intact. +#[tokio::test(start_paused = true)] +async fn a_partial_sdt_revision_keeps_the_last_good_snapshot() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone()); + capture.select(1); + let mut feed = |section: Vec| { + capture.section(0x0011, section).unwrap(); + capture.flush(Timestamp::ZERO, true).unwrap(); + catalog + .snapshot() + .ext + .mpegts + .si + .get(&0x0011) + .map(|tables| tables[&0x42].track.clone()) + }; + + feed(sdt_actual(0, 0, 1, &[(1, b"One")])); + let track = feed(sdt_actual(0, 1, 1, &[(2, b"Two")])).expect("the SDT is advertised"); + let mut corrupt = sdt_actual(1, 0, 1, &[(1, b"One")]); + let name = corrupt.len() - 4 - 3; + corrupt[name] ^= 0x01; + assert_eq!(feed(corrupt), Some(track.clone())); + assert_eq!(feed(sdt_actual(1, 1, 1, &[(2, b"Two")])), Some(track.clone())); + assert_eq!( + feed(sdt_actual(1, 1, 1, &[(2, b"Two")])), + Some(track.clone()), + "the repeated section did not commit the partial revision" + ); + assert_eq!(feed(sdt_actual(1, 0, 1, &[(1, b"Uno")])), Some(track.clone())); + capture.finish(Timestamp::ZERO).unwrap(); + + let groups = read_si_groups(&consumer, &track).await; + assert_eq!(groups.len(), 2, "no snapshot for the partial revision"); + assert_sdt_lists_only(&groups[0][0], 0, 1, b"One"); + assert_sdt_lists_only(&groups[1][0], 1, 1, b"Uno"); +} + +/// A selection filters EIT schedule actual (0x50..=0x5F) by service, as it does +/// present/following. +#[tokio::test(start_paused = true)] +async fn a_selection_filters_eit_schedule() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new( + &mut broadcast, + crate::catalog::Config::default().with_catalog(crate::catalog::hang::Catalog::::default()), + ) + .unwrap(); + let mut capture = crate::container::ts::si::Capture::new(broadcast, catalog.clone()); + capture.select(1); + let schedule = |service: u16| { + let body = [0x00, 0x01, 0x00, 0x02, 0x00, 0x50, service as u8]; + make_long_section(0x50, service, 0, 0, 0, &body) + }; + for service in [1, 2] { + capture.section(0x0012, schedule(service)).unwrap(); + } + capture.finish(Timestamp::ZERO).unwrap(); + + let si = catalog.snapshot().ext.mpegts.si.clone(); + assert_eq!( + read_si_sections(&consumer, &si[&0x0012][&0x50].track).await, + vec![Bytes::from(schedule(1))], + "only the selected service's schedule" + ); +} + +/// A selected program's reduced SI survives export: the TS parses, and importing it +/// again finds the same single-service SDT and the same EIT. +#[tokio::test(start_paused = true)] +async fn a_selected_programs_si_survives_export() { + let mut input = crate::container::ts::import::test::two_programs(); + input.extend(two_service_si()); + let mut rig = si_rig_selecting(2); + rig.import.decode(&BytesMut::from(&input[..])).unwrap(); + rig.import.finish().unwrap(); + + let ts = drain_with( + Export::with_ts( + crate::source::announced(&rig.consumer), + crate::catalog::CatalogFormat::Hang, + ) + .await + .unwrap(), + ) + .await; + assert_packet_aligned(&ts); + let mut reader = TsPacketReader::new(Cursor::new(ts.as_ref())); + while reader.read_ts_packet().unwrap().is_some() {} + + let mut again = si_rig(); + again.import.decode(&BytesMut::from(ts.as_ref())).unwrap(); + again.import.finish().unwrap(); + let si = again.catalog.snapshot().ext.mpegts.si.clone(); + let sdt = read_si_sections(&again.consumer, &si[&0x0011][&0x42].track).await; + assert_eq!(sdt.len(), 1, "one SDT actual section"); + assert_sdt_lists_only(&sdt[0], 0, 2, b"Two"); + assert_eq!( + read_si_sections(&again.consumer, &si[&0x0012][&0x4E].track).await, + vec![eit_pf(2, 0), eit_pf(2, 1)], + "only program 2's EIT" + ); +} + /// #2842: SDT other sections from two networks that reuse a transport_stream_id /// must not collide. The identity reads original_network_id (bytes 8..10) for /// table_id 0x46, so both survive as separate sub-tables; a revision within one @@ -5718,3 +6070,240 @@ async fn resume_after_a_finish() { async fn resume_after_a_drop() { resume_after(false).await; } + +/// The stats count only output that was returned. A frame the muxer refuses fails the export +/// with the span before it queued but never returned, and the resume discards it. +#[tokio::test(start_paused = true)] +async fn export_stats_skip_output_a_failure_discards() { + let origin = crate::source::produce_origin(); + let source = crate::Source::new(origin.consume(), "live"); + let publish = || { + let mut broadcast = origin.publish("live", Default::default()).unwrap(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + (broadcast, catalog) + }; + let units = |stats: Stats| stats.streams.values().map(|row| row.units).sum::() as usize; + + let (mut broadcast, mut catalog) = publish(); + let mut track = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); + let ended = source.broadcast().await.unwrap(); + let mut export = Export::new(source.clone()) + .await + .unwrap() + .with_max_age(RECORDING_MAX_AGE); + for ms in (0..200).step_by(20) { + write_aac(&mut track, ms); + } + let mut frames = drain_frames(&mut export).await; + // One byte past what an ADTS header can frame. + track + .write(Frame { + timestamp: Timestamp::from_millis(200).unwrap(), + duration: None, + payload: Bytes::from(vec![0; 8185]), + keyframe: true, + }) + .unwrap(); + let (rest, end) = drain_to_end(&mut export).await; + frames.extend(rest); + assert!(end.is_err(), "an unframeable AAC frame fails the export"); + assert!(pes_count(&frames) < 10, "the span before the failure stayed queued"); + assert_eq!(units(export.stats()), pes_count(&frames)); + + drop((broadcast, catalog, track)); + let (mut broadcast, mut catalog) = publish(); + let mut track = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); + source.returned(&ended).await.unwrap(); + export.resume().await.unwrap(); + for ms in (0..200).step_by(20) { + write_aac(&mut track, ms); + } + frames.extend(drain_frames(&mut export).await); + track.finish().unwrap(); + catalog.finish().unwrap(); + let (rest, end) = drain_to_end(&mut export).await; + frames.extend(rest); + end.unwrap(); + assert_eq!( + units(export.stats()), + pes_count(&frames), + "the discarded span never counts" + ); +} + +/// Export 25 fps video and two AAC tracks for [`TICKS`] video frames, the video and the first +/// audio track stopping after `stop` while the second carries on. Returns the stats sampled +/// after `sample` and at the end, and the frames rendered in between. +/// +/// Drained after every write, like [`export_twice`], so the export reads every frame and a row +/// stops only because its track did. +async fn export_liveness(sample: u64, stop: u64) -> (Stats, Stats, Vec) { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + + let track = broadcast + .create_track("video.avc1", hang::container::track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let mut cfg = VideoConfig::new(H264 { + profile: 0x42, + constraints: 0xc0, + level: 0x1f, + inline: false, + }); + cfg.container = Container::Legacy; + cfg.description = + Some(crate::codec::h264::build_avcc(&[Bytes::from_static(SPS)], &[Bytes::from_static(PPS)]).unwrap()); + catalog + .modify() + .unwrap() + .video + .renditions + .insert(track.name().to_string(), cfg); + let mut video = Producer::new(track, HangContainer::Legacy(crate::container::Kind::Data)); + let mut audio = [ + aac_rendition(&mut broadcast, &mut catalog, "primary.aac"), + aac_rendition(&mut broadcast, &mut catalog, "secondary.aac"), + ]; + + let mut export = Export::new(crate::source::announced(&consumer)).await.unwrap(); + assert!(export.stats().is_empty(), "no row before the program tables"); + let (mut sampled, mut frames) = (None, Vec::new()); + let mut audio_index = 0; + for tick in 0..TICKS { + if tick < stop { + let keyframe = tick % GOP == 0; + let slice = if keyframe { + vec![0x65u8; 3_000] + } else { + vec![0x41u8; 400] + }; + video + .write(Frame { + timestamp: Timestamp::from_micros(tick * VIDEO_US).unwrap(), + duration: None, + payload: length_prefixed(&[&slice]), + keyframe, + }) + .unwrap(); + } + while audio_index * AUDIO_US < (tick + 1) * VIDEO_US { + let running = if tick < stop { &mut audio[..] } else { &mut audio[1..] }; + for track in running { + track + .write(Frame { + timestamp: Timestamp::from_micros(audio_index * AUDIO_US).unwrap(), + duration: None, + payload: Bytes::from_iter((0..180u16).map(|i| (i ^ audio_index as u16) as u8)), + keyframe: audio_index % AUDIO_GROUP == 0, + }) + .unwrap(); + } + audio_index += 1; + } + + let out = drain_frames(&mut export).await; + if sampled.is_some() { + frames.extend(out); + } + if tick + 1 == sample { + sampled = Some(export.stats()); + } + } + (sampled.expect("sampled mid-run"), export.stats(), frames) +} + +/// A healthy export advances every elementary stream's row, each quiet for no longer than +/// its own frame spacing plus the mux buffer, and leaves the import-only counters at zero. +#[tokio::test(start_paused = true)] +async fn export_stats_advance_every_stream() { + let (mid, end, _) = export_liveness(TICKS / 2, TICKS).await; + + let tracks: Vec<&str> = end.streams.values().map(|row| row.track).collect(); + assert_eq!(tracks, [".aac", ".aac", ".avc3"], "one row per elementary stream"); + for (pid, row) in &end.streams { + let before = &mid.streams[pid]; + assert!(before.units > 0, "PID {pid:#x} wrote nothing by mid-run"); + assert!( + row.units > before.units, + "PID {pid:#x} stopped advancing: {before:?} -> {row:?}" + ); + let quiet = row.quiet.expect("the output carries a PCR"); + assert!(quiet < Duration::from_millis(200), "PID {pid:#x} quiet for {quiet:?}"); + assert_eq!((row.resyncs, row.discarded, row.unconfirmed), (0, 0, 0)); + } +} + +/// The #3533 shape: video and the primary audio stop reaching the exporter while the +/// secondary audio carries on. Their rows freeze and their quiet time grows on the output's +/// own PCR, which keeps running on the surviving track, as do the PSI and that track's row. +#[tokio::test(start_paused = true)] +async fn export_stats_catch_a_track_that_stops() { + let stop = TICKS / 2; + // Sampled once the mux buffer has written out what the stopped tracks sent last. + let (sampled, end, after) = export_liveness(stop + 5, stop).await; + let stalled = Duration::from_micros((TICKS - stop) * VIDEO_US); + + let advanced = |pid: &u16| end.streams[pid].units > sampled.streams[pid].units; + let (running, stopped): (Vec, Vec) = end.streams.keys().copied().partition(advanced); + let [running] = running[..] else { + panic!("expected one stream to keep advancing, got {running:?}: {end:?}"); + }; + assert_eq!(end.streams[&running].track, ".aac"); + assert_eq!(stopped.len(), 2, "video and the primary audio stopped"); + + let quiet = |pid: u16| end.streams[&pid].quiet.expect("the output carries a PCR"); + assert!(quiet(running) < Duration::from_millis(200), "{:?}", quiet(running)); + for pid in stopped { + // The mux buffer holds the last span back, so the output's clock trails the media. + assert!( + quiet(pid) > stalled - Duration::from_millis(200), + "PID {pid:#x} quiet for only {:?} of a {stalled:?} stall", + quiet(pid) + ); + } + assert!( + count_pid(&after, 0x0000) >= 3, + "the PAT kept repeating through the stall" + ); + assert!(count_pid(&after, running) > 0, "the surviving track kept flowing"); +} + +/// A stopped track's silence counts the whole of a media gap on the surviving track, not just +/// the one second of clock the exporter backfills: the output's PCR jumps across the rest +/// unflagged, and that jump is time the stopped track was silent. +#[tokio::test(start_paused = true)] +async fn export_stats_count_a_gap_longer_than_the_backfill() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let mut catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut stopped = aac_rendition(&mut broadcast, &mut catalog, "a.aac"); + let mut running = aac_rendition(&mut broadcast, &mut catalog, "b.aac"); + + let mut export = Export::new(crate::source::announced(&consumer)).await.unwrap(); + for ms in (0..2_000).step_by(20) { + write_aac(&mut stopped, ms); + write_aac(&mut running, ms); + drain_frames(&mut export).await; + } + // Five seconds with nothing on either track, then only the second resumes. + for ms in (7_000..8_000).step_by(20) { + write_aac(&mut running, ms); + drain_frames(&mut export).await; + } + + let stats = export.stats(); + let mut rows: Vec<_> = stats.streams.values().collect(); + rows.sort_by_key(|row| row.units); + let [stopped, running] = rows[..] else { + panic!("expected two rows: {stats:?}"); + }; + assert!(stopped.units < running.units, "{stats:?}"); + let quiet = stopped.quiet.expect("the output carries a PCR"); + assert!( + quiet > Duration::from_millis(5_800), + "stopped at 2 s, quiet for only {quiet:?} at 8 s" + ); + let quiet = running.quiet.expect("the output carries a PCR"); + assert!(quiet < Duration::from_millis(200), "{quiet:?}"); +} diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 299ad4d037a..c44b833ceb0 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -6,8 +6,8 @@ //! catalog entries. Elementary streams we don't decode are carried verbatim, one //! MoQ track per PID, described in the `mpegts` catalog section: PES-framed streams //! ride the normal PES reassembly, while section-framed streams (SCTE-35 and -//! other private sections, which are not PES) are intercepted before the mpeg2ts -//! reader and reassembled. TS adds PAT/PMT discovery, PES reassembly, the +//! other private sections, which are not PES) are reassembled as sections, as the +//! PAT and PMT are. TS adds PAT/PMT discovery, PES reassembly, the //! private-section path, and the 90 kHz -> microsecond PTS conversion. The service //! layer is captured too: the identity parsed from the PAT as a //! [`Program`](catalog::Program) record in the `mpegts` catalog section, and the @@ -15,18 +15,14 @@ //! (see [`si`](super::si)), so both survive the round-trip. use std::collections::{BTreeMap, HashMap, HashSet}; -use std::io::Read; -use std::sync::{Arc, Mutex}; use anyhow::Context; -use bytes::BytesMut; use mpeg2ts::es::StreamType; -use mpeg2ts::pes::PesHeader; -use mpeg2ts::ts::payload::Pes; -use mpeg2ts::ts::{Pid, ReadTsPacket, TsPacket, TsPacketReader, TsPayload}; +use mpeg2ts::ts::{Pid, TsPacket}; use super::adts; use super::catalog; +use super::psi::{self, PesStart, Pmt}; use crate::catalog::hang::CatalogExt; use crate::codec::{aac, ac3, eac3, h264, h265, legacy, mp2, opus}; use moq_net::Timestamp; @@ -42,8 +38,12 @@ use moq_net::Timestamp; /// PID, when the catalog `E` carries the [`mpegts`](super::Mpegts) section: PES-framed /// streams ride the normal PES reassembly, section-framed streams (SCTE-35, marked /// by a program-level 'CUEI' registration descriptor, and other private sections) -/// are intercepted before the reader and reassembled. With a base `Catalog<()>` -/// they're logged and dropped instead. +/// are reassembled as sections. With a base `Catalog<()>` they're logged and dropped +/// instead. +/// +/// The PAT and every PMT are reassembled the same way, so a table that spans packets or +/// sits behind a nonzero pointer_field is read whole. A PAT or PMT section whose CRC-32 +/// fails is dropped and counted in [`Stats::crc_error`]; the last good table stays in force. /// /// The selected container applies only to decoded media renditions. Verbatim tracks in the /// `mpegts` catalog section continue to use the legacy Hang container. @@ -58,15 +58,13 @@ pub struct Import { /// snapshot rather than a half-converged one. See [`Reserved`](crate::catalog::Reserved). initial_reservation: Option>, - /// Shared, refillable byte source the persistent reader pulls whole packets - /// from. Kept beside the reader so [`decode`](Self::decode) can append bytes - /// between reads without recreating the reader (which holds PAT/PMT state). - feed: Feed, - reader: TsPacketReader, - - /// PMT PIDs announced by the PAT. With `streams` (the ES PIDs a PMT registers) - /// these are the only PIDs the reader can route; see [`Self::decode`]. - pmt_pids: HashSet, + /// The PAT, reassembled off PID 0. + pat: PatReader, + /// The PMT PIDs PATs have named for the imported program, each with its reassembler. + /// Several programs' PMTs may share one. + pmt_sections: HashMap, + /// PAT and PMT sections dropped for a CRC mismatch, keyed by PID. + crc_errors: BTreeMap, /// Per elementary-stream-PID codec routing. streams: HashMap>, /// Counters from routes a later PMT replaced, keyed by PID. @@ -92,20 +90,13 @@ pub struct Import { /// True once a PMT with at least one supported stream has been parsed. initialized: bool, - /// Whole-packet accumulator. Bytes are routed one TS packet at a time - /// (section-framed verbatim PIDs diverted, the rest fed to the reader); a - /// trailing partial packet is kept here for the next call. + /// Whole-packet accumulator. Bytes are routed one TS packet at a time; a trailing + /// partial packet is kept here for the next call. scratch: Vec, - /// Sync lock. 0x47 is the packet sync byte but also occurs freely in payload (TS - /// has no byte stuffing), so a lone 0x47 isn't a boundary. False until a candidate - /// is confirmed by the next packet's sync byte; once true we stride 188 at a time - /// and trust the per-packet check. Persists across `decode` calls so a candidate - /// pending confirmation at a buffer tail is re-confirmed, not trusted blindly. - synced: bool, - /// Section-framed verbatim PIDs, intercepted before the reader. Private sections - /// (SCTE-35 table_id 0xFC and others) are not PES, so the reader would - /// `Pes::read_from` and abort. Keyed by PID. SCTE-35 is detected via the PMT - /// 'CUEI' registration descriptor. + framer: Framer, + /// Section-framed verbatim PIDs. Private sections (SCTE-35 table_id 0xFC and others) + /// are not PES, so they are reassembled as sections rather than PES-parsed. Keyed by + /// PID. SCTE-35 is detected via the PMT 'CUEI' registration descriptor. sections: HashMap>, /// Whether the catalog can carry the `mpegts` section, sampled once at construction. /// A base `Catalog<()>` can't, so its undecoded PIDs route to `Stream::Ignored`. @@ -119,9 +110,9 @@ pub struct Import { /// Whether the PMT program-level descriptors have been recorded yet (set once; /// PMT `program_info` is stable for the program's life). program_recorded: bool, - /// Reassemblers for the standalone SI PIDs ([`catalog::SI_PIDS`]), keyed by PID. - /// Intercepted before the reader (like SCTE-35 sections) so the service layer - /// survives the round-trip. Only populated with `mpegts` catalog support. + /// Reassemblers for the standalone SI PIDs ([`catalog::SI_PIDS`]), keyed by PID, so + /// the service layer survives the round-trip. Only populated with `mpegts` catalog + /// support. si_sections: HashMap, /// The SI store: buffers each sub-table to a complete generation, publishes the /// per-`(PID, table_id)` snapshot tracks, and advertises them in the catalog. @@ -142,7 +133,6 @@ pub struct Import { impl Import { pub fn new(broadcast: moq_net::broadcast::Producer, reserved: crate::catalog::Reserved) -> Self { - let feed = Feed::default(); let container = hang::catalog::Container::default(); // A long-lived producer handle for catalog edits (mpegts sections, later PMTs); the passed // reservation gates the initial publish and is dropped once the first PMT is parsed. @@ -157,9 +147,9 @@ impl Import { catalog, container, initial_reservation: Some(reserved), - reader: TsPacketReader::new(feed.clone()), - feed, - pmt_pids: HashSet::new(), + pat: PatReader::default(), + pmt_sections: HashMap::new(), + crc_errors: BTreeMap::new(), streams: HashMap::new(), retired_stats: BTreeMap::new(), liveness: Liveness::default(), @@ -170,7 +160,7 @@ impl Import { published: false, initialized: false, scratch: Vec::new(), - synced: false, + framer: Framer::default(), sections: HashMap::new(), supports_mpegts, es_descriptors: HashMap::new(), @@ -189,9 +179,11 @@ impl Import { /// /// Without this, a PAT that lists more than one program fails the import with /// [`MultipleProgramsError`] rather than merging them onto one clock. A PAT that does not - /// list `program` fails it too. + /// list `program` fails it too. The SI describes only this program's service: other + /// services' EIT actual is dropped and the SDT actual lists this service alone. pub fn with_program(mut self, program: u16) -> Self { self.program = Some(program); + self.si.select(program); self } @@ -222,51 +214,11 @@ impl Import { pub fn decode(&mut self, data: &[u8]) -> anyhow::Result<()> { self.scratch.extend_from_slice(data); - // Route one whole packet at a time. Section-framed verbatim PIDs are - // intercepted here (the reader would PES-parse their sections and abort); - // every other packet is fed to the reader. Per-packet so a PMT is parsed - // (and any section PID registered) before the packets that follow it in the - // same chunk route. + // Route one whole packet at a time, so a PMT is parsed (and any PID it declares + // registered) before the packets that follow it in the same chunk route. let mut off = 0; - while off + TsPacket::SIZE <= self.scratch.len() { - // A TS packet starts with the 0x47 sync byte. Once synced we trust it and stride - // 188 at a time; until then (or after a miss) we must (re)acquire the lock. - if !self.synced || self.scratch[off] != 0x47 { - self.synced = false; - // 0x47 also occurs freely in payload (TS has no byte stuffing), so a lone one - // isn't a boundary. Scan (SIMD via memchr) for a candidate whose next packet - // also begins with 0x47, confirming the 188 stride before locking onto it. - // Striding past a false candidate would route one bogus packet; jumping a flat - // 188 instead would only re-align on exact multiples and could desync forever. - loop { - let Some(rel) = memchr::memchr(0x47, &self.scratch[off..]) else { - // No sync byte left: the buffer is junk, drop it. - off = self.scratch.len(); - break; - }; - off += rel; - match self.scratch.get(off + TsPacket::SIZE) { - // Next packet also starts with a sync byte: lock onto this candidate. - Some(&0x47) => { - self.synced = true; - break; - } - // The byte 188 ahead isn't a sync byte: this 0x47 was payload, keep scanning. - Some(_) => off += 1, - // Can't confirm yet (candidate is near the buffer tail). Stay unsynced so - // it's re-confirmed next call (with the trailing bytes) instead of trusted. - None => break, - } - } - // Unsynced means the buffer had no confirmable sync byte, or the candidate is - // pending confirmation; either way there's nothing to route until more arrives. - if !self.synced { - break; - } - continue; - } - let pkt: [u8; TsPacket::SIZE] = self.scratch[off..off + TsPacket::SIZE].try_into().unwrap(); - off += TsPacket::SIZE; + while let Some(at) = self.framer.next(&self.scratch, &mut off) { + let pkt: [u8; TsPacket::SIZE] = self.scratch[at..at + TsPacket::SIZE].try_into().unwrap(); let pid = (((pkt[1] & 0x1f) as u16) << 8) | pkt[2] as u16; // Every packet paces the multiplex, null stuffing and retransmissions included. if self.supports_mpegts { @@ -281,8 +233,8 @@ impl Import { continue; } // Read the clock's own flag before routing, so the break lands between the media - // either side of it. The PCR PID can be a dedicated one the routing gate below - // drops, and the flag rides an adaptation-only packet as readily as a payload one. + // either side of it. The PCR PID can be a dedicated one nothing below routes, and + // the flag rides an adaptation-only packet as readily as a payload one. // A packet the demodulator flagged corrupt (`transport_error_indicator`) is not // read: its adaptation field is as untrustworthy as its payload, and taking a bit // out of it would break every track in the program on line noise. @@ -302,9 +254,8 @@ impl Import { self.liveness.delivered(pid, units); continue; } - // Intercept the standalone SI PIDs before the routing gate below drops them: - // they carry the service layer, not media, and are not fed to the reader - // (which only routes the PAT/PMT/ES PIDs it learns). + // The standalone SI PIDs carry the service layer, not media, and no PAT or PMT + // names them. if self.supports_mpegts && catalog::SI_PIDS.contains(&pid) { self.si_section(pid, &pkt)?; continue; @@ -336,33 +287,22 @@ impl Import { Continuation::Contiguous => {} } } - // PIDs we don't decode and don't carry (`Stream::Ignored`: a base catalog's - // undecoded streams, or an ambiguous 0x86 PID without CUEI) are dropped here, - // not fed to the PES reader, which aborts on private sections (spec section 7: - // never fatal). - if let Ok(p) = Pid::new(pid) - && matches!(self.streams.get(&p), Some(Stream::Ignored)) - { - continue; - } - // Feed the reader only PIDs it can route: the PAT, the PMT PIDs it - // announces, and the ES PIDs a PMT registers. A live capture joins - // mid-stream, so PES arrive before their PSI; feeding those would make - // the reader abort on an unknown PID. Drop them until the layout is - // learned (PSI repeats), then normal demux resumes. - if pid != Pid::PAT - && !Pid::new(pid).is_ok_and(|p| self.pmt_pids.contains(&p) || self.streams.contains_key(&p)) - { + let Ok(pid) = Pid::new(pid) else { continue; - } - { - let mut state = self.feed.lock(); - state.data.clear(); - state.data.extend_from_slice(&pkt); - state.pos = 0; - } - while let Some(packet) = self.reader.read_ts_packet()? { - self.handle_packet(packet)?; + }; + if pid.as_u16() == Pid::PAT { + self.pat_packet(&pkt)?; + } else if self.pmt_sections.contains_key(&pid) { + self.pmt_packet(pid, &pkt)?; + } else { + // PIDs we don't decode and don't carry (`Stream::Ignored`: a base catalog's + // undecoded streams, or an ambiguous 0x86 PID without CUEI) are dropped, as is + // every PID before the PSI that declares it: a live capture joins mid-stream, + // so PES arrive before their PMT. + match self.streams.get(&pid) { + None | Some(Stream::Ignored) => {} + Some(_) => self.pes_packet(pid, &pkt)?, + } } } @@ -378,81 +318,121 @@ impl Import { Ok(()) } - fn handle_packet(&mut self, packet: TsPacket) -> anyhow::Result<()> { - let pid = packet.header.pid; - match packet.payload { - Some(TsPayload::Pmt(pmt)) => { - // PMTs of several programs may share one PID; only the chosen one maps streams. - if self.program.is_some_and(|program| program != pmt.program_num) { - return Ok(()); - } - // Which PID speaks for the program clock, so a `discontinuity_indicator` there - // can be read as a timebase reset rather than a counter jump. - if self.pcr_pid != pmt.pcr_pid { - self.pcr_pid = pmt.pcr_pid; - // A new clock: the intervals straddling the switch measure nothing. - self.liveness.discontinuity(); - if self.mux_rate.discontinuity() { - self.record_mux_rate()?; - } - } + /// Feed one packet on PID 0 to the PAT's reassembler, applying every whole table. + fn pat_packet(&mut self, pkt: &[u8; TsPacket::SIZE]) -> anyhow::Result<()> { + let crc_error = self.crc_errors.entry(Pid::PAT).or_default(); + if let Some(pat) = self.pat.push(pkt, crc_error) { + self.handle_pat(&pat)?; + } + Ok(()) + } - // SCTE-35 is announced by a program-level registration descriptor with - // format_identifier 'CUEI' (ITU-T J.181). The stream itself uses - // stream_type 0x86, which mpeg2ts maps to a DTS audio variant, so - // detection keys off the CUEI descriptor, not the stream type alone. - let cuei = pmt - .program_info - .iter() - .any(|d| d.tag == 0x05 && d.data.len() >= 4 && &d.data[0..4] == b"CUEI"); - - // Record the program-level descriptors once (PMT program_info is stable); - // export re-emits them verbatim, including the original CUEI. - if self.supports_mpegts && !self.program_recorded && !pmt.program_info.is_empty() { - let program = to_descriptors(&pmt.program_info); - if let Some(mpegts) = self.catalog.modify()?.ext.mpegts_mut() { - mpegts.program_descriptors = program; - } - self.program_recorded = true; - } + /// Feed one packet on a PMT PID to its reassembler, applying every good PMT section. + fn pmt_packet(&mut self, pid: Pid, pkt: &[u8; TsPacket::SIZE]) -> anyhow::Result<()> { + let mut sections = Vec::new(); + self.pmt_sections.entry(pid).or_default().push(pkt, &mut sections); + for section in sections { + // Other tables may share the PID; only a PMT is this path's to check. + if section.first() != Some(&psi::PMT_TABLE_ID) { + continue; + } + if !psi::crc_ok(§ion) { + self.crc_error(pid); + continue; + } + if let Some(pmt) = Pmt::parse(§ion) { + self.handle_pmt(pmt)?; + } + } + Ok(()) + } - for es in &pmt.es_info { - let stream_type = es.stream_type as u8; - // Stash ES descriptors so a decoded media track can record them once its - // (lazily created) track exists; verbatim streams record their own. - if self.supports_mpegts { - self.es_descriptors - .insert(es.elementary_pid.as_u16(), to_descriptors(&es.descriptors)); - } - // Section-framed private data is intercepted before the reader (which - // aborts on private sections): private sections (0x05) and CUEI-marked - // SCTE-35 (0x86). Everything else routes through ensure_stream (a decoded - // codec, PES-framed verbatim, or dropped). - if stream_type == 0x05 || (cuei && matches!(es.stream_type, StreamType::Dts8ChannelLosslessAudio)) { - self.ensure_section(es.elementary_pid, stream_type, &es.descriptors)?; - } else { - self.ensure_stream(es.elementary_pid, es.stream_type, &es.descriptors)?; - } - } + /// A PAT or PMT section failed its CRC: it is dropped whole, and the table already in + /// force stays. + fn crc_error(&mut self, pid: Pid) { + *self.crc_errors.entry(pid.as_u16()).or_default() += 1; + tracing::warn!(pid = pid.as_u16(), "dropped a PSI section with a bad CRC"); + } - // Every stream in the initial program is registered now; release the reservation - // so the catalog publishes once each rendition's config resolves, not before. - self.initial_reservation = None; + fn handle_pmt(&mut self, pmt: Pmt) -> anyhow::Result<()> { + // PMTs of several programs may share one PID; only the chosen one maps streams. + if self.program.is_some_and(|program| program != pmt.program_number) { + return Ok(()); + } + // Which PID speaks for the program clock, so a `discontinuity_indicator` there + // can be read as a timebase reset rather than a counter jump. + if self.pcr_pid != pmt.pcr_pid { + self.pcr_pid = pmt.pcr_pid; + // A new clock: the intervals straddling the switch measure nothing. + self.liveness.discontinuity(); + if self.mux_rate.discontinuity() { + self.record_mux_rate()?; } - Some(TsPayload::PesStart(pes)) => self.handle_pes_start(pid, pes)?, - Some(TsPayload::PesContinuation(bytes)) => self.handle_pes_continuation(pid, &bytes)?, - Some(TsPayload::Pat(pat)) => self.handle_pat(&pat)?, - _ => {} } + + // SCTE-35 is announced by a program-level registration descriptor with + // format_identifier 'CUEI' (ITU-T J.181). The stream itself uses + // stream_type 0x86, which is also a DTS audio variant, so detection keys + // off the CUEI descriptor, not the stream type alone. + let cuei = pmt + .program_info + .iter() + .any(|d| d.tag == 0x05 && d.data.len() >= 4 && &d.data[0..4] == b"CUEI"); + + // Record the program-level descriptors once (PMT program_info is stable); + // export re-emits them verbatim, including the original CUEI. + if self.supports_mpegts && !self.program_recorded && !pmt.program_info.is_empty() { + if let Some(mpegts) = self.catalog.modify()?.ext.mpegts_mut() { + mpegts.program_descriptors = pmt.program_info.clone(); + } + self.program_recorded = true; + } + + for es in &pmt.streams { + // Stash ES descriptors so a decoded media track can record them once its + // (lazily created) track exists; verbatim streams record their own. + if self.supports_mpegts { + self.es_descriptors.insert(es.pid.as_u16(), es.descriptors.clone()); + } + // Section-framed private data is reassembled as sections, never PES-parsed: + // private sections (0x05) and CUEI-marked SCTE-35 (0x86). Everything else + // routes through ensure_stream (a decoded codec, PES-framed verbatim, or + // dropped). + if es.stream_type == 0x05 || (cuei && es.stream_type == StreamType::Dts8ChannelLosslessAudio as u8) { + self.ensure_section(es.pid, es.stream_type, &es.descriptors)?; + } else { + self.ensure_stream(es.pid, es.stream_type, &es.descriptors)?; + } + } + + // Every stream in the initial program is registered now; release the reservation + // so the catalog publishes once each rendition's config resolves, not before. + self.initial_reservation = None; Ok(()) } - fn ensure_stream( - &mut self, - pid: Pid, - stream_type: StreamType, - descriptors: &[mpeg2ts::ts::Descriptor], - ) -> anyhow::Result<()> { + /// Route one packet on an elementary stream PID into its PES reassembly. + fn pes_packet(&mut self, pid: Pid, pkt: &[u8; TsPacket::SIZE]) -> anyhow::Result<()> { + let payload = match payload(pkt) { + Payload::None => return Ok(()), + Payload::Bytes(payload) => payload, + // No payload can be found in it, so like a corrupt packet it breaks the PES. + Payload::Malformed => { + self.pending.remove(&pid); + if let Some(stream) = self.streams.get_mut(&pid) { + stream.desync(); + } + return Ok(()); + } + }; + if pkt[1] & 0x40 != 0 { + self.handle_pes_start(pid, PesStart::parse(payload)?) + } else { + self.handle_pes_continuation(pid, payload) + } + } + + fn ensure_stream(&mut self, pid: Pid, stream_type: u8, descriptors: &[catalog::Descriptor]) -> anyhow::Result<()> { // A later PMT can remap a PID that was section-framed (intercepted in // `decode`) to a PES codec/verbatim stream. Drop the stale section route first, // or it would keep intercepting the PID and the new stream would never get data. @@ -466,8 +446,8 @@ impl Import { return Ok(()); } - let stream = match stream_type { - StreamType::H264 => { + let stream = match StreamType::from_u8(stream_type).ok() { + Some(StreamType::H264) => { let track = self .broadcast .unique_track(".avc3", self.catalog.track_info(hang::catalog::PRIORITY.video))?; @@ -477,7 +457,7 @@ impl Import { unwrap: PtsUnwrap::default(), } } - StreamType::H265 => { + Some(StreamType::H265) => { let track = self .broadcast .unique_track(".hev1", self.catalog.track_info(hang::catalog::PRIORITY.video))?; @@ -489,7 +469,7 @@ impl Import { } // Only ADTS-framed AAC (0x0F). 0x11 is LATM/LOAS, which uses a different // framing and syncword, so it falls through to the ignored arm below. - StreamType::AdtsAac => Stream::Aac(Box::new(AacStream { + Some(StreamType::AdtsAac) => Stream::Aac(Box::new(AacStream { import: None, broadcast: self.broadcast.clone(), reserved: Some(self.reserve()), @@ -503,15 +483,15 @@ impl Import { // Legacy broadcast audio, carried verbatim. Both MP2 stream types // (0x03 MPEG-1, 0x04 MPEG-2 half rate) share one parser; sample rate and // channels always come from the frame header, not the PMT. - StreamType::Mpeg1Audio | StreamType::Mpeg2HalvedSampleRateAudio => { + Some(StreamType::Mpeg1Audio | StreamType::Mpeg2HalvedSampleRateAudio) => { self.legacy_stream(pid, &mp2::DESCRIPTOR) } - StreamType::DolbyDigitalUpToSixChannelAudio => self.legacy_stream(pid, &ac3::DESCRIPTOR), - StreamType::DolbyDigitalPlusUpTo16ChannelAudioForAtsc => self.legacy_stream(pid, &eac3::DESCRIPTOR), + Some(StreamType::DolbyDigitalUpToSixChannelAudio) => self.legacy_stream(pid, &ac3::DESCRIPTOR), + Some(StreamType::DolbyDigitalPlusUpTo16ChannelAudioForAtsc) => self.legacy_stream(pid, &eac3::DESCRIPTOR), // Opus rides private-data PES (0x06), distinguished from other private streams // by an 'Opus' registration descriptor. Channels and the (always 48 kHz) rate // come from the descriptors, so the importer is built up front. - StreamType::Mpeg2PacketizedData if registration_format(descriptors) == Some(*b"Opus") => { + Some(StreamType::Mpeg2PacketizedData) if registration_format(descriptors) == Some(*b"Opus") => { let config = opus_config(descriptors)?; let track = self .broadcast @@ -523,19 +503,19 @@ impl Import { unwrap: PtsUnwrap::default(), })) } - StreamType::Mpeg1Video | StreamType::Mpeg2Video => Stream::Clock, - // A codec we don't decode. Carry it verbatim as PES when the catalog supports - // the `mpegts` section. 0x86 is excluded: it's ambiguous (DTS audio, or a - // non-conformant SCTE-35 mux without CUEI, which is sections the PES reader - // would abort on), so drop it rather than risk feeding sections to the reader. - other => { - if self.supports_mpegts && !matches!(other, StreamType::Dts8ChannelLosslessAudio) { - let descriptors = to_descriptors(descriptors); + Some(StreamType::Mpeg1Video | StreamType::Mpeg2Video) => Stream::Clock, + // A codec we don't decode, named or not. Carry it verbatim as PES when the + // catalog supports the `mpegts` section. 0x86 is excluded: it's ambiguous (DTS + // audio, or a non-conformant SCTE-35 mux without CUEI, which is sections a PES + // parse would abort on), so drop it rather than risk PES-parsing sections. + _ => { + if self.supports_mpegts && stream_type != StreamType::Dts8ChannelLosslessAudio as u8 { + let descriptors = descriptors.to_vec(); match VerbatimStream::new( self.broadcast.clone(), self.catalog.clone(), pid.as_u16(), - stream_type as u8, + stream_type, descriptors, ) { Ok(stream) => Stream::Verbatim(Box::new(stream)), @@ -545,7 +525,7 @@ impl Import { } } } else { - tracing::warn!(?other, pid = pid.as_u16(), "unsupported TS stream type, dropping"); + tracing::warn!(stream_type, pid = pid.as_u16(), "unsupported TS stream type, dropping"); Stream::Ignored } } @@ -580,12 +560,7 @@ impl Import { /// Register a section-framed verbatim PID (SCTE-35 or other private sections): /// intercepted (see [`Self::decode`]) with a verbatim track when the catalog /// carries the `mpegts` section, dropped as `Ignored` when it can't. - fn ensure_section( - &mut self, - pid: Pid, - stream_type: u8, - descriptors: &[mpeg2ts::ts::Descriptor], - ) -> anyhow::Result<()> { + fn ensure_section(&mut self, pid: Pid, stream_type: u8, descriptors: &[catalog::Descriptor]) -> anyhow::Result<()> { if self.sections.contains_key(&pid.as_u16()) { return Ok(()); } @@ -594,7 +569,7 @@ impl Import { self.continuity.remove(&pid); if !self.supports_mpegts { // Always route to Ignored, replacing any prior codec on this PID (a later PMT - // can reassign it), so a private section never reaches the PES reader. Warn once. + // can reassign it), so a private section is never PES-parsed. Warn once. let previous = self.retire_stream(pid); self.streams.insert(pid, Stream::Ignored); if !matches!(previous, Some(Stream::Ignored)) { @@ -607,7 +582,7 @@ impl Import { } // A prior PMT may have routed this PID to Ignored; drop it so the PID has one route. self.retire_stream(pid); - let descriptors = to_descriptors(descriptors); + let descriptors = descriptors.to_vec(); let stream = SectionStream::new( self.broadcast.clone(), self.catalog.clone(), @@ -621,7 +596,7 @@ impl Import { tracing::debug!( pid = pid.as_u16(), stream_type, - "private section stream detected; intercepting before the reader" + "private section stream detected; reassembling its sections" ); Ok(()) } @@ -638,7 +613,7 @@ impl Import { Some(stream) } - fn handle_pes_start(&mut self, pid: Pid, pes: Pes) -> anyhow::Result<()> { + fn handle_pes_start(&mut self, pid: Pid, pes: PesStart) -> anyhow::Result<()> { // A new PES start means the previous one for this PID is complete. if self.pending.contains_key(&pid) { self.flush(pid)?; @@ -661,32 +636,31 @@ impl Import { // flushes on the next PES, so a SCTE-35 section arriving during this // frame must be timestamped with this frame's PTS ("now"), not the // previous one's. - if pes.header.pts.is_some() { - self.last_pts = unwrap_pts(&mut self.media_unwrap, pes.header.pts.map(|t| t.as_u64()))?; + if pes.pts.is_some() { + self.last_pts = unwrap_pts(&mut self.media_unwrap, pes.pts)?; } } if is_clock { // A clock-only stream never flushes, but sections are stamped with its PTS, so it // anchors the catalog here like a flushed PES would. - if let Some(pts) = pes.header.pts { - self.catalog.anchor(Timestamp::from_scale(pts.as_u64(), 90_000)?)?; + if let Some(pts) = pes.pts { + self.catalog.anchor(Timestamp::from_scale(pts, 90_000)?)?; } // Nothing is published, but each PES is a picture the source delivered. self.liveness.delivered(pid.as_u16(), 1); return Ok(()); } - let data_len = pes_data_len(&pes.header, pes.pes_packet_len); let mut pending = Pending { - pts: pes.header.pts.map(|t| t.as_u64()), - dts: pes.header.dts.map(|t| t.as_u64()), - stream_id: pes.header.stream_id.as_u8(), + pts: pes.pts, + dts: pes.dts, + stream_id: pes.stream_id, data: Vec::with_capacity(pes.data.len()), - data_len, + data_len: pes.data_len, }; - pending.data.extend_from_slice(&pes.data); - let complete = matches!(data_len, Some(len) if pending.data.len() >= len); + pending.data.extend_from_slice(pes.data); + let complete = matches!(pes.data_len, Some(len) if pending.data.len() >= len); self.pending.insert(pid, pending); if complete { @@ -822,17 +796,17 @@ impl Import { Ok(()) } - /// Pick the imported program from a PAT and learn its PMT PID, so the routing gate in - /// `decode` lets that PMT through and no other. + /// Pick the imported program from a whole PAT and learn its PMT PID, so `decode` + /// reassembles that PMT and no other. /// /// Every PAT is checked, so a program added mid-stream ends an unselected import, and a /// selected program that disappears ends a selected one; media already published stays. - fn handle_pat(&mut self, pat: &mpeg2ts::ts::payload::Pat) -> anyhow::Result<()> { + pub(super) fn handle_pat(&mut self, pat: &psi::Pat) -> anyhow::Result<()> { // program_number 0 is the network PID association, not a program. - let programs: Vec<_> = pat.table.iter().filter(|entry| entry.program_num != 0).collect(); - let numbers = || programs.iter().map(|entry| entry.program_num).collect::>(); + let programs: Vec<_> = pat.programs.iter().filter(|entry| entry.program_number != 0).collect(); + let numbers = || programs.iter().map(|entry| entry.program_number).collect::>(); let entry = match self.program { - Some(program) => match programs.iter().find(|entry| entry.program_num == program) { + Some(program) => match programs.iter().find(|entry| entry.program_number == program) { Some(entry) => entry, None => anyhow::bail!( "transport stream has no program {program}; its PAT lists {}", @@ -845,25 +819,21 @@ impl Import { _ => return Err(MultipleProgramsError { programs: numbers() }.into()), }, }; - self.pmt_pids.insert(entry.program_map_pid); + self.pmt_sections.entry(entry.pmt_pid).or_default(); self.record_program_identity(pat.transport_stream_id, entry) } /// Capture the transport/service identity (TSID, service number, PMT PID) from the /// PAT into the catalog service record, once. No-op without `mpegts` support. - fn record_program_identity( - &mut self, - transport_stream_id: u16, - entry: &mpeg2ts::ts::ProgramAssociation, - ) -> anyhow::Result<()> { + fn record_program_identity(&mut self, transport_stream_id: u16, entry: &psi::Association) -> anyhow::Result<()> { if !self.supports_mpegts || self.identity_recorded { return Ok(()); } if let Some(mpegts) = self.catalog.modify()?.ext.mpegts_mut() { let program = mpegts.program.get_or_insert_with(Default::default); program.transport_stream_id = transport_stream_id; - program.program_number = entry.program_num; - program.pmt_pid = entry.program_map_pid.as_u16(); + program.program_number = entry.program_number; + program.pmt_pid = entry.pmt_pid.as_u16(); } self.identity_recorded = true; Ok(()) @@ -874,10 +844,11 @@ impl Import { /// /// Every section is captured, whatever its `table_id`: the SDT PID also carries /// the BAT, the EIT PID carries now/next and the schedule, and a table we don't - /// recognize is exactly as worth preserving as one we do. The store buffers each - /// sub-table to a complete generation and commits it atomically, so a plain - /// repetition (SI repeats every couple of seconds) publishes nothing and a torn - /// multi-section transition is never visible. + /// recognize is exactly as worth preserving as one we do. A selected program is the + /// exception: its SI describes that service alone ([`with_program`](Self::with_program)). + /// The store buffers each sub-table to a complete generation and commits it atomically, + /// so a plain repetition (SI repeats every couple of seconds) publishes nothing and a + /// torn multi-section transition is never visible. fn si_section(&mut self, pid: u16, pkt: &[u8]) -> anyhow::Result<()> { let mut sections = Vec::new(); self.si_sections.entry(pid).or_default().push(pkt, &mut sections); @@ -915,8 +886,8 @@ impl Import { Ok(()) } - /// Snapshot what every elementary stream has delivered, and the audio frame sync lost on - /// the way. + /// Snapshot what every elementary stream has delivered, the audio frame sync lost on the + /// way, and the PAT and PMT sections dropped for a bad CRC. /// /// Cheap: it reads counters the demuxer already keeps, so a caller can poll it per /// chunk and report the delta. @@ -936,7 +907,15 @@ impl Import { for (pid, stats) in &mut streams { (stats.units, stats.quiet) = self.liveness.stream(*pid); } - Stats { streams } + Stats { + streams, + crc_error: self.crc_errors.values().sum(), + } + } + + /// [`Stats::crc_error`] per PID, for merging importers that read the same PSI. + pub(super) fn crc_errors(&self) -> &BTreeMap { + &self.crc_errors } /// Abort every track with `err` instead of finishing, so subscribers see the @@ -974,24 +953,35 @@ fn list_programs(programs: &[u16]) -> String { } /// What each demuxed elementary stream delivered, and the audio frame sync it lost or could -/// not verify, keyed by elementary stream PID. +/// not verify, keyed by elementary stream PID; and the PSI sections the stream as a whole +/// lost to corruption. /// /// Snapshot it with [`Import::stats`]. Every count is cumulative for the life of the /// importer, so what an operator alarms on is the rate: a feed that resyncs once an hour is /// healthy, one that resyncs every second is losing audio, and one whose access units stop /// advancing while the mux keeps flowing has lost that stream. +/// +/// [`Export::stats`](super::Export::stats) returns the same rows for the streams it writes, +/// so one schema reads both edges. Only `units` and `quiet` move there: the exporter builds +/// every frame header itself, so it has no frame sync to lose. #[derive(Clone, Debug, Default, PartialEq, Eq)] #[non_exhaustive] pub struct Stats { - /// One row per elementary stream PID the importer carries, ordered by PID. A PID the - /// importer drops (an undecoded stream without the `mpegts` catalog section) has none. + /// One row per elementary stream PID the importer carries or the exporter writes, ordered + /// by PID. A PID the importer drops (an undecoded stream without the `mpegts` catalog + /// section) has none. pub streams: BTreeMap, + /// PAT and PMT sections dropped because their CRC-32 did not match. Each one left the + /// table already in force standing, or, before any, delayed the program's start to the + /// next good repetition. Counted for the stream rather than per elementary stream, since + /// the PSI PIDs have no row of their own. + pub crc_error: u64, } impl Stats { - /// Whether no elementary stream has been registered yet. + /// Whether no elementary stream has been registered yet and nothing has been lost. pub fn is_empty(&self) -> bool { - self.streams.is_empty() + self.streams.is_empty() && self.crc_error == 0 } } @@ -1000,13 +990,16 @@ impl Stats { #[non_exhaustive] pub struct StreamStats { /// The current or most recent MoQ track suffix for this PID (`.avc3`, `.mp2`, `.ts`, ...), - /// or empty for MPEG-1/2 video, which is read for its clock and not published. + /// or empty for MPEG-1/2 video, which is read for its clock and not published. At export, + /// the suffix an import of the output would give the PID. pub track: &'static str, /// Access units the stream delivered: frames published for decoded media, PES payloads or - /// sections carried verbatim, and PES read on MPEG-1/2 video. + /// sections carried verbatim, and PES read on MPEG-1/2 video. At export, the PES or + /// sections written for each frame. pub units: u64, /// Transport time since the stream last delivered an access unit, or since the PMT - /// declared it, measured on the program clock (PCR). `None` until the first PCR arrives. + /// declared it, measured on the program clock (PCR): the one read at import, the one + /// written at export. `None` until the first PCR. /// /// No threshold applies: a sparse stream such as SCTE-35 is legitimately quiet for /// seconds, so how long is too long is for whoever alarms to decide. @@ -1069,17 +1062,6 @@ impl Pending { } } -/// Convert mpeg2ts PMT descriptors to the catalog's verbatim form. -fn to_descriptors(descriptors: &[mpeg2ts::ts::Descriptor]) -> Vec { - descriptors - .iter() - .map(|d| catalog::Descriptor { - tag: d.tag, - data: bytes::Bytes::copy_from_slice(&d.data), - }) - .collect() -} - /// Create a verbatim track and record it in the `mpegts` catalog section as a /// [`Track`](catalog::Track) with a `verbatim` carriage record. Shared by the /// section- and PES-framed paths. @@ -1427,7 +1409,7 @@ impl Continuity { /// Classify one 188-byte packet, recording what the next call needs. fn observe(&mut self, pkt: &[u8; 188]) -> Continuation { // transport_error_indicator: the demodulator flagged this packet as corrupt, so its - // payload can't be trusted (and we don't validate CRC-32). Forgetting the counter + // payload can't be trusted (and only a PAT or PMT has its CRC-32 checked). Forgetting the counter // keeps the next clean packet from also looking like a gap. if pkt[1] & 0x80 != 0 { self.last_cc = None; @@ -1476,9 +1458,9 @@ impl Continuity { /// Measured on the program clock, the PCR summed interval by interval, so it runs straight /// through the 33-bit wrap, a signalled time-base reset and a corrupt PCR instead of jumping /// with them. Not the media clock: that follows the video PTS, and stops with the very stream -/// this catches. +/// this catches. [`Export`](super::Export) runs the same meter on the PCR it writes. #[derive(Default)] -struct Liveness { +pub(super) struct Liveness { /// The last PCR in 27 MHz ticks, forgotten at a reset so no interval spans one. pcr: Option, /// PCR ticks elapsed since the first PCR, if one has arrived. @@ -1490,32 +1472,44 @@ struct Liveness { impl Liveness { /// A PCR arrived on the program's clock PID. - fn pcr(&mut self, pcr: u64) { + /// + /// Stepped one interval at a time, so a corrupt PCR, or the clock stepping back without a + /// flag, costs one interval rather than inventing hours of silence on every PID. The bound + /// is the mux-rate meter's. + pub(super) fn pcr(&mut self, pcr: u64) { + self.step(pcr, super::mux_rate::MAX_INTERVAL); + } + + /// A PCR the caller wrote itself. Its clock breaks only where the caller flags a + /// discontinuity, so every forward step counts, including the jump across a media gap + /// longer than the backfill covers. + pub(super) fn written_pcr(&mut self, pcr: u64) { + self.step(pcr, super::mux_rate::PCR_WRAP); + } + + fn step(&mut self, pcr: u64, max: u64) { let elapsed = self.elapsed.get_or_insert(0); if let Some(last) = self.pcr.replace(pcr) { let step = (pcr + super::mux_rate::PCR_WRAP - last) % super::mux_rate::PCR_WRAP; - // Stepped one interval at a time, so a corrupt PCR, or the clock stepping back - // without a flag, costs one interval rather than inventing hours of silence on - // every PID. The bound is the mux-rate meter's. - if step <= super::mux_rate::MAX_INTERVAL { + if step <= max { *elapsed += step; } } } /// The clock restarted or changed PID: the next interval measures nothing. - fn discontinuity(&mut self) { + pub(super) fn discontinuity(&mut self) { self.pcr = None; } /// The PMT declared `pid`. Its silence counts from here until it delivers. - fn register(&mut self, pid: u16) { + pub(super) fn register(&mut self, pid: u16) { let now = self.elapsed.unwrap_or(0); self.streams.entry(pid).or_insert((0, now)); } /// `pid` delivered `units` access units just now. - fn delivered(&mut self, pid: u16, units: u64) { + pub(super) fn delivered(&mut self, pid: u16, units: u64) { if units == 0 { return; } @@ -1527,7 +1521,7 @@ impl Liveness { } /// `pid`'s access units and how long it has been quiet. See [`StreamStats`]. - fn stream(&self, pid: u16) -> (u64, Option) { + pub(super) fn stream(&self, pid: u16) -> (u64, Option) { let Some(&(units, last)) = self.streams.get(&pid) else { return (0, None); }; @@ -1538,6 +1532,116 @@ impl Liveness { } } +/// Locks onto the 188-byte packet grid of a TS byte stream. +/// +/// 0x47 is the packet sync byte but also occurs freely in payload (TS has no byte +/// stuffing), so a lone 0x47 isn't a boundary. +#[derive(Default)] +pub(super) struct Framer { + /// False until a candidate is confirmed by the next packet's sync byte; once true we + /// stride 188 at a time and trust the per-packet check. Persists across calls so a + /// candidate pending confirmation at a buffer tail is re-confirmed, not trusted blindly. + synced: bool, +} + +impl Framer { + /// Where the next whole packet in `buf` at or after `*off` starts, advancing `*off` past + /// it. `None` once no confirmed packet remains, with `*off` at the first byte the next + /// call still needs. + pub(super) fn next(&mut self, buf: &[u8], off: &mut usize) -> Option { + while *off + TsPacket::SIZE <= buf.len() { + if self.synced && buf[*off] == 0x47 { + let at = *off; + *off += TsPacket::SIZE; + return Some(at); + } + self.synced = false; + // Scan (SIMD via memchr) for a candidate whose next packet also begins with 0x47, + // confirming the 188 stride before locking onto it. Striding past a false + // candidate would route one bogus packet; jumping a flat 188 instead would only + // re-align on exact multiples and could desync forever. + loop { + let Some(rel) = memchr::memchr(0x47, &buf[*off..]) else { + // No sync byte left: the buffer is junk, drop it. + *off = buf.len(); + break; + }; + *off += rel; + match buf.get(*off + TsPacket::SIZE) { + // Next packet also starts with a sync byte: lock onto this candidate. + Some(&0x47) => { + self.synced = true; + break; + } + // The byte 188 ahead isn't a sync byte: this 0x47 was payload, keep scanning. + Some(_) => *off += 1, + // Can't confirm yet (candidate is near the buffer tail). Stay unsynced so + // it's re-confirmed next call (with the trailing bytes) instead of trusted. + None => break, + } + } + // Unsynced means the buffer had no confirmable sync byte, or the candidate is + // pending confirmation; either way there's nothing to route until more arrives. + if !self.synced { + return None; + } + } + None + } +} + +/// Reads whole PATs off PID 0: reassembles its sections, drops any whose CRC fails, and +/// collects the rest until every section of a version is in. +#[derive(Default)] +pub(super) struct PatReader { + sections: SectionReassembler, + table: psi::PatAssembler, +} + +impl PatReader { + /// Consume one packet on PID 0, counting each section dropped for a bad CRC into + /// `crc_error`. Returns the whole PAT if this packet completed one. + pub(super) fn push(&mut self, pkt: &[u8; TsPacket::SIZE], crc_error: &mut u64) -> Option { + let mut sections = Vec::new(); + self.sections.push(pkt, &mut sections); + let mut pat = None; + for section in sections { + if section.first() != Some(&psi::PAT_TABLE_ID) { + continue; + } + if !psi::crc_ok(§ion) { + *crc_error += 1; + tracing::warn!(pid = Pid::PAT, "dropped a PSI section with a bad CRC"); + continue; + } + pat = self.table.section(§ion).or(pat); + } + pat + } +} + +/// Where a packet's payload is. +enum Payload<'a> { + /// Adaptation field only, or reserved `adaptation_field_control`. + None, + Bytes(&'a [u8]), + /// The adaptation field claims to run past the packet. + Malformed, +} + +fn payload(pkt: &[u8; TsPacket::SIZE]) -> Payload<'_> { + let afc = (pkt[3] >> 4) & 0x3; + if afc & 0x1 == 0 { + return Payload::None; + } + let off = if afc & 0x2 != 0 { 5 + pkt[4] as usize } else { 4 }; + match pkt.get(off..) { + Some([]) => Payload::None, + Some(payload) => Payload::Bytes(payload), + None => Payload::Malformed, + } +} + /// Byte-level reassembler for MPEG-TS private sections on one PID. /// /// Private sections (SCTE-35 table_id 0xFC and others) are not PES. This handles @@ -1572,23 +1676,15 @@ impl SectionReassembler { } let pusi = pkt[1] & 0x40 != 0; - let afc = (pkt[3] >> 4) & 0x3; - let has_payload = afc & 0x1 != 0; - - let mut off = 4; - if afc & 0x2 != 0 { - let af_len = pkt[4] as usize; - off = 5 + af_len; - } - - if !has_payload { - return; - } - - if off >= pkt.len() { - return; - } - let payload = &pkt[off..]; + let payload = match payload(pkt) { + Payload::None => return, + Payload::Bytes(payload) => payload, + // A section can't be found in it, so like a counter gap it costs the partial. + Payload::Malformed => { + self.acc.clear(); + return; + } + }; if pusi { // pointer_field: payload[1..1+ptr] is the tail of the section already in @@ -2434,7 +2530,7 @@ impl OpusStream { /// The 4-byte registration `format_identifier` from a PMT registration descriptor /// (tag 0x05), if present. Identifies the codec of a private-data (0x06) stream. -fn registration_format(descriptors: &[mpeg2ts::ts::Descriptor]) -> Option<[u8; 4]> { +fn registration_format(descriptors: &[catalog::Descriptor]) -> Option<[u8; 4]> { descriptors .iter() .find(|d| d.tag == 0x05) @@ -2449,7 +2545,7 @@ fn registration_format(descriptors: &[mpeg2ts::ts::Descriptor]) -> Option<[u8; 4 /// stereo and the Vorbis family 1 mapping above that. Higher codes (0x81 explicitly-coded /// layouts, reserved values) aren't supported, so they fall back to stereo rather than being /// read as a raw 129..=255 count. -fn opus_config(descriptors: &[mpeg2ts::ts::Descriptor]) -> crate::Result { +fn opus_config(descriptors: &[catalog::Descriptor]) -> crate::Result { let channels = descriptors .iter() .find(|d| d.tag == 0x7f && d.data.first() == Some(&0x80)) @@ -2787,17 +2883,6 @@ impl LegacyStream { } } -/// Replicates [`PesHeader::optional_header_len`] (which is crate-private) to -/// compute the declared PES payload length for bounded packets. -fn pes_data_len(header: &PesHeader, pes_packet_len: u16) -> Option { - if pes_packet_len == 0 { - // Unbounded (the usual case for video); flush on the next PES start. - return None; - } - let optional = 3 + header.pts.map_or(0, |_| 5) + header.dts.map_or(0, |_| 5) + header.escr.map_or(0, |_| 6); - pes_packet_len.checked_sub(optional).map(|n| n as usize) -} - /// Swallow a [`MissingKeyframe`](crate::container::MissingKeyframe) from a video /// decode: a TS capture can join mid-GOP, so the deltas before the first keyframe /// have no group to anchor and are simply dropped rather than aborting the demux. @@ -2880,38 +2965,6 @@ impl PtsUnwrap { } } -/// A cloneable, refillable [`Read`] source backed by a shared buffer. -/// -/// [`Import`] loads one whole TS packet at a time; the reader consumes it and -/// reaches end-of-stream before the next refill. -#[derive(Clone, Default)] -struct Feed(Arc>); - -#[derive(Default)] -struct FeedState { - data: BytesMut, - pos: usize, -} - -impl Feed { - fn lock(&self) -> std::sync::MutexGuard<'_, FeedState> { - self.0.lock().unwrap() - } -} - -impl Read for Feed { - fn read(&mut self, out: &mut [u8]) -> std::io::Result { - let mut state = self.lock(); - let n = out.len().min(state.data.len() - state.pos); - if n == 0 { - return Ok(0); - } - out[..n].copy_from_slice(&state.data[state.pos..state.pos + n]); - state.pos += n; - Ok(n) - } -} - #[cfg(test)] pub(super) mod test { use std::collections::BTreeMap; @@ -5295,6 +5348,232 @@ pub(super) mod test { assert!(err.contains("no program 3") && err.contains("1, 2"), "{err}"); } + /// `section` on `pid` as the packets a mux would send, counting from `*cc`: the first + /// opens it behind a pointer_field skipping `tail` (the end of an earlier section), and + /// the rest carry it on. + pub(in crate::container::ts) fn psi_packets(pid: u16, cc: &mut u8, tail: &[u8], section: &[u8]) -> Vec { + let mut payload = vec![tail.len() as u8]; + payload.extend_from_slice(tail); + payload.extend_from_slice(section); + let mut out = Vec::new(); + for (i, chunk) in payload.chunks(184).enumerate() { + let pusi = if i == 0 { 0x40 } else { 0 }; + out.extend_from_slice(&[0x47, pusi | (pid >> 8) as u8, pid as u8, 0x10 | *cc]); + out.extend_from_slice(chunk); + out.resize(out.len().next_multiple_of(188), 0xff); + *cc = (*cc + 1) & 0x0f; + } + out + } + + /// A PAT section listing `(program_number, pmt_pid)`. + pub(in crate::container::ts) fn pat_section(programs: &[(u16, u16)]) -> Vec { + let body: Vec = programs + .iter() + .flat_map(|&(number, pid)| [(number >> 8) as u8, number as u8, 0xe0 | (pid >> 8) as u8, pid as u8]) + .collect(); + super::psi::section(0x00, 1, 0, 0, 0, &body) + } + + /// A PMT section listing `(stream_type, pid, es_descriptors)`, the clock on the first. + fn pmt_section(program: u16, version: u8, streams: &[(u8, u16, &[u8])]) -> Vec { + let pcr = streams.first().map_or(0x1fff, |&(_, pid, _)| pid); + let mut body = vec![0xe0 | (pcr >> 8) as u8, pcr as u8, 0xf0, 0x00]; + for &(stream_type, pid, descriptors) in streams { + body.extend_from_slice(&[ + stream_type, + 0xe0 | (pid >> 8) as u8, + pid as u8, + 0xf0, + descriptors.len() as u8, + ]); + body.extend_from_slice(descriptors); + } + super::psi::section(0x02, program, version, 0, 0, &body) + } + + /// `section` with one bit of its CRC flipped. + pub(in crate::container::ts) fn corrupt(mut section: Vec) -> Vec { + *section.last_mut().unwrap() ^= 0x01; + section + } + + const MP2: u8 = StreamType::Mpeg1Audio as u8; + + /// A PAT listing fifty programs, which takes two packets, then program 50's PMT (one + /// MP2 stream on `0x0061`) and one PES on it. + pub(in crate::container::ts) fn fifty_programs() -> Vec { + let programs: Vec<_> = (1..=50).map(|n| (n, 0x0100 + n)).collect(); + let pat = pat_section(&programs); + let mut data = psi_packets(0, &mut 0, &[], &pat); + assert_eq!(data.len(), 2 * 188, "the PAT spans two packets"); + data.extend(psi_packets( + 0x0132, + &mut 0, + &[], + &pmt_section(50, 0, &[(MP2, 0x0061, &[])]), + )); + data.extend(mp2_pes(0x0061, 0, 90_000, [0xAA, 0xBB])); + data + } + + /// A PAT split into two sections, each in its own packet, listing program 1 then program + /// 2, then each program's PMT (one MP2 stream) and one PES on it. + pub(in crate::container::ts) fn two_section_pat() -> Vec { + let mut cc = 0; + let mut data = Vec::new(); + for (number, program) in [(0, 1u16), (1, 2)] { + let entry = [0x00, program as u8, 0xe0 | program as u8, 0x00]; + let section = super::psi::section(0x00, 1, 0, number, 1, &entry); + data.extend(psi_packets(0, &mut cc, &[], §ion)); + } + for (program, pid, fills) in [(1u16, 0x0061, [0xAA, 0xBB]), (2, 0x0071, [0xCC, 0xDD])] { + let pmt = pmt_section(program, 0, &[(MP2, pid, &[])]); + data.extend(psi_packets(program << 8, &mut 0, &[], &pmt)); + data.extend(mp2_pes(pid, 0, 90_000, fills)); + } + data + } + + fn psi_import() -> (crate::catalog::Producer, super::Import) { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let import = super::Import::new(broadcast, catalog.reserve()); + (catalog, import) + } + + #[test] + fn a_pmt_spanning_two_packets_is_read_whole() { + let (catalog, mut import) = psi_import(); + // A private descriptor long enough to push the second stream into the next packet. + let mut descriptor = vec![0x80, 200]; + descriptor.resize(202, 0x00); + let pmt = pmt_section(1, 0, &[(MP2, 0x0061, &descriptor), (MP2, 0x0062, &[])]); + let mut data = psi_packets(0, &mut 0, &[], &pat_section(&[(1, 0x0100)])); + data.extend(psi_packets(0x0100, &mut 0, &[], &pmt)); + assert_eq!(data.len(), 3 * 188, "the PMT spans two packets"); + data.extend(mp2_pes(0x0061, 0, 90_000, [0xAA, 0xBB])); + data.extend(mp2_pes(0x0062, 0, 90_000, [0xCC, 0xDD])); + + import.decode(&data).unwrap(); + import.finish().unwrap(); + assert_eq!(catalog.snapshot().audio.renditions.len(), 2, "both streams publish"); + assert_eq!(import.stats().crc_error, 0); + } + + #[test] + fn a_pat_behind_a_nonzero_pointer_field_is_read() { + let (catalog, mut import) = psi_import(); + let mut data = psi_packets(0, &mut 0, &[0xab; 7], &pat_section(&[(1, 0x0100)])); + data.extend(psi_packets( + 0x0100, + &mut 0, + &[], + &pmt_section(1, 0, &[(MP2, 0x0061, &[])]), + )); + data.extend(mp2_pes(0x0061, 0, 90_000, [0xAA, 0xBB])); + + import.decode(&data).unwrap(); + import.finish().unwrap(); + assert_eq!(catalog.snapshot().audio.renditions.len(), 1); + } + + #[test] + fn a_program_listed_in_a_pats_second_packet_is_selected() { + let (catalog, import) = psi_import(); + let mut import = import.with_program(50); + import.decode(&fifty_programs()).unwrap(); + import.finish().unwrap(); + assert_eq!(catalog.snapshot().audio.renditions.len(), 1, "program 50 publishes"); + + // Unselected, the whole PAT is what refuses the input. + let (_, mut import) = psi_import(); + let err = import.decode(&fifty_programs()).unwrap_err(); + let err = err.downcast_ref::().unwrap(); + assert_eq!(err.programs, (1..=50).collect::>()); + } + + /// A corrupt PAT and a corrupt PMT between good repetitions are each dropped and counted + /// once, while the layout they would have changed holds; a later good PMT revision still + /// applies. + #[test] + fn corrupt_psi_between_good_repetitions_is_dropped_and_counted() { + let (catalog, mut import) = psi_import(); + let (mut pat_cc, mut pmt_cc) = (0, 0); + let pat = pat_section(&[(1, 0x0100)]); + let pmt = pmt_section(1, 0, &[(MP2, 0x0061, &[])]); + let two = [(MP2, 0x0061, &[][..]), (MP2, 0x0062, &[][..])]; + + let mut data = psi_packets(0, &mut pat_cc, &[], &pat); + data.extend(psi_packets(0x0100, &mut pmt_cc, &[], &pmt)); + data.extend(mp2_pes(0x0061, 0, 90_000, [0xAA, 0xBB])); + import.decode(&data).unwrap(); + let units = import.stats().streams[&0x0061].units; + + // Read, this PAT would end the unselected import with a second program, and this PMT + // would add a stream. + let mut data = psi_packets(0, &mut pat_cc, &[], &corrupt(pat_section(&[(1, 0x0100), (2, 0x0200)]))); + data.extend(psi_packets(0x0100, &mut pmt_cc, &[], &corrupt(pmt_section(1, 1, &two)))); + data.extend(psi_packets(0, &mut pat_cc, &[], &pat)); + data.extend(psi_packets(0x0100, &mut pmt_cc, &[], &pmt)); + data.extend(mp2_pes(0x0061, 1, 180_000, [0xAA, 0xBB])); + import.decode(&data).unwrap(); + + assert_eq!( + catalog.snapshot().audio.renditions.len(), + 1, + "the corrupt PMT added nothing" + ); + assert!( + import.stats().streams[&0x0061].units > units, + "the stream kept delivering" + ); + assert_eq!(import.crc_errors(), &BTreeMap::from([(0x0000, 1), (0x0100, 1)])); + assert_eq!(import.stats().crc_error, 2); + + let mut data = psi_packets(0x0100, &mut pmt_cc, &[], &pmt_section(1, 2, &two)); + data.extend(mp2_pes(0x0062, 0, 180_000, [0xCC, 0xDD])); + import.decode(&data).unwrap(); + import.finish().unwrap(); + assert_eq!( + catalog.snapshot().audio.renditions.len(), + 2, + "the good revision applied" + ); + assert_eq!(import.stats().crc_error, 2, "good sections count nothing"); + } + + /// The positive control: with no good PAT, nothing is learned, so nothing publishes. + #[test] + fn a_feed_whose_only_pat_is_corrupt_publishes_nothing() { + let (catalog, mut import) = psi_import(); + let mut data = psi_packets(0, &mut 0, &[], &corrupt(pat_section(&[(1, 0x0100)]))); + data.extend(psi_packets( + 0x0100, + &mut 0, + &[], + &pmt_section(1, 0, &[(MP2, 0x0061, &[])]), + )); + data.extend(mp2_pes(0x0061, 0, 90_000, [0xAA, 0xBB])); + + import.decode(&data).unwrap(); + import.finish().unwrap(); + assert!(catalog.snapshot().audio.renditions.is_empty()); + assert_eq!(import.stats().crc_error, 1); + assert!(!import.stats().is_empty(), "a dropped section is something to report"); + } + + #[test] + fn adaptation_field_past_the_packet_drops_partial() { + let section = fake_section(0xfc, 300); + let mut malformed = packet(false, 1, 0, §ion[183..]); + // afc 0b11 with an adaptation_field_length reaching past the packet. + malformed[3] = 0x31; + malformed[4] = 200; + let rest = packet(false, 2, 0, §ion[183..]); + assert!(run(&[packet(true, 0, 0, §ion[..183]), malformed, rest]).is_empty()); + } + /// Two MP2 renditions, the first of which the PMT designates as the PCR PID. const PCR_PID: u16 = 0x0061; const PEER_PID: u16 = 0x0062; diff --git a/rs/moq-mux/src/container/ts/mod.rs b/rs/moq-mux/src/container/ts/mod.rs index 05858df90f1..3d5fdb454dd 100644 --- a/rs/moq-mux/src/container/ts/mod.rs +++ b/rs/moq-mux/src/container/ts/mod.rs @@ -23,6 +23,7 @@ mod export; mod import; mod mux_rate; mod programs; +mod psi; mod si; // The `mpegts` catalog section (per-track PID + descriptors plus verbatim carriage diff --git a/rs/moq-mux/src/container/ts/programs.rs b/rs/moq-mux/src/container/ts/programs.rs index 5b2b21c9b82..335ae6a161b 100644 --- a/rs/moq-mux/src/container/ts/programs.rs +++ b/rs/moq-mux/src/container/ts/programs.rs @@ -1,9 +1,12 @@ //! Every program of a multiplex, each imported as its own broadcast. +use std::collections::BTreeMap; + use anyhow::Context; -use mpeg2ts::ts::{ReadTsPacket, TsPacket, TsPacketReader, TsPayload}; +use mpeg2ts::ts::{Pid, TsPacket}; -use super::{Ext, Import, Stats}; +use super::import::{Framer, PatReader}; +use super::{Ext, Import, Stats, psi}; use crate::catalog; /// Imports every program the first PAT lists, each as its own broadcast on `origin`, with its @@ -17,8 +20,10 @@ pub struct Programs { origin: moq_net::origin::Producer, name: moq_net::PathOwned, config: catalog::Config, - /// Input held until it contains a whole PAT. + /// Input not yet read for the PAT: a trailing partial packet, or what follows the PAT. pending: Vec, + /// Reads the PAT out of the input. + scan: PatScan, /// Empty until the PAT arrives. programs: Vec, } @@ -39,6 +44,7 @@ impl Programs { name: name.as_path().to_owned(), config, pending: Vec::new(), + scan: PatScan::default(), programs: Vec::new(), } } @@ -49,13 +55,10 @@ impl Programs { return self.feed(data); } self.pending.extend_from_slice(data); - let Some(programs) = pat_programs(&self.pending).filter(|programs| !programs.is_empty()) else { - // Only a packet's worth of tail can still hold the start of the PAT. - let keep = self.pending.len().saturating_sub(TsPacket::SIZE - 1); - self.pending.drain(..keep); + let Some(pat) = self.scan.scan(&mut self.pending) else { return Ok(()); }; - for program in programs { + for program in pat.program_numbers() { let name = program_broadcast(self.name.as_str(), program); let mut broadcast = self .origin @@ -68,7 +71,10 @@ impl Programs { .clone() .with_catalog(catalog::hang::Catalog::::default()); let catalog = catalog::Producer::new(&mut broadcast, config)?; - let import = Import::new(broadcast.clone(), catalog.reserve()).with_program(program); + let mut import = Import::new(broadcast.clone(), catalog.reserve()).with_program(program); + // Seeded with the scanner's PAT rather than replaying its packets, so every section + // reaches the importer and nothing the scanner counted is counted again. + import.handle_pat(&pat)?; broadcast .announce(Default::default()) .with_context(|| format!("failed to announce broadcast {name}"))?; @@ -90,11 +96,22 @@ impl Programs { } /// Every program's counters in one map: PIDs are unique across a multiplex. + /// + /// Every importer reads the same PAT, and programs sharing a PMT PID read the same PMT + /// sections, so a dropped section counts once however many read it. pub fn stats(&self) -> Stats { let mut stats = Stats::default(); + let mut crc_errors = BTreeMap::::new(); for program in &self.programs { stats.streams.extend(program.import.stats().streams); + for (&pid, &count) in program.import.crc_errors() { + let merged = crc_errors.entry(pid).or_default(); + *merged = (*merged).max(count); + } } + // The importers only see the PATs after the one that started them. + *crc_errors.entry(Pid::PAT).or_default() += self.scan.crc_error; + stats.crc_error = crc_errors.values().sum(); stats } @@ -126,32 +143,37 @@ fn program_broadcast(name: &str, program: u16) -> String { } } -/// The program numbers listed by the first whole PAT in `data`, in PAT order, or `None` if -/// there is none yet. -fn pat_programs(data: &[u8]) -> Option> { - let mut off = 0; - while let Some(rel) = memchr::memchr(0x47, &data[off..]) { - off += rel; - let packet = data.get(off..off + TsPacket::SIZE)?; - // PID 0 opening a section. The section CRC rejects a sync byte found in payload. - if packet[1] & 0x5f == 0x40 - && packet[2] == 0 - && let Ok(Some(TsPacket { - payload: Some(TsPayload::Pat(pat)), - .. - })) = TsPacketReader::new(packet).read_ts_packet() - { - return Some( - pat.table - .iter() - .map(|entry| entry.program_num) - .filter(|&program| program != 0) - .collect(), - ); +/// Finds the first whole PAT listing a program, before any importer exists to read one. +#[derive(Default)] +struct PatScan { + framer: Framer, + pat: PatReader, + /// PAT sections dropped for a bad CRC before the first whole PAT. + crc_error: u64, +} + +impl PatScan { + /// Read `pending` up to the first whole PAT listing a program, returning it with `pending` + /// trimmed to start just after the packet that completed it. Otherwise only a trailing + /// partial packet is kept: the reader holds a partial PAT itself. + fn scan(&mut self, pending: &mut Vec) -> Option { + let mut off = 0; + let mut found = None; + while let Some(at) = self.framer.next(pending, &mut off) { + let pkt: &[u8; TsPacket::SIZE] = pending[at..at + TsPacket::SIZE].try_into().unwrap(); + if (u16::from(pkt[1] & 0x1f) << 8 | u16::from(pkt[2])) != Pid::PAT { + continue; + } + if let Some(pat) = self.pat.push(pkt, &mut self.crc_error) + && !pat.program_numbers().is_empty() + { + found = Some(pat); + break; + } } - off += 1; + pending.drain(..off); + found } - None } #[cfg(test)] @@ -161,7 +183,9 @@ mod test { use super::*; use crate::catalog::hang::Container; use crate::container::Consumer; - use crate::container::ts::import::test::two_programs; + use crate::container::ts::import::test::{ + corrupt, fifty_programs, pat_section, psi_packets, two_programs, two_section_pat, + }; #[test] fn program_broadcasts_keep_the_catalog_suffix_last() { @@ -170,6 +194,11 @@ mod test { assert_eq!(program_broadcast("event", 1), "event/1"); } + /// The PAT in a whole buffer, as one [`PatScan`] reads it. + fn pat_programs(data: &[u8]) -> Option> { + Some(PatScan::default().scan(&mut data.to_vec())?.program_numbers()) + } + #[test] fn pat_programs_reads_the_first_whole_pat() { let data = two_programs(); @@ -181,6 +210,118 @@ mod test { assert_eq!(pat_programs(&data[..100]), None); } + /// A PAT too big for one packet is read whole, and each importer is handed all of it, so + /// the program listed in its second packet publishes. + #[tokio::test] + async fn a_pat_spanning_two_packets_starts_every_program() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin.clone(), "event.hang", catalog::Config::default()); + let input = fifty_programs(); + for chunk in input.chunks(100) { + programs.decode(chunk).unwrap(); + } + programs.finish().unwrap(); + assert_eq!(programs.programs.len(), 50); + + let consumer = crate::Source::new(origin.consume(), "event/50.hang") + .broadcast() + .await + .unwrap(); + let catalog = hang::catalog::Catalog::<()>::subscribe(&consumer) + .await + .unwrap() + .next() + .await + .unwrap() + .expect("a catalog"); + assert_eq!(catalog.audio.renditions.len(), 1, "program 50 publishes its stream"); + } + + /// A corrupt PAT starts nothing and is counted; the good repetition after it starts the + /// program without counting again. + #[test] + fn a_corrupt_pat_starts_no_program() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", catalog::Config::default()); + // A null packet after each PAT confirms its sync byte. + let mut null = vec![0x47, 0x1f, 0xff, 0x10]; + null.resize(188, 0xff); + let mut cc = 0; + let mut data = psi_packets(0, &mut cc, &[], &corrupt(pat_section(&[(1, 0x0100)]))); + data.extend_from_slice(&null); + programs.decode(&data).unwrap(); + assert!(programs.programs.is_empty()); + assert_eq!(programs.stats().crc_error, 1); + + let mut data = psi_packets(0, &mut cc, &[], &pat_section(&[(1, 0x0100)])); + data.extend_from_slice(&null); + programs.decode(&data).unwrap(); + assert_eq!(programs.programs.len(), 1); + assert_eq!(programs.stats().crc_error, 1); + } + + /// How many audio renditions the catalog `path` publishes lists. + async fn renditions(origin: &moq_net::origin::Producer, path: &str) -> usize { + let consumer = crate::Source::new(origin.consume(), path).broadcast().await.unwrap(); + let catalog = hang::catalog::Catalog::<()>::subscribe(&consumer) + .await + .unwrap() + .next() + .await + .unwrap() + .expect("a catalog"); + catalog.audio.renditions.len() + } + + /// A PAT in two sections starts both programs, each importer seeded with the whole table + /// rather than only the section that completed it. + #[tokio::test] + async fn a_pat_in_two_sections_starts_every_program() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin.clone(), "event.hang", catalog::Config::default()); + programs.decode(&two_section_pat()).unwrap(); + programs.finish().unwrap(); + assert_eq!(programs.programs.len(), 2); + assert_eq!(renditions(&origin, "event/1.hang").await, 1); + assert_eq!(renditions(&origin, "event/2.hang").await, 1); + } + + /// A partial PAT holds only itself, not the multiplex that follows it. + #[test] + fn a_partial_pat_does_not_hold_the_multiplex() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", catalog::Config::default()); + programs.decode(&fifty_programs()[..188]).unwrap(); + let mut null = vec![0x47, 0x1f, 0xff, 0x10]; + null.resize(188, 0xff); + for _ in 0..1000 { + programs.decode(&null).unwrap(); + } + assert!(programs.programs.is_empty()); + assert!( + programs.pending.len() < 2 * 188, + "held {} bytes", + programs.pending.len() + ); + } + + /// A corrupt PAT section sharing a packet with the good one that starts the programs is + /// counted once. + #[test] + fn a_corrupt_pat_beside_a_good_one_counts_once() { + let origin = crate::source::produce_origin(); + let mut programs = Programs::new(origin, "event.hang", catalog::Config::default()); + let mut sections = corrupt(pat_section(&[(1, 0x0100)])); + sections.extend(pat_section(&[(1, 0x0100)])); + let mut data = psi_packets(0, &mut 0, &[], §ions); + let mut null = vec![0x47, 0x1f, 0xff, 0x10]; + null.resize(188, 0xff); + data.extend_from_slice(&null); + programs.decode(&data).unwrap(); + assert_eq!(programs.programs.len(), 1); + assert_eq!(programs.stats().crc_error, 1); + } + /// The input is held until the PAT is whole, then each program publishes as its own /// broadcast carrying only that program, its catalog clock anchored on its own first frame /// apart on one shared clock. diff --git a/rs/moq-mux/src/container/ts/psi.rs b/rs/moq-mux/src/container/ts/psi.rs new file mode 100644 index 00000000000..21883e7602a --- /dev/null +++ b/rs/moq-mux/src/container/ts/psi.rs @@ -0,0 +1,423 @@ +//! PAT, PMT, and PES header parsing for the importer's demux. +//! +//! Sections arrive whole from the importer's reassembler, so a table spanning packets or +//! sitting behind a nonzero pointer_field parses like any other. A PAT or PMT section is +//! used only when its CRC-32/MPEG-2 matches: a corrupt one is dropped whole, and the last +//! good table stays in force until a repetition replaces it. + +use std::collections::BTreeMap; + +use anyhow::Context; +use mpeg2ts::ts::Pid; + +use super::catalog::Descriptor; + +/// The checksum closing every long-form section. +pub(super) const CRC: crc::Crc = crc::Crc::::new(&crc::CRC_32_MPEG_2); + +/// Whether a section's trailing CRC-32/MPEG-2 matches the bytes before it. +pub(super) fn crc_ok(section: &[u8]) -> bool { + let Some(split) = section.len().checked_sub(4) else { + return false; + }; + let (body, crc) = section.split_at(split); + CRC.checksum(body) == u32::from_be_bytes(crc.try_into().expect("four bytes")) +} + +/// The table_id of a PAT section. +pub(super) const PAT_TABLE_ID: u8 = 0x00; +/// The table_id of a PMT section. +pub(super) const PMT_TABLE_ID: u8 = 0x02; + +/// The long-form section header a PAT or PMT carries (ISO/IEC 13818-1 2.4.4.4). +struct Header<'a> { + /// transport_stream_id on a PAT, program_number on a PMT. + extension: u16, + version: u8, + /// Clear on a table announced ahead of use, which is not in force yet. + current: bool, + number: u8, + last: u8, + /// Everything between the header and the CRC. + body: &'a [u8], +} + +impl<'a> Header<'a> { + /// A whole section as the reassembler delivers it: exactly `3 + section_length` bytes. + fn parse(section: &'a [u8]) -> Option { + // section_syntax_indicator: PAT and PMT always use the long form. + if section.len() < 12 || section[1] & 0x80 == 0 { + return None; + } + Some(Self { + extension: u16::from_be_bytes([section[3], section[4]]), + version: (section[5] >> 1) & 0x1f, + current: section[5] & 0x01 != 0, + number: section[6], + last: section[7], + body: §ion[8..section.len() - 4], + }) + } +} + +/// A 13-bit PID behind three reserved bits. +fn pid(bytes: [u8; 2]) -> Pid { + Pid::new(u16::from_be_bytes(bytes) & 0x1fff).expect("13-bit PID") +} + +/// A 12-bit length behind four reserved or unused bits. +fn length(bytes: [u8; 2]) -> usize { + usize::from(u16::from_be_bytes(bytes) & 0x0fff) +} + +/// One program the PAT lists. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Association { + /// 0 names the network PID rather than a program. + pub program_number: u16, + pub pmt_pid: Pid, +} + +/// A whole program association table, every section of its version. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Pat { + pub transport_stream_id: u16, + pub programs: Vec, +} + +impl Pat { + /// The program numbers listed, without the network PID's. + pub fn program_numbers(&self) -> Vec { + self.programs + .iter() + .map(|entry| entry.program_number) + .filter(|&number| number != 0) + .collect() + } +} + +/// Collects PAT sections until every section of one version has arrived. +/// +/// A PAT that needs more than one section is complete only once all of them are in, so a +/// program is never missed for having been listed in a section still on its way. +#[derive(Default)] +pub(super) struct PatAssembler { + /// The version, transport_stream_id, and last_section_number being collected. + table: Option<(u8, u16, u8)>, + sections: BTreeMap>, +} + +impl PatAssembler { + /// Fold in one CRC-checked PAT section, returning the whole table once it is complete. + /// + /// Every repetition of a complete table returns it again, as the PAT repeats on the wire. + pub fn section(&mut self, section: &[u8]) -> Option { + let header = Header::parse(section)?; + if section[0] != PAT_TABLE_ID || !header.current || header.number > header.last { + return None; + } + let (entries, []) = header.body.as_chunks::<4>() else { + return None; + }; + let programs = entries + .iter() + .map(|entry| Association { + program_number: u16::from_be_bytes([entry[0], entry[1]]), + pmt_pid: pid([entry[2], entry[3]]), + }) + .collect(); + + let table = (header.version, header.extension, header.last); + if self.table != Some(table) { + self.table = Some(table); + self.sections.clear(); + } + self.sections.insert(header.number, programs); + if self.sections.len() <= usize::from(header.last) { + return None; + } + Some(Pat { + transport_stream_id: header.extension, + programs: self.sections.values().flatten().copied().collect(), + }) + } +} + +/// One elementary stream a PMT lists. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Es { + /// The raw stream_type, kept whole so a type this crate has no name for still routes. + pub stream_type: u8, + pub pid: Pid, + pub descriptors: Vec, +} + +/// A program map table. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct Pmt { + pub program_number: u16, + /// `None` when the PMT names PID 0x1FFF: the program carries no clock of its own. + pub pcr_pid: Option, + pub program_info: Vec, + pub streams: Vec, +} + +impl Pmt { + /// Parse one CRC-checked PMT section. `None` for a table not yet in force, one that + /// claims more than one section (a PMT never does), or one whose lengths overrun it. + pub fn parse(section: &[u8]) -> Option { + let header = Header::parse(section)?; + if section[0] != PMT_TABLE_ID || !header.current || header.number != 0 || header.last != 0 { + return None; + } + let body = header.body; + if body.len() < 4 { + return None; + } + let pcr_pid = Some(pid([body[0], body[1]])).filter(|pid| pid.as_u16() != 0x1fff); + let info_len = length([body[2], body[3]]); + let program_info = descriptors(body.get(4..4 + info_len)?)?; + + let mut rest = &body[4 + info_len..]; + let mut streams = Vec::new(); + while !rest.is_empty() { + if rest.len() < 5 { + return None; + } + let len = length([rest[3], rest[4]]); + streams.push(Es { + stream_type: rest[0], + pid: pid([rest[1], rest[2]]), + descriptors: descriptors(rest.get(5..5 + len)?)?, + }); + rest = &rest[5 + len..]; + } + Some(Self { + program_number: header.extension, + pcr_pid, + program_info, + streams, + }) + } +} + +/// A descriptor loop, or `None` if a descriptor runs past it. +fn descriptors(mut data: &[u8]) -> Option> { + let mut out = Vec::new(); + while !data.is_empty() { + let [tag, len, ..] = *data else { + return None; + }; + let body = data.get(2..2 + usize::from(len))?; + out.push(Descriptor { + tag, + data: bytes::Bytes::copy_from_slice(body), + }); + data = &data[2 + usize::from(len)..]; + } + Some(out) +} + +/// The start of a PES packet: its header, and the payload bytes that follow it in this +/// TS packet. +pub(super) struct PesStart<'a> { + pub stream_id: u8, + /// Raw 90 kHz. + pub pts: Option, + /// Raw 90 kHz. + pub dts: Option, + /// The payload length a bounded PES declares, or `None` for an unbounded one (the + /// usual case for video). + pub data_len: Option, + pub data: &'a [u8], +} + +impl<'a> PesStart<'a> { + /// Parse the PES header at the start of a payload-unit-start packet's payload. + /// + /// Optional fields this demux has no use for (ESCR, ES rate, trick mode, copy info, the + /// PES CRC, extensions, stuffing) are skipped by `PES_header_data_length`. A payload that + /// is not a PES header at all is an error, as it was when an external reader parsed it. + pub fn parse(payload: &'a [u8]) -> anyhow::Result { + anyhow::ensure!(payload.len() >= 6, "PES header truncated"); + anyhow::ensure!(payload[..3] == [0, 0, 1], "missing PES start code prefix"); + let stream_id = payload[3]; + let packet_len = usize::from(u16::from_be_bytes([payload[4], payload[5]])); + + // Streams that carry no optional header (ISO/IEC 13818-1 table 2-21). + if matches!(stream_id, 0xbc | 0xbe | 0xbf | 0xf0 | 0xf1 | 0xf2 | 0xf8 | 0xff) { + return Ok(Self { + stream_id, + pts: None, + dts: None, + data_len: Some(packet_len).filter(|&len| len != 0), + data: &payload[6..], + }); + } + + anyhow::ensure!(payload.len() >= 9, "PES header truncated"); + anyhow::ensure!(payload[6] & 0xc0 == 0x80, "unexpected PES header marker bits"); + anyhow::ensure!(payload[6] & 0x30 == 0, "scrambled PES is not supported"); + let (pts_flag, dts_flag) = (payload[7] & 0x80 != 0, payload[7] & 0x40 != 0); + anyhow::ensure!(pts_flag || !dts_flag, "PES DTS without a PTS"); + let header_len = usize::from(payload[8]); + let optional = payload.get(9..9 + header_len).context("PES header truncated")?; + + let (pts, dts) = match (pts_flag, dts_flag) { + (false, _) => (None, None), + (true, false) => (Some(timestamp(optional.get(..5), 0b0010)?), None), + (true, true) => ( + Some(timestamp(optional.get(..5), 0b0011)?), + Some(timestamp(optional.get(5..10), 0b0001)?), + ), + }; + Ok(Self { + stream_id, + pts, + dts, + data_len: Some(packet_len) + .filter(|&len| len != 0) + .and_then(|len| len.checked_sub(3 + header_len)), + data: &payload[9 + header_len..], + }) + } +} + +/// A 33-bit PTS or DTS behind its 4-bit prefix and three marker bits. +fn timestamp(bytes: Option<&[u8]>, prefix: u8) -> anyhow::Result { + let bytes = bytes.context("PES header truncated")?; + anyhow::ensure!(bytes[0] >> 4 == prefix, "unexpected PES timestamp prefix"); + anyhow::ensure!( + bytes[0] & 1 == 1 && bytes[2] & 1 == 1 && bytes[4] & 1 == 1, + "unexpected PES timestamp marker bit" + ); + Ok((u64::from(bytes[0] >> 1 & 0x07) << 30) + | (u64::from(bytes[1]) << 22) + | (u64::from(bytes[2] >> 1) << 15) + | (u64::from(bytes[3]) << 7) + | u64::from(bytes[4] >> 1)) +} + +/// A long-form section with a valid CRC around `body`. +#[cfg(test)] +pub(super) fn section(table_id: u8, extension: u16, version: u8, number: u8, last: u8, body: &[u8]) -> Vec { + let len = 5 + body.len() + 4; + let mut out = vec![table_id, 0xb0 | (len >> 8) as u8, len as u8]; + out.extend_from_slice(&extension.to_be_bytes()); + out.extend_from_slice(&[0xc1 | (version << 1), number, last]); + out.extend_from_slice(body); + let crc = CRC.checksum(&out); + out.extend_from_slice(&crc.to_be_bytes()); + out +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn crc_matches_mpeg2() { + // The single-program PAT ffmpeg writes: TSID 1, program 1 on PID 0x1000. + let pat = [ + 0x00, 0xb0, 0x0d, 0x00, 0x01, 0xc1, 0x00, 0x00, 0x00, 0x01, 0xf0, 0x00, 0x2a, 0xb1, 0x04, 0xb2, + ]; + assert!(crc_ok(&pat)); + let mut flipped = pat; + flipped[9] ^= 0x01; + assert!(!crc_ok(&flipped)); + assert!(!crc_ok(&pat[..3])); + } + + #[test] + fn pat_waits_for_every_section() { + let mut pat = PatAssembler::default(); + let first = section(0, 7, 3, 0, 1, &[0x00, 0x01, 0xe1, 0x00]); + let second = section(0, 7, 3, 1, 1, &[0x00, 0x02, 0xe2, 0x00]); + assert_eq!(pat.section(&first), None); + let whole = pat.section(&second).expect("both sections are in"); + assert_eq!(whole.transport_stream_id, 7); + assert_eq!(whole.program_numbers(), [1, 2]); + // A repetition of either section repeats the whole table. + assert_eq!(pat.section(&first), Some(whole)); + + // A new version starts over rather than mixing with the old one's sections. + let next = section(0, 7, 4, 0, 1, &[0x00, 0x03, 0xe3, 0x00]); + assert_eq!(pat.section(&next), None); + } + + #[test] + fn pat_skips_a_table_not_yet_in_force() { + let mut next = section(0, 1, 0, 0, 0, &[0x00, 0x01, 0xe1, 0x00]); + next[5] &= !0x01; + assert_eq!(PatAssembler::default().section(&next), None); + } + + #[test] + fn pmt_keeps_unnamed_stream_types_and_descriptors() { + let body = [ + 0xe1, 0x00, // PCR PID 0x100 + 0xf0, 0x06, 0x05, 0x04, b'C', b'U', b'E', b'I', // program_info: CUEI registration + 0x2d, 0xe1, 0x01, 0xf0, 0x00, // MPEG-H 3D audio, no descriptors + 0x1b, 0xe1, 0x00, 0xf0, 0x03, 0x0a, 0x01, 0x00, // H.264 with a language descriptor + ]; + let pmt = Pmt::parse(§ion(0x02, 9, 0, 0, 0, &body)).unwrap(); + assert_eq!(pmt.program_number, 9); + assert_eq!(pmt.pcr_pid, Some(Pid::new(0x100).unwrap())); + assert_eq!(pmt.program_info[0].tag, 0x05); + assert_eq!(&pmt.program_info[0].data[..], b"CUEI"); + assert_eq!(pmt.streams.len(), 2); + assert_eq!(pmt.streams[0].stream_type, 0x2d); + assert_eq!(pmt.streams[1].pid, Pid::new(0x100).unwrap()); + assert_eq!(pmt.streams[1].descriptors[0].tag, 0x0a); + } + + #[test] + fn pmt_refuses_an_overrun() { + let body = [0xff, 0xff, 0xf0, 0x00, 0x1b, 0xe1, 0x00, 0xf0, 0x09, 0x0a]; + assert_eq!(Pmt::parse(§ion(0x02, 1, 0, 0, 0, &body)), None); + // PCR PID 0x1FFF: no clock. + let body = [0xff, 0xff, 0xf0, 0x00]; + assert_eq!(Pmt::parse(§ion(0x02, 1, 0, 0, 0, &body)).unwrap().pcr_pid, None); + } + + #[test] + fn pes_header_reads_timestamps_and_skips_optional_fields() { + // PTS 0x1_2345_6789 and DTS 90000, then an ES rate field and three stuffing bytes. + let mut payload = vec![0, 0, 1, 0xe0, 0x00, 0x00, 0x80, 0xd0, 16]; + payload.extend_from_slice(&[0x39, 0x8d, 0x15, 0xcf, 0x13]); + payload.extend_from_slice(&[0x11, 0x00, 0x05, 0xbf, 0x21]); + payload.extend_from_slice(&[0x80, 0x00, 0x01, 0xff, 0xff, 0xff]); + payload.extend_from_slice(b"data"); + let pes = PesStart::parse(&payload).unwrap(); + assert_eq!(pes.stream_id, 0xe0); + assert_eq!(pes.pts, Some(0x1_2345_6789)); + assert_eq!(pes.dts, Some(90_000)); + assert_eq!(pes.data_len, None); + assert_eq!(pes.data, b"data"); + } + + #[test] + fn bounded_pes_length_counts_header_stuffing() { + // PTS only, one stuffing byte, four payload bytes: 3 + 6 + 4 = 13. + let mut payload = vec![0, 0, 1, 0xc0, 0x00, 13, 0x80, 0x80, 6]; + payload.extend_from_slice(&[0x21, 0x00, 0x01, 0x00, 0x01, 0xff]); + payload.extend_from_slice(b"mp2!"); + let pes = PesStart::parse(&payload).unwrap(); + assert_eq!(pes.pts, Some(0)); + assert_eq!(pes.data_len, Some(4)); + assert_eq!(pes.data, b"mp2!"); + } + + #[test] + fn pes_without_optional_header() { + let payload = [0, 0, 1, 0xbf, 0x00, 0x02, 0xaa, 0xbb]; + let pes = PesStart::parse(&payload).unwrap(); + assert_eq!((pes.pts, pes.data_len, pes.data), (None, Some(2), &[0xaa, 0xbb][..])); + } + + #[test] + fn not_a_pes_header() { + assert!(PesStart::parse(&[0xfc, 0x30, 0x11, 0x00, 0x00, 0x00, 0x00]).is_err()); + assert!(PesStart::parse(&[0, 0, 1, 0xe0, 0, 0, 0x80, 0x40, 0]).is_err()); + } +} diff --git a/rs/moq-mux/src/container/ts/si.rs b/rs/moq-mux/src/container/ts/si.rs index eb934cc49be..b00f611ee4c 100644 --- a/rs/moq-mux/src/container/ts/si.rs +++ b/rs/moq-mux/src/container/ts/si.rs @@ -15,6 +15,16 @@ //! new snapshot group, so a torn multi-section transition (half old version, half //! new) is unrepresentable on the track. [`Snapshot`] is the export half: it //! reduces a track's frames back into the current section set. +//! +//! An import that selects a program ([`Capture::select`]) carries SI for that +//! service alone. Other services' EIT actual sub-tables are dropped on arrival, and +//! each SDT actual snapshot is rebuilt as one section holding only the selected +//! service's entry, with a fresh CRC. An SDT actual section whose own CRC fails is +//! dropped, and a revision commits only once every section has arrived, so the last +//! good snapshot stays in force until then. An SDT that does not list the +//! service carries no SDT actual, so a revision that drops it retires the earlier +//! one. Network-wide tables (NIT, BAT, SDT other, EIT other, TDT/TOT) pass through +//! verbatim, as does everything when no program is selected. use std::collections::{BTreeMap, HashMap}; use std::time::Duration; @@ -23,6 +33,11 @@ use bytes::Bytes; use moq_net::Timestamp; use super::catalog; +use super::psi::{self, CRC}; + +const SDT_PID: u16 = 0x0011; +const SDT_ACTUAL: u8 = 0x42; +const EIT_PID: u16 = 0x0012; /// Coalesce snapshot cuts: a junction revises many sub-tables inside a second /// (every service's now/next rolls on the hour), and one group carrying all of them @@ -106,6 +121,37 @@ pub(super) fn split_sections(payload: &Bytes) -> Vec { out } +/// Rebuild an SDT sub-table as one section listing only `service`, wherever its +/// entry sat among `sections`. The header (transport_stream_id, version_number, +/// original_network_id) is kept, the numbering reset to section 0 of 0, and the +/// length and CRC recomputed. `None` when no section lists the service. +fn sdt_service<'a>(sections: impl IntoIterator, service: u16) -> Option> { + for section in sections { + // The service loop runs from the 11-byte header to the CRC. + let end = section.len().saturating_sub(4); + let mut off = 11; + while off + 5 <= end { + let len = 5 + ((usize::from(section[off + 3] & 0x0f) << 8) | usize::from(section[off + 4])); + if off + len > end { + break; + } + if u16::from_be_bytes([section[off], section[off + 1]]) == service { + let mut out = section[..11].to_vec(); + out.extend_from_slice(§ion[off..off + len]); + let section_length = out.len() - 3 + 4; + out[1] = (out[1] & 0xf0) | (section_length >> 8) as u8; + out[2] = section_length as u8; + out[6] = 0; + out[7] = 0; + out.extend_from_slice(&CRC.checksum(&out).to_be_bytes()); + return Some(out); + } + off += len; + } + } + None +} + /// One committed generation of a sub-table: the sections currently in force. /// /// Keyed by `section_number`, sparse on purpose: DVB EIT schedule segments skip @@ -151,16 +197,21 @@ impl Pending { /// One `(PID, table_id)` entry: its snapshot track and the state of every /// sub-table riding it. struct Entry { - track: moq_net::track::Producer, - name: String, + /// Created with the first snapshot that carries anything, so a table filtered + /// to nothing never gets a track. + track: Option, interval: Option, + /// The selected service an SDT actual snapshot is reduced to. + service: Option, active: BTreeMap, /// In-flight generations, keyed by sub-table and version. pending: HashMap<(Identity, u8), Pending>, /// A commit changed `active` since the last cut. dirty: bool, - /// The entry appears in the catalog's `mpegts.si` map (deferred until the first - /// snapshot group exists, so a subscriber never finds an empty track). + /// The newest snapshot group carries at least one sub-table. + populated: bool, + /// The entry appears in the catalog's `mpegts.si` map: exactly while it is + /// `populated`, so a subscriber never finds an empty track. advertised: bool, /// Host-clock micros of the last cut ([`DEBOUNCE`]); `None` until the first. last_cut: Option, @@ -213,6 +264,12 @@ impl Entry { if let Some(prior) = pending.sections.get(&number) { if *prior == section { + // A reduced SDT reads a missing service as removal, so a dense SDT + // missing a section (dropped for a bad CRC, or lost) must not commit as + // complete-as-observed; it waits for the cycle that completes it. + if self.service.is_some() { + return; + } // The same section came round again before the set completed: the // transmission cycle wrapped, so what we hold is the whole sub-table as // this mux transmits it. For EIT schedule this is the *only* commit @@ -279,19 +336,44 @@ impl Entry { } } - /// Cut a snapshot group: the complete active set, one frame per sub-table. + /// Cut a snapshot group: the complete active set, one frame per sub-table, an + /// SDT reduced to the selected service. Nothing is cut until a snapshot carries + /// something, and that snapshot creates the entry's track on `broadcast`. /// `pts` stamps the frames; `now` is the host clock for the debounce. - fn cut(&mut self, pts: Timestamp, now: u128) -> anyhow::Result<()> { - let mut group = self.track.append_group()?; - for generation in self.active.values() { - let mut payload = Vec::new(); - for section in generation.sections.values() { - payload.extend_from_slice(section); - } + fn cut( + &mut self, + broadcast: &moq_net::broadcast::Producer, + (pid, table_id): (u16, u8), + pts: Timestamp, + now: u128, + ) -> anyhow::Result<()> { + let frames: Vec> = self + .active + .values() + .filter_map(|generation| match self.service { + Some(service) => sdt_service(generation.sections.values(), service), + None => Some(generation.sections.values().flatten().copied().collect()), + }) + .collect(); + self.dirty = false; + self.populated = !frames.is_empty(); + let track = match &mut self.track { + Some(track) => track, + None if frames.is_empty() => return Ok(()), + // Deterministic, greppable name; fall back to a unique suffix on the + // (pathological) collision with an existing track. + None => self.track.insert( + match broadcast.create_track(format!("{pid:#06x}-{table_id:#04x}.si"), None) { + Ok(track) => track, + Err(_) => broadcast.unique_track(".si", None)?, + }, + ), + }; + let mut group = track.append_group()?; + for payload in frames { group.write_frame(pts, payload)?; } group.finish()?; - self.dirty = false; self.last_cut = Some(now); Ok(()) } @@ -309,6 +391,9 @@ pub(super) struct Capture { entries: BTreeMap<(u16, u8), Entry>, /// Host clock driving the cut debounce (see [`DEBOUNCE`]). clock: crate::Clock, + /// The selected program_number, which is the service_id of the only service + /// carried; `None` captures every service. + program: Option, } impl Capture { @@ -318,11 +403,19 @@ impl Capture { catalog, entries: BTreeMap::new(), clock: crate::Clock::new(), + program: None, } } + /// Carry only the service numbered `program`: its EIT actual, and an SDT actual + /// listing it alone. SI keys services by `service_id`, assumed equal to the PAT + /// `program_number`. + pub fn select(&mut self, program: u16) { + self.program = Some(program); + } + /// Fold one completed section from `pid` into its `(pid, table_id)` entry, - /// creating the entry (and its track) on first sight of the pair. + /// creating the entry on first sight of the pair. pub fn section(&mut self, pid: u16, section: Vec) -> anyhow::Result<()> { let Some(&table_id) = section.first() else { return Ok(()); @@ -335,43 +428,47 @@ impl Capture { if long_form(§ion) && section[5] & 0x01 == 0 { return Ok(()); } + // EIT actual (now/next and schedule) is keyed by service_id: another + // service's sub-table describes a program this import does not carry. + if let Some(program) = self.program + && pid == EIT_PID + && matches!(table_id, 0x4E | 0x50..=0x5F) + && !matches!(identity(§ion), Identity::Sub { ext, .. } if ext == program) + { + return Ok(()); + } - if !self.entries.contains_key(&(pid, table_id)) { - // Deterministic, greppable name; fall back to a unique suffix on the - // (pathological) collision with an existing track. - let name = format!("{pid:#06x}-{table_id:#04x}.si"); - let track = match self.broadcast.create_track(name, None) { - Ok(track) => track, - Err(_) => self.broadcast.unique_track(".si", None)?, - }; - self.entries.insert( - (pid, table_id), - Entry { - name: track.name().to_string(), - track, - interval: catalog::si_interval(table_id), - active: BTreeMap::new(), - pending: HashMap::new(), - dirty: false, - advertised: false, - last_cut: None, - }, - ); + let service = self.program.filter(|_| (pid, table_id) == (SDT_PID, SDT_ACTUAL)); + // The rebuilt SDT gets a fresh CRC, which would certify corruption the + // source's CRC flags; dropping the section keeps the last good generation. + if service.is_some() && !psi::crc_ok(§ion) { + return Ok(()); } - let entry = self.entries.get_mut(&(pid, table_id)).unwrap(); + let entry = self.entries.entry((pid, table_id)).or_insert_with(|| Entry { + track: None, + interval: catalog::si_interval(table_id), + service, + active: BTreeMap::new(), + pending: HashMap::new(), + dirty: false, + populated: false, + advertised: false, + last_cut: None, + }); entry.section(section); Ok(()) } - /// Cut snapshot groups for the dirty entries and advertise any entry with its - /// first snapshot in the catalog (one lock for the whole batch). + /// Cut snapshot groups for the dirty entries, then advertise each entry whose + /// snapshot now carries something and retract each whose snapshot went empty + /// (one catalog lock for the whole batch). /// /// `pts` is the media clock, used to stamp the frames; the cut debounce runs on /// the host clock ([`DEBOUNCE`]), and a first snapshot is never delayed. `force` /// overrides the debounce, for end of stream. pub fn flush(&mut self, pts: Timestamp, force: bool) -> anyhow::Result<()> { let now = self.clock.now().as_micros(); - for entry in self.entries.values_mut() { + for (key, entry) in self.entries.iter_mut() { if !entry.dirty { continue; } @@ -381,27 +478,33 @@ impl Capture { { continue; } - entry.cut(pts, now)?; + entry.cut(&self.broadcast, *key, pts, now)?; } - if self.entries.values().any(|e| !e.advertised && e.last_cut.is_some()) { + if self.entries.values().any(|e| e.advertised != e.populated) { let mut guard = self.catalog.modify()?; let Some(mpegts) = guard.ext.mpegts_mut() else { anyhow::bail!("catalog extension no longer carries an mpegts section"); }; for ((pid, table_id), entry) in self.entries.iter_mut() { - if entry.advertised || entry.last_cut.is_none() { + // Either flag implies the track exists. + let Some(track) = &entry.track else { continue }; + if entry.advertised == entry.populated { continue; } - mpegts.si.entry(*pid).or_default().insert( - *table_id, - catalog::SiEntry { - track: entry.name.clone(), - interval: entry.interval, - ..Default::default() - }, - ); - entry.advertised = true; + if entry.populated { + mpegts.si.entry(*pid).or_default().insert( + *table_id, + catalog::SiEntry { + track: track.name().to_string(), + interval: entry.interval, + ..Default::default() + }, + ); + } else { + retract(&mut mpegts.si, *pid, *table_id, track.name()); + } + entry.advertised = entry.populated; } } Ok(()) @@ -410,8 +513,8 @@ impl Capture { /// Flush everything and finish every track. pub fn finish(&mut self, pts: Timestamp) -> anyhow::Result<()> { self.flush(pts, true)?; - for entry in self.entries.values_mut() { - entry.track.finish()?; + for track in self.entries.values_mut().filter_map(|entry| entry.track.as_mut()) { + track.finish()?; } Ok(()) } @@ -421,7 +524,9 @@ impl Capture { pub fn abort(mut self, err: moq_net::Error) { self.unadvertise(); for (_, entry) in std::mem::take(&mut self.entries) { - let _ = entry.track.abort(err.clone()); + if let Some(track) = entry.track { + let _ = track.abort(err.clone()); + } } } @@ -440,26 +545,34 @@ impl Capture { return; }; for ((pid, table_id), entry) in self.entries.iter_mut() { - if !entry.advertised { + let Some(track) = entry.track.as_ref().filter(|_| entry.advertised) else { continue; - } + }; entry.advertised = false; - if let Some(tables) = mpegts.si.get_mut(pid) { - // Remove only a mapping this capture still owns: a second capture on - // the same broadcast (same key, unique fallback track name) may have - // overwritten it, and it will not re-advertise, so blindly removing - // here would strip the live capture's table for good. - if tables.get(table_id).is_some_and(|e| e.track == entry.name) { - tables.remove(table_id); - } - if tables.is_empty() { - mpegts.si.remove(pid); - } - } + retract(&mut mpegts.si, *pid, *table_id, track.name()); } } } +/// Remove the `(pid, table_id)` mapping from the catalog's `si` map, and the PID +/// once nothing is left on it. +/// +/// Only a mapping still naming `track` goes: a second capture on the same broadcast +/// (same key, unique fallback track name) may have overwritten it, and it will not +/// re-advertise, so blindly removing here would strip the live capture's table for +/// good. +fn retract(si: &mut BTreeMap>, pid: u16, table_id: u8, track: &str) { + let Some(tables) = si.get_mut(&pid) else { + return; + }; + if tables.get(&table_id).is_some_and(|e| e.track == track) { + tables.remove(&table_id); + } + if tables.is_empty() { + si.remove(&pid); + } +} + impl Drop for Capture { fn drop(&mut self) { self.unadvertise(); diff --git a/rs/moq-mux/src/container/ts/stats.rs b/rs/moq-mux/src/container/ts/stats.rs index 9853721168e..519c62f180a 100644 --- a/rs/moq-mux/src/container/ts/stats.rs +++ b/rs/moq-mux/src/container/ts/stats.rs @@ -1,11 +1,12 @@ -//! Reporting on the per-stream counters [`Import::stats`](super::Import::stats) returns. +//! Reporting on the counters [`Import::stats`](super::Import::stats) and +//! [`Export::stats`](super::Export::stats) return. use std::collections::BTreeSet; use super::{Stats, StreamStats}; -/// Logs what moved in an importer's [`Stats`] from one sample to the next, so every front -/// door that runs a TS importer reports the same lines. +/// Logs what moved in an importer's or exporter's [`Stats`] from one sample to the next, so +/// every front door that runs one reports the same lines. /// /// Feed it a snapshot every [`INTERVAL`](Self::INTERVAL) of wall time. The caller owns the /// timer, since what counts as "now" differs between a CLI, a server, and a test. @@ -27,7 +28,7 @@ impl Log { /// Compare a snapshot taken [`INTERVAL`](Self::INTERVAL) after the last. /// /// Logs a stream whose access units did not advance since the last sample, once per - /// silence, and any stream whose frame-sync counters moved. + /// silence, any stream whose frame-sync counters moved, and PSI sections dropped since. pub fn sample(&mut self, latest: Stats) { self.sync(&latest); for (pid, stream) in &latest.streams { @@ -49,17 +50,24 @@ impl Log { /// Compare the snapshot taken after [`Import::finish`](super::Import::finish). /// - /// It covers less than an interval, so only the frame sync is compared: the drain at end - /// of input can publish a frame nothing vouched for. + /// It covers less than an interval, so only the losses are compared: the drain at end of + /// input can publish a frame nothing vouched for. pub fn finish(&self, latest: &Stats) { self.sync(latest); } - /// Report the streams whose frame-sync counters moved, one line each. + /// Report the streams whose frame-sync counters moved, one line each, and the PSI + /// sections dropped for a bad CRC if that count moved. /// - /// The importer already warns on each individual resync; this is the running total, which - /// is what an operator turns into a rate. + /// The importer already warns on each individual resync and dropped section; this is the + /// running total, which is what an operator turns into a rate. fn sync(&self, latest: &Stats) { + if self.previous.as_ref().map_or(0, |previous| previous.crc_error) != latest.crc_error { + tracing::info!( + crc_error = latest.crc_error, + "PAT or PMT sections dropped for a bad CRC" + ); + } let lost = |stream: &StreamStats| (stream.resyncs, stream.discarded, stream.unconfirmed); for (pid, stream) in &latest.streams { let previous = self.previous.as_ref().and_then(|previous| previous.streams.get(pid)); @@ -156,4 +164,29 @@ mod test { }); assert!(!logs_contain("stopped delivering"), "the stream kept counting"); } + + /// Dropped PSI sections are reported when the count moves, with the running total, and + /// not again while it holds. + #[test] + #[tracing_test::traced_test] + fn reports_crc_errors_when_they_move() { + let sample = |crc_error: u64| Stats { + crc_error, + ..Default::default() + }; + + let mut log = Log::default(); + log.sample(sample(0)); + log.sample(sample(2)); + log.sample(sample(2)); + log.finish(&sample(3)); + + logs_assert(|lines: &[&str]| { + let dropped: Vec<_> = lines.iter().filter(|line| line.contains("bad CRC")).collect(); + match dropped.as_slice() { + [first, second] if first.contains("crc_error=2") && second.contains("crc_error=3") => Ok(()), + _ => Err(format!("expected one line per move, got {dropped:?}")), + } + }); + } } diff --git a/rs/moq-net/Cargo.toml b/rs/moq-net/Cargo.toml index f5b82dcb394..7c09818a0d6 100644 --- a/rs/moq-net/Cargo.toml +++ b/rs/moq-net/Cargo.toml @@ -16,8 +16,8 @@ categories = ["multimedia", "network-programming", "web-programming"] ignored = ["getrandom"] [features] -# Exposes the wire codecs to the `fuzz/` harness and the announce and varint benches through the -# hidden `fuzz` module. Off by default and hidden from the docs. +# Exposes the wire codecs to the `fuzz/` harness and the announce, varint, and frame benches +# through the hidden `fuzz` module. Off by default and hidden from the docs. fuzz = [] [dependencies] @@ -81,6 +81,12 @@ name = "varint" harness = false required-features = ["fuzz"] +# Reaches the private frame ingest through the hidden `fuzz` module. +[[bench]] +name = "frame" +harness = false +required-features = ["fuzz"] + [[bench]] name = "group" harness = false diff --git a/rs/moq-net/benches/frame.rs b/rs/moq-net/benches/frame.rs new file mode 100644 index 00000000000..59a1a6ef919 --- /dev/null +++ b/rs/moq-net/benches/frame.rs @@ -0,0 +1,43 @@ +//! Frame receive throughput: what allocating a frame's declared size up front saves +//! over growing its buffer with the bytes received, swept over frame size and the +//! number of streams receiving at once. +//! +//! `upfront` allocates every declared size, `grow` only what has arrived, and +//! `budget` is the session default, which allocates up front until the frames in +//! flight would pass it. Every stream reads a packet-sized chunk per turn. +//! +//! Run with `cargo bench -p moq-net --features fuzz --bench frame`. + +use criterion::{BatchSize, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main}; +use moq_net::fuzz::FrameRecv; + +/// Roughly one QUIC packet's payload. +const CHUNK: usize = 1200; + +const SIZES: [usize; 3] = [16 * 1024, 256 * 1024, 2 * 1024 * 1024]; +const STREAMS: [usize; 3] = [1, 16, 64]; + +const BUDGETS: [(&str, Option); 3] = [("upfront", Some(usize::MAX)), ("grow", Some(0)), ("budget", None)]; + +fn bench(c: &mut Criterion) { + let mut group = c.benchmark_group("frame_recv"); + for size in SIZES { + for streams in STREAMS { + group.throughput(Throughput::Bytes((size * streams) as u64)); + for (name, budget) in BUDGETS { + let id = format!("size={size}/streams={streams}"); + group.bench_function(BenchmarkId::new(name, id), |b| { + b.iter_batched( + || FrameRecv::new(streams, size, CHUNK, budget), + FrameRecv::run, + BatchSize::PerIteration, + ) + }); + } + } + } + group.finish(); +} + +criterion_group!(benches, bench); +criterion_main!(benches); diff --git a/rs/moq-net/src/client.rs b/rs/moq-net/src/client.rs index e3baa329d87..237b0bd6203 100644 --- a/rs/moq-net/src/client.rs +++ b/rs/moq-net/src/client.rs @@ -276,6 +276,7 @@ impl Client { authority: self.setup_authority.clone(), peer_setup_stream: None, peer_declared: None, + early_unis: Vec::new(), })?; tracing::debug!(version = ?v, "connected"); @@ -426,6 +427,7 @@ impl Client { authority: None, peer_setup_stream: None, peer_declared: Some(peer_declared), + early_unis: Vec::new(), })?; (None, crate::driver::Protocol::Ietf(protocol), goaway) } diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 4d08357d6f6..8b0605a9edc 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -367,10 +367,13 @@ mod tests { let mut group = crate::group::Info { sequence: 0 }.produce(); let mut consumer = group.consume(); let frame = group - .create_frame_owned(crate::frame::Info { - size: size as u64, - timestamp: crate::Timestamp::ZERO, - }) + .create_frame_owned( + crate::frame::Info { + size: size as u64, + timestamp: crate::Timestamp::ZERO, + }, + &Default::default(), + ) .unwrap(); let payload = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap(); diff --git a/rs/moq-net/src/driver.rs b/rs/moq-net/src/driver.rs index 9fb2c7baf13..97d642561ca 100644 --- a/rs/moq-net/src/driver.rs +++ b/rs/moq-net/src/driver.rs @@ -72,10 +72,17 @@ impl Driver { } impl Protocol { + pub(crate) fn local_close(&self) -> std::sync::Arc { + match self { + Self::Lite(driver) => driver.local_close.clone(), + Self::Ietf(driver) => driver.local_close.clone(), + } + } + fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { match self { Self::Lite(driver) => driver.poll(waiter), - Self::Ietf(driver) => waiter.poll_future(driver.task.as_mut()), + Self::Ietf(driver) => waiter.poll_future(std::pin::Pin::new(driver)), } } diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 36c0a4be23f..340be0d12e2 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -13,10 +13,11 @@ use bytes::Buf; use crate::{ - Hops, Path, PathOwned, Pattern, + Hops, Path, PathOwned, Pattern, broadcast, cache, coding, coding::{Decode, Encode, VarInt}, - ietf, lite, + frame, group, ietf, lite, path::Relative, + track, }; /// One fuzz target body: it returns whether the input decoded, which is what @@ -694,18 +695,30 @@ pub fn seeds() -> Vec { // FETCH is the one arm the sweep above cannot reach: its body ends in a parameter // count, and a uniform fill never lands a zero there. Build it from the encoder // instead, which also means a field change breaks the build rather than the seed. + // Draft-20 dropped the joining forms, so each draft gets whichever layout it has. for (index, version) in IETF_VERSIONS.iter().enumerate() { - let fetch = ietf::Fetch { - request_id: ietf::RequestId(0), - subscriber_priority: 128, - group_order: ietf::GroupOrder::Ascending, - fetch_type: ietf::FetchType::AbsoluteJoining { + let fetch_types = [ + ietf::FetchType::AbsoluteJoining { subscriber_request_id: ietf::RequestId(0), group_id: 0, }, - }; - - let Ok(encoded) = fetch.encode_bytes(*version) else { + ietf::FetchType::Filtered { + namespace: crate::Path::new("a"), + track: "b".into(), + filter: ietf::Filter::Relative(1), + }, + ]; + let Some(encoded) = fetch_types.into_iter().find_map(|fetch_type| { + let fetch = ietf::Fetch { + request_id: ietf::RequestId(0), + subscriber_priority: 128, + group_order: ietf::GroupOrder::Ascending, + fetch_type, + range_filters: false, + fill_timeout: false, + }; + fetch.encode_bytes(*version).ok() + }) else { continue; }; @@ -841,6 +854,147 @@ pub fn seeds() -> Vec { seeds } +/// Frames arriving on concurrent group streams, as the frame bench times them: one +/// frame per stream, read a chunk per stream per poll turn so the streams interleave +/// the way a session's do. +/// +/// `budget` is what the session may allocate up front, so `Some(usize::MAX)` allocates +/// every declared size when the first byte lands and `Some(0)` grows every buffer with +/// the bytes received. +pub struct FrameRecv { + streams: Vec, frame::ProducerOwned)>>, + groups: Vec, + // Keeps the groups' track alive. + _track: track::Producer, + _broadcast: broadcast::Producer, +} + +impl FrameRecv { + /// `streams` frames of `size` bytes, delivered `chunk` bytes at a time. + /// `None` gives the session's default budget. + pub fn new(streams: usize, size: usize, chunk: usize, budget: Option) -> Self { + // Unbounded, so eviction never aborts a group mid-frame. + let mut info = broadcast::Info::new(); + info.pool = cache::Pool::unbounded(); + let broadcast = info.produce(); + let track = broadcast.create_track("frames", None).unwrap(); + let budget = budget.map(frame::Budget::new).unwrap_or_default(); + let payload = bytes::Bytes::from(vec![0u8; size]); + + let mut groups = Vec::with_capacity(streams); + let streams = (0..streams) + .map(|_| { + let mut group = track.append_group().unwrap(); + let info = frame::Info { + size: size as u64, + timestamp: crate::Timestamp::ZERO, + }; + let frame = group.create_frame_owned(info, &budget).unwrap(); + groups.push(group); + let stream = Chunks { + payload: payload.clone(), + chunk, + ready: true, + }; + Some((coding::Reader::new(stream, lite::Version::Lite05), frame)) + }) + .collect(); + + Self { + streams, + groups, + _track: track, + _broadcast: broadcast, + } + } + + /// Receive every frame, round-robin a chunk at a time, then drop them. + pub fn run(mut self) { + let mut cx = std::task::Context::from_waker(std::task::Waker::noop()); + let mut open = self.streams.len(); + while open > 0 { + for slot in &mut self.streams { + let Some((reader, frame)) = slot else { continue }; + if let std::task::Poll::Ready(res) = reader.poll_read_frame(&mut cx, frame) { + res.unwrap(); + let (_, frame) = slot.take().unwrap(); + frame.finish().unwrap(); + open -= 1; + } + } + } + for group in &self.groups { + group.finish().unwrap(); + } + } +} + +/// A stream that hands over one chunk of its payload, then blocks until polled again. +struct Chunks { + payload: bytes::Bytes, + chunk: usize, + ready: bool, +} + +impl web_transport_trait::poll::RecvStream for Chunks { + type Error = NoError; + + fn poll_read( + &mut self, + cx: &mut std::task::Context<'_>, + dst: &mut [u8], + ) -> std::task::Poll, NoError>> { + let max = dst.len(); + self.poll_read_chunk(cx, max).map_ok(|chunk| { + chunk.map(|chunk| { + dst[..chunk.len()].copy_from_slice(&chunk); + chunk.len() + }) + }) + } + + // Zero-copy, like the QUIC stacks, so only the frame buffer's own copy is timed. + fn poll_read_chunk( + &mut self, + _cx: &mut std::task::Context<'_>, + max: usize, + ) -> std::task::Poll, NoError>> { + self.ready = !self.ready; + if self.ready { + return std::task::Poll::Pending; + } + if self.payload.is_empty() { + return std::task::Poll::Ready(Ok(None)); + } + let n = self.chunk.min(max).min(self.payload.len()); + std::task::Poll::Ready(Ok(Some(self.payload.split_to(n)))) + } + + fn stop(&mut self, _code: u32) {} + + fn poll_closed(&mut self, _cx: &mut std::task::Context<'_>) -> std::task::Poll> { + std::task::Poll::Pending + } +} + +/// [`Chunks`] never fails. +#[derive(Debug)] +struct NoError; + +impl std::fmt::Display for NoError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("no error") + } +} + +impl std::error::Error for NoError {} + +impl web_transport_trait::Error for NoError { + fn session_error(&self) -> Option<(u32, String)> { + None + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/rs/moq-net/src/ietf/fetch.rs b/rs/moq-net/src/ietf/fetch.rs index 7bf6d6d3619..91ca47c3532 100644 --- a/rs/moq-net/src/ietf/fetch.rs +++ b/rs/moq-net/src/ietf/fetch.rs @@ -4,8 +4,9 @@ use crate::{ Path, coding::{Decode, DecodeError, Encode, EncodeError}, ietf::{ - GroupOrder, Location, Parameters, RequestId, + Filter, GroupOrder, Location, Opaque, Parameters, RequestId, namespace::{decode_namespace, encode_namespace}, + subscribe::has_range_filters, }, }; @@ -13,9 +14,13 @@ use super::Message; use super::Version; +/// What a FETCH asks for. +/// +/// Through draft-19 a Fetch Type tag picks one of the first three. Draft-20 dropped the +/// tag and the joining forms, leaving [`Self::Filtered`]. #[derive(Debug, Clone, PartialEq, Eq)] pub enum FetchType<'a> { - // + /// An inclusive range of a track, through draft-19. Standalone { namespace: Path<'a>, track: Cow<'a, str>, @@ -30,6 +35,13 @@ pub enum FetchType<'a> { subscriber_request_id: RequestId, group_id: u64, }, + /// A track, with the range as a LOCATION_FILTER, from draft-20 on. Unfiltered is + /// everything from `{0, 0}` up to Largest Object. + Filtered { + namespace: Path<'a>, + track: Cow<'a, str>, + filter: Filter, + }, } impl Encode for FetchType<'_> { @@ -63,6 +75,8 @@ impl Encode for FetchType<'_> { subscriber_request_id.encode(w, version)?; group_id.encode(w, version)?; } + // Draft-20 has no Fetch Type tag to write. + FetchType::Filtered { .. } => return Err(EncodeError::Version), } Ok(()) } @@ -111,12 +125,24 @@ pub struct Fetch<'a> { pub subscriber_priority: u8, pub group_order: GroupOrder, pub fetch_type: FetchType<'a>, + /// Whether the request carried a Range Filter (0x25-0x28). We advertise no + /// MAX_FILTER_RANGES, so the request is refused rather than served unfiltered. + /// Never encoded; we send no range filters. + pub range_filters: bool, + /// Whether the request carried FILL_TIMEOUT (0x0A), a budget for waiting on upstream + /// that ends in Timed-Out gaps we cannot write, so the request is refused rather than + /// left to wait without it. Never encoded; we send no fill timeout. + pub fill_timeout: bool, } impl Message for Fetch<'_> { const ID: u64 = 0x16; fn encode_msg(&self, w: &mut W, version: Version) -> Result<(), EncodeError> { + // GROUP_ORDER allows only Ascending or Descending, so no preference is an absent + // parameter rather than a 0 the peer must treat as a protocol violation. + let group_order = (self.group_order != GroupOrder::Any).then_some(self.group_order); + self.request_id.encode(w, version)?; if version == Version::Draft17 { 0u64.encode(w, version)?; // required_request_id_delta = 0 (draft-17 only, removed in draft-18 per #1615) @@ -129,11 +155,29 @@ impl Message for Fetch<'_> { self.fetch_type.encode(w, version)?; 0u8.encode(w, version)?; // no parameters } - _ => { + Version::Draft15 | Version::Draft16 | Version::Draft17 | Version::Draft18 | Version::Draft19 => { self.fetch_type.encode(w, version)?; encode_params!(w, version, 0x20 => self.subscriber_priority, - 0x22 => self.group_order, + 0x22 => group_order, + ); + } + _ => { + // The joining forms and the Fetch Type tag are gone in draft-20. + let FetchType::Filtered { + namespace, + track, + filter, + } = &self.fetch_type + else { + return Err(EncodeError::Version); + }; + encode_namespace(w, namespace, version)?; + track.encode(w, version)?; + encode_params!(w, version, + 0x20 => self.subscriber_priority, + 0x21 => *filter, + 0x22 => group_order, ); } } @@ -146,38 +190,102 @@ impl Message for Fetch<'_> { let _required_request_id_delta = u64::decode(buf, version)?; } - match version { + // The token is ignored: the session's grant is what authorizes the request, and + // INCLUDE_PROPERTIES only shapes a FETCH_OK we don't send on draft-20. + let (fetch_type, subscriber_priority, group_order, range_filters, fill_timeout) = match version { Version::Draft14 => { let subscriber_priority = u8::decode(buf, version)?; let group_order = GroupOrder::decode(buf, version)?; let fetch_type = FetchType::decode(buf, version)?; - let _params = Parameters::decode(buf, version)?; - Ok(Self { - request_id, - subscriber_priority, - group_order, - fetch_type, - }) + Parameters::skip(buf, version)?; + (fetch_type, Some(subscriber_priority), Some(group_order), false, false) } - _ => { + Version::Draft15 | Version::Draft16 | Version::Draft17 | Version::Draft18 | Version::Draft19 => { let fetch_type = FetchType::decode(buf, version)?; decode_params!(buf, version, + 0x03 => _authorization_token: Vec, + 0x0A => fill_timeout: Option, 0x20 => subscriber_priority: Option, 0x22 => group_order: Option, + 0x25 => subgroup_filter: Vec, + 0x26 => object_id_filter: Vec, + 0x27 => priority_filter: Vec, + 0x28 => object_property_filter: Vec, ); + let range_filters = [ + subgroup_filter, + object_id_filter, + priority_filter, + object_property_filter, + ] + .iter() + .any(|filter| !filter.is_empty()); + + // An unknown message parameter is a protocol violation, so each stays rejected + // on the drafts that predate it: FILL_TIMEOUT arrived in draft-18. + let has_fill_timeout = !matches!(version, Version::Draft15 | Version::Draft16 | Version::Draft17); + if (fill_timeout.is_some() && !has_fill_timeout) || (range_filters && !has_range_filters(version)) { + return Err(DecodeError::InvalidValue); + } - let subscriber_priority = subscriber_priority.unwrap_or(128); - // No preference: the publisher picks the order. - let group_order = group_order.unwrap_or(GroupOrder::Any); - - Ok(Self { - request_id, + ( + fetch_type, subscriber_priority, group_order, + range_filters, + fill_timeout.is_some(), + ) + } + // Draft-20 names the track up front and moves the range into LOCATION_FILTER. + _ => { + let namespace = decode_namespace(buf, version)?; + let track = Cow::::decode(buf, version)?; + decode_params!(buf, version, + 0x03 => _authorization_token: Vec, + 0x0A => fill_timeout: Option, + 0x20 => subscriber_priority: Option, + 0x21 => filter: Option, + 0x22 => group_order: Option, + 0x25 => subgroup_filter: Vec, + 0x26 => object_id_filter: Vec, + 0x27 => priority_filter: Vec, + 0x28 => object_property_filter: Vec, + 0x35 => _include_properties: Option, + ); + let range_filters = [ + subgroup_filter, + object_id_filter, + priority_filter, + object_property_filter, + ] + .iter() + .any(|filter| !filter.is_empty()); + + let fetch_type = FetchType::Filtered { + namespace, + track, + // An absent LOCATION_FILTER fetches the whole track. + filter: filter.unwrap_or(Filter::Unfiltered), + }; + ( fetch_type, - }) + subscriber_priority, + group_order, + range_filters, + fill_timeout.is_some(), + ) } - } + }; + + Ok(Self { + request_id, + subscriber_priority: subscriber_priority.unwrap_or(128), + // No preference: the publisher picks the order. + group_order: group_order.unwrap_or(GroupOrder::Any), + fetch_type, + range_filters, + fill_timeout, + }) } } @@ -235,11 +343,8 @@ impl Message for FetchOk { let group_order = GroupOrder::decode(buf, version)?; let end_of_track = bool::decode(buf, version)?; let end_location = Location::decode(buf, version)?; - let params = Parameters::decode(buf, version)?; let properties = super::Properties { - max_cache_duration: params - .get_varint(super::ParameterVarInt::from(0x04)) - .map(std::time::Duration::from_millis), + max_cache_duration: Parameters::skip(buf, version)?.map(std::time::Duration::from_millis), ..Default::default() }; Ok(Self { @@ -574,6 +679,8 @@ mod tests { start: Location { group: 0, object: 0 }, end: Location { group: 10, object: 5 }, }, + range_filters: false, + fill_timeout: false, }; let encoded = encode_message(&msg, Version::Draft14); @@ -595,6 +702,8 @@ mod tests { start: Location { group: 0, object: 0 }, end: Location { group: 10, object: 5 }, }, + range_filters: false, + fill_timeout: false, }; let encoded = encode_message(&msg, Version::Draft15); @@ -634,6 +743,8 @@ mod tests { start: Location { group: 0, object: 0 }, end: Location { group: 10, object: 5 }, }, + range_filters: false, + fill_timeout: false, }; let encoded = encode_message(&msg, Version::Draft16); @@ -655,6 +766,8 @@ mod tests { start: Location { group: 0, object: 0 }, end: Location { group: 10, object: 5 }, }, + range_filters: false, + fill_timeout: false, }; let encoded = encode_message(&msg, Version::Draft17); @@ -730,6 +843,8 @@ mod tests { start: Location { group: 0, object: 0 }, end: Location { group: 10, object: 5 }, }, + range_filters: false, + fill_timeout: false, }; let encoded = encode_message(&msg, Version::Draft18); @@ -779,6 +894,111 @@ mod tests { ]; assert_eq!(encode_message(&msg, Version::Draft18), expected); } + + /// The head of a draft-20 FETCH (Figure 16) up to its Number of Parameters: Request + /// ID 1, Track Namespace ("live"), Track Name ("video"). There is no Fetch Type. + const FETCH_HEAD: &[u8] = &[ + 0x01, 0x01, 0x04, b'l', b'i', b'v', b'e', 0x05, b'v', b'i', b'd', b'e', b'o', + ]; + + /// A draft-20 FETCH names the track up front and carries its range in LOCATION_FILTER. + /// Reading a Fetch Type there instead takes the namespace's field count for one. + #[test] + fn test_fetch_v20_decodes_every_parameter() { + #[rustfmt::skip] + let body = [FETCH_HEAD, &[ + 0x07, // Number of Parameters + 0x03, 0x03, 0x03, 0x00, 0xAA, // AUTHORIZATION TOKEN + 0x07, 0x64, // FILL_TIMEOUT (0x0A) = 100, a varint + 0x16, 0x40, // SUBSCRIBER_PRIORITY (0x20) = 64 + 0x01, 0x03, 0x04, 0x00, 0x02, // LOCATION_FILTER (0x21): groups 4 through 6 + 0x01, 0x01, // GROUP_ORDER (0x22) = Ascending + 0x04, 0x02, 0x00, 0x05, // OBJECTID_FILTER (0x26): SetID 0, from 5 + 0x0F, 0x00, // INCLUDE_PROPERTIES (0x35) = 0 + ]].concat(); + + for version in [Version::Draft20, Version::Draft21, Version::Draft22] { + let fetch: Fetch = decode_message(&body, version).unwrap_or_else(|e| panic!("{version}: {e}")); + assert_eq!(fetch.request_id, RequestId(1)); + assert_eq!(fetch.subscriber_priority, 64); + assert_eq!(fetch.group_order, GroupOrder::Ascending); + assert!(fetch.range_filters, "{version}"); + assert!(fetch.fill_timeout, "{version}"); + assert_eq!( + fetch.fetch_type, + FetchType::Filtered { + namespace: Path::new("live"), + track: "video".into(), + filter: Filter::Absolute { + start: Location { group: 4, object: 0 }, + end: Some(crate::ietf::EndLocation { group: 6, object: None }), + }, + }, + "{version}" + ); + } + } + + /// With no LOCATION_FILTER a draft-20 FETCH is the whole track, and that is what we + /// write back: the parameter is omitted rather than sent empty. + #[test] + fn test_fetch_v20_wire() { + let fetch = Fetch { + request_id: RequestId(1), + subscriber_priority: 128, + group_order: GroupOrder::Descending, + fetch_type: FetchType::Filtered { + namespace: Path::new("live"), + track: "video".into(), + filter: Filter::Unfiltered, + }, + range_filters: false, + fill_timeout: false, + }; + + #[rustfmt::skip] + let expected = [FETCH_HEAD, &[ + 0x02, // Number of Parameters + 0x20, 0x80, // SUBSCRIBER_PRIORITY = 128 + 0x02, 0x02, // GROUP_ORDER (0x22) = Descending + ]].concat(); + assert_eq!(encode_message(&fetch, Version::Draft20), expected); + assert_eq!(decode_message::(&expected, Version::Draft20).unwrap(), fetch); + + // The tagged forms have no spelling from draft-20 on, and this one none before it. + let mut buf = Vec::new(); + assert!(fetch.encode_msg(&mut buf, Version::Draft19).is_err()); + } + + /// FILL_TIMEOUT arrived in draft-18 and the Range Filters in draft-19; each is still + /// an unknown parameter before then. + #[test] + fn test_fetch_parameters_follow_their_draft() { + #[rustfmt::skip] + let joining = |params: &[u8]| [&[ + 0x01, // Request ID + 0x02, 0x03, 0x00, // Relative Joining: subscription 3, offset 0 + ][..], params].concat(); + + let token = joining(&[0x01, 0x03, 0x03, 0x03, 0x00, 0xAA]); + let fill_timeout = joining(&[0x01, 0x0A, 0x20]); + let range_filter = joining(&[0x01, 0x25, 0x00]); + + for (version, body, ok) in [ + (Version::Draft15, &token, true), + (Version::Draft16, &fill_timeout, false), + (Version::Draft18, &fill_timeout, true), + (Version::Draft18, &range_filter, false), + (Version::Draft19, &range_filter, true), + ] { + let decoded = decode_message::(body, version); + assert_eq!(decoded.is_ok(), ok, "{version}: {body:x?}"); + if let Ok(fetch) = decoded { + assert_eq!(fetch.range_filters, body == &range_filter, "{version}"); + assert_eq!(fetch.fill_timeout, body == &fill_timeout, "{version}"); + } + } + } } /// The Object serialization on a fetch stream (draft-20 section 11.4.4), which a fill's diff --git a/rs/moq-net/src/ietf/group.rs b/rs/moq-net/src/ietf/group.rs index 0196865a1aa..e3798baf005 100644 --- a/rs/moq-net/src/ietf/group.rs +++ b/rs/moq-net/src/ietf/group.rs @@ -702,9 +702,10 @@ mod tests { assert!(flags.has_end); } - /// Regression: a publisher-emitted Draft18 GroupHeader byte must satisfy the - /// subscriber's uni-stream classifier mask `(byte & 0x90) == 0x10`. Otherwise - /// the uni stream is dropped as UnexpectedStream and the data plane stalls. + /// Regression: a publisher-emitted Draft18 GroupHeader byte must match the + /// SUBGROUP_HEADER form `(byte & 0x90) == 0x10` and decode as flags, which is how + /// the uni-stream classifier recognizes it. Otherwise the uni stream is an unknown + /// type, which closes the session. #[test] fn test_draft18_group_header_passes_stream_classifier() { let header = GroupHeader { @@ -719,10 +720,14 @@ mod tests { header.encode(&mut buf, Version::Draft18).unwrap(); let type_byte = buf[0] as u64; - // The check in session.rs::run_uni_group. assert_eq!( type_byte & 0x90, 0x10, + "draft-18 SUBGROUP_HEADER type 0x{type_byte:02x} is outside the SUBGROUP_HEADER form", + ); + // The check in session.rs::UniType::classify. + assert!( + GroupFlags::decode(type_byte, Version::Draft18).is_ok(), "draft-18 SUBGROUP_HEADER type 0x{type_byte:02x} not recognized by uni-stream classifier", ); } diff --git a/rs/moq-net/src/ietf/parameters.rs b/rs/moq-net/src/ietf/parameters.rs index 5fb8ad044aa..6e12cca870e 100644 --- a/rs/moq-net/src/ietf/parameters.rs +++ b/rs/moq-net/src/ietf/parameters.rs @@ -213,6 +213,45 @@ impl Encode for Parameters { } impl Parameters { + /// Consume a draft-14 message parameter block, returning its first MAX_CACHE_DURATION + /// (0x04), the one parameter we act on. + /// + /// Draft-14 section 9.2 has a receiver ignore unrecognized parameters and allow their + /// duplicates, and lets AUTHORIZATION TOKEN repeat, so unlike [`Parameters::decode`], + /// which SETUP uses, a repeat is not refused. + pub fn skip(r: &mut R, version: Version) -> Result, DecodeError> { + let count = u64::decode(r, version)?; + if count > MAX_PARAMS { + return Err(DecodeError::TooMany); + } + + let mut cache_duration = None; + for _ in 0..count { + // Parity frames a Key-Value-Pair: even is one varint, odd is length prefixed. + let kind = u64::decode(r, version)?; + match kind % 2 { + 0 => { + let value = u64::decode(r, version)?; + if kind == 0x04 { + cache_duration.get_or_insert(value); + } + } + _ => { + let len = usize::try_from(u64::decode(r, version)?).map_err(|_| DecodeError::BoundsExceeded)?; + if len > MAX_KVP_VALUE_LEN { + return Err(DecodeError::BoundsExceeded); + } + if r.remaining() < len { + return Err(DecodeError::Short); + } + r.advance(len); + } + } + } + + Ok(cache_duration) + } + pub fn get_varint(&self, kind: ParameterVarInt) -> Option { self.vars.get(&kind).copied() } @@ -247,6 +286,14 @@ pub trait Param: Sized { fn param_present(&self) -> bool { true } + + /// Fold a repeat of this parameter into the value already decoded. + /// + /// A parameter may appear once unless its definition says otherwise, so the default + /// refuses the repeat. + fn param_repeat(self, _next: Self) -> Result { + Err(DecodeError::Duplicate) + } } impl Param for u8 { @@ -355,6 +402,13 @@ impl Param for Option { self.is_some() } + fn param_repeat(self, next: Self) -> Result { + match (self, next) { + (Some(prev), Some(next)) => Ok(Some(prev.param_repeat(next)?)), + _ => Err(DecodeError::Duplicate), + } + } + fn param_encode(&self, w: &mut W, version: Version) -> Result<(), EncodeError> { match self { Some(v) => v.param_encode(w, version), @@ -367,6 +421,45 @@ impl Param for Option { } } +/// A parameter whose definition lets it repeat, such as AUTHORIZATION TOKEN (0x03) or a +/// Range Filter (0x25-0x29). Every instance is kept, in wire order. +impl Param for Vec { + fn param_present(&self) -> bool { + !self.is_empty() + } + + fn param_encode(&self, w: &mut W, version: Version) -> Result<(), EncodeError> { + // `encode_params!` writes the key once, so only a single instance fits behind it. + match self.as_slice() { + [value] => value.param_encode(w, version), + _ => Err(EncodeError::Unsupported), + } + } + + fn param_decode(r: &mut R, version: Version) -> Result { + Ok(vec![T::param_decode(r, version)?]) + } + + fn param_repeat(mut self, next: Self) -> Result { + self.extend(next); + Ok(self) + } +} + +/// A length-prefixed parameter value, consumed without being interpreted. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Opaque(pub Vec); + +impl Param for Opaque { + fn param_encode(&self, w: &mut W, version: Version) -> Result<(), EncodeError> { + self.0.encode(w, version) + } + + fn param_decode(r: &mut R, version: Version) -> Result { + Ok(Self(Vec::::decode(r, version)?)) + } +} + /// Encode message parameters with compile-time sorted keys. /// /// Keys must be listed in ascending order (enforced at compile time). @@ -431,7 +524,8 @@ macro_rules! encode_params { /// for parameters where `T::default()` is an acceptable fallback. /// /// Unknown parameters cause `DecodeError::InvalidValue`. -/// Duplicate parameters cause `DecodeError::Duplicate`. +/// Duplicate parameters cause `DecodeError::Duplicate`, unless the type allows a repeat +/// (see [`Param::param_repeat`]), such as `Vec`. /// /// ```ignore /// decode_params!(r, version, @@ -489,10 +583,11 @@ macro_rules! decode_params { // the macro captures it as an expression, which is not a legal pattern. $( if _key == $key { - if $name.is_some() { - return Err($crate::coding::DecodeError::Duplicate); - } - $name = Some(<$ty as $crate::ietf::Param>::param_decode($r, _version)?); + let _value = <$ty as $crate::ietf::Param>::param_decode($r, _version)?; + $name = Some(match $name.take() { + None => _value, + Some(_prev) => $crate::ietf::Param::param_repeat(_prev, _value)?, + }); continue; } )* @@ -977,4 +1072,46 @@ mod tests { ); } } + + /// A parameter the draft lets repeat, such as AUTHORIZATION TOKEN, keeps every + /// instance instead of failing the message as a duplicate. + #[test] + fn test_param_repeat_allowed() { + for version in [Version::Draft15, Version::Draft16, Version::Draft17, Version::Draft20] { + let mut buf = BytesMut::new(); + 2usize.encode(&mut buf, version).unwrap(); + 0x03u64.encode(&mut buf, version).unwrap(); + Opaque(vec![0xAA]).param_encode(&mut buf, version).unwrap(); + // The second key: absolute before draft-16, a zero delta after. + let second: u64 = if version == Version::Draft15 { 0x03 } else { 0 }; + second.encode(&mut buf, version).unwrap(); + Opaque(vec![0xBB]).param_encode(&mut buf, version).unwrap(); + + let mut bytes = buf.freeze(); + let tokens = (|| -> Result, DecodeError> { + decode_params!(&mut bytes, version, 0x03 => tokens: Vec); + Ok(tokens) + })() + .unwrap_or_else(|e| panic!("{version}: {e}")); + assert_eq!(tokens, vec![Opaque(vec![0xAA]), Opaque(vec![0xBB])], "{version}"); + assert!(!bytes.has_remaining(), "{version}"); + } + } + + /// Draft-14 lets AUTHORIZATION TOKEN repeat and has unknown parameters, duplicates + /// included, ignored. The block is consumed whole either way. + #[test] + fn test_skip_allows_draft14_repeats() { + #[rustfmt::skip] + let block = [ + 0x04, // Number of Parameters + 0x03, 0x01, 0xAA, // AUTHORIZATION TOKEN + 0x03, 0x01, 0xBB, // and again + 0x3E, 0x05, // an unknown varint parameter + 0x3E, 0x06, // and again + ]; + let mut buf = &block[..]; + Parameters::skip(&mut buf, Version::Draft14).unwrap(); + assert!(buf.is_empty()); + } } diff --git a/rs/moq-net/src/ietf/publish.rs b/rs/moq-net/src/ietf/publish.rs index 7a5b24164bf..f617fc0ee9c 100644 --- a/rs/moq-net/src/ietf/publish.rs +++ b/rs/moq-net/src/ietf/publish.rs @@ -309,7 +309,7 @@ impl Message for Publish<'_> { }; let forward = bool::decode(r, version)?; // parameters - let _params = Parameters::decode(r, version)?; + Parameters::skip(r, version)?; Ok(Self { request_id, @@ -334,6 +334,7 @@ impl Message for Publish<'_> { // letting the request reach its NOT_SUPPORTED response. decode_params!(r, version, 0x02 => object_delivery_timeout: Option, + 0x03 => _authorization_token: Vec, 0x06 => subgroup_delivery_timeout: Option, 0x08 => _expires: Option, 0x09 => largest_location: Option, @@ -437,7 +438,7 @@ impl Message for PublishOk { let filter = Filter::decode(r, version)?; // no parameters - let _params = Parameters::decode(r, version)?; + Parameters::skip(r, version)?; Ok(Self { request_id, diff --git a/rs/moq-net/src/ietf/publish_namespace.rs b/rs/moq-net/src/ietf/publish_namespace.rs index 863826dfeb9..ae090688bd3 100644 --- a/rs/moq-net/src/ietf/publish_namespace.rs +++ b/rs/moq-net/src/ietf/publish_namespace.rs @@ -36,7 +36,24 @@ impl PublishNamespace<'_> { let _required_request_id_delta = u64::decode(r, version)?; } let track_namespace = decode_namespace(r, version)?; - let cluster = decode_cluster_params(r, version, negotiated)?; + + // The token is ignored: the session's grant is what authorizes the request. + decode_params!(r, version, + 0x03 => _authorization_token: Vec, + cluster::HOP_PATH => hops: Option, + cluster::ROUTE_COST => cost: Option, + ); + + let cluster = match negotiated { + true => Some(cluster::Advert { + hops: hops.ok_or(DecodeError::InvalidValue)?, + cost: cost.unwrap_or(0), + }), + // An endpoint must not append these on a session that did not negotiate the + // extension, so either is a violation. + false if hops.is_some() || cost.is_some() => return Err(DecodeError::InvalidValue), + false => None, + }; Ok(Self { request_id, @@ -127,7 +144,9 @@ impl Message for PublishNamespaceUpdate { } _ => RequestId::decode(r, version)?, }; + // The token is ignored: the session's grant is what authorizes the request. decode_params!(r, version, + 0x03 => _authorization_token: Vec, cluster::HOP_PATH => hops: Option, cluster::ROUTE_COST => cost: Option, ); @@ -157,28 +176,21 @@ pub(super) fn encode_cluster_params( Ok(()) } -/// Read the Parameters field of an advertisement. See [`encode_cluster_params`]. +/// Read the Parameters field of a NAMESPACE on a session that negotiated the extension. +/// See [`encode_cluster_params`]. pub(super) fn decode_cluster_params( r: &mut R, version: Version, - negotiated: bool, -) -> Result, DecodeError> { - if !negotiated { - // An endpoint must not append these on a session that did not negotiate the - // extension, and we know no other parameter here, so any is a violation. - decode_params!(r, version,); - return Ok(None); - } - +) -> Result { decode_params!(r, version, cluster::HOP_PATH => hops: Option, cluster::ROUTE_COST => cost: Option, ); - Ok(Some(cluster::Advert { + Ok(cluster::Advert { hops: hops.ok_or(DecodeError::InvalidValue)?, cost: cost.unwrap_or(0), - })) + }) } /// PublishNamespaceOk message (0x07) diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 34b24f39c97..686a91aed1a 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -518,6 +518,22 @@ where } .maybe_boxed() } + // SUBSCRIBE_TRACKS asks for every track under a prefix via PUBLISH, which we + // never send. The body is already framed off and draft-17+ replies carry no + // Request ID, so there is nothing to decode before refusing. + ietf::SUBSCRIBE_TRACKS_ID + if !matches!( + this.version, + Version::Draft14 | Version::Draft15 | Version::Draft16 | Version::Draft17 + ) => + { + async move { + if let Err(err) = this.reject_subscribe_tracks(stream).await { + tracing::debug!(%err, "subscribe_tracks refusal failed"); + } + } + .maybe_boxed() + } ietf::TrackStatus::ID => { let msg = ietf::TrackStatus::decode_msg(&mut data, this.version)?; if !data.is_empty() { @@ -554,6 +570,21 @@ where tracing::info!(id = %request_id, broadcast = %absolute, track = %track_name, "subscribe started"); + // Legal requests we can't honor are refused one at a time, never by closing the + // session. A subscription that forwards nothing is only useful to a subscriber + // that later turns forwarding on, and serving a Range Filter unfiltered would + // deliver objects the subscriber excluded. + if !msg.forward { + return self + .reject_subscribe(stream, request_id, &Error::Unsupported, "FORWARD=0 not supported") + .await; + } + if msg.range_filters { + return self + .reject_subscribe(stream, request_id, &Error::Unsupported, "range filters not supported") + .await; + } + // Stats (subscriptions, viewer refcount, groups/frames/bytes) are counted in // the model, through the tagged `origin::Consumer` the broadcast resolves from. @@ -1128,8 +1159,8 @@ where async fn run_fetch_stream(mut self, mut stream: Stream, msg: ietf::Fetch<'_>) -> Result<(), Error> { let priority = super::priority::from_wire(msg.subscriber_priority); - // Draft-20 moved the range into LOCATION_FILTER and replaced joining FETCH with - // subscription fills; this still reads the older layout. + // Draft-20 moved the range into LOCATION_FILTER, which decodes to + // `FetchType::Filtered` but is not served yet. if Filter::is_draft20(self.version) { return self .reject_fetch( @@ -1141,6 +1172,31 @@ where .await; } + // Serving a Range Filter unfiltered would deliver objects the subscriber excluded. + if msg.range_filters { + return self + .reject_fetch( + stream, + msg.request_id, + &Error::Unsupported, + "range filters not supported", + ) + .await; + } + + // FILL_TIMEOUT=0 asks for cache only, and any budget ends in Timed-Out gaps we + // don't write, so waiting on upstream regardless would ignore what was asked. + if msg.fill_timeout { + return self + .reject_fetch( + stream, + msg.request_id, + &Error::Unsupported, + "FILL_TIMEOUT not supported", + ) + .await; + } + let (track, start, end, timescale, joined) = match msg.fetch_type { FetchType::Standalone { namespace, @@ -1209,6 +1265,12 @@ where true, ) } + // Refused above: only draft-20 decodes this form. + FetchType::Filtered { .. } => { + return self + .reject_fetch(stream, msg.request_id, &Error::Unsupported, "not supported") + .await; + } }; // One group per FETCH: on a relay, a range costs a serial upstream fetch per missing @@ -1426,6 +1488,21 @@ where Ok(()) } + async fn reject_subscribe_tracks(&self, mut stream: Stream) -> Result<(), Error> { + stream.writer.encode(&ietf::RequestError::ID).await?; + stream + .writer + .encode(&ietf::RequestError { + request_id: None, + error_code: request::to_code(&Error::Unsupported, request::Kind::SubscribeNamespace, self.version), + reason_phrase: "SUBSCRIBE_TRACKS is not supported".into(), + retry_interval: 0, + }) + .await?; + let _ = stream.writer.close().await; + Ok(()) + } + /// Reject a FETCH, ending the request stream. See [`Self::reject_subscribe`] for why the /// close is not optional. async fn reject_fetch( @@ -3123,6 +3200,8 @@ mod serve_tests { filter, fill, properties_wanted: true, + forward: true, + range_filters: false, } } @@ -3295,6 +3374,120 @@ mod serve_tests { } } + /// Dispatch one request stream, returning the refusal code it wrote. + /// + /// `handle_stream` returning `Ok` is what keeps the session open: the dispatch loop + /// closes the session over an `Err`. + async fn refusal(version: Version, id: u64, body: Vec) -> u64 { + let h = serve(version); + let stream = Stream::open(&mut h.session.clone(), version).await.unwrap(); + let mark = h.log.writes.lock().unwrap().len(); + h.publisher + .clone() + .handle_stream(id, bytes::Bytes::from(body), stream) + .unwrap_or_else(|e| panic!("{version}: the request closed the session: {e}")) + .await; + assert!(h.log.resets().is_empty(), "{version}: refusal was reset"); + + let mut buf = bytes::Bytes::from(h.log.writes.lock().unwrap()[mark..].to_vec()); + match version { + Version::Draft14 => { + // SUBSCRIBE_ERROR and FETCH_ERROR share their layout. + let _id = u64::decode(&mut buf, version).unwrap(); + ietf::SubscribeError::decode(&mut buf, version).unwrap().error_code + } + _ => { + assert_eq!(u64::decode(&mut buf, version).unwrap(), ietf::RequestError::ID); + ietf::RequestError::decode(&mut buf, version).unwrap().error_code + } + } + } + + /// Legal requests we don't serve are refused NOT_SUPPORTED one at a time, and the + /// session stays open for the next one. + #[tokio::test] + async fn legal_requests_we_do_not_serve_are_refused_per_request() { + const NOT_SUPPORTED: u64 = 0x3; + + for version in [ + Version::Draft14, + Version::Draft15, + Version::Draft16, + Version::Draft17, + Version::Draft18, + Version::Draft19, + Version::Draft20, + Version::Draft21, + Version::Draft22, + ] { + let mut paused = subscribe(Filter::NextObject, None); + paused.forward = false; + let mut body = bytes::BytesMut::new(); + paused.encode_msg(&mut body, version).unwrap(); + assert_eq!( + refusal(version, ietf::Subscribe::ID, body.to_vec()).await, + NOT_SUPPORTED, + "{version}: SUBSCRIBE with FORWARD=0" + ); + } + + // Draft-20 bytes from the figures: Request ID 0x2B, Track Namespace ("room"), + // Track Name ("video"), then the parameters. + let head: &[u8] = &[ + 0x2B, 0x01, 0x04, b'r', b'o', b'o', b'm', 0x05, b'v', b'i', b'd', b'e', b'o', + ]; + + #[rustfmt::skip] + let cases: [(&str, u64, &[u8]); 3] = [ + ("SUBSCRIBE with a Range Filter", ietf::Subscribe::ID, &[ + 0x02, // Number of Parameters + 0x03, 0x03, 0x03, 0x00, 0xAA, // AUTHORIZATION TOKEN + 0x23, 0x02, 0x00, 0x05, // OBJECTID_FILTER (0x26): SetID 0, from 5 + ]), + ("FETCH", ietf::Fetch::ID, &[ + 0x03, // Number of Parameters + 0x0A, 0x00, // FILL_TIMEOUT = 0 + 0x17, 0x01, 0x01, // LOCATION_FILTER (0x21): from one group back + 0x14, 0x01, // INCLUDE_PROPERTIES (0x35) = 1 + ]), + ("TRACK_STATUS with parameters", ietf::TrackStatus::ID, &[ + 0x02, // Number of Parameters + 0x03, 0x03, 0x03, 0x00, 0xAA, // AUTHORIZATION TOKEN + 0x32, 0x00, // INCLUDE_PROPERTIES (0x35) = 0 + ]), + ]; + + // A standalone FETCH we would otherwise serve, carrying FILL_TIMEOUT=0: cache only, + // with gaps reported as Timed-Out, which we can't write. + #[rustfmt::skip] + let fill_timeout = [ + 0x2B, // Request ID + 0x01, // Standalone + 0x01, 0x04, b'r', b'o', b'o', b'm', 0x05, b'v', b'i', b'd', b'e', b'o', + 0x00, 0x00, // Start Location + 0x00, 0x00, // End Location: the whole group 0 + 0x01, // Number of Parameters + 0x0A, 0x00, // FILL_TIMEOUT = 0 + ]; + for version in [Version::Draft18, Version::Draft19] { + assert_eq!( + refusal(version, ietf::Fetch::ID, fill_timeout.to_vec()).await, + NOT_SUPPORTED, + "{version}: FETCH with FILL_TIMEOUT" + ); + } + + for version in [Version::Draft20, Version::Draft21, Version::Draft22] { + for (label, id, params) in cases { + assert_eq!( + refusal(version, id, [head, params].concat()).await, + NOT_SUPPORTED, + "{version}: {label}" + ); + } + } + } + /// The draft's canonical current-group join: a Next Object subscription plus a /// StartGroup=1 fill. The published head arrives exactly once, on a fetch stream, /// and the subscription starts past the snapshot, so nothing is duplicated and @@ -3469,6 +3662,8 @@ mod serve_tests { subscriber_request_id: RequestId(REQUEST_ID), group_offset: 0, }, + range_filters: false, + fill_timeout: false, }, ) .await?; @@ -3874,6 +4069,8 @@ mod serve_tests { start, end, }, + range_filters: false, + fill_timeout: false, }, ) .await @@ -4114,6 +4311,8 @@ mod serve_tests { subscriber_priority: 128, group_order: GroupOrder::Ascending, fetch_type, + range_filters: false, + fill_timeout: false, }, ) .await @@ -4155,6 +4354,8 @@ mod serve_tests { subscriber_request_id: RequestId(REQUEST_ID), group_id: 5, }, + range_filters: false, + fill_timeout: false, }, ) .await @@ -5916,6 +6117,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: true, + forward: true, + range_filters: false, }, ) .await @@ -5939,6 +6142,8 @@ mod tests { subscriber_priority: 128, group_order: GroupOrder::Descending, fetch_type, + range_filters: false, + fill_timeout: false, }, ) .await @@ -6264,6 +6469,8 @@ mod range_tests { filter, fill: None, properties_wanted: true, + forward: true, + range_filters: false, } } diff --git a/rs/moq-net/src/ietf/session.rs b/rs/moq-net/src/ietf/session.rs index 75c5d0e25f4..354b554e9f0 100644 --- a/rs/moq-net/src/ietf/session.rs +++ b/rs/moq-net/src/ietf/session.rs @@ -67,17 +67,22 @@ pub struct Config { /// What that pre-read SETUP declared, so the session does not have to parse it /// twice. `None` when [`Self::peer_setup_stream`] is. pub peer_declared: Option, + + /// Uni streams that arrived before that pre-read SETUP (see [`PeerSetup::early`]), + /// classified by the session before it accepts any more. + pub early_unis: Vec>, } pub(crate) struct Driver { - pub task: MaybeSendBox<'static, Result<(), Error>>, - pub withdrawal: crate::session::Withdrawal, + pub(crate) withdrawal: crate::session::Withdrawal, + pub(crate) local_close: std::sync::Arc, + future: MaybeSendBox<'static, Result<(), Error>>, } impl std::future::Future for Driver { type Output = Result<(), Error>; fn poll(mut self: std::pin::Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> std::task::Poll { - self.task.as_mut().poll(cx) + self.future.as_mut().poll(cx) } } @@ -100,6 +105,7 @@ where authority, peer_setup_stream, peer_declared, + early_unis, } = config; // GOAWAY wiring: the public Session holds one half (drain trigger, received @@ -113,7 +119,9 @@ where let namespaces = subscribe.as_ref().map(subscribe_prefixes).unwrap_or_default(); let withdrawal = crate::session::Withdrawal::default(); - let closing = withdrawal.clone(); + let withdrawing = withdrawal.clone(); + let local_close = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let closing = local_close.clone(); let driver = async move { // Our own Hop ID, taken from whichever origin the caller actually supplied so // every session out of this process stamps the same one and cross-session loop @@ -162,7 +170,7 @@ where version, ); let (tasks, mut task_set) = TaskSet::new(); - publisher.withdrawal = closing.clone(); + publisher.withdrawal = withdrawing.clone(); let subscriber = Subscriber::new( runtime.clone(), @@ -215,10 +223,14 @@ where let mut unis = std::pin::pin!(err_only(run_unis( adapter.clone(), subscriber.clone(), - None, - false, - version, - goaway, + UniSetup { + peer: None, + read: false, + early: Vec::new(), + version, + goaway, + local_close: closing.clone() + }, ))); let mut dispatch = std::pin::pin!(err_only(run_dispatch( dispatch_session, @@ -290,7 +302,9 @@ where Poll::Pending }) .await; - if let Err(err) = &res { + if closing.load(std::sync::atomic::Ordering::Relaxed) { + subscriber.close(); + } else if let Err(err) = &res { // Every track this session was receiving ends with its error. subscriber.abort(err); } @@ -323,7 +337,7 @@ where version, ); let (tasks, mut task_set) = TaskSet::new(); - publisher.withdrawal = closing.clone(); + publisher.withdrawal = withdrawing.clone(); let subscriber = Subscriber::new( runtime.clone(), @@ -361,10 +375,14 @@ where let mut unis = std::pin::pin!(err_only(run_unis( session.clone(), subscriber.clone(), - Some(peer_setup.clone()), - setup_read, - version, - goaway, + UniSetup { + peer: Some(peer_setup.clone()), + read: setup_read, + early: early_unis, + version, + goaway, + local_close: closing.clone() + }, ))); let mut dispatch = std::pin::pin!(err_only(run_dispatch( session.clone(), @@ -432,7 +450,9 @@ where Poll::Pending }) .await; - if let Err(err) = &res { + if closing.load(std::sync::atomic::Ordering::Relaxed) { + subscriber.close(); + } else if let Err(err) = &res { // Every track this session was receiving ends with its error. subscriber.abort(err); } @@ -461,8 +481,9 @@ where Ok(( Driver { - task: driver, withdrawal, + local_close, + future: driver, }, goaway_handle, )) @@ -481,6 +502,11 @@ pub struct PeerSetup { /// The Setup Options it declared (see [`cluster`] and [`solicit`]). pub declared: peer::Peer, + + /// Uni streams that arrived before the SETUP, type peeked but unread. The drafts say + /// to buffer early data until the control streams arrive, so the session classifies + /// these once it starts. QUIC stream credit bounds how many can pile up. + pub early: Vec>, } /// The Hop ID this session declares and detects loops against. @@ -498,23 +524,32 @@ fn self_origin(publish: Option<&origin::Consumer>, subscribe: Option<&origin::Pr /// Server (draft-17+): read the peer's SETUP off its uni stream before starting the /// session, returning that stream plus what it declared. /// -/// Blocks on the peer's Setup Stream; any other uni stream racing ahead of it is -/// `STOP_SENDING`-ed and skipped (group data needs a prior subscribe, so nothing -/// legitimate precedes the SETUP at connect). Pass the returned reader to [`start`] -/// as its `peer_setup_stream` so GOAWAY monitoring continues without re-reading it. +/// Blocks on the peer's Setup Stream. Any other uni stream racing ahead of it (padding, +/// or group data for a subscription the peer already holds) is held in +/// [`PeerSetup::early`] for the session to classify. Pass the returned reader to +/// [`start`] as its `peer_setup_stream` so GOAWAY monitoring continues without +/// re-reading it. pub async fn accept_setup( session: &mut S, version: Version, ) -> Result, Error> { let outer_version = crate::Version::Ietf(version); + let mut early = Vec::new(); loop { let recv = session.accept_uni().await.map_err(Error::from_transport)?; let mut reader: Reader = Reader::new(recv, outer_version); - if reader.decode_peek::().await? != setup::SETUP_V17 { - // Not the SETUP (group data this early is unexpected). Reject and keep waiting. - reader.abort(&Error::UnexpectedStream); + let kind = match reader.decode_peek::().await { + Ok(kind) => kind, + Err(err) if died_before_header(&err) => { + tracing::debug!(%err, "dropping uni stream that died before its type"); + continue; + } + Err(err) => return Err(err), + }; + if kind != setup::SETUP_V17 { + early.push(reader); continue; } @@ -537,6 +572,7 @@ pub async fn accept_setup( path, token, declared, + early, }); } } @@ -642,39 +678,108 @@ async fn run_setup( Ok(()) } -/// Accept incoming uni streams and dispatch each to a handler. -/// -/// For v17, this also handles the SETUP stream (0x2F00) and GOAWAY. -/// For v14-16, all uni streams are group data. -async fn run_unis( - mut session: S, - subscriber: Subscriber, - // Where to record the peer's MoQ Cluster options once its SETUP arrives. `None` - // for draft-14..16, whose SETUP rides the control stream instead. - peer_setup: Option, - // Whether the peer's SETUP was already consumed before this loop started. - setup_read: bool, +/// The PADDING stream type (draft-18+): bytes a peer sends to probe for bandwidth. +const PADDING: u64 = 0x132B3E28; + +/// What a unidirectional stream's type names on the negotiated draft. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UniType { + /// The peer's SETUP, which then carries its GOAWAY (draft-17+). + Setup, + /// A SUBGROUP_HEADER, carrying one subgroup of a subscription. + Subgroup, + /// A FETCH_HEADER, carrying a fetch response. + Fetch, + /// Data to discard (draft-18+). + Padding, +} + +impl UniType { + /// `None` is a type the draft does not define, which MUST close the session + /// (draft-21 section 6.4.1, and its equivalent in every draft we negotiate). + fn classify(kind: u64, version: Version) -> Option { + // Draft-14-17 use SUBGROUP_HEADER types 0x10-0x1D and 0x30-0x3D; draft-18 adds + // 0x40 (FIRST_OBJECT), also covering 0x50-0x5D and 0x70-0x7D. A reserved + // SUBGROUP_ID_MODE (0b11) or a bit the draft lacks is invalid, which MUST close the + // session too (draft-21 section 11.3.1). + if ietf::GroupFlags::decode(kind, version).is_ok() { + return Some(Self::Subgroup); + } + + match kind { + FetchHeader::TYPE => Some(Self::Fetch), + setup::SETUP_V17 => match version { + // SETUP rides the bidi control stream. + Version::Draft14 | Version::Draft15 | Version::Draft16 => None, + _ => Some(Self::Setup), + }, + PADDING => match version { + Version::Draft14 | Version::Draft15 | Version::Draft16 | Version::Draft17 => None, + _ => Some(Self::Padding), + }, + _ => None, + } + } +} + +/// Setup and lifecycle state for the incoming uni stream dispatcher. +struct UniSetup { + peer: Option, + read: bool, + /// Streams accepted before the session started, handled before accepting more. + early: Vec>, version: Version, goaway: crate::goaway::Protocol, -) -> Result<(), Error> + local_close: std::sync::Arc, +} + +/// Accept incoming uni streams, including SETUP and GOAWAY on draft-17+. +async fn run_unis(mut session: S, subscriber: Subscriber, setup: UniSetup) -> Result<(), Error> where S: crate::transport::poll::Boxable, { + let UniSetup { + peer: peer_setup, + read: setup_read, + early, + version, + goaway, + local_close, + } = setup; let outer_version = crate::Version::Ietf(version); let mut tasks = TaskSet::owned(); // A gated server accept already read the peer's one SETUP off its own uni stream, // so anything arriving here is a second one. let mut seen_setup = setup_read; + let mut early = early.into_iter(); loop { - let recv = tasks - .drive(|waiter| { - let mut cx = waiter.context(); - session.poll_accept_uni(&mut cx) - }) - .await - .map_err(Error::from_transport)?; - let mut reader: Reader = Reader::new(recv, outer_version); + let mut reader = match early.next() { + Some(reader) => reader, + None => { + let recv = tasks + .drive(|waiter| { + let mut cx = waiter.context(); + session.poll_accept_uni(&mut cx) + }) + .await; + let recv = match recv { + Ok(recv) => recv, + Err(err) => { + let err = Error::from_transport(err); + // Settle tracks and open groups before the owned receive tasks drop + // their cancellation guards, which would otherwise record Cancel. + if local_close.load(std::sync::atomic::Ordering::Relaxed) { + subscriber.close(); + } else { + subscriber.abort(&err); + } + return Err(err); + } + }; + Reader::new(recv, outer_version) + } + }; // A stream that dies before its type varint is that stream's failure, not the // session's. RESET_STREAM is how a peer drops a group, and QUIC does not order // the reset behind the data, so one can beat the first byte even of a stream @@ -695,85 +800,103 @@ where .await { Ok(kind) => kind, - Err( - err @ (Error::Cancel - | Error::Stream(_) - | Error::Remote(_) - | Error::Transport(_) - | Error::Decode(DecodeError::Short)), - ) => { + Err(err) if died_before_header(&err) => { tracing::debug!(%err, "dropping uni stream that died before its type"); continue; } Err(err) => return Err(err), }; - // v17+: SETUP arrives on a uni stream, then becomes the GOAWAY channel. - // We accept it in the background without blocking; the one thing that does - // need it (the MoQ Cluster negotiation) waits on `peer_setup` instead, so a - // slow SETUP delays announcements rather than the whole session. - if kind == setup::SETUP_V17 { - // Exactly one SETUP per endpoint. A second would let a peer restate its - // declared identity mid-session, silently re-attributing every route - // already built from the first. - if std::mem::replace(&mut seen_setup, true) { - return Err(Error::ProtocolViolation); - } + let Some(ty) = UniType::classify(kind, version) else { + tracing::warn!(kind, "unknown uni stream type"); + return Err(Error::UnexpectedStream); + }; - let peer_setup = peer_setup.clone(); - let mut session = session.clone(); - let goaway = goaway.clone(); - tasks.push(async move { - // The negotiation gates the announce and dispatch loops, so a SETUP we - // cannot read must end the session rather than leave them parked on a - // slot nothing will ever fill. - let msg = match reader.decode::().await { - Ok(msg) => msg, - Err(err) => { - tracing::warn!(%err, "setup decode error"); - session.close(SessionError::ProtocolViolation.to_code(), "invalid setup"); - return; - } - }; + match ty { + // SETUP then becomes the GOAWAY channel. We accept it in the background + // without blocking; the one thing that does need it (the MoQ Cluster + // negotiation) waits on `peer_setup` instead, so a slow SETUP delays + // announcements rather than the whole session. + UniType::Setup => { + // Exactly one SETUP per endpoint. A second would let a peer restate its + // declared identity mid-session, silently re-attributing every route + // already built from the first. + if std::mem::replace(&mut seen_setup, true) { + return Err(Error::ProtocolViolation); + } - if let Some(peer_setup) = peer_setup { - let peer = match decode_peer_setup(msg.parameters, version) { - Ok(peer) => peer, + let peer_setup = peer_setup.clone(); + let mut session = session.clone(); + let goaway = goaway.clone(); + tasks.push(async move { + // The negotiation gates the announce and dispatch loops, so a SETUP we + // cannot read must end the session rather than leave them parked on a + // slot nothing will ever fill. + let msg = match reader.decode::().await { + Ok(msg) => msg, Err(err) => { - tracing::warn!(%err, "setup parameter decode error"); - session.close(SessionError::ProtocolViolation.to_code(), "invalid setup parameters"); + tracing::warn!(%err, "setup decode error"); + session.close(SessionError::ProtocolViolation.to_code(), "invalid setup"); return; } }; - peer_setup.set(peer); - } - // Monitor for GOAWAY after setup completes. - if let Err(err) = run_goaway(reader.with_version(version), version, goaway).await { - tracing::warn!(%err, "goaway error"); - } - }); - - continue; - } + if let Some(peer_setup) = peer_setup { + let peer = match decode_peer_setup(msg.parameters, version) { + Ok(peer) => peer, + Err(err) => { + tracing::warn!(%err, "setup parameter decode error"); + session.close(SessionError::ProtocolViolation.to_code(), "invalid setup parameters"); + return; + } + }; + peer_setup.set(peer); + } - // Poll one child handler for each group stream. - let mut sub = subscriber.clone(); - tasks.push(async move { - let mut reader = reader.with_version(version); - if let Err(err) = run_uni_group(&mut sub, &mut reader).await { - tracing::debug!(%err, "uni stream error"); - // This handler stops only the stream, so it cannot claim the session closed. - let reset = match StreamError::from(&err) { - StreamError::Session(_) => StreamError::Internal, - reset => reset, - }; - reader.abort(reset); + // Monitor for GOAWAY after setup completes. + if let Err(err) = run_goaway(reader.with_version(version), version, goaway).await { + tracing::warn!(%err, "goaway error"); + } + }); } - }); + UniType::Subgroup => { + let mut sub = subscriber.clone(); + tasks.push(async move { + let mut reader = reader.with_version(version); + let res = sub.recv_group(&mut reader).await; + stop_on_error(&mut reader, res); + }); + } + // A fill fetch stream carries the head of the group a draft-20 subscription + // joined part way through. One answering no fill of ours is refused inside. + UniType::Fetch => { + let mut sub = subscriber.clone(); + tasks.push(async move { + let mut reader = reader.with_version(version); + let res = sub.recv_fill(&mut reader).await; + stop_on_error(&mut reader, res); + }); + } + // The receiver MUST discard padding. We read it to the end rather than cancel, + // so a peer probing for bandwidth gets the throughput it is measuring. + UniType::Padding => { + tasks.push(async move { + while let Ok(Some(_)) = std::future::poll_fn(|cx| reader.poll_read_chunk(cx, usize::MAX)).await {} + }); + } + } } } +/// Whether reading an incoming stream's header failed because the stream died, which is +/// that stream's failure and not the session's. Bytes that arrive and do not parse are not. +fn died_before_header(err: &Error) -> bool { + matches!( + err, + Error::Cancel | Error::Stream(_) | Error::Remote(_) | Error::Transport(_) | Error::Decode(DecodeError::Short) + ) +} + /// Receive QUIC datagrams, each an OBJECT_DATAGRAM for one of our subscriptions. /// /// A transport without datagrams never delivers one, so this parks. A transport failure @@ -791,30 +914,19 @@ where } } -async fn run_uni_group( - subscriber: &mut Subscriber, - stream: &mut Reader, -) -> Result<(), Error> -where - S: crate::transport::poll::Boxable, -{ - let kind: u64 = stream.decode_peek().await?; - - // SUBGROUP_HEADER type bytes match the form 0b0XX1XXXX (spec §11.4.2): - // draft-14-17 use 0x10-0x1D and 0x30-0x3D, draft-18 adds 0x40 (FIRST_OBJECT) - // extending the form to also cover 0x50-0x5D and 0x70-0x7D. Per-version and - // per-bit validation (e.g., FIRST_OBJECT must be 0 on draft-17) is done in - // `GroupFlags::decode`. - if kind <= 0xff && (kind & 0x90) == 0x10 { - return subscriber.recv_group(stream).await; - } +/// Stop a data stream whose handler failed. The handler owns only the stream, so it +/// cannot claim the session closed. +fn stop_on_error(reader: &mut Reader, res: Result<(), Error>) { + let Err(err) = res else { + return; + }; - match kind { - // A fill fetch stream carries the head of the group a draft-20 subscription joined - // part way through. One answering no fill of ours is refused inside. - FetchHeader::TYPE => subscriber.recv_fill(stream).await, - _ => Err(Error::UnexpectedStream), - } + tracing::debug!(%err, "uni stream error"); + let reset = match StreamError::from(&err) { + StreamError::Session(_) => StreamError::Internal, + reset => reset, + }; + reader.abort(reset); } /// Accept incoming bidi streams and dispatch to the correct handler based on message type. @@ -871,13 +983,7 @@ where // header that does not parse included, still fails the session. let (id, data) = match header { Ok(header) => header, - Err( - err @ (Error::Cancel - | Error::Stream(_) - | Error::Remote(_) - | Error::Transport(_) - | Error::Decode(DecodeError::Short)), - ) => { + Err(err) if died_before_header(&err) => { tracing::debug!(%err, "dropping bidi stream that died before its header"); continue; } @@ -886,11 +992,12 @@ where match id { // Publisher handles: Subscribe, Fetch, SubscribeNamespace (0x50 modern / - // 0x11 legacy), TrackStatus + // 0x11 legacy), SubscribeTracks, TrackStatus ietf::Subscribe::ID | ietf::Fetch::ID | ietf::SubscribeNamespace::ID | ietf::SubscribeNamespaceLegacy::ID + | ietf::SUBSCRIBE_TRACKS_ID | ietf::TrackStatus::ID => { tasks.push(publisher.handle_stream(id, data, stream)?); } @@ -1046,6 +1153,7 @@ mod tests { }, ..Default::default() }), + early_unis: Vec::new(), }) .expect("start the session"); @@ -1098,6 +1206,7 @@ mod tests { peer_setup_stream: None, // The requests wait on the peer's SETUP (MoQ Hidden). peer_declared: Some(peer::Peer::default()), + early_unis: Vec::new(), }) .expect("start the session"); let _driver = tokio::spawn(driver); @@ -1149,6 +1258,7 @@ mod tests { authority: None, peer_setup_stream: None, peer_declared, + early_unis: Vec::new(), }) .expect("start the session"); let _driver = tokio::spawn(driver); @@ -1259,6 +1369,7 @@ mod tests { // Pre-settled, so nothing waits on a SETUP the dead stream will never // carry and the dispatch loop actually runs. peer_declared: Some(peer::Peer::default()), + early_unis: Vec::new(), }) .expect("start the session"); @@ -1297,6 +1408,7 @@ mod tests { authority: Some(String::from_utf8(AUTHORITY.to_vec()).unwrap()), peer_setup_stream: None, peer_declared: Some(peer::Peer::default()), + early_unis: Vec::new(), }) .expect("start the session"); @@ -1360,13 +1472,28 @@ mod tests { writes.clone() } - async fn dispatch_uni(payload: Vec, retired_alias: Option) -> crate::lite::test_transport::Log { - const VERSION: Version = Version::Draft19; - - let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); + /// Run the uni dispatch loop over one incoming stream. Returns what reached the wire, + /// and the loop's result if that stream ended it. + async fn dispatch_uni( + version: Version, + payload: Vec, + retired_alias: Option, + ) -> (crate::lite::test_transport::Log, Option>) { // The peer opens one uni stream and then goes quiet, so // the loop is still running when the assertion is taken. let session = crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_incoming_unis(vec![payload]); + drive_unis(version, session, None, retired_alias).await + } + + /// Run the uni dispatch loop until it stops a stream or ends. `accepted` is a SETUP + /// already read by [`accept_setup`], with the streams that raced ahead of it. + async fn drive_unis( + version: Version, + session: crate::lite::test_transport::ScriptedSession, + accepted: Option>, + retired_alias: Option, + ) -> (crate::lite::test_transport::Log, Option>) { + let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let log = session.log.clone(); let (tasks, _task_set) = TaskSet::new(); @@ -1380,7 +1507,7 @@ mod tests { peer_setup.clone(), crate::Hop::new(1).unwrap(), None, - VERSION, + version, tasks, Default::default(), ); @@ -1390,13 +1517,26 @@ mod tests { // Held so the trigger side stays alive for as long as the loop runs. let (_goaway, goaway) = crate::goaway::Handle::new(false); - // The peer's SETUP has not arrived yet, which is the state a group stream racing - // ahead of it lands in. - let mut unis = std::pin::pin!(run_unis(session, subscriber, Some(peer_setup), false, VERSION, goaway)); + // Without `accepted`, the peer's SETUP has not arrived yet, which is the state a + // group stream racing ahead of it lands in. + let read = accepted.is_some(); + let early = accepted.map(|accepted| accepted.early).unwrap_or_default(); + let mut unis = std::pin::pin!(run_unis( + session, + subscriber, + UniSetup { + peer: Some(peer_setup), + read, + early, + version, + goaway, + local_close: Default::default() + } + )); for _ in 0..100 { if let std::task::Poll::Ready(result) = futures::poll!(unis.as_mut()) { - panic!("the dispatch loop ended over one rejected stream: {result:?}"); + return (log, Some(result)); } if !log.stops().is_empty() { break; @@ -1404,36 +1544,118 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(1)).await; } - log + (log, None) } /// A late group must reach the dispatch loop and stop with CANCELLED. #[tokio::test(start_paused = true)] async fn a_group_for_a_retired_alias_is_stopped_with_cancelled() { - let log = dispatch_uni(subgroup_header(Version::Draft19, 7, 0).await, Some(7)).await; + let (log, result) = + dispatch_uni(Version::Draft19, subgroup_header(Version::Draft19, 7, 0).await, Some(7)).await; assert_eq!( log.stops(), vec![crate::ietf::error::CANCELLED], "the group stream must be stopped with the cancelled code", ); + assert!(result.is_none(), "one dropped group ended the session: {result:?}"); assert_eq!(log.closes(), vec![], "one dropped group may not close the session"); } /// A non-zero subgroup is refused on its own stream, never by closing the session. #[tokio::test(start_paused = true)] async fn a_non_zero_subgroup_is_stopped_without_closing_the_session() { - let log = dispatch_uni(subgroup_header(Version::Draft19, 7, 1).await, None).await; + let (log, result) = dispatch_uni(Version::Draft19, subgroup_header(Version::Draft19, 7, 1).await, None).await; assert_eq!(log.stops(), vec![crate::ietf::error::INTERNAL_ERROR]); + assert!(result.is_none(), "one refused subgroup ended the session: {result:?}"); assert_eq!(log.closes(), vec![], "one refused subgroup may not close the session"); } + /// A stream type encoded for `version`, followed by a few bytes of body. + fn uni_stream(version: Version, kind: u64) -> Vec { + let mut buf = Vec::new(); + kind.encode(&mut buf, version).unwrap(); + buf.extend_from_slice(&[0; 4]); + buf + } + + /// Padding is read and dropped: no STOP_SENDING, and the session stays up. #[tokio::test(start_paused = true)] - async fn unknown_uni_type_does_not_claim_the_session_closed() { - let log = dispatch_uni(vec![0], None).await; - assert_eq!(log.stops(), vec![crate::ietf::error::INTERNAL_ERROR]); - assert!(log.closes().is_empty()); + async fn a_padding_stream_is_discarded() { + for version in [ + Version::Draft18, + Version::Draft19, + Version::Draft20, + Version::Draft21, + Version::Draft22, + ] { + let (log, result) = dispatch_uni(version, uni_stream(version, PADDING), None).await; + + assert!( + log.stops().is_empty(), + "{version:?}: padding was stopped: {:?}", + log.stops() + ); + assert!(result.is_none(), "{version:?}: padding ended the session: {result:?}"); + assert_eq!(log.closes(), vec![], "{version:?}"); + } + } + + /// A server waiting on the client's SETUP holds the padding and group streams that beat + /// it, then classifies them once the session starts. The drafts say to buffer early + /// data; rejecting it reached the wire as INTERNAL_ERROR. + #[tokio::test(start_paused = true)] + async fn uni_streams_before_setup_are_held_until_it_lands() { + const VERSION: Version = Version::Draft19; + + let mut setup = Vec::new(); + setup::Setup { + parameters: ietf::Parameters::default().encode_bytes(VERSION).unwrap(), + } + .encode(&mut setup, crate::Version::Ietf(VERSION)) + .unwrap(); + + let mut session = crate::lite::test_transport::ScriptedSession::new(Vec::new()).with_incoming_unis(vec![ + uni_stream(VERSION, PADDING), + subgroup_header(VERSION, 7, 0).await, + setup, + ]); + let accepted = accept_setup(&mut session, VERSION).await.expect("accept the SETUP"); + assert_eq!(accepted.early.len(), 2, "both early streams are held"); + assert!(session.log.stops().is_empty(), "stopped {:?}", session.log.stops()); + + // The group answers a retired subscription, so the classifier reaching it shows + // as CANCELLED; the padding is read to its end and stops nothing. + let (log, result) = drive_unis(VERSION, session, Some(accepted), Some(7)).await; + assert_eq!(log.stops(), vec![crate::ietf::error::CANCELLED]); + assert!(result.is_none(), "an early stream ended the session: {result:?}"); + assert_eq!(log.closes(), vec![]); + } + + /// An unknown or invalid stream type MUST close the session, so it stops nothing on its own: + /// the session close takes the stream with it. + #[tokio::test(start_paused = true)] + async fn an_unknown_uni_type_closes_the_session() { + for (version, kind) in [ + (Version::Draft19, 0), + // Padding and uni SETUP arrived in later drafts, so earlier ones do not know them. + (Version::Draft17, PADDING), + (Version::Draft16, setup::SETUP_V17), + // SUBGROUP_HEADER types with the reserved SUBGROUP_ID_MODE (0b11). + (Version::Draft19, 0x56), + (Version::Draft14, 0x16), + // FIRST_OBJECT (0x40) arrived in draft-18. + (Version::Draft17, 0x50), + ] { + let (log, result) = dispatch_uni(version, uni_stream(version, kind), None).await; + + let Some(Err(err)) = result else { + panic!("{version:?}: type {kind:#x} did not end the session: {result:?}"); + }; + assert_eq!(SessionError::from(&err), SessionError::ProtocolViolation, "{version:?}"); + assert!(log.stops().is_empty(), "{version:?}: stopped {:?}", log.stops()); + } } /// A peer's advertisement of `room/host`, then two namespace-keyed withdrawals of it. @@ -1518,6 +1740,7 @@ mod tests { authority: None, peer_setup_stream: None, peer_declared: None, + early_unis: Vec::new(), }) .expect("start the session"); let driver = tokio::spawn(driver); diff --git a/rs/moq-net/src/ietf/subscribe.rs b/rs/moq-net/src/ietf/subscribe.rs index dd9c1e8b829..1cacc147667 100644 --- a/rs/moq-net/src/ietf/subscribe.rs +++ b/rs/moq-net/src/ietf/subscribe.rs @@ -5,7 +5,7 @@ use std::borrow::Cow; use crate::{ Path, coding::*, - ietf::{Fill, Filter, GroupOrder, Location, Param, Parameters, Properties, RequestId}, + ietf::{Fill, Filter, GroupOrder, Location, Opaque, Parameters, Properties, RequestId}, }; use super::Message; @@ -13,31 +13,6 @@ use super::namespace::{decode_namespace, encode_namespace}; use super::Version; -/// The INCLUDE_PROPERTIES parameter (0x35), draft-20's opt-out from Track Properties. -/// -/// Length prefixed despite holding a single byte. The Key-Value-Pair rule keys the framing -/// off the parameter id's parity and 0x35 is odd, so a Length is present even though the -/// parameter's own section calls the value a uint8. Parity is what the generic parser uses -/// to skip a parameter it does not know, so following the prose instead would desync every -/// parameter after this one. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct IncludeProperties(pub bool); - -impl Param for IncludeProperties { - fn param_encode(&self, w: &mut W, version: Version) -> Result<(), EncodeError> { - vec![u8::from(self.0)].encode(w, version) - } - - fn param_decode(r: &mut R, version: Version) -> Result { - match Vec::::decode(r, version)?[..] { - // The draft allows exactly 0 or 1; anything else is a protocol violation. - [0] => Ok(Self(false)), - [1] => Ok(Self(true)), - _ => Err(DecodeError::InvalidValue), - } - } -} - /// Subscribe message (0x03) /// Sent by the subscriber to request all future objects for the given track. #[derive(Clone, Debug)] @@ -53,6 +28,26 @@ pub struct Subscribe<'a> { pub fill: Option, /// Whether the subscriber wants Track Properties on the response (draft-20). pub properties_wanted: bool, + /// Whether Objects are forwarded (FORWARD). We only serve forwarding subscriptions, + /// so a subscriber that asks for none is refused. + pub forward: bool, + /// Whether the request carried a Range Filter (0x25-0x28). We advertise no + /// MAX_FILTER_RANGES, so the request is refused rather than served unfiltered. + /// Never encoded; we send no range filters. + pub range_filters: bool, +} + +/// NEW_GROUP_REQUEST (0x32) arrived in draft-16. +fn has_new_group_request(version: Version) -> bool { + !matches!(version, Version::Draft14 | Version::Draft15) +} + +/// The Range Filters (0x25-0x29) arrived in draft-19. +pub(super) fn has_range_filters(version: Version) -> bool { + !matches!( + version, + Version::Draft14 | Version::Draft15 | Version::Draft16 | Version::Draft17 | Version::Draft18 + ) } impl Message for Subscribe<'_> { @@ -72,13 +67,9 @@ impl Message for Subscribe<'_> { let group_order = GroupOrder::decode(r, version)?; let forward = bool::decode(r, version)?; - if !forward { - return Err(DecodeError::Unsupported); - } - let filter = Filter::decode(r, version)?; - let _params = Parameters::decode(r, version)?; + Parameters::skip(r, version)?; Ok(Self { request_id, @@ -89,11 +80,17 @@ impl Message for Subscribe<'_> { filter, fill: None, properties_wanted: true, + forward, + range_filters: false, }) } _ => { + // The token is ignored: the session's grant is what authorizes the request. + // NEW_GROUP_REQUEST is ignored too, as the draft lets a publisher without + // dynamic groups do. decode_params!(r, version, 0x02 => _object_delivery_timeout: Option, + 0x03 => _authorization_token: Vec, 0x04 => rendezvous_timeout: Option, 0x06 => _subgroup_delivery_timeout: Option, 0x10 => forward: Option, @@ -101,18 +98,34 @@ impl Message for Subscribe<'_> { 0x21 => filter: Option, 0x22 => group_order: Option, 0x23 => fill: Option, - 0x35 => include_properties: Option, + 0x25 => subgroup_filter: Vec, + 0x26 => object_id_filter: Vec, + 0x27 => priority_filter: Vec, + 0x28 => object_property_filter: Vec, + 0x32 => new_group_request: Option, + 0x35 => include_properties: Option, ); - // FILL_PARAMETERS and INCLUDE_PROPERTIES are draft-20 additions. An unknown - // message parameter is a protocol violation, so they stay rejected on the - // drafts that predate them rather than being quietly tolerated. - if (fill.is_some() || include_properties.is_some()) && !Filter::is_draft20(version) { + let range_filters = [ + subgroup_filter, + object_id_filter, + priority_filter, + object_property_filter, + ] + .iter() + .any(|filter| !filter.is_empty()); + + // An unknown message parameter is a protocol violation, so each stays rejected + // on the drafts that predate it rather than being quietly tolerated. + if ((fill.is_some() || include_properties.is_some()) && !Filter::is_draft20(version)) + || (range_filters && !has_range_filters(version)) + || (new_group_request.is_some() && !has_new_group_request(version)) + { return Err(DecodeError::InvalidValue); } // Defaults to 1, so an absent parameter means the subscriber wants them. - let properties_wanted = include_properties.is_none_or(|p| p.0); + let properties_wanted = include_properties.unwrap_or(true); // RENDEZVOUS_TIMEOUT arrived in draft-17; 0x04 means MAX_CACHE_DURATION in // draft-15, which is a publisher parameter with no business in a SUBSCRIBE. @@ -126,10 +139,6 @@ impl Message for Subscribe<'_> { // We still have to parse it, or the parameter alone would kill the session. let _ = rendezvous_timeout; - if forward == Some(false) { - return Err(DecodeError::Unsupported); - } - let subscriber_priority = subscriber_priority.unwrap_or(128); let group_order = group_order.unwrap_or(GroupOrder::Descending); // An absent LOCATION_FILTER means the subscription is unfiltered. @@ -144,6 +153,9 @@ impl Message for Subscribe<'_> { filter, fill, properties_wanted, + // Absent means 1. + forward: forward.unwrap_or(true), + range_filters, }) } } @@ -161,7 +173,7 @@ impl Message for Subscribe<'_> { Version::Draft14 => { self.subscriber_priority.encode(w, version)?; self.group_order.encode(w, version)?; - true.encode(w, version)?; // forward + self.forward.encode(w, version)?; self.filter.encode(w, version)?; 0u8.encode(w, version)?; // no parameters @@ -175,11 +187,10 @@ impl Message for Subscribe<'_> { // INCLUDE_PROPERTIES defaults to 1, so only the opt-out is worth bytes. It // arrived in draft-20, and an older peer would read it as an unknown // parameter, which is a protocol violation. - let include_properties = - (!self.properties_wanted && Filter::is_draft20(version)).then_some(IncludeProperties(false)); + let include_properties = (!self.properties_wanted && Filter::is_draft20(version)).then_some(false); encode_params!(w, version, - 0x10 => true, + 0x10 => self.forward, 0x20 => self.subscriber_priority, 0x21 => self.filter, 0x22 => self.group_order, @@ -279,10 +290,7 @@ impl Message for SubscribeOk { largest = Some(Location::decode(r, version)?); } - let params = Parameters::decode(r, version)?; - properties.max_cache_duration = params - .get_varint(super::ParameterVarInt::from(0x04)) - .map(std::time::Duration::from_millis); + properties.max_cache_duration = Parameters::skip(r, version)?.map(std::time::Duration::from_millis); } _ => { // GROUP_ORDER is only legal here through draft-15, but keep accepting it so a @@ -439,7 +447,7 @@ impl Message for SubscribeUpdate { let end_group = u64::decode(r, version)?; let subscriber_priority = u8::decode(r, version)?; let forward = bool::decode(r, version)?; - let _parameters = Parameters::decode(r, version)?; + Parameters::skip(r, version)?; Ok(Self { request_id, @@ -455,12 +463,19 @@ impl Message for SubscribeUpdate { let subscription_request_id = Some(RequestId::decode(r, version)?); decode_params!(r, version, 0x02 => _object_delivery_timeout: Option, + 0x03 => _authorization_token: Vec, 0x06 => _subgroup_delivery_timeout: Option, 0x10 => forward: Option, 0x20 => subscriber_priority: Option, 0x21 => _filter: Option, + 0x32 => new_group_request: Option, ); + // NEW_GROUP_REQUEST arrived in draft-16. + if new_group_request.is_some() && !has_new_group_request(version) { + return Err(DecodeError::InvalidValue); + } + let subscriber_priority = subscriber_priority.unwrap_or(128); let forward = forward.unwrap_or(true); @@ -479,18 +494,38 @@ impl Message for SubscribeUpdate { if matches!(version, Version::Draft17) { let _required_request_id_delta = u64::decode(r, version)?; } + // Nothing reads an update's Range Filters yet; they are consumed so a legal + // update does not fail the session. TRACK_PROPERTY_FILTER (0x29) is legal + // only on an update to SUBSCRIBE_TRACKS, which the message alone can't tell. decode_params!(r, version, 0x02 => _object_delivery_timeout: Option, + 0x03 => _authorization_token: Vec, 0x06 => _subgroup_delivery_timeout: Option, 0x10 => forward: Option, 0x20 => subscriber_priority: Option, 0x21 => _filter: Option, 0x23 => fill: Option, + 0x25 => subgroup_filter: Vec, + 0x26 => object_id_filter: Vec, + 0x27 => priority_filter: Vec, + 0x28 => object_property_filter: Vec, + 0x29 => track_property_filter: Vec, + 0x32 => _new_group_request: Option, ); - // FILL_PARAMETERS is a draft-20 addition, so an earlier peer sending one is - // still the protocol violation it was. - if fill.is_some() && !Filter::is_draft20(version) { + let range_filters = [ + subgroup_filter, + object_id_filter, + priority_filter, + object_property_filter, + track_property_filter, + ] + .iter() + .any(|filter| !filter.is_empty()); + + // FILL_PARAMETERS and the Range Filters postdate draft-17, so an earlier peer + // sending one is still the protocol violation it was. + if (fill.is_some() && !Filter::is_draft20(version)) || (range_filters && !has_range_filters(version)) { return Err(DecodeError::InvalidValue); } @@ -537,6 +572,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: true, + forward: true, + range_filters: false, }; let encoded = encode_message(&msg, Version::Draft14); @@ -559,6 +596,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: true, + forward: true, + range_filters: false, }; let encoded = encode_message(&msg, Version::Draft15); @@ -649,6 +688,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: true, + forward: true, + range_filters: false, }; for version in [Version::Draft17, Version::Draft18, Version::Draft19, Version::Draft20] { @@ -678,6 +719,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: true, + forward: true, + range_filters: false, }; let encoded = encode_message(&msg, Version::Draft14); @@ -861,6 +904,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: wanted, + forward: true, + range_filters: false, }; let encoded = encode_message(&msg, version); @@ -983,6 +1028,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: true, + forward: true, + range_filters: false, }; let encoded = encode_message(&msg, Version::Draft17); @@ -1041,6 +1088,8 @@ mod tests { filter: Filter::NextObject, fill: None, properties_wanted: true, + forward: true, + range_filters: false, }; let encoded = encode_message(&msg, Version::Draft18); @@ -1251,6 +1300,170 @@ mod tests { let v18 = encode_message(&v18_msg, Version::Draft18); assert_eq!(v17.len(), v18.len() + 1); } + + /// The head of a SUBSCRIBE (draft-20 Figure 10) up to its Number of Parameters: + /// Request ID 1, Track Namespace ("live"), Track Name ("video"). Every value is + /// below 128, so each varint is one byte in the drafts before and after 17 alike. + const SUBSCRIBE_HEAD: &[u8] = &[ + 0x01, 0x01, 0x04, b'l', b'i', b'v', b'e', 0x05, b'v', b'i', b'd', b'e', b'o', + ]; + + fn subscribe_body(params: &[u8]) -> Vec { + [SUBSCRIBE_HEAD, params].concat() + } + + /// Every parameter draft-20 lets a SUBSCRIBE carry that we don't act on still decodes, + /// with what we have to refuse recorded rather than failing the session. + #[test] + fn subscribe_decodes_every_draft20_parameter() { + #[rustfmt::skip] + let body = subscribe_body(&[ + 0x06, // Number of Parameters + 0x03, 0x03, 0x03, 0x00, 0xAA, // AUTHORIZATION TOKEN: USE_VALUE, type 0, one byte + 0x00, 0x03, 0x03, 0x00, 0xBB, // a second one, which the draft allows + 0x0D, 0x00, // FORWARD (0x10) = 0, a uint8 + 0x15, 0x03, 0x00, 0x00, 0x01, // SUBGROUP_FILTER (0x25): SetID 0, range 0-1 + 0x0D, 0x07, // NEW_GROUP_REQUEST (0x32), a varint + 0x03, 0x00, // INCLUDE_PROPERTIES (0x35) = 0, a uint8 + ]); + + for version in [Version::Draft20, Version::Draft21, Version::Draft22] { + let mut buf = bytes::Bytes::from(body.clone()); + let msg = Subscribe::decode_msg(&mut buf, version).unwrap_or_else(|e| panic!("{version}: {e}")); + assert!(buf.is_empty(), "{version}: trailing bytes"); + assert!(!msg.forward, "{version}"); + assert!(msg.range_filters, "{version}"); + assert!(!msg.properties_wanted, "{version}"); + } + } + + /// INCLUDE_PROPERTIES is a uint8, one raw byte with no Length. Reading a Length would + /// swallow the parameter after it, or fail on the value. + #[test] + fn include_properties_is_a_uint8() { + let msg = Subscribe { + request_id: RequestId(1), + track_namespace: crate::Path::new("live"), + track_name: "video".into(), + subscriber_priority: 128, + group_order: GroupOrder::Descending, + filter: Filter::Unfiltered, + fill: None, + properties_wanted: false, + forward: true, + range_filters: false, + }; + + #[rustfmt::skip] + let expected = subscribe_body(&[ + 0x04, // Number of Parameters + 0x10, 0x01, // FORWARD = 1 + 0x10, 0x80, // SUBSCRIBER_PRIORITY (0x20) = 128, a raw byte + 0x02, 0x02, // GROUP_ORDER (0x22) = Descending + 0x13, 0x00, // INCLUDE_PROPERTIES (0x35) = 0 + ]); + assert_eq!(encode_message(&msg, Version::Draft20), expected); + + // Anything but 0 or 1 is a protocol violation. + let body = subscribe_body(&[0x01, 0x35, 0x02]); + assert!(decode_message::(&body, Version::Draft20).is_err()); + } + + /// FORWARD=0 is legal on every draft. It decodes so the request can be refused, + /// rather than closing the session. + #[test] + fn subscribe_decodes_forward_zero() { + for version in [ + Version::Draft14, + Version::Draft15, + Version::Draft16, + Version::Draft17, + Version::Draft18, + Version::Draft19, + Version::Draft20, + ] { + let msg = Subscribe { + request_id: RequestId(1), + track_namespace: crate::Path::new("live"), + track_name: "video".into(), + subscriber_priority: 128, + group_order: GroupOrder::Descending, + filter: Filter::NextObject, + fill: None, + properties_wanted: true, + forward: false, + range_filters: false, + }; + let decoded: Subscribe = decode_message(&encode_message(&msg, version), version).unwrap(); + assert!(!decoded.forward, "{version}"); + } + } + + /// AUTHORIZATION TOKEN is a Key-Value-Pair with an odd type through draft-16, so its + /// value is length prefixed there too. + #[test] + fn subscribe_accepts_a_token_on_every_draft() { + #[rustfmt::skip] + let body = subscribe_body(&[ + 0x01, // Number of Parameters + 0x03, 0x03, 0x03, 0x00, 0xAA, // AUTHORIZATION TOKEN + ]); + for version in [ + Version::Draft15, + Version::Draft16, + Version::Draft17, + Version::Draft18, + Version::Draft19, + Version::Draft20, + ] { + let mut body = body.clone(); + if version == Version::Draft17 { + // Required Request ID Delta, draft-17 only. + body.insert(1, 0x00); + } + let mut buf = bytes::Bytes::from(body); + Subscribe::decode_msg(&mut buf, version).unwrap_or_else(|e| panic!("{version}: {e}")); + assert!(buf.is_empty(), "{version}: trailing bytes"); + } + } + + /// A parameter from a later draft is still an unknown parameter, which the draft makes + /// a protocol violation. + #[test] + fn subscribe_refuses_parameters_before_their_draft() { + for (version, params) in [ + // NEW_GROUP_REQUEST arrived in draft-16. + (Version::Draft15, &[0x01, 0x32, 0x00][..]), + // The Range Filters arrived in draft-19. + (Version::Draft18, &[0x01, 0x25, 0x00][..]), + // INCLUDE_PROPERTIES arrived in draft-20. + (Version::Draft19, &[0x01, 0x35, 0x00][..]), + ] { + let body = subscribe_body(params); + assert!( + decode_message::(&body, version).is_err(), + "{version}: {params:x?}" + ); + } + } + + /// REQUEST_UPDATE (draft-20 Figure 12) carries a token, NEW_GROUP_REQUEST and the + /// Range Filters, TRACK_PROPERTY_FILTER included, since the update may be for a + /// SUBSCRIBE_TRACKS. + #[test] + fn request_update_decodes_every_draft20_parameter() { + #[rustfmt::skip] + let body = [ + 0x02, // Request ID + 0x04, // Number of Parameters + 0x03, 0x03, 0x03, 0x00, 0xAA, // AUTHORIZATION TOKEN + 0x22, 0x00, // SUBGROUP_FILTER (0x25), empty: remove the filter + 0x04, 0x04, 0x00, 0x02, 0x00, 0x01, // TRACK_PROPERTY_FILTER (0x29): SetID 0, type 2, range 0-1 + 0x09, 0x00, // NEW_GROUP_REQUEST (0x32) + ]; + let decoded: SubscribeUpdate = decode_message(&body, Version::Draft20).unwrap(); + assert_eq!(decoded.request_id, RequestId(2)); + } } #[cfg(test)] diff --git a/rs/moq-net/src/ietf/subscribe_namespace.rs b/rs/moq-net/src/ietf/subscribe_namespace.rs index bf57ee0bb79..252ba94b6be 100644 --- a/rs/moq-net/src/ietf/subscribe_namespace.rs +++ b/rs/moq-net/src/ietf/subscribe_namespace.rs @@ -15,9 +15,7 @@ use super::Version; /// /// moq-lite does not implement PUBLISH replication through a CDN, which is the /// only thing that SUBSCRIBE_TRACKS enables (subscribing to all tracks under a -/// prefix). If a peer sends this we fail the session loudly rather than -/// silently ignoring it, since ignoring would leave the peer waiting forever -/// for a REQUEST_OK. +/// prefix), so each one is refused with REQUEST_ERROR NOT_SUPPORTED. pub const SUBSCRIBE_TRACKS_ID: u64 = 0x51; /// True for the drafts that use the legacy 0x11 SUBSCRIBE_NAMESPACE message. @@ -76,7 +74,11 @@ impl Message for SubscribeNamespace<'_> { } let request_id = RequestId::decode(r, version)?; let namespace = decode_namespace(r, version)?; - decode_params!(r, version, HIDDEN_PARAM => hidden: Option); + // The token is ignored: the session's grant is what authorizes the request. + decode_params!(r, version, + 0x03 => _authorization_token: Vec, + HIDDEN_PARAM => hidden: Option, + ); Ok(Self { request_id, @@ -135,7 +137,11 @@ impl Message for SubscribeNamespaceLegacy<'_> { _ => 0x01, }; - decode_params!(r, version, HIDDEN_PARAM => hidden: Option); + // The token is ignored: the session's grant is what authorizes the request. + decode_params!(r, version, + 0x03 => _authorization_token: Vec, + HIDDEN_PARAM => hidden: Option, + ); Ok(Self { request_id, @@ -241,7 +247,7 @@ impl Namespace<'_> { // The base form has no Parameters field at all, so there is nothing to read // (and nothing to reject) unless the extension is on. let cluster = match negotiated { - true => super::publish_namespace::decode_cluster_params(r, version, true)?, + true => Some(super::publish_namespace::decode_cluster_params(r, version)?), false => None, }; diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index 6165ea47a5f..f5cdd1c3bdc 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -165,11 +165,21 @@ impl State { if let Some(request) = track.pending.take() { request.reject(err.clone()); } + if let Some(producer) = track.producer { + let _ = producer.abort_session(err.clone()); + } if let Fill::Ready { producer, .. } = &*track.fill.read() { let _ = producer.clone().abort(err.clone()); } + } + } + fn close(&mut self) { + for (_, mut track) in self.subscribes.drain() { + if let Some(request) = track.pending.take() { + request.reject(Error::Cancel); + } if let Some(producer) = track.producer { - let _ = producer.abort(err.clone()); + let _ = producer.close(); } } } @@ -483,6 +493,8 @@ pub(super) struct Subscriber { // Set once the peer sends a GOAWAY; new SUBSCRIBEs are then rejected with // Error::GoingAway (the peer told us to stop opening streams). going_away: crate::goaway::GoingAway, + // What this session may allocate up front for objects still arriving. + frames: frame::Budget, } /// The prefixes to issue SUBSCRIBE_NAMESPACE for: `origin`'s permitted scope, @@ -589,6 +601,7 @@ where tasks, version, going_away, + frames: Default::default(), } } @@ -597,6 +610,10 @@ where self.state.lock().abort(err); } + pub fn close(&self) { + self.state.lock().close(); + } + /// Leave `alias` in the state a cancelled subscription leaves behind: bound to a /// subscription, then retired. /// @@ -2097,6 +2114,8 @@ where filter: join.filter, fill: join.fill, properties_wanted: true, + forward: true, + range_filters: false, }) .await?; Ok(()) @@ -2165,6 +2184,8 @@ where ), group_order: GroupOrder::Ascending, fetch_type, + range_filters: false, + fill_timeout: false, }) .await?; Ok::<(), Error>(()) @@ -2371,7 +2392,7 @@ where let producer = crate::recv::Group::new(producer); let res = { - let mut ingest = GroupIngest::new(self.runtime.clone(), &group, timescale, self.version, start); + let mut ingest = GroupIngest::new(self, &group, timescale, start); let mut writing = producer.clone(); kio::wait(|waiter| { if let Poll::Ready(err) = track.poll_closed(waiter) { @@ -2707,6 +2728,7 @@ struct GroupIngest { prior_object: Option, start: u64, phase: IngestPhase, + budget: frame::Budget, } enum IngestPhase { @@ -2727,22 +2749,22 @@ enum IngestPhase { } impl GroupIngest { - fn new( - runtime: crate::time::Clock, + fn new( + subscriber: &Subscriber, group: &ietf::GroupHeader, timescale: Option, - version: Version, start: u64, ) -> Self { Self { - runtime, + runtime: subscriber.runtime.clone(), has_extensions: group.flags.has_extensions, has_end: group.flags.has_end, timescale, - version, + version: subscriber.version, prior_object: None, start, phase: IngestPhase::Delta, + budget: subscriber.frames.clone(), } } } @@ -3034,9 +3056,9 @@ where } } - // `create_frame_owned` is the allocation chokepoint and rejects an oversized `size` - // before allocating, so no pre-check is needed. - let mut frame = producer.create_frame_owned(frame::Info { size, timestamp })?; + // `create_frame_owned` is the allocation chokepoint: it rejects an oversized `size` + // and allocates up front only within the budget, so no pre-check is needed. + let mut frame = producer.create_frame_owned(frame::Info { size, timestamp }, &self.frames)?; if let Err(err) = std::future::poll_fn(|cx| stream.poll_read_frame(cx, &mut frame)).await { let _ = frame.abort(err.clone()); return Err(err); @@ -3106,6 +3128,8 @@ where object: 0, }, }, + range_filters: false, + fill_timeout: false, }) .await?; self.read_group_fetch_response(&mut stream).await @@ -3543,17 +3567,18 @@ impl GroupIngest { self.phase = IngestPhase::Status { timestamp: *timestamp }; continue; } - // `create_frame_owned` is the allocation chokepoint and rejects an - // oversized `size` before allocating, so no pre-check is needed. + // `create_frame_owned` is the allocation chokepoint: it rejects an + // oversized `size` and allocates up front only within the budget, so + // no pre-check is needed. let timestamp = timestamp.unwrap_or_else(|| crate::Timestamp::from(self.runtime.now())); - let frame = group.create_frame_owned(frame::Info { size, timestamp })?; + let frame = group.create_frame_owned(frame::Info { size, timestamp }, &self.budget)?; self.phase = IngestPhase::Payload { frame }; } IngestPhase::Status { timestamp } => { let status: u64 = ready!(reader.poll_decode(&mut cx))?; if status == 0 { let timestamp = timestamp.unwrap_or_else(|| crate::Timestamp::from(self.runtime.now())); - let frame = group.create_frame_owned(frame::Info { size: 0, timestamp })?; + let frame = group.create_frame_owned(frame::Info { size: 0, timestamp }, &self.budget)?; frame.finish()?; self.phase = IngestPhase::Delta; } else if status == END_OF_GROUP && !self.has_end { @@ -7010,6 +7035,34 @@ mod stitch_tests { ); } + #[tokio::test] + async fn session_death_keeps_a_waiting_fill_heads_resume_position() { + let h = Harness::new( + Fill::Serving(Some(Timescale::MICRO)), + vec![fill_stream(SEQUENCE, &[b"head-0"])], + ); + let track = h.track.consume(); + let mut consumer = h.track.subscribe(None); + let mut fill = h.stream().await; + h.subscriber.clone().recv_fill(&mut fill).await.expect("fill"); + let mut group = consumer.recv_group().await.unwrap().expect("the fill head arrived"); + let resume = track.resume_position(); + assert_eq!( + resume, + Some(track::Position { + group: SEQUENCE, + frame: 1 + }) + ); + let err = Error::Session(crate::SessionError::App(7)); + h.subscriber.abort(&err); + assert_eq!(track.resume_position(), resume); + assert!(matches!( + group.read_frame().await, + Err(Error::Session(crate::SessionError::App(7))) + )); + } + /// The same contradiction as above, with the streams the other way round: the whole /// group lands before the fill has written its head. The model holds one producer per /// live sequence, so the fill loses the race to create it and gives up, rather than a diff --git a/rs/moq-net/src/ietf/track.rs b/rs/moq-net/src/ietf/track.rs index 2c600c331cf..00793702c4d 100644 --- a/rs/moq-net/src/ietf/track.rs +++ b/rs/moq-net/src/ietf/track.rs @@ -7,11 +7,11 @@ use num_enum::{IntoPrimitive, TryFromPrimitive}; use crate::{ Path, coding::*, - ietf::{Filter, GroupOrder, Parameters, RequestId}, + ietf::{Filter, GroupOrder, RequestId, Subscribe}, }; use super::Message; -use super::namespace::{decode_namespace, encode_namespace}; +use super::namespace::encode_namespace; use super::Version; @@ -51,31 +51,14 @@ impl Message for TrackStatus<'_> { Ok(()) } + /// Every draft defines TRACK_STATUS as identical to SUBSCRIBE, so it decodes as one and + /// keeps only what names the track. We refuse the request, so the rest goes unread. fn decode_msg(r: &mut R, version: Version) -> Result { - let request_id = RequestId::decode(r, version)?; - if version == Version::Draft17 { - let _required_request_id_delta = u64::decode(r, version)?; - } - let track_namespace = decode_namespace(r, version)?; - let track_name = Cow::::decode(r, version)?; - - match version { - Version::Draft14 => { - let _subscriber_priority = u8::decode(r, version)?; - let _group_order = GroupOrder::decode(r, version)?; - let _forward = bool::decode(r, version)?; - let _filter_type = u64::decode(r, version)?; - let _params = Parameters::decode(r, version)?; - } - _ => { - decode_params!(r, version,); - } - } - + let subscribe = Subscribe::decode_msg(r, version)?; Ok(Self { - request_id, - track_namespace, - track_name, + request_id: subscribe.request_id, + track_namespace: subscribe.track_namespace, + track_name: subscribe.track_name, }) } } @@ -197,4 +180,44 @@ mod tests { assert_eq!(decoded.track_namespace.as_str(), "test/ns"); assert_eq!(decoded.track_name, "video"); } + + /// TRACK_STATUS is identical to SUBSCRIBE on every draft, so it carries whatever + /// fields and parameters a SUBSCRIBE can. A peer that sends them must still reach + /// our refusal rather than having its session closed. + #[test] + fn test_track_status_carries_subscribe_fields() { + // Request ID 1, Track Namespace ("live"), Track Name ("video"). + let head: &[u8] = &[ + 0x01, 0x01, 0x04, b'l', b'i', b'v', b'e', 0x05, b'v', b'i', b'd', b'e', b'o', + ]; + + #[rustfmt::skip] + let cases: [(Version, &[u8]); 3] = [ + (Version::Draft14, &[ + 0x80, // Subscriber Priority + 0x02, // Group Order + 0x00, // Forward + 0x03, 0x05, 0x01, // Filter Type AbsoluteStart, at {5, 1} + 0x00, // Number of Parameters + ]), + (Version::Draft15, &[ + 0x02, // Number of Parameters + 0x10, 0x00, // FORWARD = 0 + 0x20, 0x01, // SUBSCRIBER_PRIORITY = 1 + ]), + (Version::Draft20, &[ + 0x02, // Number of Parameters + 0x03, 0x03, 0x03, 0x00, 0xAA, // AUTHORIZATION TOKEN + 0x32, 0x00, // INCLUDE_PROPERTIES (0x35) = 0 + ]), + ]; + + for (version, rest) in cases { + let body = [head, rest].concat(); + let mut buf = bytes::Bytes::from(body); + let msg = TrackStatus::decode_msg(&mut buf, version).unwrap_or_else(|e| panic!("{version}: {e}")); + assert!(buf.is_empty(), "{version}: trailing bytes"); + assert_eq!(msg.track_name, "video", "{version}"); + } + } } diff --git a/rs/moq-net/src/ietf/version.rs b/rs/moq-net/src/ietf/version.rs index 34f68362af9..723ce095500 100644 --- a/rs/moq-net/src/ietf/version.rs +++ b/rs/moq-net/src/ietf/version.rs @@ -53,8 +53,8 @@ mod tests { use super::*; use crate::coding::Encode; use crate::ietf::{ - Fetch, FetchType, Fill, Filter, GroupFlags, GroupHeader, GroupOrder, Location, Message, Properties, Publish, - RequestId, Subscribe, SubscribeOk, + EndLocation, Fetch, FetchType, Fill, Filter, GroupFlags, GroupHeader, GroupOrder, Location, Message, + Properties, Publish, RequestId, Subscribe, SubscribeOk, }; use crate::{Path, Timescale}; @@ -95,6 +95,8 @@ mod tests { range_filters: false, }), properties_wanted: false, + forward: true, + range_filters: false, }; let subscribe_ok = SubscribeOk { @@ -119,12 +121,19 @@ mod tests { request_id: RequestId(3), subscriber_priority: 64, group_order: GroupOrder::Ascending, - fetch_type: FetchType::Standalone { + fetch_type: FetchType::Filtered { namespace: Path::new("broadcast"), track: "video".into(), - start: Location { group: 1, object: 0 }, - end: Location { group: 2, object: 0 }, + filter: Filter::Absolute { + start: Location { group: 1, object: 0 }, + end: Some(EndLocation { + group: 2, + object: Some(0), + }), + }, }, + range_filters: false, + fill_timeout: false, }; let group = GroupHeader { diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index f27f44bec95..64a767eefed 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -2,12 +2,12 @@ use crate::runtime::Timers as _; use crate::{SessionError, announce, frame, group, origin, track}; use std::{ collections::{BTreeSet, HashMap}, - ops::Bound, + ops::{Bound, ControlFlow}, sync::{ Arc, atomic::{AtomicU64, AtomicUsize, Ordering}, }, - task::{Poll, ready}, + task::{Context, Poll, ready}, time::Duration, }; @@ -15,7 +15,7 @@ use web_transport_trait::Stats; use crate::{ Error, Hop, Hops, - coding::{Encode, Stream, Writer}, + coding::{Decode, DecodeError, Encode, Stream, Writer}, lite::{ self, priority::{Priority, PriorityHandle, PriorityQueue}, @@ -243,9 +243,9 @@ enum ControlState { stream: Stream, }, Announce(AnnounceServe), - Subscribe(SubscribeServe), - Fetch(FetchServe), - TrackInfo(TrackInfoServe), + Subscribe(RequestServe>), + Fetch(RequestServe), + TrackInfo(RequestServe), Probe(ProbeServe), /// Decoding the peer's GOAWAY, surfaced through [`crate::Session::draining`]. Goaway { @@ -296,11 +296,17 @@ impl Control { ControlState::Announce(AnnounceServe::new(self.shared.clone(), stream)) } lite::ControlType::Subscribe => { - ControlState::Subscribe(SubscribeServe::new(self.shared.clone(), stream)) + ControlState::Subscribe(RequestServe::new(self.shared.clone(), stream)) } - lite::ControlType::Fetch => ControlState::Fetch(FetchServe::new(self.shared.clone(), stream)?), + // The Track Stream and FETCH are lite-05+ only. + lite::ControlType::Fetch | lite::ControlType::Track + if !self.shared.version.has_track_stream() => + { + return Poll::Ready(Err(Error::UnexpectedStream)); + } + lite::ControlType::Fetch => ControlState::Fetch(RequestServe::new(self.shared.clone(), stream)), lite::ControlType::Track => { - ControlState::TrackInfo(TrackInfoServe::new(self.shared.clone(), stream)?) + ControlState::TrackInfo(RequestServe::new(self.shared.clone(), stream)) } lite::ControlType::Probe => { ControlState::Probe(ProbeServe::new(self.shared.clone(), self.runtime.clone(), stream)) @@ -947,68 +953,119 @@ impl AnnounceRun { } } -/// Serves one TRACK stream: resolve the track, answer with its TRACK_INFO, FIN. -struct TrackInfoServe { +/// Whether an error means the stream was walked away from, rather than a fault worth a warning. +fn is_cancel(err: &Error) -> bool { + // TODO better classify WebTransport errors. + matches!( + err, + Error::Cancel + | Error::Stream(crate::StreamError::Cancel) + | Error::Session(crate::SessionError::Cancel) + | Error::Transport(_) + ) +} + +/// One kind of request stream (TRACK, SUBSCRIBE, FETCH), served by [`RequestServe`]. +trait Request: Sized { + /// Names the request in logs. + const KIND: &'static str; + /// Whether we answer the requester's FIN with our own. Otherwise we reset: the reply + /// would end short, and a FIN there would read as a whole one. + const GRACEFUL: bool; + /// The message that opens the stream. + type Message: Decode + std::fmt::Debug; + /// What the requester may send after the request; [`NoUpdate`] for nothing. + type Update: Decode + std::fmt::Debug; + + /// The broadcast and track the request names. + fn target(msg: &Self::Message) -> (&crate::Path<'static>, &str); + + /// Start answering, once the broadcast resolves. + fn start(shared: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result; + + /// Apply an update from the requester. + fn update(&mut self, update: Self::Update); + + /// Write the reply: `Continue` after each unit of work, so the requester is checked + /// between them even while work stays ready, and `Break` once all of it is buffered. + fn poll( + &mut self, + shared: &Shared, + writer: &mut Writer, + waiter: &kio::Waiter, + ) -> Poll, Error>>; +} + +/// The update of a request that takes none: any byte after the request is a violation. +#[derive(Debug)] +enum NoUpdate {} + +impl Decode for NoUpdate { + fn decode(_: &mut B, _: Version) -> Result { + Err(DecodeError::ExpectedEnd) + } +} + +/// Serves one request stream: decode the request, resolve its broadcast, drive the +/// kind's reply, then FIN. +/// +/// It owns the reader for the whole wait, so however the reply is waiting, the requester +/// closing its send direction (FIN or reset) ends it. An abandoned request then releases +/// its upstream lookup instead of pinning it for an answer nobody reads. Updates are not a +/// close: they reach the reply, held until it starts. +struct RequestServe> { shared: Arc>, + // Taken to abort. stream: Option>, - state: TrackInfoState, + state: RequestState, + // The latest update to arrive before the reply started. Each one replaces the last. + update: Option, // Log context, filled in after the decode. absolute: crate::PathOwned, track: String, } -enum TrackInfoState { +enum RequestState { Decode, - /// Resolving the split-horizon origin (waits on the peer's SETUP). - Hop { - msg: lite::Track<'static>, - }, - /// Resolving the broadcast (may wait on a dynamic handler). - Request { - msg: lite::Track<'static>, - requesting: origin::Requesting, - }, - /// Waiting for the track's info. - Query { - querying: track::Querying, + /// Resolving the split-horizon origin (waits on the peer's SETUP), then the + /// broadcast (may wait on a dynamic handler). + Resolve { + msg: M, + requesting: Option, }, - /// TRACK_INFO buffered: flush, FIN, and wait for the acknowledgement. + Serve(R), + /// FIN and wait for the acknowledgement. Finish { finished: bool, }, } -impl TrackInfoServe { - fn new(shared: Arc>, stream: Stream) -> Result { - // The Track Stream is lite-05+ only. - if !shared.version.has_track_stream() { - return Err(Error::UnexpectedStream); - } - Ok(Self { +impl> RequestServe { + fn new(shared: Arc>, stream: Stream) -> Self { + Self { shared, stream: Some(stream), - state: TrackInfoState::Decode, + state: RequestState::Decode, + update: None, absolute: Default::default(), track: Default::default(), - }) + } } fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { match ready!(self.poll_serve(waiter)) { - Ok(()) => Poll::Ready(Ok(())), - // A decode error propagates so the control loop logs it; anything past - // the decode is answered on the stream instead. - Err(err) if matches!(self.state, TrackInfoState::Decode) => Poll::Ready(Err(err)), + Ok(()) => { + tracing::info!(broadcast = %self.absolute, track = %self.track, "{} complete", R::KIND); + Poll::Ready(Ok(())) + } + // A decode error propagates so the control loop logs it; anything past the + // decode is answered on the stream instead. + Err(err) if matches!(self.state, RequestState::Decode) => Poll::Ready(Err(err)), Err(err) => { - match &err { - Error::Cancel - | Error::Stream(crate::StreamError::Cancel) - | Error::Session(crate::SessionError::Cancel) - | Error::Transport(_) => { - tracing::debug!(broadcast = %self.absolute, track = %self.track, "track info cancelled") - } - err => { - tracing::warn!(broadcast = %self.absolute, track = %self.track, %err, "track info error") + match is_cancel(&err) { + true => tracing::info!(broadcast = %self.absolute, track = %self.track, "{} cancelled", R::KIND), + false => { + tracing::warn!(broadcast = %self.absolute, track = %self.track, %err, "{} error", R::KIND) } } self.stream.take().expect("stream present").writer.abort(&err); @@ -1018,223 +1075,209 @@ impl TrackInfoServe { } fn poll_serve(&mut self, waiter: &kio::Waiter) -> Poll> { + let mut cx = waiter.context(); loop { - // While the lookup runs the requester sends nothing, so its send side closing, - // by FIN or reset, means it stopped waiting. Drop the lookup then, so the - // abandonment reaches our own upstream instead of pinning it for an answer - // nobody reads. (Once answered, its FIN is the normal end; `Finish` waits on it.) - if matches!( - self.state, - TrackInfoState::Hop { .. } | TrackInfoState::Request { .. } | TrackInfoState::Query { .. } - ) { - let stream = self.stream.as_mut().expect("stream present"); - let mut cx = waiter.context(); - if let Poll::Ready(res) = stream.reader.poll_closed(&mut cx) { - return Poll::Ready(Err(res.err().unwrap_or(Error::Cancel))); + // Once answered, the requester's FIN is the normal end, so `Finish` doesn't watch. + if matches!(self.state, RequestState::Resolve { .. } | RequestState::Serve(_)) + && let Poll::Ready(res) = self.poll_requester(&mut cx) + { + res?; + if !R::GRACEFUL { + return Poll::Ready(Err(Error::Cancel)); } + // Drop the reply mid-flight: anything half-sent is cancelled rather than + // completed for nobody. + self.state = RequestState::Finish { finished: false }; } + + let stream = self.stream.as_mut().expect("stream present"); match &mut self.state { - TrackInfoState::Decode => { - let stream = self.stream.as_mut().expect("stream present"); - let mut cx = waiter.context(); - let msg = ready!(stream.reader.poll_decode::(&mut cx))?; - self.absolute = self.shared.origin.absolute(&msg.broadcast).to_owned(); - self.track = msg.track.to_string(); - tracing::debug!(broadcast = %self.absolute, track = %self.track, "track info requested"); - self.state = TrackInfoState::Hop { msg }; + RequestState::Decode => { + let msg = ready!(stream.reader.poll_decode::(&mut cx))?; + let (broadcast, track) = R::target(&msg); + self.absolute = self.shared.origin.absolute(broadcast).to_owned(); + self.track = track.to_string(); + tracing::info!(broadcast = %self.absolute, track = %self.track, request = ?msg, "{} started", R::KIND); + self.state = RequestState::Resolve { msg, requesting: None }; } - TrackInfoState::Hop { .. } => { - // The peer requested this exact path, so it has already seen an - // announcement for it. `request_broadcast` resolves it immediately, or - // is served on demand by the route covering it (an `origin::Dynamic`). - let origin = ready!(self.shared.poll_serving_origin(waiter)); - let TrackInfoState::Hop { msg } = std::mem::replace(&mut self.state, TrackInfoState::Decode) else { + RequestState::Resolve { msg, requesting } => { + if requesting.is_none() { + // The peer requested this exact path, so it has already seen an + // announcement for it. `request_broadcast` resolves it immediately, is + // served on demand by the route covering it (an `origin::Dynamic`), or + // errors when there is none. + let origin = ready!(self.shared.poll_serving_origin(waiter)); + *requesting = Some(origin.request_broadcast(R::target(msg).0).into_inner()); + } + let broadcast = ready!(requesting.as_ref().expect("requesting").poll_ok(waiter))?; + // Any placeholder but `Decode`, so a failed start is answered on the stream. + let RequestState::Resolve { msg, .. } = + std::mem::replace(&mut self.state, RequestState::Finish { finished: false }) + else { unreachable!() }; - let requesting = origin.request_broadcast(&msg.broadcast).into_inner(); - self.state = TrackInfoState::Request { msg, requesting }; - } - TrackInfoState::Request { msg, requesting } => { - let broadcast = ready!(requesting.poll_ok(waiter))?; - let querying = broadcast.track(&msg.track)?.query().into_inner(); - self.state = TrackInfoState::Query { querying }; + let mut request = R::start(&self.shared, msg, broadcast)?; + if let Some(update) = self.update.take() { + request.update(update); + } + self.state = RequestState::Serve(request); } - TrackInfoState::Query { querying } => { - let info = ready!(querying.poll_ok(waiter))?; - - // TRACK_INFO only flows on Lite05+ (the encode errors otherwise), where every - // track is timed, so the model's timescale and retention bound go on the wire - // verbatim. - let stream = self.stream.as_mut().expect("stream present"); - stream.writer.buffer(&lite::TrackInfo { - priority: info.priority, - max_age: info.max_age, - timescale: info.timescale, - })?; - self.state = TrackInfoState::Finish { finished: false }; + RequestState::Serve(request) => { + if ready!(request.poll(&self.shared, &mut stream.writer, waiter))?.is_break() { + self.state = RequestState::Finish { finished: false }; + } } - TrackInfoState::Finish { finished } => { - let stream = self.stream.as_mut().expect("stream present"); - let mut cx = waiter.context(); + RequestState::Finish { finished } => { if !*finished { ready!(stream.writer.poll_flush(&mut cx))?; stream.writer.finish()?; *finished = true; } - return stream.writer.poll_closed(&mut cx); + return stream.writer.poll_close(&mut cx); } } } } + + /// Read what the requester sends while we answer: ready once it closes its send + /// direction, `Ok` for a FIN. Each update reaches the reply, or waits for it to start. + fn poll_requester(&mut self, cx: &mut Context<'_>) -> Poll> { + let stream = self.stream.as_mut().expect("stream present"); + loop { + let Some(update) = ready!(stream.reader.poll_decode_maybe::(cx))? else { + return Poll::Ready(Ok(())); + }; + match &mut self.state { + RequestState::Serve(request) => request.update(update), + _ => self.update = Some(update), + } + } + } } -/// Serves one SUBSCRIBE stream: resolve the track, then stream its groups (and -/// best-effort datagrams) until the peer FINs or the track ends. -struct SubscribeServe { - shared: Arc>, - stream: Option>, - state: SubscribeState, - // Log context, filled in after the decode. - id: u64, - absolute: crate::PathOwned, - track: String, +/// Answers a TRACK stream with the track's TRACK_INFO. +struct TrackInfoServe { + querying: track::Querying, } -enum SubscribeState { - Decode, - /// Resolving the split-horizon origin (waits on the peer's SETUP). - Hop { - msg: lite::Subscribe<'static>, - }, - /// Resolving the broadcast (may wait on a dynamic handler). - Request { - msg: lite::Subscribe<'static>, - requesting: origin::Requesting, - }, +impl Request for TrackInfoServe { + const KIND: &'static str = "track info"; + const GRACEFUL: bool = false; + type Message = lite::Track<'static>; + type Update = NoUpdate; + + fn target(msg: &Self::Message) -> (&crate::Path<'static>, &str) { + (&msg.broadcast, &msg.track) + } + + fn start(_: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result { + let querying = broadcast.track(&msg.track)?.query().into_inner(); + Ok(Self { querying }) + } + + fn update(&mut self, update: NoUpdate) { + match update {} + } + + fn poll( + &mut self, + _: &Shared, + writer: &mut Writer, + waiter: &kio::Waiter, + ) -> Poll, Error>> { + let info = ready!(self.querying.poll_ok(waiter))?; + + // TRACK_INFO only flows on Lite05+ (the encode errors otherwise), where every + // track is timed, so the model's timescale and retention bound go on the wire + // verbatim. + writer.buffer(&lite::TrackInfo { + priority: info.priority, + max_age: info.max_age, + timescale: info.timescale, + })?; + Poll::Ready(Ok(ControlFlow::Break(()))) + } +} + +/// Answers a SUBSCRIBE stream: streams the track's groups (and best-effort datagrams) +/// until the track ends. +enum SubscribeServe { /// Waiting for the model subscription to be confirmed. Confirm { msg: lite::Subscribe<'static>, subscribing: track::Subscribing, + update: Option, }, /// Streaming groups and datagrams. Boxed: by far the largest state, and the enum /// is moved on every transition. Run(Box>), /// The track finished: draining the in-flight group streams before the FIN. - Drain { - children: kio::Tasks>, - }, - /// FIN and wait for the acknowledgement. - Finish { - finished: bool, - }, + Drain { children: kio::Tasks> }, } -impl SubscribeServe { - fn new(shared: Arc>, stream: Stream) -> Self { - Self { - shared, - stream: Some(stream), - state: SubscribeState::Decode, - id: 0, - absolute: Default::default(), - track: Default::default(), - } - } +impl Request for SubscribeServe { + const KIND: &'static str = "subscribed"; + // Ending a subscription early is still a valid end. + const GRACEFUL: bool = true; + type Message = lite::Subscribe<'static>; + type Update = lite::SubscribeUpdate; - fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { - match ready!(self.poll_serve(waiter)) { - Ok(()) => { - tracing::info!(id = self.id, broadcast = %self.absolute, track = %self.track, "subscribed complete"); - Poll::Ready(Ok(())) - } - // A decode error propagates so the control loop logs it. - Err(err) if matches!(self.state, SubscribeState::Decode) => Poll::Ready(Err(err)), - Err(err) => { - match &err { - // TODO better classify WebTransport errors. - Error::Cancel - | Error::Stream(crate::StreamError::Cancel) - | Error::Session(crate::SessionError::Cancel) - | Error::Transport(_) => { - tracing::info!(id = self.id, broadcast = %self.absolute, track = %self.track, "subscribed cancelled") - } - err => { - tracing::warn!(id = self.id, broadcast = %self.absolute, track = %self.track, %err, "subscribed error") - } - } - self.stream.take().expect("stream present").writer.abort(&err); - Poll::Ready(Ok(())) - } - } + fn target(msg: &Self::Message) -> (&crate::Path<'static>, &str) { + (&msg.broadcast, &msg.track) } - fn poll_serve(&mut self, waiter: &kio::Waiter) -> Poll> { - loop { - match &mut self.state { - SubscribeState::Decode => { - let stream = self.stream.as_mut().expect("stream present"); - let mut cx = waiter.context(); - let msg = ready!(stream.reader.poll_decode::(&mut cx))?; - - self.id = msg.id; - self.absolute = self.shared.origin.absolute(&msg.broadcast).to_owned(); - self.track = msg.track.to_string(); - tracing::info!(id = self.id, broadcast = %self.absolute, track = %self.track, "subscribed started"); + fn start(shared: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result { + let subscription = crate::track::Subscription { + priority: msg.priority, + max_age: serving_max_age(shared.version, msg.max_age), + ..Bounds::from(&msg).positions() + }; - self.state = SubscribeState::Hop { msg }; - } - SubscribeState::Hop { .. } => { - // We just received a subscribe for this exact path, so by definition the - // peer has already seen an announcement for it. `request_broadcast` - // resolves an announced broadcast immediately; if it isn't announced it - // is served by the route covering it (or resolves to an error - // when there is none). - let origin = ready!(self.shared.poll_serving_origin(waiter)); - let SubscribeState::Hop { msg } = std::mem::replace(&mut self.state, SubscribeState::Decode) else { - unreachable!() - }; - let requesting = origin.request_broadcast(&msg.broadcast).into_inner(); - self.state = SubscribeState::Request { msg, requesting }; - } - SubscribeState::Request { requesting, .. } => { - // Waits for the covering route to serve the path if it is not local; - // resolves immediately otherwise (including an unroutable/dropped error). - let broadcast = ready!(requesting.poll_ok(waiter))?; - let SubscribeState::Request { msg, .. } = - std::mem::replace(&mut self.state, SubscribeState::Decode) - else { - unreachable!() - }; + // One subscriber for the whole subscription: the run loop polls its groups and its + // best-effort datagrams from this single cursor, so a group-only or datagram-only + // track opens exactly one subscription (no duplicate demand). + let subscribing = broadcast.track(&msg.track)?.subscribe(subscription).into_inner(); + Ok(Self::Confirm { + msg, + subscribing, + update: None, + }) + } - let subscription = crate::track::Subscription { - priority: msg.priority, - max_age: serving_max_age(self.shared.version, msg.max_age), - ..Bounds::from(&msg).positions() - }; + fn update(&mut self, update: lite::SubscribeUpdate) { + match self { + Self::Confirm { update: pending, .. } => *pending = Some(update), + Self::Run(run) => run.update(update), + // The track is over: nothing left to steer. + Self::Drain { .. } => {} + } + } - // One subscriber for the whole subscription: the run loop polls its - // groups and its best-effort datagrams from this single cursor, so a - // group-only or datagram-only track opens exactly one subscription (no - // duplicate demand). - let track_consumer = broadcast.track(&msg.track)?; - let subscribing = track_consumer.subscribe(subscription).into_inner(); - self.state = SubscribeState::Confirm { msg, subscribing }; - } - SubscribeState::Confirm { subscribing, .. } => { + fn poll( + &mut self, + shared: &Shared, + writer: &mut Writer, + waiter: &kio::Waiter, + ) -> Poll, Error>> { + loop { + match self { + Self::Confirm { subscribing, .. } => { let track = ready!(subscribing.poll_ok(waiter))?; - let SubscribeState::Confirm { msg, .. } = - std::mem::replace(&mut self.state, SubscribeState::Decode) - else { + let Self::Confirm { msg, update, .. } = std::mem::replace( + self, + Self::Drain { + children: Default::default(), + }, + ) else { unreachable!() }; - let stream = self.stream.as_mut().expect("stream present"); // Per-frame timestamps require a wire format that carries them. Lite05+ // prefixes every frame with a zigzag-delta timestamp at the track's // timescale; older drafts have no wire field, so `None` here means // "don't emit the prefix" (the frames still carry timestamps in the // model, just not on this wire). - let timescale = if self.shared.version.has_track_stream() { + let timescale = if shared.version.has_track_stream() { Some(track.info().timescale) } else { None @@ -1244,14 +1287,14 @@ impl SubscribeServe { // live in TRACK_INFO, and the resolved range arrives as // SUBSCRIBE_START/END from the run loop. Older drafts still acknowledge // with SUBSCRIBE_OK here. - if !self.shared.version.has_track_stream() { + if !shared.version.has_track_stream() { let info = lite::SubscribeOk { priority: msg.priority, max_age: Duration::ZERO, start_group: None, end_group: None, }; - stream.writer.buffer(&lite::SubscribeResponse::Ok(info))?; + writer.buffer(&lite::SubscribeResponse::Ok(info))?; } // Track-level subscriber priority. SUBSCRIBE_UPDATE messages broadcast @@ -1261,83 +1304,47 @@ impl SubscribeServe { let track_priority_tx = kio::Producer::new(msg.priority); let sub = Subscription { - session: self.shared.session.clone(), + session: shared.session.clone(), id: msg.id, track_name: Arc::from(track.name()), - priority: self.shared.priority.clone(), + priority: shared.priority.clone(), track_priority: track_priority_tx.consume(), track_priority_seen: msg.priority, - version: self.shared.version, + version: shared.version, timescale, opens: Default::default(), }; - let run = TrackRun::new(sub, track, Bounds::from(&msg), track_priority_tx); - self.state = SubscribeState::Run(Box::new(run)); + let mut run = TrackRun::new(sub, track, Bounds::from(&msg), track_priority_tx); + if let Some(update) = update { + run.update(update); + } + *self = Self::Run(Box::new(run)); } - SubscribeState::Run(run) => { - let stream = self.stream.as_mut().expect("stream present"); - match ready!(run.poll(stream, waiter))? { - // Peer FIN'd: they're done with this subscription. Drop any - // in-flight group machines (don't drain) so half-sent groups get - // cancelled rather than completed pointlessly. - TrackEnd::PeerFin => self.state = SubscribeState::Finish { finished: false }, - // The live edge reached the boundary; SUBSCRIBE_END was already - // sent (or the version predates the track stream). Drain in-flight - // group machines, then FIN. - TrackEnd::Finished => { - let SubscribeState::Run(run) = std::mem::replace(&mut self.state, SubscribeState::Decode) - else { - unreachable!() - }; - self.state = SubscribeState::Drain { children: run.children }; - } + Self::Run(run) => { + // The live edge reached the boundary; SUBSCRIBE_END was already sent (or + // the version predates the track stream). Drain the in-flight group + // machines, then FIN. + if ready!(run.poll_step(writer, waiter))?.is_continue() { + return Poll::Ready(Ok(ControlFlow::Continue(()))); } + let children = std::mem::take(&mut run.children); + *self = Self::Drain { children }; } - SubscribeState::Drain { children } => { + Self::Drain { children } => { ready!(children.poll(waiter)); - self.state = SubscribeState::Finish { finished: false }; - } - SubscribeState::Finish { finished } => { - let stream = self.stream.as_mut().expect("stream present"); - let mut cx = waiter.context(); - if !*finished { - ready!(stream.writer.poll_flush(&mut cx))?; - stream.writer.finish()?; - *finished = true; - } - return stream.writer.poll_closed(&mut cx); + return Poll::Ready(Ok(ControlFlow::Break(()))); } } } } } -/// Serves one FETCH stream: a single cached group, streamed in order, then FIN. -struct FetchServe { - shared: Arc>, - stream: Option>, - state: FetchState, - // Log context, filled in after the decode. - absolute: crate::PathOwned, - track: String, - group: u64, -} - +/// Answers a FETCH stream: a single cached group, streamed in order. // A state machine's enum is its storage: one transient instance per stream, so the // big variant is the working state, not padding held in bulk. #[allow(clippy::large_enum_variant)] -enum FetchState { - Decode, - /// Resolving the split-horizon origin (waits on the peer's SETUP). - Hop { - msg: lite::Fetch<'static>, - }, - /// Resolving the broadcast (may wait on a dynamic handler). - Request { - msg: lite::Fetch<'static>, - requesting: origin::Requesting, - }, +enum FetchServe { /// Waiting for the fetched group. Fetch { msg: lite::Fetch<'static>, @@ -1355,99 +1362,47 @@ enum FetchState { batch: Box, batch_pos: usize, }, - /// FIN and wait for the acknowledgement. - Finish { - finished: bool, - }, } -impl FetchServe { - fn new(shared: Arc>, stream: Stream) -> Result { - // FETCH is lite-05+ only; older drafts have no dedicated FETCH stream. - if !shared.version.has_track_stream() { - return Err(Error::UnexpectedStream); - } - Ok(Self { - shared, - stream: Some(stream), - state: FetchState::Decode, - absolute: Default::default(), - track: Default::default(), - group: 0, - }) +impl Request for FetchServe { + const KIND: &'static str = "fetch"; + // The response carries no header, so a run cut short would read as whole. + const GRACEFUL: bool = false; + type Message = lite::Fetch<'static>; + type Update = NoUpdate; + + fn target(msg: &Self::Message) -> (&crate::Path<'static>, &str) { + (&msg.broadcast, &msg.track) + } + + fn start(_: &Shared, msg: Self::Message, broadcast: crate::broadcast::Consumer) -> Result { + let fetching = broadcast + .track(&msg.track)? + .fetch_group( + msg.group, + group::Fetch { + priority: msg.priority, + frame_start: msg.start_frame, + ..Default::default() + }, + ) + .into_inner(); + Ok(Self::Fetch { msg, fetching }) } - fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { - match ready!(self.poll_serve(waiter)) { - Ok(()) => { - tracing::info!(broadcast = %self.absolute, track = %self.track, group = %self.group, "fetch complete"); - Poll::Ready(Ok(())) - } - // A decode error propagates so the control loop logs it. - Err(err) if matches!(self.state, FetchState::Decode) => Poll::Ready(Err(err)), - Err(err) => { - match &err { - Error::Cancel - | Error::Stream(crate::StreamError::Cancel) - | Error::Session(crate::SessionError::Cancel) - | Error::Transport(_) => { - tracing::info!(broadcast = %self.absolute, track = %self.track, group = %self.group, "fetch cancelled") - } - err => { - tracing::warn!(broadcast = %self.absolute, track = %self.track, group = %self.group, %err, "fetch error") - } - } - self.stream.take().expect("stream present").writer.abort(&err); - Poll::Ready(Ok(())) - } - } + fn update(&mut self, update: NoUpdate) { + match update {} } - fn poll_serve(&mut self, waiter: &kio::Waiter) -> Poll> { + fn poll( + &mut self, + shared: &Shared, + writer: &mut Writer, + waiter: &kio::Waiter, + ) -> Poll, Error>> { loop { - match &mut self.state { - FetchState::Decode => { - let stream = self.stream.as_mut().expect("stream present"); - let mut cx = waiter.context(); - let msg = ready!(stream.reader.poll_decode::(&mut cx))?; - - self.absolute = self.shared.origin.absolute(&msg.broadcast).to_owned(); - self.track = msg.track.to_string(); - self.group = msg.group; - tracing::info!(broadcast = %self.absolute, track = %self.track, group = %self.group, "fetch started"); - - self.state = FetchState::Hop { msg }; - } - FetchState::Hop { .. } => { - // The peer fetched this exact path, so it has already seen an - // announcement for it. `request_broadcast` resolves it immediately, or - // is served on demand by the route covering it (an `origin::Dynamic`). - let origin = ready!(self.shared.poll_serving_origin(waiter)); - let FetchState::Hop { msg } = std::mem::replace(&mut self.state, FetchState::Decode) else { - unreachable!() - }; - let requesting = origin.request_broadcast(&msg.broadcast).into_inner(); - self.state = FetchState::Request { msg, requesting }; - } - FetchState::Request { requesting, .. } => { - let broadcast = ready!(requesting.poll_ok(waiter))?; - let FetchState::Request { msg, .. } = std::mem::replace(&mut self.state, FetchState::Decode) else { - unreachable!() - }; - let track = broadcast.track(&msg.track)?; - let fetching = track - .fetch_group( - msg.group, - group::Fetch { - priority: msg.priority, - frame_start: msg.start_frame, - ..Default::default() - }, - ) - .into_inner(); - self.state = FetchState::Fetch { msg, fetching }; - } - FetchState::Fetch { msg, fetching } => { + match self { + Self::Fetch { msg, fetching } => { let mut group = ready!(kio::Task::poll(fetching, waiter))?; // The response carries no header, so a short run is indistinguishable @@ -1464,13 +1419,13 @@ impl FetchServe { // FETCH is gated to lite-05+, which learned the track timescale via // TRACK_INFO. - let timescale = if self.shared.version.has_track_stream() { + let timescale = if shared.version.has_track_stream() { Some(group.timescale()) } else { None }; - self.state = FetchState::Serve { + *self = Self::Serve { group, timescale, prev_ts: 0, @@ -1480,7 +1435,7 @@ impl FetchServe { batch_pos: 0, }; } - FetchState::Serve { + Self::Serve { group, timescale, prev_ts, @@ -1489,12 +1444,11 @@ impl FetchServe { batch, batch_pos, } => { - let stream = self.stream.as_mut().expect("stream present"); let mut cx = waiter.context(); loop { - ready!(stream.writer.poll_flush(&mut cx))?; + ready!(writer.poll_flush(&mut cx))?; if let Some(pending) = chunk { - ready!(stream.writer.poll_write(&mut cx, pending))?; + ready!(writer.poll_write(&mut cx, pending))?; if !bytes::Buf::has_remaining(pending) { *chunk = None; } @@ -1506,7 +1460,7 @@ impl FetchServe { } else if *batch_pos < batch.len() { let batched = &mut batch.filled_mut()[*batch_pos]; buffer_frame_info( - &mut stream.writer, + writer, batched.timestamp, batched.payload.len() as u64, *timescale, @@ -1522,38 +1476,22 @@ impl FetchServe { match group.poll_read_frames(waiter, batch) { Poll::Ready(Ok(count)) if count > 0 => { *batch_pos = 0; - continue; + return Poll::Ready(Ok(ControlFlow::Continue(()))); } - Poll::Ready(Ok(_)) => break, + Poll::Ready(Ok(_)) => return Poll::Ready(Ok(ControlFlow::Break(()))), Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), Poll::Pending => {} } match ready!(group.poll_next_frame(waiter))? { Some(next) => { - buffer_frame_info( - &mut stream.writer, - next.timestamp, - next.size, - *timescale, - prev_ts, - )?; + buffer_frame_info(writer, next.timestamp, next.size, *timescale, prev_ts)?; *frame = Some(next); + return Poll::Ready(Ok(ControlFlow::Continue(()))); } - None => break, + None => return Poll::Ready(Ok(ControlFlow::Break(()))), } } } - self.state = FetchState::Finish { finished: false }; - } - FetchState::Finish { finished } => { - let stream = self.stream.as_mut().expect("stream present"); - let mut cx = waiter.context(); - if !*finished { - ready!(stream.writer.poll_flush(&mut cx))?; - stream.writer.finish()?; - *finished = true; - } - return stream.writer.poll_closed(&mut cx); } } } @@ -2345,16 +2283,6 @@ impl Subscription { } } -/// What ended a subscription's run loop. -enum TrackEnd { - /// The peer FIN'd the subscribe stream: drop the in-flight group machines so - /// half-sent groups get cancelled rather than completed pointlessly. - PeerFin, - /// The live edge reached the track's boundary: drain the in-flight group - /// machines, then FIN. - Finished, -} - /// A subscription's run loop: one subscriber cursor serving groups, datagrams, /// and SUBSCRIBE_UPDATE messages, with an in-flight group machine per group. struct TrackRun { @@ -2420,125 +2348,127 @@ impl TrackRun { } } + /// Apply a SUBSCRIBE_UPDATE. + /// /// `end_group` is a serving cap, not a subscription terminator: groups past the /// cap are held in the producer's cache until the subscriber raises the cap (or - /// unsets it) via SUBSCRIBE_UPDATE, then served in order. Only a peer FIN actually - /// ends the subscription. This is what lets relays pause an upstream subscription - /// across consumer churn without tearing it down. - fn poll(&mut self, stream: &mut Stream, waiter: &kio::Waiter) -> Poll> { + /// unsets it), then served in order. Only a peer FIN actually ends the + /// subscription. This is what lets relays pause an upstream subscription across + /// consumer churn without tearing it down. + fn update(&mut self, upd: lite::SubscribeUpdate) { + if let Ok(mut value) = self.track_priority_tx.write() { + *value = upd.priority; + } + // Feed the full update into the model subscriber so the producer's + // aggregate reflects it (and a relay re-forwards it upstream). + let bounds = Bounds::from(&upd); + let _ = self.track.update(crate::track::Subscription { + priority: upd.priority, + max_age: serving_max_age(self.ctx.version, upd.max_age), + ..bounds.positions() + }); + if let Some(start_group) = upd.start_group { + self.track.start_at(start_group); + } + self.track + .end_at(upd.end_group.map_or(Bound::Unbounded, Bound::Included)); + self.start_frame = bounds.start_frame(); + self.end_frame = bounds.end_frame(); + } + + /// Serve until the live edge reaches the track's boundary. + #[cfg(test)] + fn poll(&mut self, writer: &mut Writer, waiter: &kio::Waiter) -> Poll> { + while ready!(self.poll_step(writer, waiter))?.is_continue() {} + Poll::Ready(Ok(())) + } + + /// Serve one ready group or datagram (`Continue`), or `Break` once the live edge + /// reaches the track's boundary. Returning per item lets the caller watch the + /// requester while work stays ready. + fn poll_step( + &mut self, + writer: &mut Writer, + waiter: &kio::Waiter, + ) -> Poll, Error>> { let mut cx = waiter.context(); - loop { - // Deliver the buffered range messages before selecting more work. - ready!(stream.writer.poll_flush(&mut cx))?; + // Deliver the buffered range messages before selecting more work. + ready!(writer.poll_flush(&mut cx))?; - // Drive the in-flight group machines; completions just retire. - let _ = self.children.poll(waiter); + // Drive the in-flight group machines; completions just retire. + let _ = self.children.poll(waiter); - // Control first: SUBSCRIBE_UPDATE/FIN messages are rare, so they can't - // starve the data path, while a deep group backlog polled first could - // defer an unsubscribe or priority change indefinitely. Partial message - // bytes persist in the reader's buffer across turns. - if let Poll::Ready(upd) = stream.reader.poll_decode_maybe::(&mut cx) { - let Some(upd) = upd? else { - return Poll::Ready(Ok(TrackEnd::PeerFin)); - }; - if let Ok(mut value) = self.track_priority_tx.write() { - *value = upd.priority; - } - // Feed the full update into the model subscriber so the producer's - // aggregate reflects it (and a relay re-forwards it upstream). - let bounds = Bounds::from(&upd); - let _ = self.track.update(crate::track::Subscription { - priority: upd.priority, - max_age: serving_max_age(self.ctx.version, upd.max_age), - ..bounds.positions() - }); - if let Some(start_group) = upd.start_group { - self.track.start_at(start_group); + // The first group waits for the source to resolve where its feed starts; datagrams + // keep flowing meanwhile. + if let Some(group) = self.first.take() { + match self.track.poll_start(waiter) { + Poll::Ready(source) => { + self.start(group, source, writer)?; + return Poll::Ready(Ok(ControlFlow::Continue(()))); } - self.track - .end_at(upd.end_group.map_or(Bound::Unbounded, Bound::Included)); - self.start_frame = bounds.start_frame(); - self.end_frame = bounds.end_frame(); - continue; - } - - // The first group waits for the source to resolve where its feed starts; datagrams - // keep flowing meanwhile. - if let Some(group) = self.first.take() { - match self.track.poll_start(waiter) { - Poll::Ready(source) => { - self.start(group, source, stream)?; - continue; - } - Poll::Pending => { - self.first = Some(group); - if self.datagrams - && let Poll::Ready(Some(datagram)) = self.track.poll_recv_datagram(waiter)? - { - self.ctx.serve_datagram(datagram); - continue; - } - return Poll::Pending; + Poll::Pending => { + self.first = Some(group); + if self.datagrams + && let Poll::Ready(Some(datagram)) = self.track.poll_recv_datagram(waiter)? + { + self.ctx.serve_datagram(datagram); + return Poll::Ready(Ok(ControlFlow::Continue(()))); } + return Poll::Pending; } } + } - // One cursor drives the whole subscription: poll the cap-aware arrival-order - // group and, when enabled, the next best-effort datagram. Groups are polled - // first so a datagram burst can't starve them; datagrams flow whenever no - // group is ready (including while groups are parked above the cap). - let emit_boundary = self.emit_range && !self.end_sent && !self.count_streams; - if let Poll::Ready(res) = poll_recv_next(&mut self.track, self.datagrams, emit_boundary, waiter) { - match res? { - Recv::Group(mut group) => { - if !position_group(&mut group, self.start_frame, self.end_frame) { - // Its head is gone, and this subscriber didn't ask for a - // partial group. Skip it rather than open a stream that can - // only be reset; the next servable group resolves the start. - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = group.sequence, "skipping group with a missing head"); - if self.emit_range && !self.start_sent { - self.skipped.insert(group.sequence); - } - continue; - } - match self.emit_range && !self.start_sent { - true => self.first = Some(group), - false => self.serve(group), + // One cursor drives the whole subscription: poll the cap-aware arrival-order + // group and, when enabled, the next best-effort datagram. Groups are polled + // first so a datagram burst can't starve them; datagrams flow whenever no + // group is ready (including while groups are parked above the cap). + let emit_boundary = self.emit_range && !self.end_sent && !self.count_streams; + if let Poll::Ready(res) = poll_recv_next(&mut self.track, self.datagrams, emit_boundary, waiter) { + match res? { + Recv::Group(mut group) => { + if !position_group(&mut group, self.start_frame, self.end_frame) { + // Its head is gone, and this subscriber didn't ask for a + // partial group. Skip it rather than open a stream that can + // only be reset; the next servable group resolves the start. + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = group.sequence, "skipping group with a missing head"); + if self.emit_range && !self.start_sent { + self.skipped.insert(group.sequence); } + return Poll::Ready(Ok(ControlFlow::Continue(()))); } - Recv::Datagram(datagram) => self.ctx.serve_datagram(datagram), - Recv::Boundary(group) => { - // The track declared its exclusive final sequence. Forward it now, - // even if trailing groups (below `group`) are still in flight, then - // keep serving them until the live edge reaches the boundary. - self.end_sent = true; - stream - .writer - .buffer(&lite::SubscribeResponse::End(lite::SubscribeEnd { group, streams: 0 }))?; + match self.emit_range && !self.start_sent { + true => self.first = Some(group), + false => self.serve(group), } - Recv::Finished if self.count_streams && !self.end_sent => { - // The count is final only once no served group is still waiting to - // open its stream. A group that gives up first is never counted, so - // the subscriber is not left waiting for it. The child's wake - // re-polls this loop. - if self.ctx.opens.pending.load(Ordering::Relaxed) > 0 { - return Poll::Pending; - } - let group = ready!(self.track.poll_finished(waiter))?; - let streams = self.ctx.opens.opened.load(Ordering::Relaxed); - self.end_sent = true; - stream - .writer - .buffer(&lite::SubscribeResponse::End(lite::SubscribeEnd { group, streams }))?; + } + Recv::Datagram(datagram) => self.ctx.serve_datagram(datagram), + Recv::Boundary(group) => { + // The track declared its exclusive final sequence. Forward it now, + // even if trailing groups (below `group`) are still in flight, then + // keep serving them until the live edge reaches the boundary. + self.end_sent = true; + writer.buffer(&lite::SubscribeResponse::End(lite::SubscribeEnd { group, streams: 0 }))?; + } + Recv::Finished if self.count_streams && !self.end_sent => { + // The count is final only once no served group is still waiting to + // open its stream. A group that gives up first is never counted, so + // the subscriber is not left waiting for it. The child's wake + // re-polls this loop. + if self.ctx.opens.pending.load(Ordering::Relaxed) > 0 { + return Poll::Pending; } - Recv::Finished => return Poll::Ready(Ok(TrackEnd::Finished)), + let group = ready!(self.track.poll_finished(waiter))?; + let streams = self.ctx.opens.opened.load(Ordering::Relaxed); + self.end_sent = true; + writer.buffer(&lite::SubscribeResponse::End(lite::SubscribeEnd { group, streams }))?; } - continue; + Recv::Finished => return Poll::Ready(Ok(ControlFlow::Break(()))), } - - return Poll::Pending; + return Poll::Ready(Ok(ControlFlow::Continue(()))); } + + Poll::Pending } } @@ -2554,7 +2484,7 @@ impl TrackRun { &mut self, group: group::Consumer, source: Option, - stream: &mut Stream, + writer: &mut Writer, ) -> Result<(), Error> { let mut start = source.map_or(group.sequence, |source| source.min(group.sequence)); // A skipped group is never served, so it cannot be where the feed starts. This stops @@ -2566,9 +2496,7 @@ impl TrackRun { self.start_sent = true; // Only the group: the subscriber derives the start frame from its own request // (see `lite::SubscribeStart`). - stream - .writer - .buffer(&lite::SubscribeResponse::Start(lite::SubscribeStart { group: start }))?; + writer.buffer(&lite::SubscribeResponse::Start(lite::SubscribeStart { group: start }))?; // SUBSCRIBE_START is an implicit drop of everything below the resolved start (the // subscriber records it as a permanent miss), so a lower group arriving late must // not be served after all. A widening SUBSCRIBE_UPDATE re-lowers the floor, @@ -3368,12 +3296,9 @@ mod serve_group_test { fn lite07_run( session: SinkSession, track: track::Subscriber, - ) -> (TrackRun, Stream, Log) { + ) -> (TrackRun, Writer, Log) { let log = Log::default(); - let stream = Stream { - writer: Writer::new(SinkSend::new(log.clone()), Version::Lite07), - reader: crate::coding::Reader::new(PendingRecv, Version::Lite07), - }; + let writer = Writer::new(SinkSend::new(log.clone()), Version::Lite07); let track_priority = kio::Producer::new(0u8); let ctx = Subscription { session, @@ -3392,7 +3317,7 @@ mod serve_group_test { end_group: None, end_frame: None, }; - (TrackRun::new(ctx, track, bounds, track_priority), stream, log) + (TrackRun::new(ctx, track, bounds, track_priority), writer, log) } fn write_group(track: &mut track::Producer, sequence: u64, millis: u64) { @@ -3409,14 +3334,14 @@ mod serve_group_test { async fn lite07_end_counts_the_streams_opened() { let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); let subscriber = track.subscribe(None); - let (mut run, mut stream, log) = lite07_run(SinkSession::new(Log::default()), subscriber); - let mut run = std::pin::pin!(kio::wait(move |waiter| run.poll(&mut stream, waiter))); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + let mut run = std::pin::pin!(kio::wait(move |waiter| run.poll(&mut writer, waiter))); write_group(&mut track, 0, 0); assert!(futures::poll!(run.as_mut()).is_pending()); write_group(&mut track, 2, 2); track.finish().unwrap(); - assert!(matches!(run.await.unwrap(), TrackEnd::Finished)); + run.await.unwrap(); // SUBSCRIBE_START at 0, then SUBSCRIBE_END at 3 with 2 streams. assert_eq!(*log.writes.lock().unwrap(), [0, 1, 0, 1, 2, 3, 2]); @@ -3426,30 +3351,22 @@ mod serve_group_test { /// source's SUBSCRIBE_START, on a subscribe stream the test can push updates onto. struct RelayRun { run: TrackRun, - stream: Stream, + writer: Writer, session: ScriptedSession, opens: Arc, } impl RelayRun { /// Subscribe to `track` from `start_group`. - async fn new(track: &mut track::Producer, start_group: u64) -> Self { + fn new(track: &mut track::Producer, start_group: u64) -> Self { track.request_start(Some(0)).unwrap(); let subscription = track::Subscription::default() .with_start(track::Position::group(start_group)) .with_max_age(Duration::from_secs(30)); let subscriber = track.subscribe(subscription); - let mut session = ScriptedSession::new(Vec::new()); - let (send, recv) = futures::future::poll_fn(|cx| { - ::poll_open_bi(&mut session, cx) - }) - .await - .unwrap(); - let stream = Stream:: { - writer: Writer::new(send, Version::Lite07), - reader: crate::coding::Reader::new(recv, Version::Lite07), - }; + let session = ScriptedSession::new(Vec::new()); + let writer = Writer::new(SinkSend::new(session.log.clone()), Version::Lite07); let track_priority = kio::Producer::new(0u8); let opens = Arc::::default(); let ctx = Subscription { @@ -3471,7 +3388,7 @@ mod serve_group_test { }; Self { run: TrackRun::new(ctx, subscriber, bounds, track_priority), - stream, + writer, session, opens, } @@ -3479,8 +3396,8 @@ mod serve_group_test { /// Drive the run until it parks, which it must. fn settle(&mut self) { - let Self { run, stream, .. } = self; - let res = kio::wait(|waiter| run.poll(stream, waiter)).now_or_never(); + let Self { run, writer, .. } = self; + let res = kio::wait(|waiter| run.poll(writer, waiter)).now_or_never(); assert!(res.is_none(), "the run ended"); } @@ -3502,7 +3419,7 @@ mod serve_group_test { #[tokio::test] async fn held_first_group_keeps_an_updated_floor() { let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); - let mut relay = RelayRun::new(&mut track, 0).await; + let mut relay = RelayRun::new(&mut track, 0); write_group(&mut track, 5, 0); relay.settle(); @@ -3516,7 +3433,7 @@ mod serve_group_test { start_frame: 0, end_frame: None, }; - relay.session.push(&update.encode_bytes(Version::Lite07).unwrap()); + relay.run.update(update); relay.settle(); track.start_at(0).unwrap(); @@ -3534,7 +3451,7 @@ mod serve_group_test { #[tokio::test] async fn held_first_group_resolves_to_the_floor_under_the_source() { let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); - let mut relay = RelayRun::new(&mut track, 5).await; + let mut relay = RelayRun::new(&mut track, 5); write_group(&mut track, 6, 6); relay.settle(); @@ -3553,7 +3470,7 @@ mod serve_group_test { #[tokio::test] async fn held_first_group_starts_past_a_skipped_head() { let mut track = track::Producer::new(Arc::new(broadcast::Info::default()), "test", None); - let mut relay = RelayRun::new(&mut track, 5).await; + let mut relay = RelayRun::new(&mut track, 5); // Group 5's first frame is gone. let mut headless = track.create_group(group::Info { sequence: 5 }).unwrap(); @@ -3578,8 +3495,8 @@ mod serve_group_test { let subscriber = track.subscribe(None); track.finish().unwrap(); - let (mut run, mut stream, log) = lite07_run(SinkSession::new(Log::default()), subscriber); - kio::wait(|waiter| run.poll(&mut stream, waiter)).await.unwrap(); + let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); + kio::wait(|waiter| run.poll(&mut writer, waiter)).await.unwrap(); assert_eq!(*log.writes.lock().unwrap(), [1, 2, 0, 0]); } @@ -3595,8 +3512,8 @@ mod serve_group_test { let subscriber = track.subscribe(None); write_group(&mut track, 0, 0); - let (mut run, mut stream, log) = lite07_run(SinkSession::gated_open_uni(gate.consume()), subscriber); - let mut run = std::pin::pin!(kio::wait(move |waiter| run.poll(&mut stream, waiter))); + let (mut run, mut writer, log) = lite07_run(SinkSession::gated_open_uni(gate.consume()), subscriber); + let mut run = std::pin::pin!(kio::wait(move |waiter| run.poll(&mut writer, waiter))); assert!(futures::poll!(run.as_mut()).is_pending()); // Group 1 lands a second later, expiring group 0 before it ever opened. @@ -3619,8 +3536,9 @@ mod serve_group_test { #[cfg(test)] mod tests { use super::*; - use crate::lite::test_transport::SinkSession; + use crate::lite::test_transport::{Close, ScriptedSession, SinkSession}; use crate::model::ProduceTest; + use futures::FutureExt; /// A peer that declares no origin in its SETUP is split-horizoned by the identity /// the caller assigned it, on the data plane and not just the announce filter. @@ -3945,58 +3863,222 @@ mod tests { assert_eq!(probes[1].bitrate, None); } - /// A requester that FINs its TRACK request before the answer has terminated the - /// transaction (moq-lite: closing the send direction ends the stream), so the - /// lookup ends rather than pinning an upstream request nobody will read. - #[tokio::test] - async fn track_info_ends_when_the_requester_finishes() { - use crate::coding::Encode; - use crate::lite::test_transport::ScriptedSession; + /// Where a request waits when its requester leaves. + #[derive(Clone, Copy, Debug)] + enum Wait { + /// On a route that would serve the broadcast on demand, but never does. + Broadcast, + /// On a broadcast whose track is requested but never answered. + Track, + } + + /// Drive `serve` until it ends, or everything parks: whether it ended. + async fn drive>(serve: &mut RequestServe) -> bool { + tokio::time::timeout(Duration::from_millis(1), kio::wait(|waiter| serve.poll(waiter))) + .await + .is_ok() + } + /// Serve an `R` request whose broadcast or track never resolves, then have the + /// requester close its send side. + async fn leave_request>( + version: Version, + request: &impl Encode, + wait: Wait, + close: Close, + ) { + let case = format!("{} over {version:?}, waiting on the {wait:?}, {close:?}", R::KIND); + let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); + // Held, never answered. + let mut _route = None; + let mut held = None; + match wait { + Wait::Broadcast => _route = Some(origin.dynamic("room", crate::origin::Route::default()).unwrap()), + Wait::Track => { + let broadcast = origin.create_broadcast("room").unwrap(); + let dynamic = broadcast.dynamic(); + broadcast.announce(Default::default()).unwrap(); + held = Some((broadcast, dynamic)); + } + } + + let peer_setup = crate::lite::PeerSetup::default(); + peer_setup.set(crate::lite::Setup::default()); + let (_, goaway) = crate::goaway::Handle::new(true); + let publisher = Publisher::new(PublisherConfig { + runtime: crate::time::Clock::tokio(), + session: ScriptedSession::new(Vec::new()), + origin: origin.consume(), + version, + peer_setup, + goaway, + peer_hop: None, + }); + + let mut script = Vec::new(); + request.encode(&mut script, version).unwrap(); + let mut session = ScriptedSession::new(script); + let (send, recv) = futures::future::poll_fn(|cx| { + ::poll_open_bi(&mut session, cx) + }) + .await + .unwrap(); + let stream = Stream:: { + writer: Writer::new(send, version), + reader: crate::coding::Reader::new(recv, version), + }; + let mut serve = RequestServe::<_, R>::new(publisher.shared.clone(), stream); + let waiter = kio::Waiter::noop(); + assert!(!drive(&mut serve).await, "{case}: nothing to answer with"); + + // The lookup reached the track, which stays wanted while the requester waits. + let track = held.as_mut().map(|(_, dynamic)| { + let track = dynamic + .requested_track() + .now_or_never() + .expect("track requested") + .unwrap(); + assert!( + track.poll_unused(&waiter).is_pending(), + "{case}: nobody wants the track" + ); + track + }); + + session.close(close); + assert!(drive(&mut serve).await, "{case}: still serving"); + drop(serve); + if let Some(track) = track { + let unused = tokio::time::timeout(Duration::from_millis(1), kio::wait(|waiter| track.poll_unused(waiter))); + assert!(unused.await.is_ok(), "{case}: the track is still wanted"); + } + + // A subscription cut short is still a whole one, so it ends with our FIN. Any + // other reply would read as whole if it ended short, so it resets. + let graceful = R::GRACEFUL && matches!(close, Close::Fin); + assert_eq!( + session.log.resets().is_empty(), + graceful, + "{case}: {:?}", + session.log.resets() + ); + } + + /// A requester that closes its send direction, by FIN or reset, has terminated the + /// transaction (moq-lite: closing the send direction ends the stream), however the + /// answer is waiting. The lookup ends then, rather than pinning an upstream request + /// nobody will read. + #[tokio::test(start_paused = true)] + async fn a_request_ends_when_the_requester_leaves() { + let broadcast = crate::Path::new("room"); + for version in [Version::Lite05, Version::Lite06, Version::Lite07] { + for wait in [Wait::Broadcast, Wait::Track] { + for close in [Close::Fin, Close::Reset] { + let track = lite::Track { + broadcast: broadcast.clone(), + track: "video".into(), + }; + leave_request::(version, &track, wait, close).await; + + let subscribe = lite::Subscribe { + id: 0, + broadcast: broadcast.clone(), + track: "video".into(), + priority: 0, + max_age: Duration::ZERO, + start_group: None, + end_group: None, + start_frame: 0, + end_frame: None, + }; + leave_request::>(version, &subscribe, wait, close).await; + + let fetch = lite::Fetch { + broadcast: broadcast.clone(), + track: "video".into(), + priority: 0, + group: 0, + start_frame: 0, + end_frame: None, + }; + leave_request::(version, &fetch, wait, close).await; + } + } + } + } + + /// SUBSCRIBE_UPDATE is not a close: an update arriving while the subscription + /// resolves keeps it waiting, and still applies once it starts. + #[tokio::test(start_paused = true)] + async fn a_subscribe_update_is_not_a_close() { let origin = crate::origin::Config::new(crate::Hop::new(1).unwrap()).produce(); let broadcast = origin.create_broadcast("room").unwrap(); - // Held, never polled: the track request stays unanswered. - let _dynamic = broadcast.dynamic(); + let mut dynamic = broadcast.dynamic(); broadcast.announce(Default::default()).unwrap(); let peer_setup = crate::lite::PeerSetup::default(); peer_setup.set(crate::lite::Setup::default()); let (_, goaway) = crate::goaway::Handle::new(true); + let version = Version::Lite07; let publisher = Publisher::new(PublisherConfig { runtime: crate::time::Clock::tokio(), session: ScriptedSession::new(Vec::new()), origin: origin.consume(), - version: Version::Lite06, + version, peer_setup, goaway, peer_hop: None, }); - let mut request = Vec::new(); - lite::Track { + let mut script = Vec::new(); + lite::Subscribe { + id: 0, broadcast: crate::Path::new("room"), track: "video".into(), + priority: 1, + max_age: Duration::ZERO, + start_group: None, + end_group: None, + start_frame: 0, + end_frame: None, } - .encode(&mut request, Version::Lite06) + .encode(&mut script, version) + .unwrap(); + let mut session = ScriptedSession::new(script); + let (send, recv) = futures::future::poll_fn(|cx| { + ::poll_open_bi(&mut session, cx) + }) + .await .unwrap(); + let stream = Stream:: { + writer: Writer::new(send, version), + reader: crate::coding::Reader::new(recv, version), + }; + let mut serve = RequestServe::<_, SubscribeServe>::new(publisher.shared.clone(), stream); + assert!(!drive(&mut serve).await); - for (finished, session) in [ - (false, ScriptedSession::new(request.clone())), - (true, ScriptedSession::eof(request.clone())), - ] { - let mut session = session; - let (send, recv) = futures::future::poll_fn(|cx| { - ::poll_open_bi(&mut session, cx) - }) - .await + let update = lite::SubscribeUpdate { + priority: 7, + max_age: Duration::ZERO, + start_group: None, + end_group: None, + start_frame: 0, + end_frame: None, + }; + session.push(&update.encode_bytes(version).unwrap()); + assert!(!drive(&mut serve).await, "an update ended the subscription"); + + // Resolve the track: the run starts at the updated priority. + let request = dynamic + .requested_track() + .now_or_never() + .expect("track requested") .unwrap(); - let stream = Stream:: { - writer: Writer::new(send, Version::Lite06), - reader: crate::coding::Reader::new(recv, Version::Lite06), - }; - let mut serve = TrackInfoServe::new(publisher.shared.clone(), stream).unwrap(); - let ended = serve.poll(&kio::Waiter::noop()).is_ready(); - assert_eq!(ended, finished, "finished={finished}"); - } + let _track = request.accept(None); + assert!(!drive(&mut serve).await, "the subscription ended"); + let RequestState::Serve(SubscribeServe::Run(run)) = &serve.state else { + panic!("the subscription is not running"); + }; + assert_eq!(*run.track_priority_tx.read(), 7, "the update was lost"); } } diff --git a/rs/moq-net/src/lite/session.rs b/rs/moq-net/src/lite/session.rs index 80989ac7c93..f9552fbe849 100644 --- a/rs/moq-net/src/lite/session.rs +++ b/rs/moq-net/src/lite/session.rs @@ -189,6 +189,7 @@ where }); let driver = Driver { + local_close: Default::default(), setup: version .has_setup_stream() .then(|| SendSetup::new(session.clone(), our_setup, version)), @@ -209,6 +210,7 @@ where /// The lite session driver: one poll function racing every protocol arm, in /// place of a task set of boxed futures. pub(crate) struct Driver { + pub(crate) local_close: std::sync::Arc, /// Advertising our capabilities, or `None` once sent (or on a version with no /// Setup Stream). setup: Option>, @@ -230,7 +232,9 @@ where { pub(crate) fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { let res = std::task::ready!(self.poll_protocol(waiter)); - if let Err(err) = &res { + if self.local_close.load(std::sync::atomic::Ordering::Relaxed) { + self.subscriber.close(); + } else if let Err(err) = &res { // Every track this session was receiving ends with its error. self.subscriber.abort(err); } diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 4e85ed56ffd..137aa17c480 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -89,6 +89,8 @@ pub(super) struct Subscriber { /// [`SourceServe`] machines. sources: kio::Queue<(PathOwned, crate::broadcast::Dynamic)>, going_away: crate::goaway::GoingAway, + /// What this session may allocate up front for frames still arriving. + frames: frame::Budget, } #[derive(Clone)] @@ -124,6 +126,7 @@ impl Subscriber { cost: config.cost, sources: kio::Queue::new(), going_away: config.going_away, + frames: Default::default(), } } @@ -487,7 +490,12 @@ impl SubscriptionCleanup { /// their cleanup independently; this records the track's terminal state. fn abort(&self, err: &Error) { for (_, entry) in self.0.lock().drain() { - let _ = entry.producer.abort(err.clone()); + let _ = entry.producer.abort_session(err.clone()); + } + } + fn close(&self) { + for (_, entry) in self.0.lock().drain() { + let _ = entry.producer.close(); } } } @@ -544,6 +552,11 @@ impl SubscriberDriver { self.cleanup.abort(err); } + pub fn close(&self) { + self.subscriber.note_end(&Error::Cancel); + self.cleanup.close(); + } + pub fn poll(&mut self, waiter: &kio::Waiter) -> Poll> { let mut i = 0; while i < self.prefixes.len() { @@ -812,7 +825,7 @@ impl GroupRecv { self.state = GroupRecvState::Serve { group: crate::recv::Group::new(group), track, - ingest: FrameIngest::new(self.subscriber.runtime.clone(), timescale), + ingest: FrameIngest::new(&self.subscriber, timescale), _reading: reading, }; } @@ -877,6 +890,7 @@ struct FrameIngest { /// zigzag-delta decode. The first frame's delta is absolute (prev = 0). prev_ts: u64, phase: IngestPhase, + budget: frame::Budget, } enum IngestPhase { @@ -891,12 +905,13 @@ enum IngestPhase { } impl FrameIngest { - fn new(runtime: crate::time::Clock, timescale: Option) -> Self { + fn new(subscriber: &Subscriber, timescale: Option) -> Self { Self { timescale, prev_ts: 0, phase: IngestPhase::Timing, - runtime, + runtime: subscriber.runtime.clone(), + budget: subscriber.frames.clone(), } } @@ -935,11 +950,12 @@ impl FrameIngest { let Some(size) = ready!(reader.poll_decode_maybe::(&mut cx))? else { return Poll::Ready(Ok(())); }; - // `create_frame_owned` is the allocation chokepoint and rejects an - // oversized `size` before allocating, so no pre-check is needed. No - // wire timestamp (pre-lite-05) means local receive time. + // `create_frame_owned` is the allocation chokepoint: it rejects an + // oversized `size` and allocates up front only within the budget, so + // no pre-check is needed. No wire timestamp (pre-lite-05) means local + // receive time. let timestamp = timestamp.unwrap_or_else(|| Timestamp::from(self.runtime.now())); - let frame = group.create_frame_owned(frame::Info { size, timestamp })?; + let frame = group.create_frame_owned(frame::Info { size, timestamp }, &self.budget)?; self.phase = IngestPhase::Payload { frame }; } IngestPhase::Payload { frame } => { @@ -4221,11 +4237,25 @@ enum FetchRunState { Ingest { stream: Stream, producer: group::Producer, - ingest: FrameIngest, + // The accepted request's demand: fetches that joined it but have not yet + // picked the group up from the cache. + joined: kio::Producer, + // Boxed so the other states stay small; one allocation per fetch. + ingest: Box, }, Done, } +impl FetchRunState { + /// The downstream request, while it waits on the publisher's answer. + fn request(&self) -> Option<&group::Request> { + match self { + Self::Open { request } | Self::Send { request, .. } | Self::Answer { request, .. } => request.as_ref(), + Self::Ingest { .. } | Self::Done => None, + } + } +} + impl FetchServeRun { fn new(serve: TrackServe, request: group::Request, timescale: Option) -> Self { let session = serve.subscriber.session.clone(); @@ -4246,6 +4276,20 @@ impl kio::Task for FetchServeRun { fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { let mut cx = waiter.context(); loop { + // A fetch nobody waits on any more is cancelled upstream, so the publisher stops + // serving it (and a relay there releases its own FETCH). Ingest has its own check. + if let Some(request) = self.state.request() + && request.poll_unused(waiter).is_ready() + { + tracing::debug!(track = %self.serve.name, group = self.group, "fetch abandoned"); + if let FetchRunState::Send { stream, .. } | FetchRunState::Answer { stream, .. } = + std::mem::replace(&mut self.state, FetchRunState::Done) + { + stream.writer.abort(&Error::Cancel); + } + return Poll::Ready(()); + } + match &mut self.state { FetchRunState::Open { request } => { tracing::info!(broadcast = %self.serve.subscriber.log_path(&self.serve.path), track = %self.serve.name, group = self.group, "fetch started"); @@ -4365,6 +4409,9 @@ impl kio::Task for FetchServeRun { // `Some`; fall back to the default scale defensively rather than // panicking. let group_info = track::Info::default().with_timescale(self.timescale.unwrap_or_default()); + // The joined fetches pick the group up from the cache only when next + // polled, so their demand outlives the request until then. + let joined = request.result.clone(); let mut producer = match request.accept(group_info) { Ok(producer) => producer, Err(err) => { @@ -4386,15 +4433,36 @@ impl kio::Task for FetchServeRun { self.state = FetchRunState::Ingest { stream, producer, - ingest: FrameIngest::new(self.serve.subscriber.runtime.clone(), self.timescale), + joined, + ingest: Box::new(FrameIngest::new(&self.serve.subscriber, self.timescale)), }; } FetchRunState::Ingest { stream, producer, + joined, ingest, } => { - let res = ready!(ingest.poll(&mut stream.reader, producer, waiter)); + let Poll::Ready(res) = ingest.poll(&mut stream.reader, producer, waiter) else { + // Still short of its end, so completion always wins. Once nobody + // wants the rest (no joined fetch left to pick it up, no reader), + // cancel upstream and abort the truncated group, never caching it + // as complete. The abort is atomic with a new reader arriving. + if joined.poll_unused(waiter).is_pending() || producer.poll_unused(waiter).is_pending() { + return Poll::Pending; + } + if !producer.abort_unused(Error::Cancel) { + continue; + } + tracing::debug!(track = %self.serve.name, group = self.group, "fetch abandoned"); + let FetchRunState::Ingest { stream, .. } = + std::mem::replace(&mut self.state, FetchRunState::Done) + else { + unreachable!() + }; + stream.writer.abort(&Error::Cancel); + return Poll::Ready(()); + }; let FetchRunState::Ingest { producer, .. } = std::mem::replace(&mut self.state, FetchRunState::Done) else { diff --git a/rs/moq-net/src/lite/test_transport.rs b/rs/moq-net/src/lite/test_transport.rs index fe9e1170de3..7903d3cda4e 100644 --- a/rs/moq-net/src/lite/test_transport.rs +++ b/rs/moq-net/src/lite/test_transport.rs @@ -559,14 +559,18 @@ impl web_transport_trait::Stats for SinkStats { /// EOF would exit, and a test usually wants to assert against the loop still running. pub struct ScriptedRecv { script: Arc>>, - /// Report EOF once the script is exhausted rather than parking, so a test can drive - /// a read loop all the way through its exit path. See [`ScriptedSession::eof`]. - eof: bool, - /// Report a reset once the script is exhausted. See [`ScriptedSession::per_stream_reset`]. - reset: bool, + /// How the peer's send side ends once the script runs out; `None` parks. + close: Arc>>, log: Log, } +/// How a scripted peer closes its send side. See [`ScriptedSession::close`]. +#[derive(Clone, Copy, Debug)] +pub enum Close { + Fin, + Reset, +} + impl poll::RecvStream for ScriptedRecv { type Error = SinkError; @@ -584,9 +588,11 @@ impl poll::RecvStream for ScriptedRecv { }; match take { - 0 if self.reset => Poll::Ready(Err(SinkError)), - 0 if self.eof => Poll::Ready(Ok(None)), - 0 => Poll::Pending, + 0 => match *self.close.lock().unwrap() { + Some(Close::Fin) => Poll::Ready(Ok(None)), + Some(Close::Reset) => Poll::Ready(Err(SinkError)), + None => Poll::Pending, + }, take => Poll::Ready(Ok(Some(take))), } } @@ -609,10 +615,8 @@ impl poll::RecvStream for ScriptedRecv { #[derive(Clone)] pub struct ScriptedSession { pub log: Log, - /// Whether an exhausted script reports EOF instead of parking. - eof: bool, - /// Whether an exhausted script reports a reset instead of parking. - reset: bool, + /// Shared with every stream, like `script`. See [`Self::close`]. + close: Arc>>, script: Arc>>, /// Per-stream scripts popped by `open_bi` in order; `None` shares `script` /// across every stream. @@ -631,8 +635,7 @@ impl ScriptedSession { pub fn new(script: Vec) -> Self { Self { log: Log::default(), - eof: false, - reset: false, + close: Default::default(), script: Arc::new(Mutex::new(script)), queue: None, open_gate: None, @@ -659,10 +662,9 @@ impl ScriptedSession { /// Parking is the right default for asserting that a loop is still running, but a /// test for what a loop does on the way *out* needs the read to actually end. pub fn eof(script: Vec) -> Self { - Self { - eof: true, - ..Self::new(script) - } + let session = Self::new(script); + session.close(Close::Fin); + session } /// Each `open_bi` replays the next script in order. An exhausted queue (or an @@ -677,19 +679,17 @@ impl ScriptedSession { /// Like [`Self::per_stream`], but an exhausted script closes the stream instead of /// parking, for a test that drives each stream to its end. pub fn per_stream_eof(scripts: Vec>) -> Self { - Self { - eof: true, - ..Self::per_stream(scripts) - } + let session = Self::per_stream(scripts); + session.close(Close::Fin); + session } /// Like [`Self::per_stream`], but an exhausted script resets the stream, as a peer /// that fails after replying would. pub fn per_stream_reset(scripts: Vec>) -> Self { - Self { - reset: true, - ..Self::per_stream(scripts) - } + let session = Self::per_stream(scripts); + session.close(Close::Reset); + session } /// Append to the shared script: the peer sending more on a stream it already opened. @@ -698,6 +698,12 @@ impl ScriptedSession { self.script.lock().unwrap().extend_from_slice(bytes); } + /// Close the peer's send side once the script runs out. Nothing is woken, so the + /// test re-polls the reader itself. + pub fn close(&self, close: Close) { + *self.close.lock().unwrap() = Some(close); + } + /// Answer each stream from `scripts`, but only once the gate opens: a peer that /// replies normally and is simply out of stream credit until then. pub fn gated_open(scripts: Vec>, gate: kio::Consumer) -> Self { @@ -719,8 +725,7 @@ impl poll::Session for ScriptedSession { }; Poll::Ready(Ok(ScriptedRecv { script: Arc::new(Mutex::new(script)), - eof: self.eof, - reset: self.reset, + close: self.close.clone(), log: self.log.clone(), })) } @@ -751,8 +756,7 @@ impl poll::Session for ScriptedSession { SinkSend::new(self.log.clone()), ScriptedRecv { script, - eof: self.eof, - reset: self.reset, + close: self.close.clone(), log: self.log.clone(), }, ))) diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index d2eab832f2a..da7f4d8cd5b 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -7,8 +7,8 @@ //! reads one frame, sharing the group's channel rather than a per-frame one. use std::ops::Range; use std::sync::Arc; -use std::sync::OnceLock; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Mutex, OnceLock}; use std::task::{Poll, ready}; use arrayvec::ArrayVec; @@ -127,40 +127,93 @@ impl Buffer { } } +/// Bytes one session may allocate up front for the frames it is still receiving. +/// +/// A peer declares each frame's size before sending it, and allocating that size up +/// front saves a reallocation and copy per doubling as the payload arrives. The +/// declaration costs the peer nothing, though, so without a bound it could commit up to +/// [`group::MAX_CACHE_BYTES`] per stream without sending a byte. A frame whose declared +/// size fits what remains is allocated up front; past that, its buffer grows with the +/// bytes received. Each frame returns its share once it completes or aborts. +#[derive(Clone)] +pub(crate) struct Budget(Arc); + +impl Budget { + /// Enough for a handful of large keyframes in flight at once. + const DEFAULT: usize = 16 * 1024 * 1024; + + pub(crate) fn new(bytes: usize) -> Self { + Self(Arc::new(AtomicUsize::new(bytes))) + } + + /// Take `size` bytes of the budget, or `None` if that is more than remains. + pub(crate) fn reserve(&self, size: usize) -> Option { + self.0 + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |left| left.checked_sub(size)) + .ok()?; + Some(Reservation { + budget: self.clone(), + size, + }) + } +} + +impl Default for Budget { + fn default() -> Self { + Self::new(Self::DEFAULT) + } +} + +/// A frame's share of a [`Budget`], returned on drop. +pub(crate) struct Reservation { + budget: Budget, + size: usize, +} + +impl Drop for Reservation { + fn drop(&mut self) { + self.budget.0.fetch_add(self.size, Ordering::Relaxed); + } +} + /// Payload storage for the single in-flight frame, shared between the writing /// [`Producer`] and any streaming [`Consumer`]s. /// -/// A whole-frame [`Bytes`] write is stored directly. Chunked writes fall back to one -/// mutable heap allocation sized to the declared frame. The producer writes through -/// the raw pointer (sole writer, guaranteed by the exclusive borrow of the parent -/// group); `written` provides happens-before for cross-thread reads. Implements -/// [AsRef]<[u8]> so it can back a [`Bytes::from_owner`]. +/// A whole-frame [`Bytes`] write is stored directly. Chunked writes are copied into a +/// heap [`Segment`], either sized to the declared frame up front or grown as bytes +/// arrive (see [`Budget`]). The producer writes through the raw pointer (sole writer, +/// guaranteed by the exclusive borrow of the parent group); `written` provides +/// happens-before for cross-thread reads. #[derive(Clone)] pub(crate) struct FrameBuf(Arc); struct FrameBufInner { - capacity: usize, + size: usize, + grow: bool, written: AtomicUsize, storage: OnceLock, } enum FrameStorage { Shared(Bytes), - Mutable(MutableFrameBuf), + Fixed(Arc), + // Replaced by a larger copy whenever a write outgrows it. A reader clones the current + // segment under the lock, so a slice it already handed out keeps the old one alive. + Growing(Mutex>), } -struct MutableFrameBuf { - // Owned heap allocation of `capacity` bytes (zero-initialized). +/// An owned, zero-initialized heap allocation that chunked writes are copied into. +struct Segment { data: *mut u8, capacity: usize, } // Safety: `data` is owned (Box-allocated, freed in Drop). The producer is the sole // writer and consumers only read bytes `< written`. -unsafe impl Send for MutableFrameBuf {} -unsafe impl Sync for MutableFrameBuf {} +unsafe impl Send for Segment {} +unsafe impl Sync for Segment {} -impl Drop for MutableFrameBuf { +impl Drop for Segment { fn drop(&mut self) { // Safety: data was obtained from `Box::into_raw` of a `Box<[u8]>` of length // `capacity` and is not aliased at drop (Arc refcount hit 0). @@ -171,30 +224,66 @@ impl Drop for MutableFrameBuf { } } -impl MutableFrameBuf { - fn new(size: usize) -> Self { - let boxed: Box<[u8]> = vec![0u8; size].into_boxed_slice(); +impl Segment { + fn new(capacity: usize) -> Self { + let boxed: Box<[u8]> = vec![0u8; capacity].into_boxed_slice(); let capacity = boxed.len(); let data = Box::into_raw(boxed) as *mut u8; Self { data, capacity } } + + /// Copy `src` in at `offset`. + /// + /// Safety: caller must be the sole writer, `offset + src.len()` must be within + /// `capacity`, and no reader may have been handed bytes at or past `offset`. + unsafe fn write(&self, offset: usize, src: &[u8]) { + debug_assert!(offset + src.len() <= self.capacity); + unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), self.data.add(offset), src.len()) }; + } +} + +/// The first `len` bytes of a [`Segment`], all written, so it can back a +/// [`Bytes::from_owner`]. +struct Filled { + segment: Arc, + len: usize, +} + +impl AsRef<[u8]> for Filled { + fn as_ref(&self) -> &[u8] { + // Safety: `len` never exceeds what the producer had published when this was + // built, and the producer never rewrites published bytes. The Arc keeps the + // allocation alive while any reference to the slice lives. + unsafe { std::slice::from_raw_parts(self.segment.data, self.len) } + } } impl FrameBuf { - /// Allocate a buffer for a frame of `size` bytes. + /// A buffer for a frame of `size` bytes, allocated at that size on the first write. /// /// The oversized-frame guard lives in [`group::Producer`], which rejects a declared /// size larger than the group's byte budget before calling this. pub(crate) fn new(size: usize) -> Self { + Self::with_growth(size, false) + } + + /// A buffer for a frame of `size` bytes that allocates only as bytes arrive. + pub(crate) fn growing(size: usize) -> Self { + Self::with_growth(size, true) + } + + fn with_growth(size: usize, grow: bool) -> Self { Self(Arc::new(FrameBufInner { - capacity: size, + size, + grow, written: AtomicUsize::new(0), storage: OnceLock::new(), })) } - pub(crate) fn capacity(&self) -> usize { - self.0.capacity + /// The frame's declared size. + pub(crate) fn size(&self) -> usize { + self.0.size } pub(crate) fn written(&self, ord: Ordering) -> usize { @@ -202,7 +291,7 @@ impl FrameBuf { } fn try_set_bytes(&self, bytes: Bytes) -> std::result::Result<(), Bytes> { - if bytes.len() != self.capacity() || self.written(Ordering::Acquire) != 0 { + if bytes.len() != self.size() || self.written(Ordering::Acquire) != 0 { return Err(bytes); } self.0 @@ -210,25 +299,11 @@ impl FrameBuf { .set(FrameStorage::Shared(bytes)) .map_err(|storage| match storage { FrameStorage::Shared(bytes) => bytes, - FrameStorage::Mutable(_) => unreachable!("try_set_bytes only installs shared storage"), + _ => unreachable!("try_set_bytes only installs shared storage"), }) } - /// The mutable buffer for multi-chunk writes, lazily allocated. - /// - /// Returns `None` once a whole-frame write has installed shared storage. - fn mutable(&self) -> Option<&MutableFrameBuf> { - match self - .0 - .storage - .get_or_init(|| FrameStorage::Mutable(MutableFrameBuf::new(self.capacity()))) - { - FrameStorage::Shared(_) => None, - FrameStorage::Mutable(buf) => Some(buf), - } - } - - /// Safety: caller must be the sole producer and `new_written` must be `<= capacity`. + /// Safety: caller must be the sole producer and `new_written` must be `<= size`. unsafe fn store_written(&self, new_written: usize) { // Release pairs with consumers' Acquire load to publish prior writes. self.0.written.store(new_written, Ordering::Release); @@ -244,50 +319,70 @@ impl FrameBuf { return; } let prev = self.written(Ordering::Relaxed); - let Some(buf) = self.mutable() else { + let storage = self.0.storage.get_or_init(|| match self.0.grow { + true => FrameStorage::Growing(Mutex::new(Arc::new(Segment::new(0)))), + false => FrameStorage::Fixed(Arc::new(Segment::new(self.size()))), + }); + // Safety (every write below): sole writer; the caller bounds-checked `src` against + // the remaining size, and consumers only read `[..written]`. + match storage { // Only reachable if the frame is already complete via shared storage, which // `Producer::write` rejects for a non-empty chunk. Nothing to copy. - return; - }; - // Safety: sole writer; the caller bounds-checked `src` against the remaining - // capacity, and consumers only read `[..written]`. - unsafe { - std::ptr::copy_nonoverlapping(src.as_ptr(), buf.data.add(prev), src.len()); - self.store_written(prev + src.len()); + FrameStorage::Shared(_) => return, + FrameStorage::Fixed(segment) => unsafe { segment.write(prev, src) }, + FrameStorage::Growing(current) => { + let mut current = current.lock().expect("mutex poisoned"); + let needed = prev + src.len(); + if current.capacity < needed { + // Doubling keeps the copies linear in the frame size, and the cap makes + // the last segment exactly the frame, so it freezes without a copy. + let next = Segment::new(needed.max(current.capacity * 2).min(self.size())); + let filled = Filled { + segment: current.clone(), + len: prev, + }; + unsafe { next.write(0, filled.as_ref()) }; + *current = Arc::new(next); + } + unsafe { current.write(prev, src) }; + } } + // Safety: sole writer, and the caller bounds-checked `src` against the size. + unsafe { self.store_written(prev + src.len()) }; } - /// Freeze the buffer into the completed payload (`size` bytes). + /// Freeze the buffer into the completed payload. /// /// Returns the shared [`Bytes`] directly for a whole-frame write (zero-copy), or /// wraps the mutable allocation otherwise. - fn freeze(&self, size: usize) -> Bytes { - match self.0.storage.get() { - Some(FrameStorage::Shared(bytes)) => bytes.clone(), - _ => self.slice(0, size), - } + fn freeze(&self) -> Bytes { + self.slice(0, self.size()) } - /// A zero-copy slice of the initialized region `[start..end]`. + /// A zero-copy slice of the written region `[start..end]`. + /// + /// `end` must not exceed a `written` value already loaded with `Acquire`, which is + /// also what makes the segment read here at least as new as those bytes. fn slice(&self, start: usize, end: usize) -> Bytes { - Bytes::from_owner(self.clone()).slice(start..end) + let segment = match self.0.storage.get() { + Some(FrameStorage::Shared(bytes)) => return bytes.slice(start..end), + Some(FrameStorage::Fixed(segment)) => segment.clone(), + Some(FrameStorage::Growing(current)) => current.lock().expect("mutex poisoned").clone(), + // Nothing written, so the range is empty. + None => return Bytes::new(), + }; + debug_assert!(end <= segment.capacity); + Bytes::from_owner(Filled { segment, len: end }).slice(start..) } -} -impl AsRef<[u8]> for FrameBuf { - fn as_ref(&self) -> &[u8] { - // Snapshot the initialized region (bytes the producer has written so far). - // Acquire pairs with the producer's Release on `written`. - let written = self.0.written.load(Ordering::Acquire); + /// Heap bytes the buffer currently holds for the payload. + #[cfg(test)] + pub(crate) fn allocated(&self) -> usize { match self.0.storage.get() { - Some(FrameStorage::Shared(bytes)) => &bytes[..written], - Some(FrameStorage::Mutable(buf)) => { - // Safety: data..data+written is initialized (zero-init at alloc + producer - // writes up to `written`). The Arc keeps the allocation alive while any - // reference to the slice lives. - unsafe { std::slice::from_raw_parts(buf.data, written) } - } - None => &[], + Some(FrameStorage::Shared(bytes)) => bytes.len(), + Some(FrameStorage::Fixed(segment)) => segment.capacity, + Some(FrameStorage::Growing(current)) => current.lock().expect("mutex poisoned").capacity, + None => 0, } } } @@ -307,7 +402,7 @@ struct Raw> { impl> Raw { fn remaining(&self) -> usize { - self.buf.capacity() - self.buf.written(Ordering::Acquire) + self.buf.size() - self.buf.written(Ordering::Acquire) } fn write(&mut self, chunk: B) -> Result<()> { @@ -318,13 +413,13 @@ impl> Raw { // Ingress payload: count the chunk's bytes as they're written. self.stats.bytes(len as u64); // Fast path: a single whole-frame write keeps the caller's allocation. - if len == self.buf.capacity() && self.buf.written(Ordering::Acquire) == 0 { + if len == self.buf.size() && self.buf.written(Ordering::Acquire) == 0 { match self.buf.try_set_bytes(chunk.into_bytes()) { Ok(()) => { - let cap = self.buf.capacity(); + let size = self.buf.size(); // Safety: `try_set_bytes` checked the buffer exactly matches the declared // size, so publishing all bytes is in bounds. - unsafe { self.buf.store_written(cap) }; + unsafe { self.buf.store_written(size) }; } Err(chunk) => self.buf.append(&chunk), } @@ -335,10 +430,10 @@ impl> Raw { } fn finish(&mut self) -> Result<()> { - if self.buf.written(Ordering::Acquire) != self.buf.capacity() { + if self.buf.written(Ordering::Acquire) != self.buf.size() { return Err(Error::WrongSize); } - let payload = self.buf.freeze(self.buf.capacity()); + let payload = self.buf.freeze(); self.group.borrow_mut().frame_commit(Frame { timestamp: self.info.timestamp, payload, @@ -448,36 +543,44 @@ impl<'a> Producer<'a> { /// Crate-private on purpose: the exclusivity the public borrow enforces (one /// live frame per group) becomes the holder's promise here. Do not open another /// frame on the group until this one is finished or aborted. -pub(crate) struct ProducerOwned(Raw); +/// +/// Holds the frame's share of the session's [`Budget`], if it got one, until then. +pub(crate) struct ProducerOwned { + raw: Raw, + _reserved: Option, +} impl std::ops::Deref for ProducerOwned { type Target = Info; fn deref(&self) -> &Self::Target { - &self.0.info + &self.raw.info } } impl ProducerOwned { - pub(crate) fn new(group: group::Producer, buf: FrameBuf, info: Info) -> Self { - Self(Raw { - group, - buf, - info, - done: false, - stats: stats::Meter::default(), - }) + pub(crate) fn new(group: group::Producer, buf: FrameBuf, info: Info, reserved: Option) -> Self { + Self { + raw: Raw { + group, + buf, + info, + done: false, + stats: stats::Meter::default(), + }, + _reserved: reserved, + } } /// Attach the parent group's ingress meter, so written chunks bump `bytes`. pub(crate) fn with_meter(mut self, meter: stats::Meter) -> Self { - self.0.stats = meter; + self.raw.stats = meter; self } /// Bytes still needed to complete the frame. pub fn remaining(&self) -> usize { - self.0.remaining() + self.raw.remaining() } /// Write a chunk of payload *without* waking consumers; pair it with [`Self::notify`]. @@ -489,22 +592,28 @@ impl ProducerOwned { /// /// `coding::Reader::poll_read_frame` owns the pairing and is the only caller. pub(crate) fn write(&mut self, chunk: B) -> Result<()> { - self.0.write(chunk) + self.raw.write(chunk) } /// Publish what has been written so far, waking consumers parked on the group. pub(crate) fn notify(&self) { - self.0.group.frame_notify(); + self.raw.group.frame_notify(); } /// Commit the frame, verifying that all bytes were written. pub fn finish(mut self) -> Result<()> { - self.0.finish() + self.raw.finish() } /// Abort the frame (and its group) with the given error. pub fn abort(mut self, err: Error) -> Result<()> { - self.0.abort(err) + self.raw.abort(err) + } + + /// Heap bytes the frame's buffer currently holds for the payload. + #[cfg(test)] + pub(crate) fn allocated(&self) -> usize { + self.raw.buf.allocated() } } @@ -749,3 +858,72 @@ where Err(state) => Err(state.abort.clone().unwrap_or(Error::Dropped)), }) } + +#[cfg(test)] +mod tests { + use super::*; + + /// Each reallocation leaves the slices already handed out intact, and the last + /// segment is exactly the frame, so the payload freezes without another copy. + #[test] + fn growing_buffer_keeps_handed_out_slices() { + let buf = FrameBuf::growing(10); + buf.append(b"abc"); + assert_eq!(buf.allocated(), 3); + let early = buf.slice(0, 3); + + buf.append(b"defg"); + assert_eq!(buf.allocated(), 7); + buf.append(b"hij"); + assert_eq!(buf.allocated(), 10, "doubling stops at the declared size"); + + assert_eq!(early, &b"abc"[..]); + assert_eq!(buf.slice(3, 7), &b"defg"[..]); + assert_eq!(buf.freeze(), &b"abcdefghij"[..]); + } + + /// A reader on another thread slicing while the writer reallocates only ever sees + /// the bytes that were written. + #[test] + fn growing_buffer_reads_across_threads() { + const SIZE: usize = 256 * 1024; + let buf = FrameBuf::growing(SIZE); + let reader = std::thread::spawn({ + let buf = buf.clone(); + move || { + let mut read = 0; + while read < SIZE { + let written = buf.written(Ordering::Acquire); + let chunk = buf.slice(read, written); + for (i, byte) in chunk.iter().enumerate() { + assert_eq!(*byte, ((read + i) % 251) as u8); + } + read = written; + std::thread::yield_now(); + } + } + }); + + let payload: Vec = (0..SIZE).map(|i| (i % 251) as u8).collect(); + for chunk in payload.chunks(1000) { + buf.append(chunk); + } + reader.join().unwrap(); + assert_eq!(buf.freeze(), payload); + } + + /// A reservation holds its bytes until it drops, and a request larger than what + /// remains takes nothing. + #[test] + fn budget_returns_reservations() { + let budget = Budget::new(100); + let a = budget.reserve(60).unwrap(); + assert!(budget.reserve(41).is_none()); + let b = budget.reserve(40).unwrap(); + assert!(budget.reserve(1).is_none()); + + drop(a); + drop(b); + assert!(budget.reserve(100).is_some()); + } +} diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index e9f3a4bbda5..e199da5f881 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -116,6 +116,8 @@ impl From for Info { pub(crate) struct Partial { timestamp: Timestamp, buf: FrameBuf, + // How much of `buf` has been charged to the cache so far. + charged: u64, } /// Shared group state. `pub(crate)` so [`frame`] handles can observe the abort flag @@ -144,7 +146,8 @@ pub(crate) struct GroupState { // its opener saw the payload, and only partly. committed: usize, - // The total size (in bytes) of all cached frames plus any in-flight frame. + // The total size (in bytes) of all cached frames plus what the in-flight frame has + // written so far. pub(crate) cache: u64, // Mirrors `cache` into the track's shared cache pool, so the group's bytes count @@ -175,9 +178,16 @@ pub(crate) struct GroupState { impl GroupState { /// Content still available to a reader of this group. + /// + /// Counts the in-flight frame at its declared size, like [`Self::content_range`]: a + /// reader that skips it misses the whole frame, however much has arrived. fn content(&self) -> stats::Content { + let unwritten = self + .partial + .as_ref() + .map_or(0, |partial| partial.buf.size() as u64 - partial.charged); stats::Content { - bytes: self.cache, + bytes: self.cache + unwritten, frames: self.next_index.saturating_sub(self.offset) as u64, groups: 1, datagrams: 0, @@ -203,7 +213,7 @@ impl GroupState { && self.committed < end && let Some(partial) = &self.partial { - bytes += partial.buf.capacity() as u64; + bytes += partial.buf.size() as u64; } stats::Content { @@ -236,7 +246,7 @@ impl GroupState { { self.charge.refresh(); let info = frame::Info { - size: p.buf.capacity() as u64, + size: p.buf.size() as u64, timestamp: p.timestamp, }; return Poll::Ready(Ok(Some((info, frame::Source::Partial(p.buf.clone()))))); @@ -282,6 +292,21 @@ impl GroupState { || self.cache.saturating_add(extra_bytes) > MAX_CACHE_BYTES } + /// Charge the in-flight frame's bytes written since its last charge, as a write access. + /// + /// Returns the coarse tick it stamped, like [`cache::Charge::add`]. + fn charge_partial(&mut self) -> Option { + let written = match &mut self.partial { + Some(partial) => { + let written = partial.buf.written(Ordering::Acquire) as u64; + written - std::mem::replace(&mut partial.charged, written) + } + None => 0, + }; + self.cache += written; + self.charge.add(written) + } + /// Drop the cached frames (and any in-flight tail) and release their pool charge. fn release(&mut self) { self.frames.clear(); @@ -592,54 +617,10 @@ impl Producer { /// if the declared size exceeds the group's byte budget (refused before allocating) /// or [`Error::TimestampMismatch`] if the timestamp can't be converted (overflow). pub fn create_frame(&mut self, frame: frame::Info) -> Result> { - let timestamp = frame - .timestamp - .convert(self.track.timescale) - .map_err(|_| Error::TimestampMismatch)?; - if frame.size > MAX_CACHE_BYTES { - return Err(Error::FrameTooLarge); - } + // The buffer allocates on its first write, so refusing the frame costs nothing. let buf = FrameBuf::new(frame.size as usize); - - let mut state = modify(&self.state)?; - if state.fin.is_some() { - return Err(Error::Closed); - } - if state.partial.is_some() { - return Err(Error::FrameOpen); - } - let next_index = state - .next_index - .checked_add(1) - .ok_or(Error::BoundsExceeded(crate::coding::BoundsExceeded))?; - if state.would_overflow(1, frame.size) { - return Err(self.abort_too_large(state)); - } - state.cache += frame.size; - let now = state.charge.add(frame.size); - state.partial = Some(Partial { - timestamp, - buf: buf.clone(), - }); - state.next_index = next_index; - // Opening the frame is enough: the header carries the timestamp, so the group's - // place in time is known before a single payload byte streams in. - state.stamp(timestamp); - drop(state); - - // With the group lock released (lock order is track then group), settle - // eviction debt if enough has been written since the track last paid. - self.cache.settle(now); - - // Ingress payload: one frame opened; its bytes are counted per chunk as the - // frame::Producer writes them. - self.stats.frames(1); + let info = self.open_frame(frame, &buf)?; let meter = self.stats.clone(); - - let info = frame::Info { - size: frame.size, - timestamp, - }; Ok(frame::Producer::new(self, buf, info).with_meter(meter)) } @@ -647,7 +628,30 @@ impl Producer { /// stream a frame across polls and cannot hold the group borrowed inside their /// state. The one-live-frame rule the borrow normally enforces becomes the /// caller's promise; see [`frame::ProducerOwned`]. - pub(crate) fn create_frame_owned(&mut self, frame: frame::Info) -> Result { + /// + /// The declared size is the peer's claim, so the buffer is allocated up front only + /// while it fits the session's `budget`; otherwise it grows with the bytes received. + pub(crate) fn create_frame_owned( + &mut self, + frame: frame::Info, + budget: &frame::Budget, + ) -> Result { + let size = frame.size as usize; + let reserved = budget.reserve(size); + let buf = match reserved { + Some(_) => FrameBuf::new(size), + None => FrameBuf::growing(size), + }; + let info = self.open_frame(frame, &buf)?; + let meter = self.stats.clone(); + Ok(frame::ProducerOwned::new(self.clone(), buf, info, reserved).with_meter(meter)) + } + + /// Open `buf` as the in-flight frame, returning its header in the track's timescale. + /// + /// Its bytes are charged to the cache as they are written, not here: the declared + /// size is only a promise until they arrive. + fn open_frame(&mut self, frame: frame::Info, buf: &FrameBuf) -> Result { let timestamp = frame .timestamp .convert(self.track.timescale) @@ -655,7 +659,6 @@ impl Producer { if frame.size > MAX_CACHE_BYTES { return Err(Error::FrameTooLarge); } - let buf = FrameBuf::new(frame.size as usize); let mut state = modify(&self.state)?; if state.fin.is_some() { @@ -668,14 +671,16 @@ impl Producer { .next_index .checked_add(1) .ok_or(Error::BoundsExceeded(crate::coding::BoundsExceeded))?; + // Only one frame is ever in flight, so the declared size is the most the cache + // can grow by before the next check. if state.would_overflow(1, frame.size) { return Err(self.abort_too_large(state)); } - state.cache += frame.size; - let now = state.charge.add(frame.size); + let now = state.charge.record_write(); state.partial = Some(Partial { timestamp, buf: buf.clone(), + charged: 0, }); state.next_index = next_index; // Opening the frame is enough: the header carries the timestamp, so the group's @@ -690,47 +695,39 @@ impl Producer { // Ingress payload: one frame opened; its bytes are counted per chunk as the // producer writes them. self.stats.frames(1); - let meter = self.stats.clone(); - let info = frame::Info { + Ok(frame::Info { size: frame.size, timestamp, - }; - Ok(frame::ProducerOwned::new(self.clone(), buf, info).with_meter(meter)) + }) } /// Wake consumers parked on the group channel (called after a partial write). pub(crate) fn frame_notify(&self) { - // The chunk that was just written is a write access: restart the retention - // clock so a straggler group streaming a large frame isn't expired - // mid-write (its bytes were already charged when the frame was created). - // `record_write` takes `&mut`, which marks the guard modified: kio only - // notifies on a mutably-accessed guard's release, and that notify is what + // The chunk that was just written is charged, and is a write access: restart the + // retention clock so a straggler group streaming a large frame isn't expired + // mid-write. `charge_partial` takes `&mut`, which marks the guard modified: kio + // only notifies on a mutably-accessed guard's release, and that notify is what // delivers the chunk to parked readers. - let now = self - .state - .write() - .ok() - .and_then(|mut state| state.charge.record_write()); - // The payload was charged when the frame opened, but a long streamed frame - // still counts as track activity for the independent expiry time gate. + let now = self.state.write().ok().and_then(|mut state| state.charge_partial()); + // A long streamed frame also counts as track activity for the independent + // expiry time gate. self.cache.settle(now); } /// Commit the in-flight frame as a completed frame (called by [`frame::Producer::finish`]). pub(crate) fn frame_commit(&mut self, frame: Frame) -> Result<()> { let mut state = modify(&self.state)?; - // Bytes were already counted against the cache (and the pool charge) when the - // frame was created; committing just moves the tail into the completed set. + // Completing the frame charges whatever it wrote since the last notify, and is a + // write access like any chunk, and the only one the payload is guaranteed to get: + // the wire ingest defers its chunk notifications to the poll boundary, so a tail + // that arrives and completes in one turn never reaches [`Self::frame_notify`]. + // Without this, a group whose payload streamed in across an idle gap would expire + // the instant it finished. + let now = state.charge_partial(); state.partial = None; state.frames.push_back(frame); state.committed = state.next_index; - // Completing the frame is a write access like any chunk, and the only one the - // payload is guaranteed to get: the wire ingest defers its chunk notifications - // to the poll boundary, so a tail that arrives and completes in one turn never - // reaches [`Self::frame_notify`]. Without this, a group whose payload streamed - // in across an idle gap would expire the instant it finished. - let now = state.charge.record_write(); drop(state); // With the group lock released (lock order is track then group), settle @@ -773,12 +770,43 @@ impl Producer { /// their buffers in memory forever; consumers that haven't drained yet surface the /// abort error instead of the leftover cache. pub fn abort(self, err: Error) -> Result<()> { - let mut guard = modify(&self.state)?; + self.close_aborted(err) + } + + /// Abort with `err` unless another handle still owns the group, so a retained clone + /// lets go without the unfinished-drop warning and never cuts off a live writer. + /// Sound without a lock: only an existing handle can mint another. + pub(crate) fn abort_if_last(&self, err: Error) -> Result<()> { + if Arc::strong_count(&self.alive) > 1 { + return Ok(()); + } + self.close_aborted(err) + } + + /// Abort with `err` only while nothing consumes the group, returning whether it is + /// closed. Consumer creation and the check share a lock, so a reader arriving after + /// [`poll_unused`](Self::poll_unused) keeps the group alive instead of reading the abort. + pub(crate) fn abort_unused(&self, err: Error) -> bool { + match self.state.write_unused() { + kio::Unused::Idle(guard) => { + self.commit_abort(guard, err); + true + } + kio::Unused::Closed => true, + kio::Unused::Used => false, + } + } + + fn close_aborted(&self, err: Error) -> Result<()> { + self.commit_abort(modify(&self.state)?, err); + Ok(()) + } + + fn commit_abort(&self, mut guard: kio::Mut<'_, GroupState>, err: Error) { guard.abort = Some(err); self.alive.aborted.store(true, Ordering::Release); guard.release(); guard.close(); - Ok(()) } /// Abort a write that would grow the group past its budget, holding the lock already @@ -902,11 +930,22 @@ impl Producer { /// Create a new consumer for the group. pub fn consume(&self) -> Consumer { + self.consumer(self.state.consume()) + } + + /// Create a consumer, or `None` once the group is aborted. Paired with + /// [`abort_unused`](Self::abort_unused): the closed check and the count share its + /// lock, so a consumer either exists in time to decline the abort or is never made. + pub(crate) fn try_consume(&self) -> Option { + self.state.weak().try_consume().map(|state| self.consumer(state)) + } + + fn consumer(&self, state: kio::Consumer) -> Consumer { Consumer { info: self.info, track: self.track.clone(), inner: ConsumerKind::Plain(Plain { - state: self.state.consume(), + state, index: 0, end: None, prefetch: Prefetch::default(), @@ -959,6 +998,11 @@ impl Producer { self.state.unused().await.map_err(|_| self.abort_reason()) } + /// Poll for the group becoming unused (every consumer dropped). + pub(crate) fn poll_unused(&self, waiter: &kio::Waiter) -> Poll<()> { + self.state.poll_unused(waiter).map(|_| ()) + } + /// The recorded abort reason, or [`Error::Dropped`] if the group closed without one. fn abort_reason(&self) -> Error { self.state.read().abort.clone().unwrap_or(Error::Dropped) @@ -1958,6 +2002,21 @@ mod test { assert!(matches!(result, Err(crate::Error::Cancel))); } + #[test] + fn abort_unused_pairs_with_try_consume() { + let producer = Info { sequence: 0 }.produce(); + + let consumer = producer.try_consume().expect("open"); + assert!( + !producer.abort_unused(crate::Error::Cancel), + "a reader declines the abort" + ); + drop(consumer); + + assert!(producer.abort_unused(crate::Error::Cancel)); + assert!(producer.try_consume().is_none(), "an aborted group mints no reader"); + } + #[test] fn abort_clears_cached_frames() { let mut producer = Info { sequence: 0 }.produce(); @@ -2552,4 +2611,91 @@ mod test { }); assert!(matches!(result, Err(Error::FrameTooLarge))); } + + fn sized(size: u64) -> frame::Info { + frame::Info { + size, + timestamp: Timestamp::ZERO, + } + } + + /// A declared size costs the peer nothing, so the pool is charged as the bytes + /// arrive. Charging the declaration would let a peer evict the cache without + /// sending anything. + #[test] + fn frame_is_charged_as_written() { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info { sequence: 0 }, track::Info::default(), cache); + let before = pool.used(); + + let mut frame = producer + .create_frame_owned(sized(MAX_CACHE_BYTES), &frame::Budget::default()) + .unwrap(); + assert_eq!(pool.used(), before, "the declared size was charged"); + + frame.write(Bytes::from(vec![0u8; 100])).unwrap(); + assert_eq!(pool.used(), before, "charged before the write was published"); + frame.notify(); + assert_eq!(pool.used(), before + 100); + assert_eq!(producer.state.read().cache, 100); + // A reader skipping the group still misses the whole declared frame. + assert_eq!(producer.state.read().content().bytes, MAX_CACHE_BYTES); + + // Aborting releases what was charged. + frame.abort(Error::Cancel).unwrap(); + assert_eq!(producer.state.read().cache, 0); + assert!(pool.used() <= before); + } + + /// Completing a frame charges whatever it wrote since the last notify. + #[test] + fn frame_commit_charges_the_tail() { + let pool = cache::Pool::unbounded(); + let cache = cache::Track::new(pool.clone(), kio::Weak::new()); + let mut producer = Producer::new(Info { sequence: 0 }, track::Info::default(), cache); + let before = pool.used(); + + let mut frame = producer + .create_frame_owned(sized(300), &frame::Budget::default()) + .unwrap(); + frame.write(Bytes::from(vec![0u8; 100])).unwrap(); + frame.notify(); + frame.write(Bytes::from(vec![0u8; 200])).unwrap(); + frame.finish().unwrap(); + assert_eq!(pool.used(), before + 300); + assert_eq!(producer.state.read().cache, 300); + } + + /// Frames within the session's budget allocate their declared size up front; past + /// it, a frame's buffer holds only what has arrived. A frame hands its share back + /// once it ends. + #[test] + fn frame_budget_bounds_upfront_allocation() { + let budget = frame::Budget::new(1024); + let mut a = Info { sequence: 0 }.produce(); + let mut b = Info { sequence: 1 }.produce(); + + let mut first = a.create_frame_owned(sized(1024), &budget).unwrap(); + first.write(Bytes::from_static(b"x")).unwrap(); + assert_eq!(first.allocated(), 1024, "a frame within budget is allocated up front"); + + let mut second = b.create_frame_owned(sized(MAX_CACHE_BYTES), &budget).unwrap(); + second.write(Bytes::from(vec![0u8; 100])).unwrap(); + assert_eq!( + second.allocated(), + 100, + "a frame past budget allocated its declared size" + ); + second.abort(Error::Cancel).unwrap(); + + first.write(Bytes::from(vec![0u8; 1023])).unwrap(); + first.finish().unwrap(); + a.finish().unwrap(); + + let mut a = Info { sequence: 2 }.produce(); + let mut third = a.create_frame_owned(sized(1024), &budget).unwrap(); + third.write(Bytes::from_static(b"x")).unwrap(); + assert_eq!(third.allocated(), 1024, "the finished frame did not return its share"); + } } diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index 12a9c1459af..5f96414526c 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -2281,17 +2281,37 @@ impl Drop for WarmGroup { } } +/// A spliced copy's newest group, held so its head outlives the copy's route (see +/// [`TrackIo::held`]). Borrowed from the route, which may still be writing it for other +/// readers, so it aborts on drop only once it is the last handle: a dead route never +/// finishes it. +struct HeldGroup(group::Producer); + +impl Drop for HeldGroup { + fn drop(&mut self) { + if !self.0.is_finished() { + let _ = self.0.abort_if_last(Error::Cancel); + } + } +} + /// Cache what `source` delivered on a new local track the origin owns: its complete /// groups, and its open live edge rebuilt from the frames already delivered. /// /// `head` is the previous park's edge. A copy spliced after a warm cache continues its /// edge group from the next frame, so its copy of that group lacks the head, which -/// `head` supplies. -fn warm_copy(source: &track::Consumer, head: Option<&WarmGroup>) -> Option { +/// `head` supplies. `held` is the copy's newest group as [`TrackIo::held`] kept it: a +/// copy that died tore its open groups out of its own cache, but not out of the held one. +fn warm_copy(source: &track::Consumer, head: Option<&WarmGroup>, held: Option<&group::Producer>) -> Option { let info = source.cached_info()?; let mut track = track::Producer::new(Arc::new(source.broadcast().clone()), source.name(), info); let head = head.map(|head| &head.0); - let groups = source.cached_groups(); + let mut groups = source.cached_groups(); + if let Some(held) = held + && !groups.iter().any(|(group, _)| group.sequence == held.sequence) + { + groups.push((held.clone(), true)); + } // Not `source.latest()`: datagrams share the sequence counter and can run past it. let latest = groups .iter() @@ -2451,6 +2471,14 @@ struct TrackIo { /// The last warm copy's newest group, outliving it so the copy spliced after it /// can continue the group (see [`WarmGroup`]). Released at the next park. head: Option, + /// The spliced copy's newest group. A route that ends abruptly releases its open + /// groups, and the takeover after it asks the next copy only for the frames past + /// the break, so without this handle no route holds the group's head for a reader + /// that arrives later. See [`TrackIo::retire`]. + held: Option, + /// The spliced copy, once it died: its segment keeps ending readers with its error + /// unless a takeover follows, which retires it. + dead: Option, /// Whether the track had a reader as of the last demand edge. used: bool, } @@ -2464,6 +2492,39 @@ impl TrackIo { self.copy = None; self.warm = None; self.head = None; + self.held = None; + self.dead = None; + } + + /// Keep the spliced copy as dead. The front only refreshes `held` between wakes, so + /// first catch a newest group the copy surfaced since, while its cache still has it. + fn bury(&mut self) { + let Some((_, copy)) = self.copy.take() else { return }; + let after = self.held.as_ref().map(|held| held.0.sequence); + if let Poll::Ready(group) = copy.poll_latest_group(after, &kio::Waiter::noop()) { + self.held = Some(HeldGroup(group)); + } + self.dead = Some(copy); + } + + /// Retire `copy` as a takeover replaces it. Mid-group, park its delivery warm, open + /// group included, so a reader arriving later still gets the group's head and the + /// takeover continues it. Between groups its own cache keeps everything finished, so + /// there is nothing to do. + fn retire(&mut self, copy: &track::Consumer) -> Result<(), Error> { + let Some(held) = self.held.take() else { + return Ok(()); + }; + if held.0.is_finished() { + return Ok(()); + } + let Some(warm) = warm_copy(copy, self.head.as_ref(), Some(&held.0)) else { + return Ok(()); + }; + self.head = None; + self.resume.park(&warm.track)?; + self.warm = Some(warm); + Ok(()) } } @@ -2489,6 +2550,7 @@ async fn run_front(task: FrontTask) { SourceClosed(u64), Info(Arc, u64, Result), Ended(Arc, u64, Result<(), Error>), + Held(Arc, group::Producer), Demand(Arc), Deadline, Table, @@ -2640,7 +2702,7 @@ async fn run_front(task: FrontTask) { // spliced until a replacement resumes past them. for io in tracks.values_mut() { if io.copy.as_ref().is_some_and(|(s, _)| *s == source) { - io.copy = None; + io.bury(); } if io.query.as_ref().is_some_and(|(s, ..)| *s == source) { io.query = None; @@ -2682,7 +2744,12 @@ async fn run_front(task: FrontTask) { if staged != source { continue; } - if let Err(err) = io.resume.takeover(©) { + let outgoing = io.copy.take().map(|(_, copy)| copy).or_else(|| io.dead.take()); + let retired = match outgoing { + Some(outgoing) => io.retire(&outgoing), + None => Ok(()), + }; + if let Err(err) = retired.and_then(|()| io.resume.takeover(©)) { // Closed means the logical track already ended. Anything // else is a boundary bug; abort rather than strand // subscribers on a track nobody serves. @@ -2696,6 +2763,7 @@ async fn run_front(task: FrontTask) { // The new segment has produced nothing yet: this is the // edge the copy is asked to advance. io.edge = io.resume.resume_position(); + io.held = None; io.copy = Some((source, copy)); } Action::Park { track: name } => { @@ -2704,9 +2772,11 @@ async fn run_front(task: FrontTask) { // Drop the source copy so its producer goes idle at once; keep // the groups it delivered on a local track so resume stays // spliced until the linger expires. - let warm = warm_copy(©, io.head.as_ref()); + let warm = warm_copy(©, io.head.as_ref(), io.held.as_ref().map(|held| &held.0)); drop(copy); io.head = None; + io.held = None; + io.dead = None; let parked = match &warm { Some(warm) => io.resume.park(&warm.track), None => io.resume.release(), @@ -2821,6 +2891,12 @@ async fn run_front(task: FrontTask) { { return Poll::Ready(Step::Ended(name.clone(), *source, result)); } + if let Some((_, copy)) = &io.copy + && let Poll::Ready(group) = + copy.poll_latest_group(io.held.as_ref().map(|held| held.0.sequence), waiter) + { + return Poll::Ready(Step::Held(name.clone(), group)); + } // Watch the demand edge in whichever direction is unmet. let edge = match io.used { true => io.resume.poll_unused(waiter), @@ -2849,6 +2925,8 @@ async fn run_front(task: FrontTask) { edge: None, warm: None, head: None, + held: None, + dead: None, used: false, }, ); @@ -2908,7 +2986,7 @@ async fn run_front(task: FrontTask) { Step::Ended(name, source, result) => { let closing = sources.get(&source).is_some_and(|s| s.is_closing()); let Some(io) = tracks.get_mut(&name) else { continue }; - io.copy = None; + io.bury(); let delivered = io.resume.resume_position() != io.edge; Event::TrackEnded { track: name, @@ -2918,6 +2996,12 @@ async fn run_front(task: FrontTask) { delivered, } } + Step::Held(name, group) => { + if let Some(io) = tracks.get_mut(&name) { + io.held = Some(HeldGroup(group)); + } + continue; + } Step::Demand(name) => { let Some(io) = tracks.get_mut(&name) else { continue }; io.used = io.resume.is_used(); @@ -6275,6 +6359,223 @@ mod tests { } } + /// A route that ends abruptly releases its open groups, and the takeover asks the + /// next route only for the frames past the break. The front keeps the head it + /// spliced, so a reader arriving afterwards still gets the whole group: whether the + /// old route dies before the new one takes over (a reconnect) or after (a cheaper + /// route preempting it). A reader joining mid-outage gets it too, and the reader + /// already mid-group is not handed it a second time. + #[tokio::test] + async fn a_takeover_keeps_the_open_group_head_for_later_readers() { + tokio::time::pause(); + for dies_first in [true, false] { + takeover_keeps_the_open_group_head(dies_first).await; + } + } + + async fn takeover_keeps_the_open_group_head(dies_first: bool) { + let producer = origin(1).produce(); + let first_server = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(5)) + .unwrap(); + let pending = producer.consume().request_broadcast("room/alice"); + let upstream = broadcast::Info::new().produce(); + let mut dynamic = upstream.dynamic(); + queued(&first_server).await.accept(&upstream); + let resolved = pending.await.unwrap(); + + async fn read(group: &mut group::Consumer) -> Vec { + let frame = tokio::time::timeout(Duration::from_secs(1), group.read_frame()) + .await + .expect("frame") + .unwrap() + .expect("group ended"); + frame.payload.to_vec() + } + + // A reader is mid-way through the open group when the route changes. + let track = resolved.track("log").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let source = dynamic.requested_track().await.unwrap().accept(None); + let mut group = source.create_group(0u64.into()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"a".as_ref()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"b".as_ref()).unwrap(); + let mut subscription = subscribing.await.unwrap().unwrap(); + let mut reading = subscription.recv_group().await.unwrap().unwrap(); + assert_eq!(read(&mut reading).await, b"a"); + assert_eq!(read(&mut reading).await, b"b"); + + // Either the first route's session ends and reconnects, announcing the same route + // afresh while the old one is still held, or a cheaper route preempts the live one. + let mut first = Some((group, source, upstream, dynamic)); + let mut joining = None; + if dies_first { + drop(first.take()); + // The reader sees its route die and stalls. Polled in place: yielding would let + // the front give up on a path with no route left before the replacement lands. + assert!(futures::FutureExt::now_or_never(subscription.recv_group()).is_none()); + let track = resolved.track("log").unwrap(); + joining = Some(tokio::spawn(async move { + let mut joined = track.subscribe(None).await.unwrap(); + let group = joined.recv_group().await.unwrap().expect("track ended"); + (joined, group) + })); + } + let cost = if dies_first { 5 } else { 0 }; + let replacement_server = producer + .dynamic("room/alice", Route::default().with_hops(hops(&[10])).with_cost(cost)) + .unwrap(); + let replacement = broadcast::Info::new().produce(); + let mut replacement_dynamic = replacement.dynamic(); + queued(&replacement_server).await.accept(&replacement); + let mut next = tokio::time::timeout(Duration::from_secs(1), replacement_dynamic.requested_track()) + .await + .expect("the front asked the replacement") + .unwrap() + .resolving_start() + .accept(None); + next.start_at(0).unwrap(); + let mut next_group = next.create_group(0u64.into()).unwrap(); + next_group.start_at(2).unwrap(); + next_group.write_frame(crate::Timestamp::ZERO, b"c".as_ref()).unwrap(); + assert_eq!(read(&mut reading).await, b"c", "dies_first={dies_first}"); + + // A preempted route still owns its open group: its writer and other readers carry on. + if let Some((group, ..)) = &mut first { + let mut independent = group.consume(); + group.write_frame(crate::Timestamp::ZERO, b"x".as_ref()).unwrap(); + for expect in [b"a", b"b", b"x"] { + assert_eq!(read(&mut independent).await, expect); + } + group.finish().unwrap(); + } + drop(first); + + let track = resolved.track("log").unwrap(); + let mut later = tokio::time::timeout(Duration::from_secs(1), track.subscribe(None)) + .await + .expect("subscribe") + .unwrap(); + let mut late = tokio::time::timeout(Duration::from_secs(1), later.recv_group()) + .await + .unwrap_or_else(|_| panic!("dies_first={dies_first}: the later reader never got the group")) + .unwrap() + .expect("track ended"); + assert_eq!(late.sequence, 0); + for expect in [b"a", b"b", b"c"] { + assert_eq!(read(&mut late).await, expect, "dies_first={dies_first}"); + } + let mut joined = None; + if let Some(joining) = joining { + let (subscription, mut group) = tokio::time::timeout(Duration::from_secs(1), joining) + .await + .expect("the reader joining mid-outage never got the group") + .unwrap(); + assert_eq!(group.sequence, 0); + for expect in [b"a", b"b", b"c"] { + assert_eq!(read(&mut group).await, expect); + } + joined = Some((subscription, group)); + } + + // The reader that was mid-group already has it. + assert!( + tokio::time::timeout(Duration::from_secs(1), subscription.recv_group()) + .await + .is_err(), + "dies_first={dies_first}: the group was handed out twice" + ); + + // The continuation still flows to both. + next_group.write_frame(crate::Timestamp::ZERO, b"d".as_ref()).unwrap(); + assert_eq!(read(&mut reading).await, b"d"); + assert_eq!(read(&mut late).await, b"d"); + if let Some((_, group)) = &mut joined { + assert_eq!(read(group).await, b"d"); + } + next_group.finish().unwrap(); + drop(( + joined, + reading, + late, + subscription, + later, + next, + replacement, + replacement_server, + first_server, + )); + } + + /// A returning reader is handed the parked open group spliced onto the source's + /// continuation, which starts past the warm head. The group's end stays unknown + /// until the source finishes it, even before a frame of it is read: readers drop a + /// group ahead of their cursor whose end resolves to an error. + #[tokio::test] + async fn returning_reader_gets_a_live_open_warm_group() { + use futures::FutureExt; + + tokio::time::pause(); + let (_server, _upstream, mut dynamic, resolved) = served_front().await; + + // The first reader leaves mid-group. + let track = resolved.track("video").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let mut source = request.resolving_start().accept(None); + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + source.start_at(0).unwrap(); + let mut group = source.create_group(0u64.into()).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"a".as_ref()).unwrap(); + let mut reading = subscription.recv_group().await.unwrap().expect("group 0"); + assert_eq!(reading.read_frame().await.unwrap().unwrap().payload.as_ref(), b"a"); + drop(reading); + drop(subscription); + tokio::time::timeout(Duration::from_secs(1), source.demand().unused()) + .await + .expect("parked") + .expect("source open"); + drop(group); + drop(source); + + // The next reader returns while the group is still live upstream. + let track = resolved.track("video").unwrap(); + let subscribing = tokio::spawn(async move { track.subscribe(None).await }); + let request = tokio::time::timeout(Duration::from_secs(1), dynamic.requested_track()) + .await + .expect("the front asked the source") + .expect("request"); + let mut source = request.resolving_start().accept(None); + let mut subscription = subscribing.await.unwrap().expect("subscribe"); + source.start_at(0).unwrap(); + let mut group = source.create_group(0u64.into()).unwrap(); + group.start_at(1).unwrap(); + group.write_frame(crate::Timestamp::ZERO, b"b".as_ref()).unwrap(); + + let mut reading = tokio::time::timeout(Duration::from_secs(1), subscription.recv_group()) + .await + .expect("group 0") + .unwrap() + .expect("track ended"); + assert_eq!(reading.sequence, 0); + let early = reading.finished().now_or_never(); + assert!( + early.is_none(), + "the live group already resolved its end before a frame was read: {early:?}" + ); + + // The end answers for this reader's cursor, not the seam it was probed from. + group.finish().unwrap(); + assert_eq!(reading.finished().await.unwrap(), 0); + assert_eq!(reading.read_frame().await.unwrap().unwrap().payload.as_ref(), b"a"); + assert_eq!(reading.finished().await.unwrap(), 1); + assert_eq!(reading.read_frame().await.unwrap().unwrap().payload.as_ref(), b"b"); + assert_eq!(reading.finished().await.unwrap(), 2); + } + #[tokio::test] async fn warm_head_survives_another_takeover_before_park() { tokio::time::pause(); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 5620702cfb9..acbd0d00f99 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -962,9 +962,9 @@ fn last_group(end: Option) -> Option { /// mid-group therefore reaches an already-handed-out group with no bookkeeping, and a /// reader that never touches the subscription again still picks the continuation up. /// -/// A copy that dies (or never arrives) stalls the group rather than erroring it, the -/// same way a dead segment stalls the track; the loss is only surfaced once no -/// replacement can arrive. +/// A copy that dies waits for a replacement route. A missing continuation subscribes +/// that route from the frame this group needs; a refusal ends the wait, as does a live +/// edge past this group when the route declares no start. pub(crate) struct Group { state: kio::Consumer, /// The logical subscription's live max age budget. @@ -985,6 +985,10 @@ pub(crate) struct Group { /// this group, and this reader's positioned copy. current: Option, + /// Demand held on a replacement copy until this group ends, since the parent + /// subscriber may be idle. + waiting: Option, + /// The segment whose copy died under us, and why. dead: Option<(u64, Error)>, @@ -992,6 +996,27 @@ pub(crate) struct Group { stale_stats: crate::stats::Meter, } +/// A group reader's own subscription on one segment's route, starting at the frame +/// it needs. A floor never widens the parent's request past it, where a live-edge +/// start would erase the parent's continuation floor from the aggregate. +struct Waiting { + segment: u64, + start: Position, + track: track::Consumer, + pending: kio::Pending, +} + +impl Waiting { + fn new(segment: u64, start: Position, track: &track::Consumer) -> Self { + Self { + segment, + start, + track: track.clone(), + pending: track.subscribe(track::Subscription::default().with_start(start)), + } + } +} + struct Current { segment: u64, cap: Option, @@ -1030,6 +1055,12 @@ impl Clone for Group { index: self.index, end: self.end, current, + // Each reader holds its own demand, so dropping the original cannot + // strand a clone still reading the tail. + waiting: self + .waiting + .as_ref() + .map(|w| Waiting::new(w.segment, w.start, &w.track)), dead: self.dead.clone(), stale_stats: self.stale_stats.clone(), } @@ -1052,6 +1083,7 @@ impl Group { index, end: None, current: None, + waiting: None, dead: None, stale_stats: Default::default(), } @@ -1195,29 +1227,74 @@ impl Group { } } + /// Ask the replacement for this group's continuation even when the parent subscriber + /// is idle. + /// + /// The demand outlives the peek: a copy whose header has arrived, even before + /// this reader first looked, still needs it for the tail. It is released only + /// once the group ends or moves routes. + fn poll_copy( + &mut self, + segment: u64, + start: Position, + track: &track::Consumer, + waiter: &kio::Waiter, + ) -> Poll>> { + // A route is reused whatever `start` asks for, since the reader's position only + // moves forward and the segments are disjoint: keeping the earlier floor asks for + // a superset of what a later read needs, and re-subscribing to narrow it would + // drop the demand in between. + if self.waiting.as_ref().is_none_or(|w| w.segment != segment) { + self.waiting = Some(Waiting::new(segment, start, track)); + } + let result = (|| { + if let Poll::Ready(Some(copy)) = track.poll_peek_group(self.sequence, waiter) { + return Poll::Ready(Ok(Some(copy))); + } + // The pending handle owns the demand; accepting it checks refusals without + // moving this group's cursor or asking for historical data via FETCH. + ready!(self.waiting.as_ref().expect("registered above").pending.poll_ok(waiter))?; + // A declared start already tells the peek whether this group is coming, even + // when a newer group's stream overtakes it. Without one, a live edge past + // this group is the only sign it never will. + if ready!(track.poll_start(waiter)).is_none() + && track + .peek_latest() + .is_some_and(|latest| latest.sequence > self.sequence) + { + return Poll::Ready(Err(Error::NotFound)); + } + track.poll_peek_group(self.sequence, waiter).map(Ok) + })(); + if !matches!(result, Poll::Pending | Poll::Ready(Ok(Some(_)))) { + self.waiting = None; + } + result + } + /// Point `current` at the route owning frame `index` of this group. /// - /// `Ready(Ok(false))` once the group can produce nothing more, and `Ready(Err(_))` - /// only when the route that died is the last word on those frames. - fn poll_current(&mut self, waiter: &kio::Waiter) -> Poll> { + /// `Ready(Err(_))` once no route can ever serve those frames: a group only ends + /// cleanly through its unbounded tail copy, so this is always a truncation. + fn poll_current(&mut self, waiter: &kio::Waiter) -> Poll> { loop { let position = Position { group: self.sequence, frame: self.index, }; let dead = self.dead.as_ref().map(|(segment, _)| *segment); - let sequence = self.sequence; let found = ready!(self.poll_covering(position, dead, waiter)); let Some((segment, track, Some(cap), bound)) = found else { + self.waiting = None; // No segment covers the position, but a latched copy still drains: // a pruned segment's cursor holds exactly the frames it owned (its // cap was its produced edge), so read it dry before giving up. Once // it ends or dies, `current` clears and the next resolve settles it. if self.current.is_some() { - return Poll::Ready(Ok(true)); + return Poll::Ready(Ok(())); } - return Poll::Ready(self.give_up()); + return Poll::Ready(Err(self.give_up())); }; // Reuse the positioned handle while the route and its bound hold: it carries @@ -1227,11 +1304,11 @@ impl Group { .as_ref() .is_some_and(|current| current.segment == segment && current.cap == cap && current.bound == bound) { - return Poll::Ready(Ok(true)); + return Poll::Ready(Ok(())); } // The route may not have delivered this group yet, so wait on its cache. - let Some(group) = ready!(track.poll_peek_group(sequence, waiter)) else { + let Some(group) = ready!(self.poll_copy(segment, position, &track, waiter))? else { // This route will never have it; fall back to whichever segment replaces it. self.dead = Some((segment, Error::NotFound)); continue; @@ -1256,34 +1333,31 @@ impl Group { group, }); self.dead = None; - return Poll::Ready(Ok(true)); + return Poll::Ready(Ok(())); } } - /// No replacement can arrive: report the loss that stalled us, or a clean end. + /// No replacement can arrive: report the loss that cut the group short, from the + /// route that died, or [`Error::Dropped`] when its segment was pruned away. /// /// The dead route stays recorded, so every later poll reaches the same verdict. /// Clearing it would re-resolve the route's reclaimed copy as one still to come /// and park, and a reader probing `poll_finished` for the loss would hang. - fn give_up(&self) -> Result { - match &self.dead { - Some((_, err)) => { - // The only place a spliced group's loss becomes visible, so say which - // frames went missing rather than leaving a stuck group to explain itself. - // An old group was skipped on purpose (something newer superseded it), so - // it is not a loss worth reporting. - if !matches!(crate::StreamError::from(err), crate::StreamError::Old) { - tracing::warn!( - group = self.sequence, - frame = self.index, - %err, - "no route can serve the rest of this group" - ); - } - Err(err.clone()) - } - None => Ok(false), + fn give_up(&self) -> Error { + let err = self.dead.as_ref().map_or(Error::Dropped, |(_, err)| err.clone()); + // The only place a spliced group's loss becomes visible, so say which frames + // went missing rather than leaving a stuck group to explain itself. An old + // group was skipped on purpose (something newer superseded it), so it is not + // a loss worth reporting. + if !matches!(crate::StreamError::from(&err), crate::StreamError::Old) { + tracing::warn!( + group = self.sequence, + frame = self.index, + %err, + "no route can serve the rest of this group" + ); } + err } /// Advance past a copy that ran out at its boundary, or report the group's end. @@ -1315,11 +1389,11 @@ impl Group { pub fn poll_read_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { loop { if self.end.is_some_and(|end| self.index >= end) { + // Raising the cap re-registers the demand on the next read. + self.waiting = None; return Poll::Ready(Ok(None)); } - if !ready!(self.poll_current(waiter))? { - return Poll::Ready(Ok(None)); - } + ready!(self.poll_current(waiter))?; let result = { let current = self.current.as_mut().expect("resolved above"); ready!(current.group.poll_read_frame(waiter)) @@ -1334,8 +1408,16 @@ impl Group { return Poll::Ready(Ok(Some(frame))); } Ok(None) if self.roll() => continue, - Ok(None) => return Poll::Ready(Ok(None)), - Err(err) if latency_expired => return Poll::Ready(Err(err)), + Ok(None) => { + self.waiting = None; + return Poll::Ready(Ok(None)); + } + // Terminal, and the reader may be retained past the error, so the + // demand outliving the group would pin upstream demand for good. + Err(err) if latency_expired => { + self.waiting = None; + return Poll::Ready(Err(err)); + } Err(err) => self.bury(err), } } @@ -1344,11 +1426,11 @@ impl Group { pub fn poll_next_frame(&mut self, waiter: &kio::Waiter) -> Poll>> { loop { if self.end.is_some_and(|end| self.index >= end) { + // Raising the cap re-registers the demand on the next read. + self.waiting = None; return Poll::Ready(Ok(None)); } - if !ready!(self.poll_current(waiter))? { - return Poll::Ready(Ok(None)); - } + ready!(self.poll_current(waiter))?; let result = { let current = self.current.as_mut().expect("resolved above"); ready!(current.group.poll_next_frame(waiter)) @@ -1363,26 +1445,36 @@ impl Group { return Poll::Ready(Ok(Some(frame))); } Ok(None) if self.roll() => continue, - Ok(None) => return Poll::Ready(Ok(None)), - Err(err) if latency_expired => return Poll::Ready(Err(err)), + Ok(None) => { + self.waiting = None; + return Poll::Ready(Ok(None)); + } + // Terminal, and the reader may be retained past the error, so the + // demand outliving the group would pin upstream demand for good. + Err(err) if latency_expired => { + self.waiting = None; + return Poll::Ready(Err(err)); + } Err(err) => self.bury(err), } } } - /// The logical group's total frame count, which only the unbounded tail copy knows: - /// its own count already includes the frames it skipped. + /// Resolve once the logical group ends, answering with the read cursor. Only the + /// unbounded tail copy knows that end; a group cut short at a seam no route can + /// serve fails instead, so a truncated group never passes for a complete one. pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { - if !ready!(self.poll_current(waiter))? { - return Poll::Ready(Ok(self.index)); - } + ready!(self.poll_current(waiter))?; let current = self.current.as_mut().expect("resolved above"); let Some(cap) = current.cap else { - return current.group.poll_finished(waiter); + let count = ready!(current.group.poll_finished(waiter)); + // A complete copy is cached, so it no longer needs the demand that pulled it. + self.waiting = None; + return Poll::Ready(count); }; - // A bounded copy can't declare the end; the continuation does, unless it - // can never arrive: then the cap is the group's end. Probed here, not + // A bounded copy can't declare the end; the continuation does, and when it + // can never arrive the group is truncated at the cap. Probed here, not // left to `poll_current`: a latched bounded copy resolves without // consulting the segment list, so this is the poll that must park on the // seam (or a caller that never drains to it would hang with no waiter @@ -1396,16 +1488,29 @@ impl Group { loop { let dead = self.dead.as_ref().map(|(segment, _)| *segment); let Some((segment, track, _, bound)) = ready!(self.poll_covering(seam, dead, waiter)) else { - return Poll::Ready(Ok(cap)); + self.waiting = None; + return Poll::Ready(Err(self.give_up())); }; - match ready!(track.poll_peek_group(self.sequence, waiter)) { + match ready!(self.poll_copy(segment, seam, &track, waiter))? { // The continuation's copy declares the count: its own count // already includes the frames it skipped. Some(continuation) => { let mut continuation = track.guard_group(continuation, self.subscription.clone(), self.anchor.clone(), bound); continuation.set_stale_meter(self.stale_stats.clone()); - return continuation.poll_finished(waiter); + // Ask from the seam, as `poll_current` reads from it. A fresh cursor + // sits at frame 0, below a continuation that starts past the head, + // and would report a live group as lagged. The probe's index is the + // seam, not this reader's, so answer with our own. + continuation.start_at(cap); + if continuation.index() != cap { + self.dead = Some((segment, Error::Lagged)); + continue; + } + let index = self.index; + let end = ready!(continuation.poll_finished(waiter)); + self.waiting = None; + return Poll::Ready(end.map(|_| index)); } // This route will never have it; wait for whatever replaces it. None => self.dead = Some((segment, Error::NotFound)), @@ -1419,6 +1524,9 @@ struct SegmentSub { id: u64, start: Option, end: Option, + /// Groups below this were handed out before the cursor subscribed: a warm cache + /// taking over a reader's cursors must not surface them again. + floor: u64, /// Where the source is asked to start; see [`Segment::ask`]. ask: Option, sub: SubState, @@ -1461,7 +1569,7 @@ impl SegmentSub { } /// Move an active cursor into terminal retention and mark the segment done. - fn complete(&mut self, end: Result) { + fn complete(&mut self, end: Result>) { let previous = std::mem::replace(&mut self.sub, SubState::Done(end)); if let SubState::Active(sub) = previous { self.terminal = Some(*sub); @@ -1470,7 +1578,7 @@ impl SegmentSub { /// The first group this segment can serve, for the underlying read cursor. fn first_group(&self) -> u64 { - self.start.map_or(0, |start| start.group) + self.start.map_or(0, |start| start.group).max(self.floor) } /// The exclusive group cap this segment can serve, for the underlying read @@ -1509,7 +1617,7 @@ enum SubState { /// cleanly, `Err` with the cause when it aborted or was dropped. An abort only /// surfaces once no switch can follow: until then a dead route stalls the /// logical track for the next switch to replace. - Done(Result), + Done(Result>), } /// A live subscription spliced across every segment of a logical track. @@ -1681,6 +1789,17 @@ impl Subscriber { for s in &mut self.segments { s.pruned = !segments.iter().any(|n| n.id == s.id); } + // A park replaces every segment with a cache of what they delivered (see + // [`Producer::park`]). Cursors still draining the replaced segments would hand + // that content out a second time, so the cache takes over from them, past what + // this reader was already handed. Groups they parked at the cap are in the + // cache too, and park again from there. + let parked = segments + .iter() + .any(|n| n.warm && !self.segments.iter().any(|s| s.id == n.id)); + if parked { + self.segments.retain(|s| !s.pruned); + } self.segments.retain(|s| !s.retired()); let nexts: Vec<_> = segments.iter().skip(1).map(|next| Some(next.track.clone())).collect(); @@ -1716,6 +1835,8 @@ impl Subscriber { id: segment.id, start: segment.start, end: segment.end, + // A cache taking over (see above) starts past what this reader has. + floor: if segment.warm { self.next_sequence } else { 0 }, ask: segment.ask, sub: SubState::Pending(sub), terminal: None, @@ -1943,7 +2064,7 @@ impl Subscriber { return Poll::Ready(Some(group)); } Ok(None) => { - let end = match sub.poll_finished(waiter) { + let end = match sub.poll_end(waiter) { Poll::Ready(end) => end, Poll::Pending => Err(Error::Dropped), }; @@ -2032,7 +2153,7 @@ impl Subscriber { } // The track ran out at or below the floor: the segment drained. Poll::Ready(Ok(None)) => { - let end = match sub.poll_finished(waiter) { + let end = match sub.poll_end(waiter) { Poll::Ready(end) => end, Poll::Pending => Err(Error::Dropped), }; @@ -2301,6 +2422,11 @@ impl Subscriber { /// Poll for the logical track finishing, returning the final segment's group /// count (one past its last sequence). pub fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { + self.poll_end(waiter) + .map(|res| res.and_then(|end| end.ok_or(Error::Closed))) + } + + pub(crate) fn poll_end(&mut self, waiter: &kio::Waiter) -> Poll>> { self.poll_sync(waiter); if let Some(err) = &self.abort { @@ -2313,7 +2439,7 @@ impl Subscriber { // only if it finished. match ready!(self.poll_final(waiter)) { Some(end) => Poll::Ready(end), - None if self.finished => Poll::Ready(Ok(0)), + None if self.finished => Poll::Ready(Ok(Some(0))), None => Poll::Ready(Err(Error::Dropped)), } } @@ -2343,7 +2469,7 @@ impl Subscriber { /// decide the end. Only the subscription is resolved here: consuming groups, or /// completing the segment, would steal them from a `recv_group` caller on the /// same subscriber. - fn poll_final(&mut self, waiter: &kio::Waiter) -> Poll>> { + fn poll_final(&mut self, waiter: &kio::Waiter) -> Poll>>> { let Some(seg) = self.segments.last_mut() else { return Poll::Ready(None); }; @@ -2352,7 +2478,7 @@ impl Subscriber { SubState::Done(end) => Poll::Ready(Some(end.clone())), // Observe only: the cursor may still hold groups, so the read path // completes the segment once it drains. - SubState::Active(sub) => Poll::Ready(Some(ready!(sub.poll_finished(waiter)))), + SubState::Active(sub) => Poll::Ready(Some(ready!(sub.poll_end(waiter)))), SubState::Pending(_) => unreachable!("poll_activate resolved above"), } } @@ -3132,6 +3258,85 @@ mod test { assert_eq!(recv(&mut sub), 1); } + /// A park that replaces a reader's segments with a cache of what they delivered + /// hands that reader nothing twice, however many times one open group (a catalog + /// taking deltas) changes route. A newcomer still gets the whole group, head first, + /// after the takeovers have pruned every segment that started at its head. + #[tokio::test] + async fn repeated_parks_mid_group_hand_the_head_out_once() { + let mut producer = Producer::new(); + let (track, consumer) = track_pair("a"); + producer.takeover(&consumer).unwrap(); + let mut sub = producer.consume().subscribe(None); + let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::ZERO, b"0".to_vec()).unwrap(); + let mut reading = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut reading), b"0"); + + let mut routes = vec![(track, consumer, group)]; + let mut heads = Vec::new(); + let rounds = MAX_SEGMENTS as u64 + 1; + for frame in 1..=rounds { + // The cache holds the whole group so far, the way the origin rebuilds it. + let (cache, cache_consumer) = track_pair("cache"); + let mut whole = cache.create_group(group::Info { sequence: 0 }).unwrap(); + for f in 0..frame { + whole.write_frame(Timestamp::ZERO, f.to_string().into_bytes()).unwrap(); + } + producer.park(&cache_consumer).unwrap(); + + let (next, next_consumer) = track_pair("next"); + producer.takeover(&next_consumer).unwrap(); + let mut continuation = next.create_group(group::Info { sequence: 0 }).unwrap(); + continuation.start_at(frame).unwrap(); + continuation + .write_frame(Timestamp::ZERO, frame.to_string().into_bytes()) + .unwrap(); + + assert_eq!(read(&mut reading), frame.to_string().as_bytes()); + recv_pending(&mut sub); + routes.push((next, next_consumer, continuation)); + // The origin finishes a cache once the takeover lands, but holds its open + // group: dropping that would clear its frames. + cache.finish().unwrap(); + heads.push(whole); + } + + let mut late = producer.consume().subscribe(None); + let mut group = late.recv_group().now_or_never().unwrap().unwrap().unwrap(); + for frame in 0..=rounds { + assert_eq!(read(&mut group), frame.to_string().as_bytes()); + } + } + + /// A group held back at the reader's cap when a park replaces its segments comes + /// from the cache once the cap rises, exactly once. + #[tokio::test] + async fn a_park_reoffers_a_group_held_at_the_cap_once() { + let (mut track_a, consumer_a) = track_pair("a"); + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(replay()); + sub.end_at(..1); + write_group(&mut track_a, 0, "0"); + write_group(&mut track_a, 1, "1"); + assert_eq!(recv(&mut sub), 0); + recv_pending(&mut sub); + + let (mut cache, cache_consumer) = track_pair("cache"); + write_group(&mut cache, 0, "0"); + write_group(&mut cache, 1, "1"); + producer.park(&cache_consumer).unwrap(); + let (mut track_b, consumer_b) = track_pair("b"); + producer.takeover(&consumer_b).unwrap(); + write_group(&mut track_b, 2, "2"); + + sub.end_at(..3); + assert_eq!(recv(&mut sub), 1); + assert_eq!(recv(&mut sub), 2); + recv_pending(&mut sub); + } + /// A parked beyond-cap group must not block in-range groups that arrive /// behind it: a relay can ingest a burst micro-reordered (newest first). #[tokio::test] @@ -4303,13 +4508,9 @@ mod test { ); } - /// A route that has run ahead of the seam but has not been given up on still parks. - /// - /// It may yet deliver the missing frames out of order, and its own progress is what - /// moved the resume point past them, so its being ahead is not evidence the frames - /// are gone. Only a route already declared dead strands the reader. + /// Latest-only recovery ends a missing continuation once the replacement is ahead. #[tokio::test] - async fn live_route_ahead_of_the_seam_still_parks() { + async fn live_route_ahead_of_the_seam_ends_the_group() { let (track_a, consumer_a) = track_pair("a"); let (mut track_b, consumer_b) = track_pair("b"); @@ -4331,12 +4532,40 @@ mod test { write_group(&mut track_b, 1, "b1"); assert_eq!(recv(&mut sub), 1); + assert!( + matches!(reading.read_frame().now_or_never(), Some(Err(Error::NotFound))), + "latest recovery cannot request an older continuation" + ); + } + + /// A declared start at or below the seam promises the group, so a newer group + /// overtaking it on the wire does not end the wait. + #[tokio::test] + async fn declared_start_waits_for_an_overtaken_group() { + let (track_a, consumer_a) = track_pair("a"); + let (mut track_b, consumer_b) = track_pair("b"); + + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(replay()); + + let mut group = track_a.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::ZERO, b"a0".to_vec()).unwrap(); + group.write_frame(Timestamp::ZERO, b"a1".to_vec()).unwrap(); + + let mut reading = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut reading), b"a0"); + assert_eq!(read(&mut reading), b"a1"); + + producer.takeover(&consumer_b).unwrap(); + track_b.start_at(0).unwrap(); + write_group(&mut track_b, 1, "b1"); + assert_eq!(recv(&mut sub), 1); assert!( reading.read_frame().now_or_never().is_none(), - "a live route may still fill the seam out of order" + "the declared start still promises group 0" ); - // Once it does, the reader picks up where it left off. let mut group = track_b.create_group(group::Info { sequence: 0 }).unwrap(); group.start_at(2).unwrap(); group.write_frame(Timestamp::ZERO, b"b2".to_vec()).unwrap(); @@ -4958,13 +5187,13 @@ mod test { assert_eq!(recv(&mut sub), 0); } - /// `finished()` on a group bounded by a mid-group takeover resolves once no - /// segment can serve the seam: the continuation owns the count, and when the - /// covering segments are pruned away the cap is the group's end. Polled - /// without draining first, which is exactly the caller the seam check must - /// park (and wake) rather than hang. + /// `finished()` on a group bounded by a mid-group takeover fails once no + /// segment can serve the seam: the continuation owns the end, and when the + /// covering segments are pruned away the group is truncated, not complete. + /// Polled without draining first, which is exactly the caller the seam check + /// must park (and wake) rather than hang. #[tokio::test] - async fn finished_resolves_for_a_pruned_bounded_group() { + async fn finished_fails_for_a_pruned_bounded_group() { let (track_a, consumer_a) = track_pair("a"); let mut producer = Producer::new(); producer.takeover(&consumer_a).unwrap(); @@ -4981,30 +5210,79 @@ mod test { drop(group); track_a.abort(Error::Dropped).unwrap(); - // While B covers the seam the count is still open: B may serve frame 2. + // Until B advances, it may still serve frame 2. + assert!(reading.finished().now_or_never().is_none()); write_group(&mut track_b, 1, "b1"); assert_eq!(recv(&mut sub), 1); - assert!( - reading.finished().now_or_never().is_none(), - "the seam is still coverable" - ); // Enough failovers prune A and B: nothing can serve the seam anymore, so - // the cap is the end. + // the group was cut short. for sequence in 2..=(1 + MAX_SEGMENTS as u64) { let (mut track, consumer) = track_pair("t"); producer.takeover(&consumer).unwrap(); write_group(&mut track, sequence, "payload"); assert_eq!(recv(&mut sub), sequence); } - assert_eq!( - reading - .finished() - .now_or_never() - .expect("the lost seam must resolve the count") - .unwrap(), - 2 - ); + let err = reading + .finished() + .now_or_never() + .expect("the lost seam must resolve") + .unwrap_err(); + assert!(matches!(err, Error::Dropped), "{err:?}"); + } + + /// A reader that drained a bounded copy to its cap before the seam was lost + /// still learns the group was truncated: `finished()` and the read cursor + /// answer for the original group, not for where the copy stopped. + #[tokio::test] + async fn finished_fails_for_a_drained_pruned_bounded_group() { + let (track_a, consumer_a) = track_pair("a"); + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + + let mut group = track_a.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::ZERO, b"f0".to_vec()).unwrap(); + group.write_frame(Timestamp::ZERO, b"f1".to_vec()).unwrap(); + let (mut track_b, consumer_b) = track_pair("b"); + producer.takeover(&consumer_b).unwrap(); + let mut reading = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); + assert_eq!(read(&mut reading), b"f0"); + assert_eq!(read(&mut reading), b"f1"); + drop(group); + track_a.abort(Error::Dropped).unwrap(); + + write_group(&mut track_b, 1, "b1"); + assert_eq!(recv(&mut sub), 1); + for sequence in 2..=(1 + MAX_SEGMENTS as u64) { + let (mut track, consumer) = track_pair("t"); + producer.takeover(&consumer).unwrap(); + write_group(&mut track, sequence, "payload"); + assert_eq!(recv(&mut sub), sequence); + } + + let mut cloned = reading.clone(); + let err = reading + .finished() + .now_or_never() + .expect("the lost seam must resolve") + .unwrap_err(); + assert!(matches!(err, Error::Dropped), "{err:?}"); + + // Reading past the cap rolls off the bounded copy; the end it reaches is + // still the truncation, never a clean `None`. + let err = reading + .read_frame() + .now_or_never() + .expect("the lost seam must resolve") + .unwrap_err(); + assert!(matches!(err, Error::Dropped), "{err:?}"); + let err = reading.finished().now_or_never().unwrap().unwrap_err(); + assert!(matches!(err, Error::Dropped), "{err:?}"); + + // Same for `next_frame`. + let next = cloned.next_frame().now_or_never().unwrap(); + assert!(matches!(next, Err(Error::Dropped)), "{:?}", next.err()); } /// A handed-out group survives its segment's prune, for every reader: the @@ -5049,13 +5327,132 @@ mod test { } } - /// `finished()` resolves when the seam's covering route skip-declared the - /// group: its segment geometrically covers the continuation, but its - /// SUBSCRIBE_START floor proves the group will never arrive, so the cap is - /// the end. Polled without draining, and woken by the successor's track (the - /// seam probe parks on the peek), not just the segment list. + #[tokio::test(start_paused = true)] + async fn group_only_read_requests_its_continuation() { + group_only_continuation(false).await; + } + + #[tokio::test(start_paused = true)] + async fn group_only_finish_requests_its_continuation() { + group_only_continuation(true).await; + } + + async fn group_only_continuation(finish: bool) { + let (track_a, consumer_a) = track_pair("a"); + let (track_b, consumer_b) = track_pair("b"); + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + let mut group = track_a.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::ZERO, b"head".to_vec()).unwrap(); + // `finished` probes the seam past a bounded copy; a read waits past a dead one. + if finish { + producer.takeover(&consumer_b).unwrap(); + } + let mut reading = sub.recv_group().await.unwrap().unwrap(); + // The parent subscriber never asks B for anything. + drop(sub); + let parked = |reading: &mut group::Consumer| match finish { + true => reading.finished().now_or_never().is_none(), + false => reading.read_frame().now_or_never().is_none(), + }; + if !finish { + assert_eq!(read(&mut reading), b"head"); + producer.takeover(&consumer_b).unwrap(); + track_a.abort(Error::Dropped).unwrap(); + } + assert!(parked(&mut reading)); + // Demand starts at the frame the reader needs: its cursor, or the seam past A's copy. + assert_eq!( + track_b.subscription().expect("group-only demand").start, + Some(Position { group: 0, frame: 1 }) + ); + + // The header arriving does not end the wait: the tail still needs the demand. + let mut copy = track_b.create_group(group::Info { sequence: 0 }).unwrap(); + copy.write_frame(Timestamp::ZERO, b"head".to_vec()).unwrap(); + assert!(parked(&mut reading)); + assert!(track_b.subscription().is_some(), "the tail still needs demand"); + + copy.write_frame(Timestamp::ZERO, b"tail".to_vec()).unwrap(); + copy.finish().unwrap(); + if finish { + reading.finished().await.unwrap(); + assert_eq!(read(&mut reading), b"head"); + } + assert_eq!(read(&mut reading), b"tail"); + assert!(reading.read_frame().await.unwrap().is_none()); + assert!(track_b.subscription().is_none(), "the finished group releases demand"); + } + + /// A replacement cached before the reader first looks still needs demand for its + /// tail, and a clone keeps that demand after the original reader is dropped. + #[tokio::test(start_paused = true)] + async fn cached_replacement_holds_demand_across_clones() { + let (track_a, consumer_a) = track_pair("a"); + let (track_b, consumer_b) = track_pair("b"); + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + let mut group = track_a.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::ZERO, b"head".to_vec()).unwrap(); + let mut reading = sub.recv_group().await.unwrap().unwrap(); + drop(sub); + assert_eq!(read(&mut reading), b"head"); + + producer.takeover(&consumer_b).unwrap(); + track_a.abort(Error::Dropped).unwrap(); + let mut copy = track_b.create_group(group::Info { sequence: 0 }).unwrap(); + copy.write_frame(Timestamp::ZERO, b"head".to_vec()).unwrap(); + assert!(reading.read_frame().now_or_never().is_none()); + assert!(track_b.subscription().is_some(), "a cached header still needs demand"); + + let mut clone = reading.clone(); + drop(reading); + assert!(track_b.subscription().is_some(), "the clone keeps its own demand"); + + copy.write_frame(Timestamp::ZERO, b"tail".to_vec()).unwrap(); + copy.finish().unwrap(); + assert_eq!(read(&mut clone), b"tail"); + assert!(clone.read_frame().await.unwrap().is_none()); + assert!(track_b.subscription().is_none(), "the finished group releases demand"); + } + + /// A reader that stops at its cap may be retained, so the cap releases the demand + /// on the replacement rather than the drop. + #[tokio::test(start_paused = true)] + async fn capped_read_releases_replacement_demand() { + let (track_a, consumer_a) = track_pair("a"); + let (track_b, consumer_b) = track_pair("b"); + let mut producer = Producer::new(); + producer.takeover(&consumer_a).unwrap(); + let mut sub = producer.consume().subscribe(None); + let mut group = track_a.create_group(group::Info { sequence: 0 }).unwrap(); + group.write_frame(Timestamp::ZERO, b"head".to_vec()).unwrap(); + let mut reading = sub.recv_group().await.unwrap().unwrap(); + drop(sub); + assert_eq!(read(&mut reading), b"head"); + + producer.takeover(&consumer_b).unwrap(); + track_a.abort(Error::Dropped).unwrap(); + let mut copy = track_b.create_group(group::Info { sequence: 0 }).unwrap(); + copy.write_frame(Timestamp::ZERO, b"head".to_vec()).unwrap(); + copy.write_frame(Timestamp::ZERO, b"mid".to_vec()).unwrap(); + reading.set_frames(..2); + assert_eq!(read(&mut reading), b"mid"); + assert!(track_b.subscription().is_some(), "the read still holds demand"); + + assert!(reading.read_frame().now_or_never().unwrap().unwrap().is_none()); + assert!(track_b.subscription().is_none(), "the cap releases demand"); + } + + /// `finished()` fails when the seam's covering route skip-declared the group: + /// its segment geometrically covers the continuation, but its SUBSCRIBE_START + /// floor proves the group will never arrive, so the group is truncated at the + /// cap. Polled without draining, and woken by the successor's track (the seam + /// probe parks on the peek), not just the segment list. #[tokio::test] - async fn finished_resolves_when_the_successor_skips_the_seam() { + async fn finished_fails_when_the_successor_skips_the_seam() { let (track_a, consumer_a) = track_pair("a"); let (mut track_b, consumer_b) = track_pair("b"); let mut producer = Producer::new(); @@ -5075,18 +5472,16 @@ mod test { assert!(reading.finished().now_or_never().is_none(), "the seam is coverable"); // B declares it starts at group 1 and produces it: group 0's continuation - // is skipped for good, so the cap is the end. + // is skipped for good, so the group was cut short. track_b.start_at(1).unwrap(); write_group(&mut track_b, 1, "b1"); assert_eq!(recv(&mut sub), 1); - assert_eq!( - reading - .finished() - .now_or_never() - .expect("a skip-declared seam must resolve the count") - .unwrap(), - 2 - ); + let err = reading + .finished() + .now_or_never() + .expect("a skip-declared seam must resolve") + .unwrap_err(); + assert!(matches!(err, Error::NotFound), "{err:?}"); } /// A reader that already latched a pruned segment's copy keeps draining it: the diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index 921c72af34e..84edd324f1e 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -224,8 +224,8 @@ pub(crate) struct TrackState { // The sequence number at which the track was finalized. final_sequence: Option, - // The last producer dropped after the boundary was declared, so a group still missing - // below it will never be produced. + // Nothing more can arrive: the last producer dropped after a declared boundary, + // or the receiving session closed locally without declaring an end. sealed: bool, // No producer remains (aborted, sealed, or dropped), so nothing protects the live @@ -634,12 +634,16 @@ impl TrackState { /// end-of-stream. The handler side (a rejection, or no [`Dynamic`] at all) lives /// in [`FetchState`]; [`Fetching`] polls both. fn poll_fetch_cached(&self, sequence: u64, frame_start: u64) -> Poll> { - if let Some(group) = self.covering_group(sequence, frame_start) { + // A group aborted between the lookup and the consume (an abandoned fetch cut + // short) is a miss, not a hit that fails on its first read. + if let Some(group) = self.covering_group(sequence, frame_start) + && let Some(consumer) = group.try_consume() + { // A cache hit refreshes the group: it resets both its age (expiry keys // off the last access) and its standing against the pool-wide average, // so the eviction walk keeps it over never-read groups. group.cache_refresh(); - return Poll::Ready(Ok(group.consume())); + return Poll::Ready(Ok(consumer)); } if let Some(err) = &self.abort { @@ -1118,11 +1122,12 @@ impl TrackState { /// An abort before the end settled wins over it: a group below the boundary was /// still open, so the track was cut off rather than ended. fn is_complete(&self) -> bool { - // `sealed` is a clean end when the last producer drops. An abort still wins - // unless that end had already settled: a group below it was still open. - let reached = self - .final_sequence - .is_some_and(|fin| self.sealed || self.max_sequence.map_or(0, |max| max.saturating_add(1)) >= fin); + // `sealed` also ends a locally closed receive track without a declared end. + // An abort still wins unless that end had already settled: a group below it was still open. + let reached = self.sealed + || self + .final_sequence + .is_some_and(|fin| self.max_sequence.map_or(0, |max| max.saturating_add(1)) >= fin); reached && (self.abort.is_none() || self.settled) } @@ -1168,11 +1173,22 @@ impl TrackState { Poll::Ready(Ok(fin)) } else if let Some(err) = &self.abort { Poll::Ready(Err(err.clone())) + } else if self.sealed { + Poll::Ready(Err(Error::Closed)) } else { Poll::Pending } } + /// Distinguish a locally closed receive track from a declared content end. + fn poll_end(&self) -> Poll>> { + if self.sealed && self.final_sequence.is_none() && self.abort.is_none() { + Poll::Ready(Ok(None)) + } else { + self.poll_finished().map(|res| res.map(Some)) + } + } + fn modify(producer: &kio::Producer) -> Result> { producer.write().map_err(|r| r.abort.clone().unwrap_or(Error::Dropped)) } @@ -1208,7 +1224,11 @@ impl TrackState { // An evicted sequence can be re-fetched; a live one is a duplicate. self.claim_sequence(sequence, frame_start)?; - let group = group::Producer::new(group::Info { sequence }, info, self.cache.clone()); + let mut group = group::Producer::new(group::Info { sequence }, info, self.cache.clone()); + // Start where the request did before the group is visible: a fetch looking it up + // in between would otherwise see it begin at 0 and get a reader that later skips + // the head it asked for. + group.start_at(frame_start)?; // A backfill exists because someone is fetching it right now: stamp that // access so the eviction walk can't kill it before the fetch resolves. // It is also invisible to arrival-order subscribers: fetched on demand, @@ -1464,6 +1484,35 @@ impl Producer { Ok(()) } + /// End a locally closed session's receive track without declaring a wire boundary. + pub(crate) fn close(self) -> Result<()> { + let mut state = self.modify()?; + state.sealed = true; + // See `commit_abort`: a takeover resumes mid-group once the open group goes away. + state.resume = state.resume_position(); + state.drop_open_groups(); + state.close_cache(); + state.close(); + Ok(()) + } + + /// Abort the groups still receiving from a failed session before ending its track. + pub(crate) fn abort_session(self, err: Error) -> Result<()> { + let state = self.modify()?; + let open: Vec<_> = state + .lookup + .values() + .filter(|slot| !slot.group.is_finished()) + .map(|slot| slot.group.clone()) + .collect(); + // Snapshot the resume boundary before aborting groups releases their frames. + commit_abort(state, err.clone()); + for group in open { + let _ = group.abort(err.clone()); + } + Ok(()) + } + /// Mark the track as finished after the last appended group. /// /// Sets the final sequence to one past the current max_sequence. @@ -1947,7 +1996,7 @@ impl Drop for Alive { state.close_cache(); } Err(state) => { - if state.final_sequence.is_some() || state.abort.is_some() { + if state.sealed || state.final_sequence.is_some() || state.abort.is_some() { return; } tracing::warn!( @@ -2610,6 +2659,31 @@ impl Consumer { })) } + /// Poll for the newest cached group once its sequence passes `after`, as a producer + /// handle: holding one keeps its frames when the track's own producer tears down + /// abruptly, which otherwise releases every unfinished group. Pending once closed. + pub(crate) fn poll_latest_group(&self, after: Option, waiter: &kio::Waiter) -> Poll { + let ConsumerKind::Plain(state) = &self.inner else { + return Poll::Pending; + }; + let res = state.poll(waiter, |state| { + match state + .latest_group + .filter(|latest| after.is_none_or(|after| *latest > after)) + { + Some(latest) => match state.lookup.get(&latest) { + Some(slot) => Poll::Ready(slot.group.clone()), + None => Poll::Pending, + }, + None => Poll::Pending, + } + }); + match res { + Poll::Ready(Ok(group)) => Poll::Ready(group), + _ => Poll::Pending, + } + } + /// Poll for a cached group by sequence, parking `waiter` until it lands. /// /// `Ready(None)` once it can never arrive: the track ended below the sequence, or @@ -2721,13 +2795,20 @@ impl Consumer { // Queue a request only when the group isn't already resolvable from the track // (cached, aborted, or past-final all resolve through `Fetching::poll` without // a queue entry). - let (fetch, unresolved) = { + let (fetch, cached) = { let state = state.read(); ( state.fetch.clone(), - state.poll_fetch_cached(sequence, options.frame_start).is_pending(), + state.poll_fetch_cached(sequence, options.frame_start), ) }; + let unresolved = cached.is_pending(); + // Hold a hit until the caller polls: the held consumer keeps the group wanted, + // so an abandoned fetch filling it can't abort it in between. + let hit = match cached { + Poll::Ready(Ok(group)) => Some(Box::new(group)), + _ => None, + }; if unresolved { let mut fetch = fetch.lock(); @@ -2768,6 +2849,7 @@ impl Consumer { sequence, frame_start: options.frame_start, result, + hit, }, stats: self.stats.clone(), }) @@ -2827,7 +2909,8 @@ impl Consumer { return Poll::Pending; }; match ready!(state.poll(waiter, |state| { - if state.is_complete() { + // A local close without a declared end is a dead route, not finished content. + if state.final_sequence.is_some() && state.is_complete() { Poll::Ready(()) } else { Poll::Pending @@ -2973,8 +3056,10 @@ impl group::Request { /// The first frame of the group the consumer wants; 0 is the whole group. /// - /// A handler serving this must [`start_at`](group::Producer::start_at) it, so the frames - /// it writes carry the indices they have in the group rather than restarting at 0. + /// The group [`accept`](Self::accept) returns already starts here, so the frames a + /// handler writes carry the indices they have in the group rather than restarting at + /// 0. A handler serving from elsewhere moves it with + /// [`start_at`](group::Producer::start_at) before the first frame. /// /// There is no end: the handler fetches through the end of the group so the result /// is cacheable for anyone (see [`group::Fetch::frame_start`]). @@ -2997,6 +3082,12 @@ impl group::Request { let res = TrackState::modify(&self.state) .and_then(|mut state| state.insert_group_request(self.sequence, self.frame_start, info.into())); self.remove(); + // A joined fetch the cached group can't cover (a wider start that joined after + // the range went on the wire) fails rather than waits. Closing the channel says + // the same, but a handler tracking the joined demand keeps it open. + if let Ok(mut outcome) = self.result.write() { + outcome.rejected = Some(Error::NotFound); + } res } @@ -3021,6 +3112,28 @@ impl group::Request { } } + /// Poll for the request becoming unused (every waiting [`Consumer::fetch_group`] + /// dropped), so a handler can stop serving and drop the request. + /// + /// Once ready, a later fetch of the group starts a fresh request rather than + /// joining this abandoned one. + pub fn poll_unused(&self, waiter: &kio::Waiter) -> Poll<()> { + loop { + // A closed channel has nothing left to serve either. + if ready!(self.result.poll_unused(waiter)).is_err() { + return Poll::Ready(()); + } + // Fetches join under the fetch lock, so re-checking under it makes the + // withdrawal atomic with a join racing the last one leaving. + let mut fetch = self.fetch.lock(); + if self.result.is_used() { + continue; + } + fetch.remove_if(&self.sequence, |pending| pending.result.same_channel(&self.result)); + return Poll::Ready(()); + } + } + /// Remove this attempt from the fetch state, unless a newer attempt for the same /// sequence has already replaced it. fn remove(&self) { @@ -3062,8 +3175,12 @@ enum FetchingKind { // This caller's own start, so a cached group that begins above it is a miss // rather than a short answer. frame_start: u64, - // The joined attempt's result channel; `None` when no handler existed to queue on. + // The joined attempt's result channel; `None` on a cache hit, or when no handler + // existed to queue on. result: Option>, + // The group already cached when the fetch was made, held so it stays wanted. + // Boxed: a group consumer dwarfs the rest of the handle. + hit: Option>, }, /// A spliced track's fetch: waits for a segment, then fetches from it. Spliced(kio::Pending), @@ -3073,14 +3190,15 @@ impl kio::Task for Fetching { type Output = Result; fn poll(&mut self, waiter: &kio::Waiter) -> Poll { - let (state, fetch, sequence, frame_start, result) = match &mut self.inner { + let (state, fetch, sequence, frame_start, result, hit) = match &mut self.inner { FetchingKind::Plain { state, fetch, sequence, frame_start, result, - } => (state, fetch, *sequence, *frame_start, result.as_ref()), + hit, + } => (state, fetch, *sequence, *frame_start, result.as_ref(), hit), FetchingKind::Spliced(spliced) => { // A fetched group is metered here (once), at the tagged handle: the // spliced source track it comes from is the origin's own, untagged. @@ -3089,22 +3207,26 @@ impl kio::Task for Fetching { } }; + // Hand back a consumer sitting where the caller asked rather than at the group's + // own start. Coverage was checked first, so this can only skip frames they excluded. + let resolve = |mut group: group::Consumer| { + group.start_at(frame_start); + group.with_meter(self.stats.meter()) + }; + if let Some(group) = hit { + return Poll::Ready(Ok(resolve((**group).clone()))); + } + // Track side: the cached group, the abort error, or past-final. The outer // error is the channel closing without any of those. - match state.poll(waiter, |state| state.poll_fetch_cached(sequence, frame_start)) { - Poll::Ready(Ok(res)) => { - return Poll::Ready(res.map(|mut group| { - // Hand back a consumer sitting where the caller asked rather than at - // the group's own start. Coverage was checked above, so this can only - // skip frames they excluded. - group.start_at(frame_start); - group.with_meter(self.stats.meter()) - })); - } - Poll::Ready(Err(closed)) => { - return Poll::Ready(Err(closed.abort.clone().unwrap_or(Error::Dropped))); - } - Poll::Pending => {} + let cached = + |waiter: &kio::Waiter| match state.poll(waiter, |state| state.poll_fetch_cached(sequence, frame_start)) { + Poll::Ready(Ok(res)) => Poll::Ready(res.map(resolve)), + Poll::Ready(Err(closed)) => Poll::Ready(Err(closed.abort.clone().unwrap_or(Error::Dropped))), + Poll::Pending => Poll::Pending, + }; + if let Poll::Ready(res) = cached(waiter) { + return Poll::Ready(res); } // Handler side. @@ -3122,13 +3244,19 @@ impl kio::Task for Fetching { // A written rejection fails every joined fetch. The channel closing without // one means the attempt was dropped unserved (its handlers went away). - match result.poll(waiter, |outcome| match &outcome.rejected { + let err = match result.poll(waiter, |outcome| match &outcome.rejected { Some(err) => Poll::Ready(err.clone()), None => Poll::Pending, }) { - Poll::Ready(Ok(err)) => Poll::Ready(Err(err)), - Poll::Ready(Err(_closed)) => Poll::Ready(Err(Error::NotFound)), - Poll::Pending => Poll::Pending, + Poll::Ready(Ok(err)) => err, + Poll::Ready(Err(_closed)) => Error::NotFound, + Poll::Pending => return Poll::Pending, + }; + // An accept caches the group before it answers the channel, which may land + // between the track check above and this one: look again before failing. + match cached(waiter) { + Poll::Ready(res) => Poll::Ready(res), + Poll::Pending => Poll::Ready(Err(err)), } } } @@ -4011,12 +4139,21 @@ impl Subscriber { } } + /// Observe a declared end or a clean local close for an origin's spliced cursor. + pub(crate) fn poll_end(&mut self, waiter: &kio::Waiter) -> Poll>> { + match &mut self.inner { + SubscriberKind::Plain(plain) => plain.poll(waiter, |state| state.poll_end()), + SubscriberKind::Spliced(spliced) => spliced.poll_end(waiter), + } + } + /// Block until the track declares its end, returning the exclusive final sequence /// (also the total group count), or the cause on an abort. /// /// Resolves as soon as the boundary is known, which may be ahead of the live edge /// when the producer finished via [`Producer::finish_at`]. This reports the declared - /// end, not that every group has arrived: drive [`Self::recv_group`] (or + /// end, not that every group has arrived. A local session close without a declared + /// end returns [`Error::Closed`]. Drive [`Self::recv_group`] (or /// [`Ordered::next_group`]) until it yields `None` to observe the track fully drained. pub async fn finished(&mut self) -> Result { kio::wait(|waiter| self.poll_finished(waiter)).await @@ -5111,10 +5248,13 @@ mod test { producer.create_group(1u64.into()).unwrap().finish().unwrap(); let mut frame = straggler - .create_frame_owned(frame::Info { - size: 3, - timestamp: Timestamp::ZERO, - }) + .create_frame_owned( + frame::Info { + size: 3, + timestamp: Timestamp::ZERO, + }, + &Default::default(), + ) .unwrap(); // The sender stalls past the retention window, then the whole payload lands in @@ -6985,6 +7125,51 @@ mod test { assert!(matches!(res, Err(Error::Timeout))); } + #[tokio::test] + async fn local_close_keeps_finished_groups_without_declaring_an_end() { + let producer = track_producer("local", None); + let mut consumer = producer.consume().subscribe(None).await.unwrap(); + producer.append_group().unwrap().finish().unwrap(); + producer.clone().close().unwrap(); + assert!(producer.final_sequence().is_none()); + assert!(matches!(consumer.finished().await, Err(Error::Closed))); + assert!(consumer.recv_group().await.unwrap().is_some()); + assert!(consumer.recv_group().await.unwrap().is_none()); + } + + #[tokio::test] + async fn local_close_cannot_mask_an_abort_before_declared_end_settles() { + let mut producer = track_producer("local", None); + let mut consumer = producer.consume().subscribe(None).await.unwrap(); + let _group = producer.append_group().unwrap(); + producer.finish_at(2).unwrap(); + producer.clone().abort(Error::Timeout).unwrap(); + assert!(producer.close().is_err()); + assert!(matches!(consumer.recv_group().await, Err(Error::Timeout))); + } + + /// A local close is a dead route to the origin, not finished content, so a + /// replacement route can take over where it left off. + #[test] + fn local_close_is_not_completion_and_keeps_the_resume_position() { + let producer = track_producer("local", None); + let consumer = producer.consume(); + producer.append_group().unwrap().finish().unwrap(); + let mut group = producer.append_group().unwrap(); + group + .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")) + .unwrap(); + group + .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"b")) + .unwrap(); + producer.close().unwrap(); + assert!(matches!( + consumer.poll_complete(&kio::Waiter::noop()), + Poll::Ready(Err(_)) + )); + assert_eq!(consumer.resume_position(), Some(Position { group: 1, frame: 2 })); + } + /// An abort short of the end keeps the groups that finished for a reader that has not /// pulled them yet: it gets them, then the abort. The open group nobody will finish /// is gone, on both cursors. @@ -7862,6 +8047,120 @@ mod test { assert!(matches!(pending.await, Err(Error::Dropped))); } + /// A request stays wanted while any joined fetch waits, and once the last leaves it + /// is withdrawn: a later fetch starts a fresh request instead of joining it. + #[tokio::test] + async fn fetch_request_unused_once_every_fetch_leaves() { + let producer = track_producer("test", None); + let dynamic = producer.dynamic(); + let consumer = producer.consume(); + + let first = consumer.fetch_group(5, None); + let req = dynamic + .requested_group() + .now_or_never() + .expect("should not block") + .unwrap(); + let second = consumer.fetch_group(5, None); + + drop(first); + assert!(req.poll_unused(&kio::Waiter::noop()).is_pending()); + drop(second); + assert!(req.poll_unused(&kio::Waiter::noop()).is_ready()); + + let retry = consumer.fetch_group(5, None); + let fresh = dynamic + .requested_group() + .now_or_never() + .expect("the retry queues a fresh request") + .unwrap(); + drop(req); + assert!(fresh.poll_unused(&kio::Waiter::noop()).is_pending()); + fresh.accept(None).unwrap().finish().unwrap(); + assert_eq!(retry.await.unwrap().sequence, 5); + } + + /// A fetch that hits the cache holds the group until polled, so a handler that aborts + /// the group once nobody wants it (an abandoned upstream fetch) leaves it alone. + #[tokio::test] + async fn fetch_hit_keeps_the_group_wanted() { + let producer = track_producer("test", None); + let dynamic = producer.dynamic(); + let consumer = producer.consume(); + + let first = consumer.fetch_group(5, None); + let req = dynamic + .requested_group() + .now_or_never() + .expect("should not block") + .unwrap(); + let mut group = req.accept(None).unwrap(); + group + .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .unwrap(); + drop(first.await.unwrap()); + + let hit = consumer.fetch_group(5, None); + assert!(!group.abort_unused(Error::Cancel), "the pending hit wants the group"); + let mut fetched = hit.await.unwrap(); + assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"head"); + } + + /// An accepted group starts where its request did before anyone can see it, so a + /// wider fetch arriving before the handler writes misses instead of being handed a + /// reader that would skip the head it asked for. + #[tokio::test] + async fn accepted_group_starts_at_the_request() { + let producer = track_producer("test", None); + let dynamic = producer.dynamic(); + let consumer = producer.consume(); + + let _tail = consumer.fetch_group(5, group::Fetch::default().with_frame_start(3)); + let req = dynamic + .requested_group() + .now_or_never() + .expect("should not block") + .unwrap(); + let _group = req.accept(None).unwrap(); + + let mut whole = consumer.fetch_group(5, None); + let req = dynamic + .requested_group() + .now_or_never() + .expect("the wider fetch misses and queues its own request") + .unwrap(); + assert_eq!(req.frame_start(), 0); + assert!(kio::Task::poll(&mut *whole, &kio::Waiter::noop()).is_pending()); + } + + /// Once a group nobody wanted is aborted, a later fetch misses rather than reading + /// the abort, and queues a fresh request. + #[tokio::test] + async fn fetch_misses_an_abandoned_group() { + let producer = track_producer("test", None); + let dynamic = producer.dynamic(); + let consumer = producer.consume(); + + let first = consumer.fetch_group(5, None); + let req = dynamic + .requested_group() + .now_or_never() + .expect("should not block") + .unwrap(); + let group = req.accept(None).unwrap(); + drop(first.await.unwrap()); + assert!(group.abort_unused(Error::Cancel)); + + let retry = consumer.fetch_group(5, None); + let req = dynamic + .requested_group() + .now_or_never() + .expect("the retry queues a fresh request") + .unwrap(); + req.accept(None).unwrap().finish().unwrap(); + assert!(retry.await.unwrap().read_frame().await.unwrap().is_none()); + } + #[tokio::test] async fn fetch_reject_does_not_poison_retry() { let producer = track_producer("test", None); @@ -7968,6 +8267,9 @@ mod test { assert!(futures::poll!(widest.as_mut()).is_pending()); assert_eq!(request.frame_start(), 2, "the in-flight range is already on the wire"); + // A handler may keep tracking the joined demand past the accept, as the lite + // subscriber does, which holds the result channel open. + let _joined = request.result.clone(); let mut group = request.accept(None).unwrap(); // A handler numbers the frames from where it was asked to start, as `serve_fetch` // does; that offset is what makes the cached group too narrow for `widest`. @@ -7975,13 +8277,14 @@ mod test { group .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"from2")) .unwrap(); - group.finish().unwrap(); // The two callers it covers resolve; the one it doesn't fails cleanly rather - // than being handed a group that starts above what it asked for. + // than being handed a group that starts above what it asked for, even while + // the group is still being written. assert_eq!(narrow.await.unwrap().index(), 5); assert_eq!(wider.await.unwrap().index(), 2); assert!(matches!(widest.await, Err(Error::NotFound))); + group.finish().unwrap(); } #[tokio::test] diff --git a/rs/moq-net/src/server.rs b/rs/moq-net/src/server.rs index a503d72bb54..777f9df25f0 100644 --- a/rs/moq-net/src/server.rs +++ b/rs/moq-net/src/server.rs @@ -535,6 +535,7 @@ where authority: None, peer_setup_stream: Some(peer_setup.stream), peer_declared: Some(peer_setup.declared), + early_unis: peer_setup.early, })?; tracing::debug!(?version, "connected"); Ok(Session::new( @@ -643,6 +644,7 @@ where authority: None, peer_setup_stream: None, peer_declared: Some(peer_declared), + early_unis: Vec::new(), })?; (None, crate::driver::Protocol::Ietf(protocol), goaway) } @@ -1245,6 +1247,7 @@ mod tests { path: None, token: None, declared: ietf::peer::Peer::default(), + early: Vec::new(), }, })), }), diff --git a/rs/moq-net/src/session.rs b/rs/moq-net/src/session.rs index efea7bfc5c8..a192b5fc2ea 100644 --- a/rs/moq-net/src/session.rs +++ b/rs/moq-net/src/session.rs @@ -279,6 +279,7 @@ impl Session { }); let supervisor = Supervisor { + local_close: protocol.local_close(), runtime: runtime.clone(), closed_watch: session.clone(), session, @@ -319,6 +320,7 @@ impl Session { /// /// Finishes once the transport reports closed; everything else is moot then. pub(crate) struct Supervisor { + local_close: Arc, runtime: crate::time::Clock, session: S, // A dedicated clone for the close watch, since each pending poll operation @@ -395,9 +397,12 @@ impl Supervisor { }) { Poll::Ready(Ok(request)) => (request, false), Poll::Ready(Err(last)) => ( - last.clone().unwrap_or_else(|| Close::Abort { - code: SessionError::Cancel.to_code(), - reason: "dropped".to_string(), + last.clone().unwrap_or_else(|| { + self.local_close.store(true, std::sync::atomic::Ordering::Relaxed); + Close::Abort { + code: SessionError::Cancel.to_code(), + reason: "dropped".to_string(), + } }), true, ), @@ -443,6 +448,7 @@ impl Supervisor { false if deadline.poll(waiter).is_ready() => Err(Error::Timeout), false => return false, }; + self.local_close.store(true, std::sync::atomic::Ordering::Relaxed); self.session.close(SessionError::Cancel.to_code(), ""); self.drain = Drain::Done(res); // The transport is closed, so no later request can change anything. diff --git a/rs/moq-net/tests/fetch_cancel.rs b/rs/moq-net/tests/fetch_cancel.rs new file mode 100644 index 00000000000..5e2d45ff7c5 --- /dev/null +++ b/rs/moq-net/tests/fetch_cancel.rs @@ -0,0 +1,154 @@ +//! A FETCH nobody wants anymore is cancelled all the way to the publisher. +//! +//! Each relay on the path holds a FETCH stream and the group behind it. When the +//! reader gives up, every hop must let go, whether the publisher has yet to answer +//! (lite has no FETCH_OK, so one that never answers would pin them for good) or is +//! partway through the group. A group cut short is aborted, never cached as whole. + +mod support; + +use std::time::Duration; + +use bytes::Bytes; +use moq_net::{Hop, Timestamp, Version, origin, track}; +use support::harness::peer; + +fn produce_origin(hop: u64) -> origin::Producer { + let (producer, driver) = origin::Producer::new(origin::Config::new(Hop::new(hop).unwrap())); + tokio::spawn(support::harness::run(driver)); + producer +} + +/// Where the reader gives up. +#[derive(Clone, Copy, Debug)] +enum Stage { + /// Before the publisher answers the FETCH. + Unanswered, + /// After the first frame, with the rest of the group still to come. + MidResponse, + /// Once every frame is read, as the FIN arrives. + Drained, +} + +/// `relays` relays between the publisher and the reader. +async fn abandoned_fetch_reaches_the_publisher(version: &str, relays: u64, stage: Stage) { + let version: Version = version.parse().unwrap(); + let ctx = format!("{version:?} over {relays} relays, {stage:?}"); + let nodes: Vec<_> = (1..=relays + 1).map(produce_origin).collect(); + let mut _pairs = Vec::new(); + for pair in nodes.windows(2) { + _pairs.push(peer(version, &pair[0], &pair[1]).await); + } + + let broadcast = nodes[0].create_broadcast("room").unwrap(); + let mut dynamic = broadcast.dynamic(); + broadcast.announce(Default::default()).unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + + let consumer = nodes[relays as usize] + .consume() + .request_broadcast("room") + .await + .unwrap(); + let track = consumer.track("video").unwrap(); + let mut waiting = Box::pin(track.fetch_group(0, None)); + + // Answer the TRACK_INFO, then take the FETCH. + let request = tokio::select! { + request = dynamic.requested_track() => request.expect("the track request reaches the publisher"), + _ = &mut waiting => panic!("nothing answered the track"), + }; + let groups = request.dynamic(); + let _track = request.accept(track::Info::default()); + let request = tokio::select! { + request = groups.requested_group() => request.expect("the fetch reaches the publisher"), + _ = &mut waiting => panic!("nothing answered the fetch"), + }; + + match stage { + Stage::Unanswered => { + drop(waiting); + tokio::time::timeout(Duration::from_secs(5), kio::wait(|waiter| request.poll_unused(waiter))) + .await + .unwrap_or_else(|_| panic!("{ctx}: the publisher's fetch is still wanted")); + } + Stage::MidResponse => { + let mut group = request.accept(None).unwrap(); + group.write_frame(Timestamp::ZERO, Bytes::from_static(b"head")).unwrap(); + + let mut fetched = waiting.await.unwrap(); + let frame = fetched.read_frame().await.unwrap().unwrap(); + assert_eq!(&frame.payload[..], b"head", "{ctx}"); + drop(fetched); + + // The publisher stops serving the group: nobody reads it any more. + tokio::time::timeout(Duration::from_secs(5), group.unused()) + .await + .unwrap_or_else(|_| panic!("{ctx}: the publisher still serves the fetch")) + .unwrap(); + + // No hop cached the head as the whole group: a fresh fetch reads it all. + group.write_frame(Timestamp::ZERO, Bytes::from_static(b"tail")).unwrap(); + group.finish().unwrap(); + let mut fetched = tokio::time::timeout(Duration::from_secs(5), track.fetch_group(0, None)) + .await + .unwrap_or_else(|_| panic!("{ctx}: the refetch stalled")) + .unwrap(); + let mut payloads = Vec::new(); + while let Some(frame) = fetched.read_frame().await.unwrap() { + payloads.push(frame.payload); + } + assert_eq!(payloads, [&b"head"[..], &b"tail"[..]], "{ctx}"); + } + Stage::Drained => { + let mut group = request.accept(None).unwrap(); + group.write_frame(Timestamp::ZERO, Bytes::from_static(b"head")).unwrap(); + group.write_frame(Timestamp::ZERO, Bytes::from_static(b"tail")).unwrap(); + + let mut fetched = waiting.await.unwrap(); + for expected in [&b"head"[..], b"tail"] { + let frame = fetched.read_frame().await.unwrap().unwrap(); + assert_eq!(&frame.payload[..], expected, "{ctx}"); + } + + // The FIN reaches the reader's node at the instant the reader leaves: the + // group is whole, so it must be cached as such rather than cancelled. + const LATENCY: Duration = Duration::from_millis(100); + _pairs.last().unwrap().client_transport.set_latency(LATENCY); + group.finish().unwrap(); + tokio::time::sleep(LATENCY).await; + drop(fetched); + tokio::time::sleep(Duration::from_secs(1)).await; + + // The publisher can no longer serve it, so only a cached copy answers. + group.abort(moq_net::Error::Cancel).unwrap(); + let mut fetched = tokio::time::timeout(Duration::from_secs(5), track.fetch_group(0, None)) + .await + .unwrap_or_else(|_| panic!("{ctx}: the refetch stalled")) + .unwrap_or_else(|err| panic!("{ctx}: the whole group was not cached: {err}")); + let mut payloads = Vec::new(); + while let Some(frame) = fetched.read_frame().await.unwrap() { + payloads.push(frame.payload); + } + assert_eq!(payloads, [&b"head"[..], &b"tail"[..]], "{ctx}"); + } + } +} + +async fn abandoned_fetches_reach_the_publisher(version: &str) { + for stage in [Stage::Unanswered, Stage::MidResponse, Stage::Drained] { + for relays in [1, 2] { + abandoned_fetch_reaches_the_publisher(version, relays, stage).await; + } + } +} + +#[tokio::test(start_paused = true)] +async fn abandoned_fetch_reaches_the_publisher_lite05() { + abandoned_fetches_reach_the_publisher("moq-lite-05").await; +} + +#[tokio::test(start_paused = true)] +async fn abandoned_fetch_reaches_the_publisher_lite06() { + abandoned_fetches_reach_the_publisher("moq-lite-06").await; +} diff --git a/rs/moq-net/tests/rejoin.rs b/rs/moq-net/tests/rejoin.rs index d49edfd6109..b6684c46ce0 100644 --- a/rs/moq-net/tests/rejoin.rs +++ b/rs/moq-net/tests/rejoin.rs @@ -17,83 +17,94 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer } -async fn read_all(group: &mut group::Consumer) -> Vec> { +async fn read_all(group: &mut group::Consumer) -> moq_net::Result>> { let mut frames = Vec::new(); - while let Some(frame) = group.read_frame().await.expect("group aborted") { + while let Some(frame) = group.read_frame().await? { frames.push(frame.payload.to_vec()); } - frames + Ok(frames) } -/// The group in flight when the last reader left comes back whole on rejoin. +/// A rejoin recovers the group reset on leave, whether or not the live edge moved past it. /// /// The relay cancels its idle upstream subscription, resetting that group mid-transfer. /// Resuming the rejoin at the frame where the reset landed would ask upstream for a tail /// whose head is gone, so the group would never reach the returning reader. -#[tokio::test] +#[tokio::test(start_paused = true)] async fn rejoin_recovers_the_group_reset_on_leave() { - for version in ["moq-lite-05", "moq-lite-06"] { - tokio::time::timeout(TEST_TIMEOUT, async { - let publisher = produce_origin(1); - let relay = produce_origin(2); - - let broadcast = publisher.create_broadcast("bench").unwrap(); - let track = broadcast.create_track("video", None).unwrap(); - broadcast.announce(Default::default()).unwrap(); - - let mut options = MockConnectOptions::new(version.parse::().unwrap()); - options.server_publish = Some(publisher.consume()); - options.client_subscribe = Some(relay.clone()); - let _pair = connect_mock(options).await; - - let consumer = relay.consume(); - consumer.routed("bench").await.unwrap(); - let remote = consumer.request_broadcast("bench").await.unwrap(); - - let ts = |ms| Timestamp::from_millis(ms).unwrap(); - let prefs = || track::Subscription::default().with_max_age(Duration::from_secs(10)); - - let mut group = track.append_group().unwrap(); - group.write_frame(ts(0), b"a0".as_ref()).unwrap(); - group.finish().unwrap(); - let mut open = track.append_group().unwrap(); - open.write_frame(ts(100), b"b0".as_ref()).unwrap(); - - let mut sub = remote.track("video").unwrap().subscribe(prefs()).await.unwrap(); - loop { - let mut group = sub.recv_group().await.unwrap().unwrap(); - group.read_frame().await.unwrap().unwrap(); - if group.sequence == 1 { - break; + for advance in [false, true] { + for version in ["moq-lite-05", "moq-lite-06"] { + tokio::time::timeout(TEST_TIMEOUT, async { + let publisher = produce_origin(1); + let relay = produce_origin(2); + + let broadcast = publisher.create_broadcast("bench").unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let mut options = MockConnectOptions::new(version.parse::().unwrap()); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay.clone()); + let _pair = connect_mock(options).await; + + let consumer = relay.consume(); + consumer.routed("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench").await.unwrap(); + + let ts = |ms| Timestamp::from_millis(ms).unwrap(); + let prefs = || track::Subscription::default().with_max_age(Duration::from_secs(10)); + + let mut group = track.append_group().unwrap(); + group.write_frame(ts(0), b"a0".as_ref()).unwrap(); + group.finish().unwrap(); + let mut open = track.append_group().unwrap(); + open.write_frame(ts(100), b"b0".as_ref()).unwrap(); + + let mut sub = remote.track("video").unwrap().subscribe(prefs()).await.unwrap(); + loop { + let mut group = sub.recv_group().await.unwrap().unwrap(); + group.read_frame().await.unwrap().unwrap(); + if group.sequence == 1 { + break; + } } - } - - // Leave mid-group; the publisher cuts the open group once demand is gone. - drop(sub); - track.demand().unused().await.unwrap(); - open.write_frame(ts(133), b"b1".as_ref()).unwrap(); - open.finish().unwrap(); - - let mut live = track.append_group().unwrap(); - live.write_frame(ts(5000), b"c0".as_ref()).unwrap(); - live.finish().unwrap(); - - let mut sub = remote.track("video").unwrap().subscribe(prefs()).await.unwrap(); - let mut rejoined = Vec::new(); - loop { - let mut group = sub.recv_group().await.unwrap().unwrap(); - let sequence = group.sequence; - rejoined.push((sequence, read_all(&mut group).await)); - if sequence == 2 { - break; + + // Leave mid-group; the publisher cuts the open group once demand is gone. + drop(sub); + track.demand().unused().await.unwrap(); + open.write_frame(ts(133), b"b1".as_ref()).unwrap(); + open.finish().unwrap(); + + if advance { + let mut live = track.append_group().unwrap(); + live.write_frame(ts(5000), b"c0".as_ref()).unwrap(); + live.finish().unwrap(); + } + + let mut sub = remote.track("video").unwrap().subscribe(prefs()).await.unwrap(); + let mut rejoined = Vec::new(); + loop { + let mut group = sub.recv_group().await.unwrap().unwrap(); + let sequence = group.sequence; + rejoined.push((sequence, read_all(&mut group).await)); + if sequence == if advance { 2 } else { 1 } { + break; + } + } + let reset = rejoined.iter().find(|(seq, _)| *seq == 1).expect("reset group"); + // The reader asks for the group it is missing, so the reset group comes back + // whole even once the live edge has moved past it. + assert_eq!( + reset.1.as_ref().unwrap(), + &vec![b"b0".to_vec(), b"b1".to_vec()], + "{version}" + ); + if advance { + assert_eq!(rejoined.last().unwrap().1.as_ref().unwrap(), &vec![b"c0".to_vec()]); } - } - assert!( - rejoined.contains(&(1, vec![b"b0".to_vec(), b"b1".to_vec()])), - "{version}: the reset group never came back whole: {rejoined:?}" - ); - }) - .await - .unwrap_or_else(|_| panic!("{version} timed out")); + }) + .await + .unwrap_or_else(|_| panic!("{version} timed out")); + } } } diff --git a/rs/moq-net/tests/subscribe_tracks.rs b/rs/moq-net/tests/subscribe_tracks.rs new file mode 100644 index 00000000000..096c5ea3d66 --- /dev/null +++ b/rs/moq-net/tests/subscribe_tracks.rs @@ -0,0 +1,148 @@ +//! A SUBSCRIBE_TRACKS is refused on its own stream, and the session keeps delivering. +//! +//! Deterministic: paused time and a single-threaded runtime over the mock transport. + +mod support; + +use std::time::Duration; + +use moq_net::transport::poll::{RecvStream as _, SendStream as _, Session as _}; +use moq_net::{Hop, Timestamp, Version}; +use support::harness::{MockConnectOptions, MockPair, connect_mock}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// SUBSCRIBE_TRACKS for prefix `room`, with no parameters. Every value fits a one-byte +/// varint, so the frame is hand-rolled. +#[rustfmt::skip] +const SUBSCRIBE_TRACKS: &[u8] = &[ + 0x51, // SUBSCRIBE_TRACKS + 0x00, 0x08, // Length + 0x01, // Request ID + 0x01, 0x04, b'r', b'o', b'o', b'm', // Track Namespace Prefix + 0x00, // Number of Parameters +]; + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); + tokio::spawn(support::harness::run(driver)); + producer +} + +/// A client publishing one track to a server subscribed to it, plus the payloads the +/// server has read so far. +struct Setup { + pair: MockPair, + track: moq_net::track::Producer, + frames: tokio::sync::mpsc::UnboundedReceiver>, + _broadcast: moq_net::broadcast::Producer, +} + +async fn setup(version: Version) -> Setup { + let publisher = produce_origin(1); + let broadcast = publisher.create_broadcast("room").unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let subscriber = produce_origin(2); + let mut options = MockConnectOptions::new(version); + options.client_publish = Some(publisher.consume()); + options.server_subscribe = Some(subscriber.clone()); + let pair = connect_mock(options).await; + + let consumer = subscriber.consume(); + tokio::time::timeout(TIMEOUT, consumer.routed("room")) + .await + .expect("announce timeout") + .expect("routed"); + let remote = tokio::time::timeout(TIMEOUT, consumer.request_broadcast("room")) + .await + .expect("resolve timeout") + .expect("broadcast resolves"); + + let (tx, frames) = tokio::sync::mpsc::unbounded_channel(); + tokio::spawn(async move { + let mut sub = remote.track("video").unwrap().subscribe(None).await.expect("subscribe"); + while let Ok(Some(mut group)) = sub.recv_group().await { + while let Ok(Some(frame)) = group.read_frame().await { + let _ = tx.send(frame.payload.to_vec()); + } + } + }); + + tokio::time::timeout(TIMEOUT, track.demand().used()) + .await + .expect("no subscriber appeared") + .unwrap(); + + Setup { + pair, + track, + frames, + _broadcast: broadcast, + } +} + +async fn deliver(setup: &mut Setup, payload: &'static [u8]) { + let mut group = setup.track.append_group().unwrap(); + group.write_frame(Timestamp::ZERO, payload).unwrap(); + group.finish().unwrap(); + let got = tokio::time::timeout(TIMEOUT, setup.frames.recv()) + .await + .expect("frame timed out") + .expect("the subscription ended"); + assert_eq!(got, payload); +} + +/// Send SUBSCRIBE_TRACKS from the server to the publishing client, returning every byte +/// of the reply once the client ends the stream. +async fn subscribe_tracks(pair: &MockPair) -> Vec { + let (mut send, mut recv) = pair.server_transport.clone().open_bi().await.expect("open_bi"); + send.write(SUBSCRIBE_TRACKS).await.expect("write"); + + let mut reply = Vec::new(); + let mut buf = [0u8; 64]; + while let Some(n) = tokio::time::timeout(TIMEOUT, recv.read(&mut buf)) + .await + .expect("reply timed out") + .expect("read") + { + reply.extend_from_slice(&buf[..n]); + } + reply +} + +/// Draft-18 defines SUBSCRIBE_TRACKS, so a limited endpoint answers NOT_SUPPORTED and +/// the subscription already on the session keeps going. +#[tokio::test(start_paused = true)] +async fn subscribe_tracks_is_refused_per_request() { + for version in [ + "moq-transport-18", + "moq-transport-19", + "moq-transport-20", + "moq-transport-21", + "moq-transport-22", + ] { + let mut setup = setup(version.parse().unwrap()).await; + deliver(&mut setup, b"before").await; + + let reply = subscribe_tracks(&setup.pair).await; + // REQUEST_ERROR (0x05), a two-byte length, then the error code: NOT_SUPPORTED (0x3). + assert_eq!(reply.first(), Some(&0x05), "{version}: not a REQUEST_ERROR: {reply:?}"); + assert_eq!(reply.get(3), Some(&0x03), "{version}: not NOT_SUPPORTED: {reply:?}"); + + deliver(&mut setup, b"after").await; + } +} + +/// Before draft-18, 0x51 is not a message at all, so it is still a protocol violation. +#[tokio::test(start_paused = true)] +async fn subscribe_tracks_before_draft_18_closes_the_session() { + let setup = setup("moq-transport-17".parse().unwrap()).await; + let (mut send, _recv) = setup.pair.server_transport.clone().open_bi().await.expect("open_bi"); + send.write(SUBSCRIBE_TRACKS).await.expect("write"); + + tokio::time::timeout(TIMEOUT, setup.pair.client.closed()) + .await + .expect("the session stayed open"); +} diff --git a/rs/moq-net/tests/subscription_end_integrity.rs b/rs/moq-net/tests/subscription_end_integrity.rs index 6a323c2fc75..621c2df3836 100644 --- a/rs/moq-net/tests/subscription_end_integrity.rs +++ b/rs/moq-net/tests/subscription_end_integrity.rs @@ -199,7 +199,7 @@ const DEATH: moq_net::SessionError = moq_net::SessionError::App(7); /// Kill the publisher's session with a group still open, and return the error the /// subscriber's `recv_group` ends with once it drains what arrived. -async fn killed(version: &str) -> Option { +async fn killed(version: &str, local: bool) -> (Option, Option) { let publisher = produce_origin(1); let broadcast = publisher.create_broadcast("bcast").unwrap(); let track = broadcast.create_track("video", None).unwrap(); @@ -224,14 +224,23 @@ async fn killed(version: &str) -> Option { .subscribe(subscription) .await .expect("subscribe"); + let mut group_end = None; loop { let mut group = match sub.recv_group().await { Ok(Some(group)) => group, - Ok(None) => return None, - Err(err) => return Some(err), + Ok(None) => return (None, group_end), + Err(err) => return (Some(err), group_end), }; - // The open group ends however it ends; only the track's end is at stake here. - while let Ok(Some(_)) = group.read_frame().await {} + loop { + match group.read_frame().await { + Ok(Some(_)) => {} + Ok(None) => break, + Err(err) => { + group_end = Some(err); + break; + } + } + } } }); @@ -240,7 +249,11 @@ async fn killed(version: &str) -> Option { group.write_frame(Timestamp::ZERO, HEAD[0]).unwrap(); settle().await; - server.abort(moq_net::Error::Session(DEATH)); + if local { + client.clone().close().await.expect("local close"); + } else { + server.abort(moq_net::Error::Session(DEATH)); + } let err = reader.await.expect("reader panicked"); drop((client, server, group, track, broadcast, publisher, subscriber)); err @@ -252,12 +265,26 @@ async fn killed(version: &str) -> Option { async fn a_session_death_ends_the_track_with_its_error() { tokio::time::pause(); for version in DEATH_VERSIONS { - let err = tokio::time::timeout(TIMEOUT, killed(version)) + let (err, group_end) = tokio::time::timeout(TIMEOUT, killed(version, false)) .await .unwrap_or_else(|_| panic!("{version}: the track never ended")); assert!( matches!(&err, Some(moq_net::Error::Session(code)) if *code == DEATH), "{version}: the track ended with {err:?}, not the session's error" ); + assert!( + matches!(&group_end, Some(moq_net::Error::Session(code)) if *code == DEATH), + "{version}: group ended with {group_end:?}" + ); + } +} + +#[tokio::test(start_paused = true)] +async fn a_local_session_close_ends_the_track_cleanly() { + for version in DEATH_VERSIONS { + let (err, _) = tokio::time::timeout(TIMEOUT, killed(version, true)) + .await + .expect("the track never ended"); + assert!(err.is_none(), "{version}: local close ended with {err:?}"); } } diff --git a/rs/moq-net/tests/track_abort_over_session.rs b/rs/moq-net/tests/track_abort_over_session.rs new file mode 100644 index 00000000000..4fbc351386f --- /dev/null +++ b/rs/moq-net/tests/track_abort_over_session.rs @@ -0,0 +1,108 @@ +//! A track aborted by its publisher while the broadcast stays announced, read by a +//! subscriber on the far side of one session. + +mod support; + +use std::time::Duration; + +use moq_net::{Hop, Timestamp, Version}; +use support::harness::{MockConnectOptions, connect_mock}; + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(Hop::new(hop).unwrap())); + tokio::spawn(support::harness::run(driver)); + producer +} + +/// A group reader alone must drive failover, without polling the next group. +async fn after_abort(version: &str, hops: u32) { + let publisher = produce_origin(1); + let relay = produce_origin(2); + let far = produce_origin(3); + let broadcast = publisher.create_broadcast("bench").unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + broadcast.announce(Default::default()).unwrap(); + let mut options = MockConnectOptions::new(version.parse::().unwrap()); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay.clone()); + let _pair = connect_mock(options).await; + let mut options = MockConnectOptions::new(version.parse::().unwrap()); + options.server_publish = Some(relay.consume()); + options.client_subscribe = Some(far.clone()); + let _pair2 = connect_mock(options).await; + let consumer = if hops == 2 { far.consume() } else { relay.consume() }; + consumer.routed("bench").await.unwrap(); + let remote = consumer.request_broadcast("bench").await.unwrap(); + let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); + let mut open = track.append_group().unwrap(); + open.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut group = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(group.read_frame().await.unwrap().unwrap().payload, b"head".as_ref()); + track.abort(moq_net::Error::Cancel).unwrap(); + drop(open); + assert!( + tokio::time::timeout(Duration::from_secs(2), group.read_frame()) + .await + .expect("group-only read hung") + .is_err() + ); +} + +macro_rules! abort_test { + ($name:ident, $version:literal, $hops:literal) => { + #[tokio::test(start_paused = true)] + async fn $name() { + after_abort($version, $hops).await; + } + }; +} +abort_test!(abort_03_one, "moq-lite-03", 1); +abort_test!(abort_03_two, "moq-lite-03", 2); +abort_test!(abort_04_one, "moq-lite-04", 1); +abort_test!(abort_04_two, "moq-lite-04", 2); +abort_test!(abort_05_one, "moq-lite-05", 1); +abort_test!(abort_05_two, "moq-lite-05", 2); +abort_test!(abort_06_one, "moq-lite-06", 1); +abort_test!(abort_06_two, "moq-lite-06", 2); + +async fn recreate(finish: bool) { + let origin = produce_origin(1); + let broadcast = origin.create_broadcast("bench").unwrap(); + broadcast.announce(Default::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let remote = origin.consume().request_broadcast("bench").await.unwrap(); + let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); + let mut open = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + open.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut group = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(group.sequence, 2); + group.read_frame().await.unwrap().unwrap(); + track.abort(moq_net::Error::Cancel).unwrap(); + let fresh = broadcast.create_track("video", None).unwrap(); + let mut next = fresh.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + next.write_frame(Timestamp::from_millis(1).unwrap(), b"next".as_ref()) + .unwrap(); + drop(open); + let result = tokio::time::timeout(Duration::from_secs(2), async { + if finish { + group.finished().await.map(|_| ()) + } else { + group.read_frame().await.map(|_| ()) + } + }) + .await + .expect("group-only failover hung"); + assert!(result.is_err()); + let next = sub.recv_group().await.unwrap().unwrap(); + assert_eq!(next.sequence, 3); +} + +#[tokio::test(start_paused = true)] +async fn recreated_track_skips_old_group() { + recreate(false).await; +} + +#[tokio::test(start_paused = true)] +async fn recreated_track_finishes_old_group() { + recreate(true).await; +} diff --git a/rs/moq-relay/Cargo.toml b/rs/moq-relay/Cargo.toml index 2f77c34c965..e4cbc5c5e72 100644 --- a/rs/moq-relay/Cargo.toml +++ b/rs/moq-relay/Cargo.toml @@ -59,6 +59,7 @@ bytesize = "2.4.2" futures = { workspace = true } http-cache-reqwest = { version = "1.0.0-alpha.6", features = ["manager-moka", "reqwest-middleware", "url-standard"], default-features = false } humantime = { workspace = true } +hyper-util = { workspace = true } moq-auth = { workspace = true, features = ["client"] } moq-net = { workspace = true } moq-stats = { workspace = true } diff --git a/rs/moq-relay/src/config.rs b/rs/moq-relay/src/config.rs index 98ad54e9373..31975a5f4f8 100644 --- a/rs/moq-relay/src/config.rs +++ b/rs/moq-relay/src/config.rs @@ -91,8 +91,9 @@ pub struct Config { /// How long accepted sessions may keep running after a shutdown signal, e.g. /// "10s" or "500ms". The first signal sends every session a GOAWAY and waits - /// this long for clients to reconnect elsewhere before force-closing them; a - /// second signal exits immediately. Zero closes them at once, with no GOAWAY + /// up to this long for clients to reconnect elsewhere before force-closing + /// them, exiting as soon as they have all left; a second signal exits + /// immediately. Zero closes them at once, with no GOAWAY /// they would have no time to act on. Defaults to 10 seconds. #[usage(skip)] #[serde(with = "crate::duration::serde_duration")] diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index bdace30947c..6399f660b9c 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -184,7 +184,7 @@ impl Connection { /// the session ([`auth::Lease::ended`]) the session closes with the reason, and /// the session's own close is reported back through the lease as the `end` event. /// Either way, a relay shutdown drains the session with a GOAWAY instead of -/// cutting it off. +/// cutting it off, and does not exit before this returns or the drain deadline. /// /// The session handle is `Send + Sync` whatever transport carries it, so this /// runs on the shared runtime even for sessions a pinned QUIC worker drives. @@ -194,6 +194,7 @@ pub async fn supervise( mut shutdown: crate::shutdown::Observer, registration: Option, ) -> anyhow::Result<()> { + let _serving = shutdown.serve(); loop { let nudged = async { match ®istration { diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index dc13fde0099..61b859d65de 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -9,7 +9,8 @@ //! - `/metrics` - this node's own traffic counters as Prometheus text //! exposition, plus the accept-loop health of its TCP listeners //! ([`with_listeners`](Internal::with_listeners)) and the per-worker health of -//! its io_uring runtime ([`with_uring`](Internal::with_uring)). A distinct plane +//! its io_uring runtime ([`with_uring`](Internal::with_uring)), and the +//! progress of a shutdown drain ([`with_shutdown`](Internal::with_shutdown)). A distinct plane //! from both the customer `web` surface and the MoQ `.stats` broadcast: the same //! atomics, but a different transport and audience (an ops scraper, not a //! customer or the dashboard/billing aggregators). The runtime counters are @@ -88,6 +89,7 @@ pub struct Internal { health: moq_tokio::accept::Health, listeners: Vec, uring: Vec, + shutdown: Option, listener: Option, addr: Option, } @@ -99,6 +101,7 @@ struct InternalState { sessions: crate::session::Registry, listeners: Vec, uring: Vec, + shutdown: Option, } impl Internal { @@ -123,6 +126,7 @@ impl Internal { health, listeners, uring: Vec::new(), + shutdown: None, listener: None, addr: None, } @@ -195,6 +199,12 @@ impl Internal { self } + /// Report the sessions a shutdown drain is still waiting on at `/metrics`. + pub fn with_shutdown(mut self, shutdown: crate::shutdown::Observer) -> Self { + self.shutdown = Some(shutdown); + self + } + /// Attach the relay cluster used to serve the `/nodes` topology snapshot. pub fn with_cluster(mut self, cluster: &crate::cluster::Cluster) -> Self { self.nodes = Some(cluster.nodes.clone()); @@ -228,6 +238,7 @@ impl Internal { sessions: self.sessions.clone(), listeners: self.listeners.clone(), uring: self.uring.clone(), + shutdown: self.shutdown.clone(), }) } @@ -287,7 +298,10 @@ async fn serve_health() -> Response { /// current cumulative snapshot; a downstream scraper derives rates and live /// counts (`open - closed`). async fn serve_metrics(State(state): State) -> Response { - let body = render_metrics(&state.stats.snapshot(), &state.listeners, &state.uring); + let mut body = render_metrics(&state.stats.snapshot(), &state.listeners, &state.uring); + if let Some(shutdown) = &state.shutdown { + render_drain(&mut body, shutdown.tally()); + } ([(http::header::CONTENT_TYPE, "text/plain; version=0.0.4")], body).into_response() } @@ -463,6 +477,21 @@ fn render_metrics( out } +/// The sessions a shutdown drain is still waiting on: 0 until the drain starts, +/// and back to 0 when every session has left, which is when the relay exits. +/// A scrape that last saw it above 0 shortly before the deadline means the +/// deadline force-closed the rest; the exit log records how many. +fn render_drain(out: &mut String, tally: crate::shutdown::Tally) { + use std::fmt::Write as _; + + let _ = writeln!( + out, + "# HELP moq_relay_draining_sessions Sessions sent a shutdown GOAWAY that have not left yet." + ); + let _ = writeln!(out, "# TYPE moq_relay_draining_sessions gauge"); + let _ = writeln!(out, "moq_relay_draining_sessions {}", tally.draining); +} + /// The accept-loop health of every listener on the node. /// /// The counters are the load-bearing half: a process out of descriptors cannot @@ -874,6 +903,7 @@ mod tests { sessions: crate::session::Registry::new(), listeners: Vec::new(), uring: Vec::new(), + shutdown: None, }; let Json(snapshot) = serve_nodes(State(state)).await; @@ -890,6 +920,7 @@ mod tests { sessions: crate::session::Registry::new(), listeners: Vec::new(), uring: Vec::new(), + shutdown: None, }; let Json(snapshot) = serve_nodes(State(state)).await; diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index cdf037dac9e..c10c65d4571 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -84,6 +84,9 @@ pub struct Relay { addr: Option, shutdown: shutdown::Observer, shutdown_trigger: shutdown::Trigger, + /// Whether [`Self::run`] drains on SIGINT/SIGTERM itself, or leaves that to + /// the embedder firing [`Self::shutdown_trigger`]. + signals: bool, /// Replacement for the default public router. `None` serves [`web::Web::routes`]. web_routes: Option, /// Replacement for the default ops router. `None` serves [`internal::Internal::routes`]. @@ -142,6 +145,7 @@ impl Relay { } let server_versions = config.listen.versions(); + let handshake_timeout = config.listen.resolved_timeout(); // Bind the QUIC workers first: they own the listen address when configured, // so the server below must not also try to bind it. @@ -268,13 +272,15 @@ impl Relay { let cluster = cluster.with_stats(stats.clone()); // Graceful shutdown: the first signal drains every accepted session with a - // GOAWAY; a second signal (or the drain window elapsing) exits. + // GOAWAY; the relay exits once they have all left, at the drain deadline, + // or on a second signal. let (shutdown_trigger, shutdown) = shutdown::Observer::new(drain_timeout); let sessions = crate::session::Registry::new(); let (ready, _) = tokio::sync::watch::channel(false); let web = web::Web::new(auth.clone(), cluster.clone(), certificates, config.web) .with_shutdown(shutdown.clone()) .with_versions(server_versions) + .with_timeout(handshake_timeout) .with_sessions(sessions.clone()) .bind()?; // `bind`, not `listen`: the TCP/Unix accept loops handshake as soon as @@ -290,6 +296,7 @@ impl Relay { let internal = internal::Internal::new(config.internal, cluster.stats.clone()) .with_cluster(&cluster) .with_sessions(sessions.clone()) + .with_shutdown(shutdown.clone()) .with_listeners(web.accept_health()) .with_listeners(server.accept_health()) .bind()?; @@ -323,6 +330,7 @@ impl Relay { addr, shutdown, shutdown_trigger, + signals: true, web_routes: None, internal_routes: None, sessions, @@ -402,9 +410,10 @@ impl Relay { &self.shutdown } - /// Starts graceful shutdown: every session drains with a GOAWAY and - /// [`Self::run`] returns once the drain window elapses. Clone it before - /// `run` consumes the relay. + /// Starts graceful shutdown: every session, including any accepted + /// afterwards, drains with a GOAWAY and [`Self::run`] returns once every + /// session has left or the drain window elapses. Clone it before `run` + /// consumes the relay. pub fn shutdown_trigger(&self) -> &shutdown::Trigger { &self.shutdown_trigger } @@ -460,18 +469,28 @@ impl Relay { self } + /// Whether [`Self::run`] starts the drain on SIGINT/SIGTERM. Defaults to + /// `true`; pass `false` when the application owns the signals and fires + /// [`Self::shutdown_trigger`] itself, e.g. after withdrawing the node from DNS. + #[must_use = "the relay with the signal choice is returned"] + pub fn with_signals(mut self, signals: bool) -> Self { + self.signals = signals; + self + } + /// Serve until something fails or shutdown completes: accept sessions, run /// the cluster, and serve both HTTP surfaces. Notifies systemd once - /// everything is up. Returns once the drain window elapses after a signal - /// or [`shutdown::Trigger::start`], with every listener released and every - /// worker joined. + /// everything is up. Returns once a drain started by a signal (see + /// [`Self::with_signals`]) or [`shutdown::Trigger::start`] ends, as soon as + /// every session has left or at the drain deadline, with every listener + /// released and every worker joined. /// /// This is also the embedding loop. Extra routes go on via [`Self::with_web`] /// / [`Self::with_internal`] before calling this; cloned handles outlive it. pub async fn run(self) -> anyhow::Result<()> { let Relay { ready, - server, + mut server, auth, admissions, cluster, @@ -479,6 +498,7 @@ impl Relay { web, shutdown, shutdown_trigger, + signals, web_routes, internal_routes, sessions, @@ -610,6 +630,7 @@ impl Relay { let has_iroh = false; let serve_shared = { let idle = quic_on_workers && server.accept_health().is_empty() && !has_iroh; + let server = &mut server; let cluster = cluster.clone(); let auth = auth.clone(); let shutdown = shutdown.clone(); @@ -630,10 +651,14 @@ impl Relay { Err(err) = quic_workers => Err(err).context("QUIC workers failed"), err = uring_failed => Err(err).context("io_uring QUIC workers failed"), Err(err) = jemalloc => Err(err).context("jemalloc profiler failed"), - res = drain(shutdown_trigger, shutdown.clone()) => res, + res = drain(shutdown_trigger, shutdown.clone(), signals) => res, else => Ok(()), }; + // Dropping the listener would leave its QUIC socket open until the + // endpoint's closing connections finish in the background, past `run`. + server.close().await; + // Explicitly, so the joins land on the blocking pool rather than on the // executor thread this future happens to be running on. #[cfg(feature = "_quic")] @@ -651,12 +676,20 @@ impl Relay { /// Two-stage shutdown: the first signal, or an embedder firing /// [`shutdown::Trigger::start`], starts the drain broadcast (every session sends -/// GOAWAY and waits for its peer to leave); a second signal, or the drain -/// window elapsing, returns from [`Relay::run`]. -async fn drain(trigger: shutdown::Trigger, mut shutdown: shutdown::Observer) -> anyhow::Result<()> { +/// GOAWAY and waits for its peer to leave). Returns from [`Relay::run`] once +/// every session has left, which the drain deadline forces, or on a second +/// signal, logging which ended it. +/// Without `signals` only the trigger and the sessions count. +async fn drain(trigger: shutdown::Trigger, mut shutdown: shutdown::Observer, signals: bool) -> anyhow::Result<()> { let window = shutdown.drain_timeout; + let signal = || async move { + match signals { + true => shutdown_signal().await, + false => std::future::pending().await, + } + }; tokio::select! { - res = shutdown_signal() => { + res = signal() => { res?; tracing::info!( ?window, @@ -667,15 +700,38 @@ async fn drain(trigger: shutdown::Trigger, mut shutdown: shutdown::Observer) -> _ = shutdown.started() => tracing::info!(?window, "shutdown requested; draining sessions"), } - // One extra second past the window so per-session force-closes fire first, - // giving every peer a proper GoawayTimeout instead of a dropped transport. - let grace = window + std::time::Duration::from_secs(1); + // The deadline fixed when the trigger fired, which may be earlier than this + // future was polled (the embedder can start the drain during startup); a + // fresh window here would keep the process up past the time sessions were + // told. Each session is force-closed at it, so `drained` resolves by then; + // the extra second only bounds a session whose close never completes. + let deadline = shutdown.deadline().context("drain started without a deadline")?; tokio::select! { - res = shutdown_signal() => { + res = signal() => { res?; - tracing::warn!("second shutdown signal; exiting immediately"); + tracing::warn!(open = shutdown.tally().live, "second shutdown signal; exiting immediately"); + return Ok(()); + } + _ = shutdown.drained() => {} + _ = tokio::time::sleep_until(deadline + std::time::Duration::from_secs(1)) => {} + } + + let elapsed = tokio::time::Instant::now().saturating_duration_since(deadline - window); + match shutdown.tally() { + shutdown::Tally { live: 0, forced: 0, .. } => { + tracing::info!(?elapsed, "drain complete: every session left; exiting") + } + shutdown::Tally { live: 0, forced, .. } => { + tracing::warn!(?elapsed, forced, "drain deadline force-closed sessions; exiting") + } + shutdown::Tally { live, forced, .. } => { + tracing::warn!( + ?elapsed, + forced, + open = live, + "drain deadline passed with sessions still open; exiting" + ) } - _ = tokio::time::sleep(grace) => tracing::info!("drain window elapsed; exiting"), } Ok(()) } @@ -714,12 +770,12 @@ async fn serve( ) -> anyhow::Result<()> { // Each QUIC worker binds here; Relay::run binds the shared listener before // readiness and passes it to the same accept loop. - let listener = server.listen().await.context("failed to bind listeners")?; - serve_listening(listener, cluster, auth, shutdown, sessions).await + let mut listener = server.listen().await.context("failed to bind listeners")?; + serve_listening(&mut listener, cluster, auth, shutdown, sessions).await } async fn serve_listening( - mut listener: moq_tokio::Listener, + listener: &mut moq_tokio::Listener, cluster: cluster::Cluster, auth: auth::Auth, shutdown: shutdown::Observer, diff --git a/rs/moq-relay/src/shutdown.rs b/rs/moq-relay/src/shutdown.rs index f7c309c9a12..cf7db816756 100644 --- a/rs/moq-relay/src/shutdown.rs +++ b/rs/moq-relay/src/shutdown.rs @@ -1,33 +1,63 @@ //! Graceful shutdown coordination: the first shutdown signal fires a broadcast //! that every accepted session observes, draining it with a GOAWAY before the //! process exits. +//! +//! The drain has one deadline, fixed when it starts. A session accepted after +//! that (a cached DNS resolve, a pool alias) is sent a GOAWAY at once, carrying +//! only the time left, so no session outlives the window the relay promised. +//! +//! The drain ends as soon as every session has left, or at that deadline. +//! Sessions are counted once established, so one still in +//! its handshake when the last established session leaves is cut off with the +//! process. -use std::time::Duration; +use std::{sync::Arc, time::Duration}; -use tokio::sync::watch; +use tokio::{sync::watch, time::Instant}; /// Fires the relay-wide shutdown broadcast. Held by `Relay::run` for the OS /// signal path; an embedder clones one to stop the relay from its own task. #[derive(Clone)] pub struct Trigger { - tx: watch::Sender, + tx: watch::Sender>, + drain_timeout: Duration, } impl Trigger { /// Start the drain: every [`Observer`] handle's [`started`](Observer::started) - /// resolves and sessions begin sending GOAWAY. + /// resolves and sessions begin sending GOAWAY, including any accepted later. + /// Calling it again keeps the first deadline. pub fn start(&self) { - let _ = self.tx.send(true); + let deadline = Instant::now() + self.drain_timeout; + self.tx.send_if_modified(|started| { + let first = started.is_none(); + if first { + *started = Some(deadline); + } + first + }); } } +/// The sessions a drain is waiting on. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Tally { + /// Established sessions still open, drained or not. + pub live: usize, + /// Sessions sent a GOAWAY that have not left yet. + pub draining: usize, + /// Sessions still open when their drain deadline passed, and so closed by it. + pub forced: usize, +} + /// A per-connection handle observing the relay-wide shutdown broadcast. /// /// Cheap to clone; each accepted session waits on [`started`](Self::started) /// and drains itself via [`drain_session`](Self::drain_session) when it fires. #[derive(Clone)] pub struct Observer { - rx: watch::Receiver, + rx: watch::Receiver>, + tally: Arc>, /// How long a drained session may keep running before it is force-closed. pub drain_timeout: Duration, } @@ -35,49 +65,97 @@ pub struct Observer { impl Observer { /// Create the trigger and its observer half. pub fn new(drain_timeout: Duration) -> (Trigger, Self) { - let (tx, rx) = watch::channel(false); - (Trigger { tx }, Self { rx, drain_timeout }) + let (tx, rx) = watch::channel(None); + (Trigger { tx, drain_timeout }, Self::from(rx, drain_timeout)) } /// A handle that never fires, for callers without shutdown coordination /// (tests, embedders that manage their own lifecycle). pub fn disabled() -> Self { - let (tx, rx) = watch::channel(false); + let (tx, rx) = watch::channel(None); // Leak-free: dropping the sender doesn't resolve `started` (it waits for - // a `true` value, not for channel closure). + // a deadline, not for channel closure). drop(tx); + Self::from(rx, crate::DEFAULT_DRAIN_TIMEOUT) + } + + fn from(rx: watch::Receiver>, drain_timeout: Duration) -> Self { Self { rx, - drain_timeout: crate::DEFAULT_DRAIN_TIMEOUT, + tally: Arc::new(watch::channel(Tally::default()).0), + drain_timeout, } } - /// Resolve once the shutdown broadcast fires. Never resolves for - /// [`disabled`](Self::disabled) handles. + /// Resolve once the shutdown broadcast fires, at once if it already has. + /// Never resolves for [`disabled`](Self::disabled) handles. pub async fn started(&mut self) { // wait_for returns Err once the sender is dropped without firing; park // forever in that case (a dropped trigger means no shutdown, not shutdown). - if self.rx.wait_for(|started| *started).await.is_err() { + if self.rx.wait_for(Option::is_some).await.is_err() { std::future::pending::<()>().await; } } + /// When the drain window ends, if [`Trigger::start`] has fired. + pub(crate) fn deadline(&self) -> Option { + *self.rx.borrow() + } + + /// Count an established session until the returned guard drops, so the drain + /// waits for it to leave. + pub(crate) fn serve(&self) -> Serving { + self.tally.send_modify(|tally| tally.live += 1); + Serving { + tally: self.tally.clone(), + } + } + + /// The sessions counted so far. + pub(crate) fn tally(&self) -> Tally { + *self.tally.borrow() + } + + /// Resolve once no [`serve`](Self::serve) guard is left. + pub(crate) async fn drained(&self) { + // The sender lives in `self`, so the channel cannot close under the wait. + let _ = self.tally.subscribe().wait_for(|tally| tally.live == 0).await; + } + + /// Count a session as draining until the returned guard drops, and as forced + /// if it is still open at `deadline`. + fn draining(&self, deadline: Instant) -> Draining { + self.tally.send_modify(|tally| tally.draining += 1); + Draining { + tally: self.tally.clone(), + deadline, + } + } + /// Drain `session` with an empty-URI GOAWAY ("reconnect to me"), waiting for /// the peer to leave. /// - /// The driver force-closes with [`moq_net::Error::GoawayTimeout`] once - /// [`drain_timeout`](Self::drain_timeout) passes, so this resolves either way, - /// on every version. A peer too old for GOAWAY (moq-lite-03 and earlier) keeps - /// being served for the same window and is then closed the same way; it simply - /// never learns why, which beats cutting it off with no warning and no grace. + /// The driver force-closes with [`moq_net::Error::GoawayTimeout`] at the + /// drain's deadline: [`drain_timeout`](Self::drain_timeout) after the trigger + /// fired, or after this call if it has not. So this resolves either way, on + /// every version. A peer too old for GOAWAY (moq-lite-03 and earlier) keeps + /// being served until then and is closed the same way; it simply never learns + /// why, which beats cutting it off with no warning and no grace. /// - /// A zero [`drain_timeout`](Self::drain_timeout) means no grace at all: the - /// session is closed at once. + /// With no time left (a zero [`drain_timeout`](Self::drain_timeout), or a + /// session accepted after the deadline) the session is closed at once. pub async fn drain_session(&self, session: &moq_net::Session) { - // No grace window, so there is nothing to drain. Close now rather than send a + let now = Instant::now(); + let deadline = self.deadline().unwrap_or(now + self.drain_timeout); + let remaining = deadline.saturating_duration_since(now); + // Dropped when this returns or is cancelled (a WebSocket driver ending + // first), so the count holds whichever way the session goes. + let _draining = self.draining(deadline); + + // No grace left, so there is nothing to drain. Close now rather than send a // GOAWAY: the peer would have no time to act on it, and a zero timeout means // "no deadline" on the wire, which is the opposite of what was asked for. - if self.drain_timeout.is_zero() { + if remaining.is_zero() { session.abort(moq_net::Error::GoingAway); return; } @@ -87,7 +165,7 @@ impl Observer { // is reachable; the deadline closes the session regardless. if let Err(err) = session .drain() - .send(moq_net::goaway::Goaway::new().with_timeout(self.drain_timeout)) + .send(moq_net::goaway::Goaway::new().with_timeout(remaining)) { tracing::warn!(%err, "failed to drain session"); } @@ -95,3 +173,78 @@ impl Observer { session.closed().await; } } + +/// An established session the drain waits on; see [`Observer::serve`]. +pub(crate) struct Serving { + tally: Arc>, +} + +impl Drop for Serving { + fn drop(&mut self) { + self.tally.send_modify(|tally| tally.live -= 1); + } +} + +/// A session sent a GOAWAY; see [`Observer::draining`]. +struct Draining { + tally: Arc>, + deadline: Instant, +} + +impl Drop for Draining { + fn drop(&mut self) { + // The driver arms its force-close after sending the GOAWAY, so it never + // fires before `deadline` and every session it closes is counted. A peer + // leaving in that sliver was still there at the deadline too. + let forced = Instant::now() >= self.deadline; + self.tally.send_modify(|tally| { + tally.draining -= 1; + tally.forced += usize::from(forced); + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test(start_paused = true)] + async fn drained_waits_for_every_session() { + let (trigger, shutdown) = Observer::new(Duration::from_secs(10)); + let first = shutdown.serve(); + let second = shutdown.serve(); + trigger.start(); + + let drained = tokio::spawn({ + let shutdown = shutdown.clone(); + async move { shutdown.drained().await } + }); + drop(first); + tokio::task::yield_now().await; + assert!(!drained.is_finished(), "drained with a session still open"); + + drop(second); + drained.await.expect("drained task"); + assert_eq!(shutdown.tally(), Tally::default()); + } + + #[tokio::test(start_paused = true)] + async fn a_session_open_at_the_deadline_is_forced() { + let window = Duration::from_secs(10); + let (trigger, shutdown) = Observer::new(window); + trigger.start(); + let deadline = shutdown.deadline().expect("started"); + + let left = shutdown.draining(deadline); + let straggler = shutdown.draining(deadline); + assert_eq!(shutdown.tally().draining, 2); + + drop(left); + tokio::time::sleep(window).await; + drop(straggler); + + let tally = shutdown.tally(); + assert_eq!(tally.draining, 0); + assert_eq!(tally.forced, 1, "only the session open at the deadline is forced"); + } +} diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index c3fbd498946..6a878d0b0f3 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -173,6 +173,9 @@ pub(crate) struct WebState { /// like a QUIC one. #[cfg_attr(not(feature = "websocket"), allow(dead_code))] pub(crate) sessions: crate::session::Registry, + /// How long a connection has to send its request headers, and a WebSocket its + /// MoQ SETUP; `None` waits forever. + pub(crate) timeout: Option, } /// The bound addresses of the public web listeners. @@ -214,6 +217,7 @@ impl Web { conn_id: AtomicU64::new(0), shutdown: crate::shutdown::Observer::disabled(), sessions: crate::session::Registry::new(), + timeout: moq_tokio::listen::Config::default().resolved_timeout(), }); Self { state, @@ -300,6 +304,18 @@ impl Web { self } + /// Bound how long a connection may take to send its HTTP request headers, and a + /// WebSocket its MoQ SETUP after the upgrade; `None`, or a timeout past the + /// clock's range, waits forever. Defaults to the + /// [`moq_tokio::listen::Config::resolved_timeout`] default. + pub fn with_timeout(mut self, timeout: Option) -> Self { + let state = Arc::get_mut(&mut self.state).expect("with_timeout called after routes were built"); + // hyper adds the header timeout to the clock unchecked, so an unreachable + // one would panic on every connection. + state.timeout = timeout.filter(|timeout| std::time::Instant::now().checked_add(*timeout).is_some()); + self + } + /// Register WebSocket sessions in the node's live table so they can be listed /// and nudged. Without it they are served but do not appear. pub fn with_sessions(mut self, sessions: crate::session::Registry) -> Self { @@ -360,6 +376,7 @@ impl Web { /// passes that router to [`crate::Relay::with_web`] instead of calling this. pub async fn serve(self, app: Router) -> anyhow::Result<()> { let Web { + state, config, health, http_listener, @@ -374,7 +391,8 @@ impl Web { let ws = config.resolved_ws(); let http = if let Some(listener) = http_listener { - let server = crate::listener::server(listener, health.clone(), WebAcceptor::plain(ws))?; + let mut server = crate::listener::server(listener, health.clone(), WebAcceptor::plain(ws))?; + header_timeout(&mut server, state.timeout); Some(server.serve(app.clone())) } else { None @@ -388,7 +406,8 @@ impl Web { config.https.key, config.https.root, )); - let server = crate::listener::server(listener, health.clone(), WebAcceptor::tls(rustls_config, ws))?; + let mut server = crate::listener::server(listener, health.clone(), WebAcceptor::tls(rustls_config, ws))?; + header_timeout(&mut server, state.timeout); Some(server.serve(app)) } else { None @@ -451,6 +470,20 @@ async fn get_cors(request: axum::extract::Request, next: axum::middleware::Next) response } +/// Close an HTTP/1 connection that hasn't sent its full request headers within +/// `timeout`, or never when `None`. +/// +/// axum-server builds hyper without a timer, and hyper enforces no header deadline +/// without one, so a client that trickles its headers would otherwise hold its +/// connection forever. +fn header_timeout(server: &mut axum_server::Server, timeout: Option) { + server + .http_builder() + .http1() + .timer(hyper_util::rt::TokioTimer::new()) + .header_read_timeout(timeout); +} + /// Build a [`rustls::ServerConfig`] for the HTTPS listener. /// /// TLS version is left at the rustls default (1.2 + 1.3) so older clients @@ -1385,6 +1418,63 @@ mod tests { serving.abort(); } + /// A timeout past the clock's range waits forever, rather than reaching hyper, + /// which would panic adding it to the clock on every connection. + #[tokio::test] + async fn unreachable_timeout_waits_forever() { + let auth_config = crate::auth::Config { + public_subscribe: vec![moq_auth::Pattern::all()], + ..Default::default() + }; + let auth = auth_config.init("test", &moq_tokio::tls::Connect::default()).unwrap(); + let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let dir = TempDir::new().unwrap(); + let (_, cert, _) = make_certs(&dir); + let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); + let web = Web::new(auth, cluster, certificates, Config::default()).with_timeout(Some(std::time::Duration::MAX)); + assert_eq!(web.state.timeout, None); + } + + /// A client that never finishes its request headers is disconnected once the + /// timeout passes. Real sockets, so this runs on the wall clock and bounds the + /// close from below. + #[tokio::test] + async fn slow_request_headers_are_timed_out() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let timeout = std::time::Duration::from_millis(200); + let mut config = Config::default(); + config.http.listen = Some("127.0.0.1:0".parse().unwrap()); + let auth_config = crate::auth::Config { + public_subscribe: vec![moq_auth::Pattern::all()], + ..Default::default() + }; + let auth = auth_config.init("test", &moq_tokio::tls::Connect::default()).unwrap(); + let cluster = cluster::Cluster::new(crate::cluster::Options::default()).unwrap(); + let dir = TempDir::new().unwrap(); + let (_, cert, _) = make_certs(&dir); + let certificates = moq_tokio::tls::Certificates::from_pem(&std::fs::read(&cert).unwrap()).unwrap(); + let web = Web::new(auth, cluster, certificates, config) + .with_timeout(Some(timeout)) + .bind() + .unwrap(); + let http = web.addrs().http.expect("the HTTP listener is configured"); + let serving = tokio::spawn(web.serve(Router::new())); + + let mut tcp = tokio::net::TcpStream::connect(http).await.unwrap(); + let start = std::time::Instant::now(); + tcp.write_all(b"GET / HTTP/1.1\r\nHost: localhost\r\n").await.unwrap(); + + let mut rest = Vec::new(); + tokio::time::timeout(std::time::Duration::from_secs(10), tcp.read_to_end(&mut rest)) + .await + .expect("slow headers held the connection open") + .ok(); + assert!(start.elapsed() >= timeout, "closed before the timeout"); + + serving.abort(); + } + /// Confirm `SetConnectionExtensions` injects the marker into request extensions /// when a peer cert was presented, and leaves them untouched otherwise. #[tokio::test] diff --git a/rs/moq-relay/src/websocket.rs b/rs/moq-relay/src/websocket.rs index 8ae772428be..9ddac84abea 100644 --- a/rs/moq-relay/src/websocket.rs +++ b/rs/moq-relay/src/websocket.rs @@ -97,6 +97,7 @@ pub(crate) async fn serve_ws( stats, shutdown: state.shutdown.clone(), socket_stats: socket_stats.map(|Extension(s)| s), + timeout: state.timeout, }; let _ = handle_socket(socket, session, lease, Some((state.sessions.clone(), request))).await; })) @@ -115,6 +116,8 @@ struct SessionInputs { shutdown: crate::shutdown::Observer, /// The kernel's view of the socket under the upgrade, captured at accept time. socket_stats: Option, + /// How long the peer has to send its MoQ SETUP after the upgrade, or `None` to wait forever. + timeout: Option, } /// Serve one upgraded WebSocket until it closes or its lease ends. @@ -148,6 +151,7 @@ where stats, mut shutdown, socket_stats, + timeout, } = session; // Wrap the WebSocket in a WebTransport compatibility layer. We have to @@ -184,12 +188,22 @@ where server = server.with_subscriber(publish); } // Keep the driver in this task so cancellation tears down the transport. - let (session, driver) = server - .accept( - tokio::time::Instant::now().into_std(), - moq_tokio::transport::Session::new(ws), - ) - .await?; + let ws = moq_tokio::transport::Session::new(ws); + let mut refused = ws.clone(); + let accept = server.accept(tokio::time::Instant::now().into_std(), ws); + let (session, driver) = match timeout { + None => accept.await?, + Some(timeout) => match tokio::time::timeout(timeout, accept).await { + Ok(accepted) => accepted?, + // A peer that upgraded and never sent SETUP; tell it why it is dropped. + Err(_) => { + use web_transport_trait::poll::Session as _; + let err = moq_net::Error::Timeout; + refused.close(moq_net::SessionError::from(&err).to_code(), &err.to_string()); + return Err(err.into()); + } + }, + }; let driver = moq_net::time::run(driver); tokio::pin!(driver); @@ -197,6 +211,7 @@ where // The handshake is done, so this is a MoQ session now: only now can a push // be serviced, and only now does the session appear in the live table. let registration = pending.map(|(sessions, request)| sessions.register(request)); + let _serving = shutdown.serve(); loop { let nudged = async { @@ -960,6 +975,8 @@ mod tests { // No descriptor to hand over: this drives the transport directly rather // than through an accepted socket. socket_stats: None, + // The keep-alive is under test, so the SETUP deadline must not end it first. + timeout: None, }; let grant = moq_auth::Grant::new( [moq_auth::Pattern::all()].into_iter().collect(), @@ -999,4 +1016,65 @@ mod tests { drop(client); } + + /// A peer that upgrades and then never sends SETUP is refused at the deadline, + /// well before the keep-alive would notice, since its pongs keep it alive. + #[tokio::test(start_paused = true)] + async fn stalled_setup_is_closed_at_the_deadline() { + use web_transport_trait::Error as _; + + let alpn = format!("{}{}", preferred_qmux_prefix(), newest_moq_alpn()); + let (client_to_server, server_incoming) = mpsc::unbounded_channel(); + let (server_to_client, client_incoming) = mpsc::unbounded_channel(); + let timeout = Duration::from_secs(10); + + let session = SessionInputs { + id: 0, + session: String::new(), + remote: "127.0.0.1:0".parse().unwrap(), + alpn: Some(alpn.clone()), + versions: moq_net::Versions::all(), + publish: None, + subscribe: None, + stats: Session::default(), + shutdown: crate::shutdown::Observer::disabled(), + socket_stats: None, + timeout: Some(timeout), + }; + let grant = moq_auth::Grant::new( + [moq_auth::Pattern::all()].into_iter().collect(), + [moq_auth::Pattern::all()].into_iter().collect(), + ); + let lease = crate::auth::Lease::new("/", moq_auth::lease::Consumer::fixed(grant)); + let start = tokio::time::Instant::now(); + let server = tokio::spawn(handle_socket( + Pipe::new(server_incoming, server_to_client, Arc::new(AtomicBool::new(false))), + session, + lease, + None, + )); + + // A live qmux peer that answers keep-alives but never speaks moq. + let client = qmux::ws::Upgraded::new(Pipe::new( + client_incoming, + client_to_server, + Arc::new(AtomicBool::new(false)), + )) + .with_alpn(&alpn) + .connect(); + + let err = server + .await + .expect("server task panicked") + .expect_err("a peer that never sent SETUP was served"); + assert!(matches!(err.downcast_ref(), Some(moq_net::Error::Timeout)), "{err:#}"); + assert_eq!(start.elapsed(), timeout); + + let closed = client.closed().await; + assert_eq!( + closed.session_error().map(|(code, _)| code), + Some(moq_net::SessionError::Timeout.to_code()), + "{closed:?}" + ); + } } diff --git a/rs/moq-relay/tests/cluster_idle.rs b/rs/moq-relay/tests/cluster_idle.rs new file mode 100644 index 00000000000..fc90ee1b7eb --- /dev/null +++ b/rs/moq-relay/tests/cluster_idle.rs @@ -0,0 +1,141 @@ +//! A cluster drops a silent peer's routes within the default `--quic-idle-timeout`. +//! +//! A crashed or partitioned peer sends no close, so the idle timeout is the whole +//! outage window for subscribes routed through it. +#![cfg(feature = "noq")] + +use std::time::{Duration, Instant}; + +use moq_relay::cluster::{self, Peer}; + +/// How long the cluster may take to drop the dead peer's route: the 10s default +/// plus slack for a slow runner, well short of 30s so a longer default fails here. +const DETECT: Duration = Duration::from_secs(20); + +/// Multi-transport client and session types make the test future large; give it a +/// big stack, as `goaway_cluster.rs` does. +fn run_cluster_test(fut: F) +where + F: std::future::Future + Send + 'static, +{ + std::thread::Builder::new() + .stack_size(32 * 1024 * 1024) + .spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("build test runtime") + .block_on(fut); + }) + .expect("spawn test thread") + .join() + .expect("test thread panicked"); +} + +/// A peer announcing one broadcast, on its own runtime so dropping the runtime +/// silences it the way a crash or a partition does: no CONNECTION_CLOSE, so only +/// the idle timeout can tell the cluster it is gone. +async fn spawn_peer() -> (u16, tokio::runtime::Runtime) { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .enable_all() + .build() + .expect("build peer runtime"); + + let (bound, port) = tokio::sync::oneshot::channel(); + runtime.spawn(async move { + let origin = moq_tokio::origin::spawn(); + let broadcast = origin.create_broadcast("peer").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + + let mut config = moq_tokio::listen::Config::default(); + config.bind = Some("127.0.0.1:0".parse().expect("parse bind")); + config.tls.generate = vec!["localhost".into()]; + let mut listener = config + .init(Default::default()) + .expect("peer init") + .listen() + .await + .expect("peer listen"); + let _ = bound.send(listener.local_addr().expect("peer addr").port()); + + while let Some(request) = listener.accept().await { + let origin = origin.clone(); + tokio::spawn(async move { + if let Ok(session) = request.with_publisher(&origin).ok().await { + let _ = session.closed().await; + } + }); + } + drop(broadcast); + }); + + let port = tokio::time::timeout(DETECT, port) + .await + .expect("peer startup timed out") + .expect("peer failed to start"); + (port, runtime) +} + +#[test] +fn silent_peer_detected_within_idle_timeout() { + run_cluster_test(silent_peer_detected()); +} + +async fn silent_peer_detected() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let (port, peer) = spawn_peer().await; + + // Both ends keep the `--quic-*` defaults. + let mut connect = moq_tokio::connect::Config::default(); + connect.bind = Some("127.0.0.1:0".parse().expect("parse bind")); + connect.tls.insecure = Some(true); + #[cfg(feature = "websocket")] + { + connect.websocket.enabled = Some(false); + } + let client = connect.init(Default::default()).expect("client init"); + + let mut config = cluster::Config::default(); + config.connect = vec![Peer::new(format!("https://127.0.0.1:{port}/"))]; + let cluster = cluster::Cluster::new(cluster::Options::new(config)) + .expect("cluster init") + .with_client(client); + let mut announced = cluster.origin.consume().announced(); + let run = tokio::spawn(cluster.clone().start().await.expect("cluster start").run()); + + tokio::time::timeout(DETECT, async { + loop { + let event = announced.next().await.expect("origin closed"); + if let moq_net::announce::Event::Start(update) | moq_net::announce::Event::Update(update) = event + && update.prefix.as_str() == "peer" + { + break; + } + } + }) + .await + .expect("the peer's broadcast never arrived"); + + tokio::task::spawn_blocking(move || peer.shutdown_timeout(Duration::from_secs(1))) + .await + .expect("peer shutdown panicked"); + let silenced = Instant::now(); + + tokio::time::timeout(DETECT, async { + loop { + let event = announced.next().await.expect("origin closed"); + if let moq_net::announce::Event::End(update) = event + && update.prefix.as_str() == "peer" + { + break; + } + } + }) + .await + .expect("the silent peer's route outlived the idle timeout"); + println!("silent peer dropped after {:?}", silenced.elapsed()); + + run.abort(); +} diff --git a/rs/moq-relay/tests/goaway_cluster.rs b/rs/moq-relay/tests/goaway_cluster.rs index 06c7c41f42f..2bacd26e84c 100644 --- a/rs/moq-relay/tests/goaway_cluster.rs +++ b/rs/moq-relay/tests/goaway_cluster.rs @@ -118,6 +118,107 @@ async fn drain_session_with_zero_timeout_closes_at_once_inner() { let _ = within("client observes the close", client_connection.closed()).await; } +#[test] +fn cluster_continues_a_group_split_by_goaway() { + run_cluster_test(cluster_continues_a_group_split_by_goaway_inner()); +} + +/// A GOAWAY that lands mid-group hands the rest of the group to the reader already +/// holding it, and the next group after it, never the split group a second time. +async fn cluster_continues_a_group_split_by_goaway_inner() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let upstream_origin = moq_tokio::origin::spawn(); + let broadcast = upstream_origin.create_broadcast("cam").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let track = broadcast.create_track("video", None).expect("create track"); + + let (port_a, mut accepted_a, _handle_a) = spawn_upstream(upstream_origin.clone()).await; + let (port_b, mut accepted_b, _handle_b) = spawn_upstream(upstream_origin.clone()).await; + + let mut client_config = moq_tokio::connect::Config::default(); + client_config.tls.insecure = Some(true); + client_config.goaway.handover = Duration::from_secs(2); + let client = client_config.init(Default::default()).expect("client init"); + + let mut cluster_config = cluster::Config::default(); + cluster_config.connect = vec![Peer::new(format!("tcp://127.0.0.1:{port_a}/"))]; + let cluster = cluster::Cluster::new(cluster::Options::new(cluster_config)) + .expect("cluster init") + .with_client(client); + let started = cluster.clone().start().await.expect("cluster start"); + let cluster_run = tokio::spawn(started.run()); + + let session_a = within("A accepts", accepted_a.recv()).await.expect("A accepts"); + let consumer = cluster.origin.consume(); + within("routed via A", consumer.routed("cam")).await.expect("routed"); + let bc = consumer.request_broadcast("cam").await.expect("broadcast resolves"); + let mut sub = within("subscribe", bc.track("video").expect("track").subscribe(None)) + .await + .expect("subscribe"); + + let write = |group: &mut moq_net::group::Producer, payload: &'static str| { + group + .write_frame(moq_net::Timestamp::ZERO, payload.as_bytes()) + .expect("write frame"); + }; + // A group's sequence and its next frame, or `None` at its end. + async fn read(group: &mut moq_net::group::Consumer) -> (u64, Option) { + let frame = within("read frame", group.read_frame()).await.expect("read"); + (group.sequence, frame.map(|frame| frame.payload)) + } + + // The first group is open, its first frame read, when A redirects to B. + let mut split = track.append_group().expect("append group"); + write(&mut split, "a"); + let mut reading = within("recv the split group", sub.recv_group()) + .await + .expect("recv") + .expect("track ended"); + assert_eq!(read(&mut reading).await, (0, Some("a".into()))); + + // The cursor replays A's route first. After the GOAWAY, A's re-prices to DRAIN, so + // the next route at any other cost is B's. + let mut announced = consumer.announced(); + within("A's route", announced.next()).await.expect("announce update"); + session_a + .drain() + .send(moq_net::goaway::Goaway::redirect(format!("tcp://127.0.0.1:{port_b}/"))) + .expect("send goaway"); + let _session_b = within("B accepts", accepted_b.recv()).await.expect("B accepts"); + within("routed via B", async { + loop { + let event = announced.next().await.expect("announce update"); + if let moq_net::announce::Event::Start(update) | moq_net::announce::Event::Update(update) = event + && update.route.cost != moq_net::origin::Cost::DRAIN + { + break; + } + } + }) + .await; + + write(&mut split, "b"); + split.finish().expect("finish"); + assert_eq!(read(&mut reading).await, (0, Some("b".into()))); + assert_eq!(read(&mut reading).await, (0, None)); + + let mut after = track.append_group().expect("append group"); + write(&mut after, "c"); + after.finish().expect("finish"); + let mut next = within("recv the next group", sub.recv_group()) + .await + .expect("recv") + .expect("track ended"); + assert_eq!( + read(&mut next).await, + (1, Some("c".into())), + "the split group came back" + ); + + cluster_run.abort(); +} + #[test] fn cluster_reconnects_on_empty_uri_goaway() { run_cluster_test(cluster_reconnects_on_empty_uri_goaway_inner()); diff --git a/rs/moq-relay/tests/runtime_uring.rs b/rs/moq-relay/tests/runtime_uring.rs index b6d5cf48bd4..640b81753f1 100644 --- a/rs/moq-relay/tests/runtime_uring.rs +++ b/rs/moq-relay/tests/runtime_uring.rs @@ -232,6 +232,94 @@ async fn uring_workers_report_link_facts() { let _ = running.await; } +/// The shutdown trigger drains sessions the io_uring workers serve as it does +/// the shared runtime's: an established session and one arriving mid-drain +/// are each sent a GOAWAY and leave, and `run` then returns at the deadline +/// with the worker threads joined and the port free. +/// +/// What an arrival is told is left of the window only reaches the wire on +/// moq-transport-17+, which the workers do not speak; `shutdown_signal.rs` +/// reads it through the shared runtime, whose supervision this path shares. +#[tokio::test] +async fn uring_workers_drain_on_the_trigger() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + if !supported() { + return; + } + + const DRAIN: Duration = Duration::from_secs(2); + + let dir = tempfile::tempdir().expect("tempdir"); + let (cert, key) = certificate(dir.path()); + + let mut config = uring_config(&cert, &key); + config.drain_timeout = DRAIN; + let relay = Relay::load(config).await.expect("load relay").with_signals(false); + let port = relay.addr().expect("workers bound an address").port(); + let trigger = relay.shutdown_trigger().clone(); + let sessions = relay.sessions().clone(); + let running = tokio::spawn(relay.run()); + + // One-shot (see `client`), so a session leaves on its GOAWAY rather than + // migrating, and closing cleanly shows it was one. + let client = client(); + let url: url::Url = format!("moql://127.0.0.1:{port}/drain").parse().expect("parse url"); + + let established = connect(client.clone(), url.clone()).await; + // moq-lite-03 has no GOAWAY, so this peer stays until the deadline closes it, + // keeping the drain open for the arrival below once `established` leaves. + let mut straggler = moq_tokio::connect::Config::default(); + straggler.tls.insecure = Some(true); + straggler.once = Some(true); + straggler.bind = Some("127.0.0.1:0".parse().expect("parse bind")); + straggler.version = vec!["moq-lite-03".parse().expect("parse version")]; + let _straggler = connect(straggler.init(Default::default()).expect("client init"), url.clone()).await; + + // A client can see its session established before the relay counts it, and a + // drain with nothing counted ends at once. + let deadline = std::time::Instant::now() + TIMEOUT; + while sessions.list(&Default::default()).len() < 2 { + assert!( + std::time::Instant::now() < deadline, + "the relay never listed both sessions" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } + trigger.start(); + let goaway = tokio::time::timeout(TIMEOUT, established.draining().expect("connected").recv()) + .await + .expect("no GOAWAY after the trigger") + .expect("session closed without a GOAWAY"); + assert_eq!(goaway.uri(), "", "expected a reconnect-to-me GOAWAY"); + tokio::time::timeout(TIMEOUT, established.closed()) + .await + .expect("the drained session never closed") + .expect("a one-shot session leaves cleanly on GOAWAY"); + + // A straggler dialing mid-drain is admitted through a worker, then told to + // leave at once. + tokio::time::sleep(DRAIN / 2).await; + let arrival = connect(client, url).await; + let goaway = tokio::time::timeout(Duration::from_secs(1), arrival.draining().expect("connected").recv()) + .await + .expect("an arrival mid-drain was not sent a GOAWAY") + .expect("arrival closed without a GOAWAY"); + assert_eq!(goaway.uri(), "", "expected a reconnect-to-me GOAWAY"); + tokio::time::timeout(TIMEOUT, arrival.closed()) + .await + .expect("the arrival never closed") + .expect("a one-shot session leaves cleanly on GOAWAY"); + + // `run` joins the worker threads before returning, so returning at all is + // the clean join; the rebind proves every worker let go of the port. + tokio::time::timeout(TIMEOUT, running) + .await + .expect("run did not return after the drain window") + .expect("run panicked") + .expect("run returned an error after the drain"); + std::net::UdpSocket::bind(("127.0.0.1", port)).expect("a worker still holds the QUIC port"); +} + /// One HTTP/1.1 GET against `addr`, returning the response body. /// /// Hand-rolled because the relay has no HTTP client among its dev diff --git a/rs/moq-relay/tests/shutdown_signal.rs b/rs/moq-relay/tests/shutdown_signal.rs index 3f45135fd03..9e706825706 100644 --- a/rs/moq-relay/tests/shutdown_signal.rs +++ b/rs/moq-relay/tests/shutdown_signal.rs @@ -11,6 +11,18 @@ //! against the drain future that deliberately waits out the window. SIGTERM was //! unaffected (only the drain future watches it), which is how the gap stayed //! hidden behind systemd while an operator's ctrl-C dropped every session. +//! +//! An embedder draining on its own schedule (withdraw from DNS, wait out the +//! TTL, then drain) turns the relay's signal handling off and fires the trigger +//! itself; a session that still arrives mid-drain is sent a GOAWAY at once, +//! carrying only what is left of the window. +//! +//! The drain ends as soon as every session has left rather than waiting out the +//! window, which a stop-time budget depends on. +//! +//! Each signal test raises or handles process signals, so they rely on +//! nextest's process-per-test isolation. `a_trigger_before_run_keeps_the_deadline` +//! fires the trigger before `run` instead of a signal. #![cfg(unix)] @@ -19,29 +31,54 @@ use std::time::Duration; use moq_relay::{Config, Relay, auth}; /// Long enough that "exited immediately" and "waited out the window" cannot be -/// confused, short enough to keep the test quick: `Relay::run` sleeps this plus -/// one second before exiting. +/// confused, short enough to keep the test quick: a session that never leaves +/// keeps `Relay::run` up this long. const DRAIN_TIMEOUT: Duration = Duration::from_secs(3); -#[test] -fn sigint_drains_sessions_before_exiting() { +/// Run `test` on a current-thread runtime with a large stack. +fn run_test + 'static>(test: fn() -> F) { // Same reason as the cluster tests: under `--all-features` a `Connection` // carries every transport backend, and holding one across awaits overflows // libtest's 2 MiB per-test stack in an unoptimized build. std::thread::Builder::new() .stack_size(32 * 1024 * 1024) - .spawn(|| { + .spawn(move || { tokio::runtime::Builder::new_current_thread() .enable_all() .build() .expect("build test runtime") - .block_on(sigint_drains_sessions_before_exiting_inner()); + .block_on(test()); }) .expect("spawn test thread") .join() .expect("test thread panicked"); } +#[test] +fn sigint_drains_sessions_before_exiting() { + run_test(sigint_drains_sessions_before_exiting_inner); +} + +#[test] +fn an_embedder_owns_the_signals() { + run_test(an_embedder_owns_the_signals_inner); +} + +#[test] +fn a_session_arriving_mid_drain_gets_what_is_left() { + run_test(a_session_arriving_mid_drain_gets_what_is_left_inner); +} + +#[test] +fn a_drain_ends_once_every_session_leaves() { + run_test(a_drain_ends_once_every_session_leaves_inner); +} + +#[test] +fn a_trigger_before_run_keeps_the_deadline() { + run_test(a_trigger_before_run_keeps_the_deadline_inner); +} + async fn sigint_drains_sessions_before_exiting_inner() { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); @@ -54,20 +91,13 @@ async fn sigint_drains_sessions_before_exiting_inner() { let relay = Relay::load(relay_config()).await.expect("load relay"); let port = relay.tcp_addr().expect("TCP listener bound").port(); let run = tokio::spawn(relay.run()); + let client = client(Vec::new()); - let mut client_config = moq_tokio::connect::Config::default(); - client_config.tls.insecure = Some(true); - let client = client_config.init(Default::default()).expect("client init"); - // One-shot: a reconnecting client would migrate on the GOAWAY and hide whether - // the original session outlived the signal. - let url: url::Url = format!("tcp://127.0.0.1:{port}/").parse().expect("parse url"); - let connection = client - .with_reconnect(false) - .connect(url) - .established() - .await - .expect("connect"); + let connection = connect(&client, port).await; let draining = connection.draining().expect("connected"); + // The one-shot client leaves on the GOAWAY; this peer is what keeps the drain + // open for its whole window. + let _straggler = straggler(port).await; let signalled = std::time::Instant::now(); // SAFETY: `raise` is async-signal-safe, and SIGINT's disposition is tokio's @@ -105,6 +135,208 @@ async fn sigint_drains_sessions_before_exiting_inner() { ); } +async fn an_embedder_owns_the_signals_inner() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + // The embedder's own handler, which is also what keeps SIGINT from killing + // the test process. + let mut interrupt = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt()).expect("register SIGINT"); + + let relay = Relay::load(relay_config()) + .await + .expect("load relay") + .with_signals(false); + let port = relay.tcp_addr().expect("TCP listener bound").port(); + let trigger = relay.shutdown_trigger().clone(); + let run = tokio::spawn(relay.run()); + let client = client(Vec::new()); + + let connection = connect(&client, port).await; + let draining = connection.draining().expect("connected"); + let _straggler = straggler(port).await; + + // SAFETY: `raise` is async-signal-safe, and SIGINT's disposition is tokio's + // handler, registered above. + assert_eq!(unsafe { libc::raise(libc::SIGINT) }, 0, "failed to raise SIGINT"); + tokio::time::timeout(Duration::from_secs(5), interrupt.recv()) + .await + .expect("the embedder never saw SIGINT"); + + // The signal is the embedder's: the relay keeps serving without a GOAWAY. + assert!( + tokio::time::timeout(Duration::from_secs(1), draining.recv()) + .await + .is_err(), + "the relay drained on a signal the embedder owns" + ); + assert!(!run.is_finished(), "the relay exited on a signal the embedder owns"); + + // Until the embedder decides. + let started = std::time::Instant::now(); + trigger.start(); + let goaway = tokio::time::timeout(Duration::from_secs(5), draining.recv()) + .await + .expect("no GOAWAY within 5s of the trigger") + .expect("session closed without a GOAWAY"); + assert_eq!(goaway.uri(), "", "expected a reconnect-to-me GOAWAY"); + + tokio::time::timeout(Duration::from_secs(15), run) + .await + .expect("relay never exited after the drain window") + .expect("relay task panicked") + .expect("relay exited with an error"); + assert!( + started.elapsed() >= DRAIN_TIMEOUT, + "relay exited after {:?}, short of the {DRAIN_TIMEOUT:?} drain window", + started.elapsed() + ); +} + +async fn a_session_arriving_mid_drain_gets_what_is_left_inner() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let relay = Relay::load(relay_config()) + .await + .expect("load relay") + .with_signals(false); + let port = relay.tcp_addr().expect("TCP listener bound").port(); + let trigger = relay.shutdown_trigger().clone(); + let run = tokio::spawn(relay.run()); + // moq-transport-17 is the first version whose GOAWAY carries its deadline on + // the wire, which is what this test reads. + let client = client(vec!["moq-transport-17".parse().expect("parse version")]); + + let established = connect(&client, port).await; + let draining = established.draining().expect("connected"); + // Keeps the relay up for the arrival below once `established` leaves. + let _straggler = straggler(port).await; + trigger.start(); + let deadline = std::time::Instant::now() + DRAIN_TIMEOUT; + let goaway = tokio::time::timeout(Duration::from_secs(5), draining.recv()) + .await + .expect("no GOAWAY within 5s of the trigger") + .expect("session closed without a GOAWAY"); + assert_eq!(goaway.uri(), "", "expected a reconnect-to-me GOAWAY"); + let timeout = goaway.timeout().expect("the GOAWAY carries its deadline"); + assert!( + timeout > DRAIN_TIMEOUT / 2 && timeout <= DRAIN_TIMEOUT, + "an established session gets the whole window, got {timeout:?}" + ); + + // A straggler dialing mid-drain (a cached DNS resolve) is still admitted, + // then told to leave at once. + tokio::time::sleep(DRAIN_TIMEOUT / 2).await; + let dialed = std::time::Instant::now(); + let arrival = connect(&client, port).await; + let goaway = tokio::time::timeout(Duration::from_secs(1), arrival.draining().expect("connected").recv()) + .await + .expect("an arrival mid-drain was not sent a GOAWAY") + .expect("arrival closed without a GOAWAY"); + assert_eq!(goaway.uri(), "", "expected a reconnect-to-me GOAWAY"); + + // Only what is left of the window, not a fresh one that would outlive the + // relay. The wire rounds up to whole milliseconds. + let left = deadline.saturating_duration_since(dialed) + Duration::from_millis(1); + let timeout = goaway.timeout().expect("the GOAWAY carries its deadline"); + assert!( + timeout <= left, + "an arrival mid-drain got {timeout:?}, past the {left:?} left of the window" + ); + + tokio::time::timeout(Duration::from_secs(15), run) + .await + .expect("relay never exited after the drain window") + .expect("relay task panicked") + .expect("relay exited with an error"); +} + +async fn a_drain_ends_once_every_session_leaves_inner() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + // A window the test would time out long before, so only an early exit passes. + let mut config = relay_config(); + config.drain_timeout = Duration::from_secs(600); + config.internal.listen = Some("127.0.0.1:0".parse().expect("parse addr")); + let relay = Relay::load(config).await.expect("load relay").with_signals(false); + let port = relay.tcp_addr().expect("TCP listener bound").port(); + let internal = relay.internal().addr().expect("internal listener bound").port(); + let trigger = relay.shutdown_trigger().clone(); + let run = tokio::spawn(relay.run()); + + let left = connect(&client(Vec::new()), port).await; + let straggler = straggler(port).await; + trigger.start(); + + // The one-shot client leaves on its GOAWAY; the straggler is still draining. + tokio::time::timeout(Duration::from_secs(5), left.closed()) + .await + .expect("the one-shot client did not leave on the GOAWAY") + .expect("the one-shot client failed"); + wait_metric(internal, "moq_relay_draining_sessions 1").await; + assert!(!run.is_finished(), "the relay exited with a session still draining"); + + drop(straggler); + tokio::time::timeout(Duration::from_secs(5), run) + .await + .expect("relay kept running after every session left") + .expect("relay task panicked") + .expect("relay exited with an error"); +} + +async fn a_trigger_before_run_keeps_the_deadline_inner() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let relay = Relay::load(relay_config()) + .await + .expect("load relay") + .with_signals(false); + let trigger = relay.shutdown_trigger().clone(); + + // Fired while `run` is still starting. The session deadline is this instant; + // `drain` must not open a fresh window when it finally observes the watch. + trigger.start(); + let deadline = std::time::Instant::now() + DRAIN_TIMEOUT; + tokio::time::sleep(DRAIN_TIMEOUT / 2).await; + + let run = tokio::spawn(relay.run()); + tokio::time::timeout(Duration::from_secs(15), run) + .await + .expect("relay never exited") + .expect("relay task panicked") + .expect("relay exited with an error"); + let over = std::time::Instant::now().saturating_duration_since(deadline); + assert!( + over <= Duration::from_secs(2), + "run returned {over:?} past the recorded deadline, not the one-second grace" + ); +} + +/// A one-shot client offering `version` (every version when empty): a +/// reconnecting one would migrate on the GOAWAY and hide whether the relay +/// closed the original session. +fn client(version: Vec) -> moq_tokio::Client { + let mut client_config = moq_tokio::connect::Config::default(); + client_config.tls.insecure = Some(true); + client_config.version = version; + client_config + .init(Default::default()) + .expect("client init") + .with_reconnect(false) +} + +/// A session that stays until the drain deadline closes it: moq-lite-03 has no +/// GOAWAY message, so the relay drains it without the peer ever knowing. +async fn straggler(port: u16) -> moq_tokio::Connection { + connect(&client(vec!["moq-lite-03".parse().expect("parse version")]), port).await +} + +/// A session to the relay on `port`. +async fn connect(client: &moq_tokio::Client, port: u16) -> moq_tokio::Connection { + let url: url::Url = format!("tcp://127.0.0.1:{port}/").parse().expect("parse url"); + client.connect(url).established().await.expect("connect") +} + /// A stream-only relay on an ephemeral loopback TCP port, fully public, with a /// short drain window, to hand [`Relay::load`]. fn relay_config() -> Config { @@ -118,3 +350,21 @@ fn relay_config() -> Config { config.drain_timeout = DRAIN_TIMEOUT; config } + +/// Wait until the internal listener on `port` reports `line` at `/metrics`. +async fn wait_metric(port: u16, line: &str) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let metrics = reqwest::get(format!("http://127.0.0.1:{port}/metrics")) + .await + .expect("scrape metrics") + .text() + .await + .expect("read metrics"); + if metrics.lines().any(|l| l == line) { + break; + } + assert!(std::time::Instant::now() < deadline, "never saw {line} in:\n{metrics}"); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} diff --git a/rs/moq-shaper/Cargo.toml b/rs/moq-shaper/Cargo.toml index a923d0f2f10..ebeb0190d72 100644 --- a/rs/moq-shaper/Cargo.toml +++ b/rs/moq-shaper/Cargo.toml @@ -20,5 +20,12 @@ doc = false anyhow = { workspace = true } clap = { workspace = true } humantime = { workspace = true } +humantime-serde = { workspace = true } rand = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +toml = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md new file mode 100644 index 00000000000..1ccad1cea2e --- /dev/null +++ b/rs/moq-shaper/README.md @@ -0,0 +1,175 @@ +# moq-shaper + +A seeded userspace UDP impairment relay, so a test can put an impaired path in front of a server. + +The shaper binds one socket, forwards every datagram to a target after a profile's treatment, and +forwards the replies back. A client dials the shaper instead of the server and reaches it through a +path with loss, a rate limit, delay, jitter, and reordering. QUIC is indifferent to the extra hop, so +this impairs a real transport with no capabilities and nothing touching the host's network, on macOS +and Linux alike. Kernel impairment (`netem` in a network namespace) is Linux only, needs +`CAP_NET_ADMIN`, and changes the host's network for everything running on it. + +It is a datagram relay, not an HTTP interceptor or a TCP proxy, neither of which can impair QUIC. + +## Usage + +```bash +moq-relay --listen 127.0.0.1:4443 & + +moq-shaper --listen 127.0.0.1:4444 --target 127.0.0.1:4443 \ + --delay 20ms --jitter 5ms --loss 0.05 --reorder 0.02 --rate 10000000 --seed 1 +``` + +Point the client at the shaper's port. The flags build one profile, applied both ways. The seed and +profile print at start, and the counters at exit (Ctrl-C or SIGTERM), which exits nonzero if the +profile never acted. + +| Flag | What it does | +| --- | --- | +| `--listen` | The address clients send to. | +| `--target` | The address every datagram is forwarded to. | +| `--seed` | Seeds every decision; the profile's own, or random, when omitted. | +| `--delay` | The base one-way delay. | +| `--jitter` | How far the delay varies: the most either way, or the sigma of a gaussian. | +| `--jitter-model` | `uniform` (the default) or `gaussian`. See [Jitter](#jitter). | +| `--loss` | The probability that a datagram is dropped. | +| `--reorder` | The probability that a datagram skips the delay, overtaking those in flight. | +| `--rate`, `--burst`, `--queue` | A token bucket: bits per second, the bytes it lets out ahead after an idle spell (1500), and the longest a datagram waits before it is dropped (100ms). | +| `--batch`, `--batch-window` | Hold datagrams until this many wait, or the window closes, then release them together. See [Batches](#batches). | +| `--shared` | Every client shares one link each way. See [Shared path](#shared-path). | +| `--profile` | A built-in profile's name, or a profile TOML file, in place of the flags that shape the path. See [Profiles](#profiles). | +| `--report` | Write the profile, seed and counters to this file as JSON at exit. See [Report](#report). | +| `--report-interval` | Print the same JSON as one line on stdout this often. | + +As a library, `Shaper::bind` takes a `Config`: where to listen and forward, the seed, and a `Profile` +per direction. It also takes a `Setup`, which is a `Config` plus the opt-in options below, each off by +default. `Shaper::verify` fails when the shaper stopped forwarding, when an impairment the profile +configures never acted and the traffic makes that silence implausible, or when a direction carried +traffic but no datagram saw one of its phases, before the first step or after one. The relay's drills +(`rs/moq-relay/tests/drills.rs`, described in `test/drill/README.md`) run every scenario through it. + +## What a seed does and does not fix + +Every decision comes from the seed: each link draws loss, reorder and jitter from its own stream, in the +same order for every datagram whether or not the knob is on. Replay a run with the same seed and the +same datagrams are dropped, the same ones skip the delay, and each draws the same jitter. + +The seed does not make the kernel's delivery clock reproducible: when a held datagram actually lands is +the host's scheduling. A test asserting on a replay should assert on the decisions, not on arrival times. + +## Jitter + +The default, `uniform`, draws each datagram's delay on its own within `jitter` either way of `delay`, so a +later datagram can overtake an earlier one. A drill wants that, and `jitter` may not exceed `delay`. + +`gaussian` treats `jitter` as a sigma, clamps the draw at zero, and never lets a datagram leave before the +one in front of it: queueing delay on a FIFO path, which stretches and compresses the spacing and never +changes the order. The sigma may exceed `delay`, but a nonzero `jitter` still needs a nonzero `delay`. Only `reorder` overtakes. This matters beyond realism: QUIC reads an overtake as loss, +retransmits and backs off, so a run meant to measure a jittery path ends up measuring the congestion +response. On a 5ms delay with 5ms of jitter, reordering alone dragged a receiver's audio buffer from +120ms to nearly two seconds. + +## Shared path + +Each client flow gets a link of its own each way by default: its own seeded stream, rate limit and queue, +so its decisions do not depend on anyone else's traffic. + +With `shared`, every flow takes one link each way, the way clients behind one access link do. A browser +page with a WebTransport session of its own and another in its audio worker is two flows on one path: +shared, a gaussian path keeps the order across both, a batch fills from both, and one rate limit serves +both. A flow's treatment then depends on how its datagrams interleave with the others'. + +## Batches + +A paced hop, a Wi-Fi access point or a cellular scheduler bunches traffic: nothing moves for a while, then +several datagrams arrive at once. A batch holds datagrams until `count` are waiting, or until `window` +after the first arrived, then sends them all when the latest would have left. No delay or jitter produces +that clump, and it is what a receiver's jitter estimate has to cope with. A `count` below two is refused, +since the datagram that fills a batch never waits. + +## Steps + +A step changes a direction's delay, jitter, loss or reorder once the run reaches `at`, counted from when +forwarding started. It changes only what it names, so a later step puts one knob back. The rate limit +never steps, since datagrams already queued behind it would keep the old rate's departures and a new +rate would reorder them. A step that leaves the profile as it was, or comes at zero, is refused. Steps +come from a profile file. `verify` holds each phase's impairments only to the traffic that phase treated. +A direction that carried traffic fails `verify` unless some datagram saw each phase: the profile before +the first step and after every step. Traffic that starts late, ends early, or goes quiet across a step +never saw the path change. + +## Profiles + +A profile file names both directions' treatment, the seed, and whether the path is shared. `up` is from +the client to the target, `down` the way back. Every field of a direction is optional and off when left +out, and unknown fields are refused. + +```toml +name = "example" +seed = 7 +shared = true + +[up] +delay = "5ms" +jitter = "5ms" +jitter_model = "gaussian" # required once the delay varies, through jitter or steps +loss = 0.02 +reorder = 0.01 # skips the delay, so it needs one + +[up.rate] +bits_per_second = 2000000 +burst = 12500 +queue = "250ms" + +[up.batch] +count = 7 +window = "160ms" + +[[up.steps]] # at 30s the delay grows, +at = "30s" +delay = "60ms" + +[[up.steps]] # and at 60s it recovers +at = "60s" +delay = "5ms" + +[down] +delay = "5ms" +``` + +`--profile` takes a file's path, or one of the built-ins in `profiles/`, embedded in the binary. The +audio-quality lane (`test/audio-quality/`) uses `bursty` and `step`, plus its own flag profiles. + +| Name | What it models | +| --- | --- | +| `near-zero` | The control. The shaper is in the path but treats nothing. | +| `mild` | A healthy wired LAN: 5ms delay, 5ms sigma, in order. | +| `bursty` | A paced hop: seven datagrams per 160ms window, released together. | +| `step` | 5ms for thirty seconds, then 60ms, with no recovery. | + +Each built-in is a shared path, as a browser page's sessions on one host would be. + +## Counters + +`Shaper::stats` counts what each direction did, summed over every client. + +| Counter | Meaning | +| --- | --- | +| `packets` | Datagrams received. | +| `lost` | Datagrams dropped by `loss`. | +| `overflowed` | Datagrams dropped because the rate limit's queue was full. | +| `throttled` | Datagrams that waited for the rate limit. | +| `delayed` | Datagrams given a nonzero delay, by the profile or by a batch holding them. | +| `reordered` | Datagrams sent ahead of the delay, overtaking any still in flight. | + +## Report + +`--report ` writes `{"profile", "seed", "up", "down"}` at exit, before the verdict, so a run that +fails it still leaves its counters behind. `profile` is the profile's name, or null when the flags built +it, and `up` and `down` hold the counters above. `--report-interval` prints the same object as one line on +stdout each period, which also survives a shaper killed before it could write the file. + +A profile that silently did nothing turns an impaired run into an unimpaired pass. `verify` catches that +for the shaper's own exit; a harness grading a run should also check the report, where `near-zero` is the +only profile for which a `delayed` of zero is the right answer. Keep the name, the seed and the counters +with the run's artifacts: the seed is what turns a failing run into one that can be looked at again. diff --git a/rs/moq-shaper/profiles/bursty.toml b/rs/moq-shaper/profiles/bursty.toml new file mode 100644 index 00000000000..307c8f2f4e5 --- /dev/null +++ b/rs/moq-shaper/profiles/bursty.toml @@ -0,0 +1,16 @@ +# A paced hop that bunches traffic: nothing moves until seven datagrams are +# waiting or the window closes, and then they all leave together. +# +# This is the shape a receiver's jitter estimate has to see, and no delay-and- +# spread profile produces it: the datagrams arrive together rather than late. +name = "bursty" +seed = 3 +shared = true + +[up.batch] +count = 7 +window = "160ms" + +[down.batch] +count = 7 +window = "160ms" diff --git a/rs/moq-shaper/profiles/mild.toml b/rs/moq-shaper/profiles/mild.toml new file mode 100644 index 00000000000..de101bb5bf9 --- /dev/null +++ b/rs/moq-shaper/profiles/mild.toml @@ -0,0 +1,15 @@ +# A healthy wired LAN: a few milliseconds of delay with the same spread again, +# queued in order. +name = "mild" +seed = 2 +shared = true + +[up] +delay = "5ms" +jitter = "5ms" +jitter_model = "gaussian" + +[down] +delay = "5ms" +jitter = "5ms" +jitter_model = "gaussian" diff --git a/rs/moq-shaper/profiles/near-zero.toml b/rs/moq-shaper/profiles/near-zero.toml new file mode 100644 index 00000000000..627f619487a --- /dev/null +++ b/rs/moq-shaper/profiles/near-zero.toml @@ -0,0 +1,7 @@ +# The control: the shaper sits in the path but treats nothing. +# +# Every other profile is graded against this one, and a harness asserting that an +# impairment applied needs a profile where `delayed == 0` is the right answer. +name = "near-zero" +seed = 1 +shared = true diff --git a/rs/moq-shaper/profiles/step.toml b/rs/moq-shaper/profiles/step.toml new file mode 100644 index 00000000000..bdd0170d2fc --- /dev/null +++ b/rs/moq-shaper/profiles/step.toml @@ -0,0 +1,24 @@ +# A path that gets worse mid-run: 5ms for the first thirty seconds, then 60ms. +# +# An estimator that only ever converges downward passes every steady profile and +# fails this one. It never recovers, which is what makes it the baseline a +# recovery profile is read against. +name = "step" +seed = 4 +shared = true + +[up] +delay = "5ms" +jitter_model = "gaussian" + +[[up.steps]] +at = "30s" +delay = "60ms" + +[down] +delay = "5ms" +jitter_model = "gaussian" + +[[down.steps]] +at = "30s" +delay = "60ms" diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 730d3a98146..4e9ca01e5ac 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -13,33 +13,43 @@ //! A profile that silently did nothing would turn an impaired run into an //! unimpaired pass, so [`Shaper::verify`] fails when an impairment the profile //! configures never acted and the traffic makes that silence implausible. +//! +//! A [`Setup`] adds opt-in options to a [`Config`]: a jitter +//! model that keeps the order, one link shared by every client, batches, steps, +//! and named profiles loaded as a [`Preset`]. The README says why each exists. use std::{ cmp::Reverse, - collections::{BinaryHeap, HashMap, hash_map}, + collections::{BTreeMap, BinaryHeap, HashMap, hash_map}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::{ - Arc, OnceLock, + Arc, Mutex, OnceLock, atomic::{AtomicU64, Ordering}, }, time::Duration, }; +mod preset; + +pub use preset::Preset; + use anyhow::Context; use rand::{RngExt, SeedableRng, rngs::Xoshiro256PlusPlus}; +use serde::{Deserialize, Serialize}; use tokio::{net::UdpSocket, sync::mpsc, task::JoinSet, time::Instant}; /// How one direction of the path treats each datagram. /// /// A datagram is first subject to `loss`, then waits for the `rate` limit, then -/// takes `delay` plus or minus `jitter` to arrive, unless `reorder` sends it -/// ahead of everything still in flight. +/// takes `delay` plus or minus `jitter`, drawn as its [`Jitter`] model says, to +/// arrive, unless `reorder` sends it ahead of everything still in flight. #[derive(Clone, Debug, Default, PartialEq)] pub struct Profile { /// The base one-way delay. pub delay: Duration, - /// The most a datagram's delay varies from `delay`, uniformly either way. + /// How far a datagram's delay varies from `delay`: the most either way when + /// [`Jitter::Uniform`], the sigma when [`Jitter::Gaussian`]. pub jitter: Duration, /// The probability that a datagram is dropped. pub loss: f64, @@ -50,7 +60,7 @@ pub struct Profile { } impl Profile { - fn validate(&self) -> anyhow::Result<()> { + fn validate(&self, model: Jitter) -> anyhow::Result<()> { anyhow::ensure!( (0.0..=1.0).contains(&self.loss), "loss {} is not a probability", @@ -66,12 +76,19 @@ impl Profile { "reorder {} needs a delay to overtake", self.reorder ); + // A gaussian clamps its draw at zero instead, as a queue cannot run early. anyhow::ensure!( - self.jitter <= self.delay, + model == Jitter::Gaussian || self.jitter <= self.delay, "jitter {:?} exceeds delay {:?}, which would need a negative delay", self.jitter, self.delay ); + // With no delay to vary, `verify` has nothing to hold the jitter to. + anyhow::ensure!( + self.jitter.is_zero() || !self.delay.is_zero(), + "jitter {:?} needs a delay to vary", + self.jitter + ); if let Some(rate) = &self.rate { anyhow::ensure!(rate.bits_per_second > 0, "a rate limit of zero passes nothing"); } @@ -79,6 +96,133 @@ impl Profile { } } +/// How a direction draws each datagram's jitter. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Jitter { + /// Uniform within `jitter` either way of `delay`, drawn for each datagram on + /// its own, so a later datagram can overtake an earlier one. + #[default] + Uniform, + /// A gaussian with `jitter` as its sigma, clamped at zero, that never leaves + /// before the datagram in front: it varies the spacing, never the order. + /// + /// This is queueing delay on a FIFO path. A jitter that overtakes hands QUIC + /// a gap it can only read as loss, so the run measures its congestion + /// response rather than the jitter. + Gaussian, +} + +/// Hold datagrams, then release them together, the way a paced hop bunches them. +/// +/// A batch closes once `count` datagrams are waiting, or `window` after the +/// first of them arrived, and everything in it leaves when the latest would. +/// That clump is what a receiver's jitter estimate sees on such a path, and no +/// delay or jitter produces it: the datagrams arrive together rather than late. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Batch { + /// How many datagrams close a batch early. + pub count: usize, + /// How long a batch waits for that many. + #[serde(with = "humantime_serde")] + pub window: Duration, +} + +/// A change to one direction's profile once the run reaches `at`. +/// +/// A step changes only what it names, so a later step puts one knob back +/// without restating the rest. The rate limit never steps: datagrams already +/// queued behind it keep the old rate's departures, so a new rate would +/// reorder them. +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Step { + /// How far into the run the change happens. + #[serde(with = "humantime_serde")] + pub at: Duration, + /// The delay from then on. + #[serde(default, with = "humantime_serde")] + pub delay: Option, + /// The jitter from then on. + #[serde(default, with = "humantime_serde")] + pub jitter: Option, + /// The loss from then on. + #[serde(default)] + pub loss: Option, + /// The reorder from then on. + #[serde(default)] + pub reorder: Option, +} + +impl Step { + /// Change what this step names in `profile`. + fn apply(&self, profile: &mut Profile) { + if let Some(delay) = self.delay { + profile.delay = delay; + } + if let Some(jitter) = self.jitter { + profile.jitter = jitter; + } + if let Some(loss) = self.loss { + profile.loss = loss; + } + if let Some(reorder) = self.reorder { + profile.reorder = reorder; + } + } +} + +/// One direction's opt-in options beyond its [`Profile`]. The default adds nothing. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Options { + /// How the profile's `jitter` is drawn. + pub jitter_model: Jitter, + /// Hold datagrams and release them together, if at all. + pub batch: Option, + /// Changes to the profile part-way through the run, earliest first. + pub steps: Vec, +} + +impl Options { + /// Check the options, and `profile` as they draw it, after every step too. + fn validate(&self, profile: &Profile) -> anyhow::Result<()> { + profile.validate(self.jitter_model)?; + if let Some(batch) = &self.batch { + // The datagram that fills a batch leaves at once, so one of one holds nothing. + anyhow::ensure!(batch.count > 1, "a batch of {} never holds anything", batch.count); + anyhow::ensure!(!batch.window.is_zero(), "a batch with no window never holds anything"); + } + + let mut profile = profile.clone(); + let mut previous: Option = None; + for step in &self.steps { + // The profile is what the run opens with, so a step at zero would + // replace it before any datagram saw it. + anyhow::ensure!( + !step.at.is_zero(), + "a step at zero replaces the profile; change the profile instead" + ); + // A link only ever looks at the next step due, so one out of order + // would be skipped without a word. + anyhow::ensure!( + previous.is_none_or(|previous| step.at > previous), + "the step at {:?} comes after the one at {previous:?}; steps go in order", + step.at + ); + // A step that leaves the profile as it was thinks it changes and does not. + let before = profile.clone(); + step.apply(&mut profile); + anyhow::ensure!(profile != before, "the step at {:?} changes nothing", step.at); + profile + .validate(self.jitter_model) + .with_context(|| format!("after the step at {:?}", step.at))?; + previous = Some(step.at); + } + Ok(()) + } +} + /// Each impairment a profile draws for: its name, how likely it is to act on /// any one datagram (zero when unconfigured), and the count showing it did. /// @@ -100,27 +244,61 @@ const IMPAIRMENTS: [Impairment; 3] = [ /// cannot, and that is when the silence means the profile is not in the path. const IMPLAUSIBLE: f64 = 1e-4; -/// The impairments `config` configures that `stats` shows implausibly never acted. -fn unapplied(config: &Config, stats: &Stats) -> Vec<&'static str> { +/// The impairments that `stats` shows implausibly never acted, given every +/// phase of both directions as the profile in force and the datagrams it treated. +/// +/// Each phase is charged only its own traffic, so the clean traffic after a step +/// that turns an impairment off never counts against the phase before it. +fn unapplied(phases: &[(Profile, u64)], stats: &Stats) -> Vec<&'static str> { IMPAIRMENTS .iter() .filter(|(_, chance, acted)| { - let silence = (1.0 - chance(&config.up)).powf(stats.up.packets as f64) - * (1.0 - chance(&config.down)).powf(stats.down.packets as f64); + let silence: f64 = phases + .iter() + .map(|(profile, packets)| (1.0 - chance(profile)).powf(*packets as f64)) + .product(); acted(&stats.up) + acted(&stats.down) == 0 && silence < IMPLAUSIBLE }) .map(|(name, ..)| *name) .collect() } +/// `profile` as the run opens and as each of `steps` leaves it, paired with the +/// datagrams `counts` says that phase treated. +fn phased(profile: &Profile, steps: &[Step], counts: &BTreeMap) -> Vec<(Profile, u64)> { + let count = |phase: usize| counts.get(&phase).copied().unwrap_or(0); + let mut profile = profile.clone(); + let mut phases = vec![(profile.clone(), count(0))]; + for (index, step) in steps.iter().enumerate() { + step.apply(&mut profile); + phases.push((profile.clone(), count(index + 1))); + } + phases +} + +/// Whether the batches `setup` configures implausibly never held a datagram. +/// +/// A batch holds every datagram but the one that fills it, so it acts on at +/// least `1 - 1 / count` of them. `batched` counts only what a batch held, not +/// what the profile delayed, so a batch that never changed a departure is caught +/// even behind a delay. +fn unbatched(setup: &Setup, stats: &Stats, batched: u64) -> bool { + let chance = |options: &Options| options.batch.map_or(0.0, |batch| 1.0 - 1.0 / batch.count as f64); + let silence = (1.0 - chance(&setup.up)).powf(stats.up.packets as f64) + * (1.0 - chance(&setup.down)).powf(stats.down.packets as f64); + batched == 0 && silence < IMPLAUSIBLE +} + /// A token-bucket rate limit with a bounded queue behind it. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Rate { /// The sustained rate. pub bits_per_second: u64, /// How many bytes may go out ahead of the sustained rate after an idle spell. pub burst: u64, /// The longest a datagram waits for the bucket; one that would wait longer is dropped. + #[serde(with = "humantime_serde")] pub queue: Duration, } @@ -146,8 +324,40 @@ pub struct Config { pub down: Profile, } +/// A [`Config`] plus the opt-in options it has no field for. +/// +/// Every option defaults to off, so a setup made from a config shapes exactly +/// as that config does. +#[derive(Clone, Debug)] +pub struct Setup { + /// Where to listen and forward, the seed, and each direction's profile. + pub config: Config, + /// Every client shares one link each way, the way clients behind one access + /// link do, rather than each getting its own. + /// + /// Shared, a jitter that keeps the order keeps it across clients, and one + /// seeded stream and one rate limit serve them all, so a client's treatment + /// depends on how its datagrams interleave with the others'. + pub shared: bool, + /// The options from a client toward the target. + pub up: Options, + /// The options from the target back toward a client. + pub down: Options, +} + +impl From for Setup { + fn from(config: Config) -> Self { + Self { + config, + shared: false, + up: Options::default(), + down: Options::default(), + } + } +} + /// What one direction did, summed over every client. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] pub struct Counters { /// Datagrams received. pub packets: u64, @@ -157,7 +367,7 @@ pub struct Counters { pub overflowed: u64, /// Datagrams that waited for the rate limit. pub throttled: u64, - /// Datagrams given a nonzero delay. + /// Datagrams given a nonzero delay, by the profile or by a batch holding them. pub delayed: u64, /// Datagrams sent ahead of the delay, overtaking any still in flight. pub reordered: u64, @@ -174,7 +384,7 @@ impl fmt::Display for Counters { } /// What both directions did. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] pub struct Stats { /// From clients toward the target. pub up: Counters, @@ -191,7 +401,7 @@ impl fmt::Display for Stats { /// A running shaper. Dropping it stops forwarding. pub struct Shaper { addr: SocketAddr, - config: Config, + setup: Setup, tally: Arc<[Tally; 2]>, /// Why forwarding stopped, if it did. failed: Arc>, @@ -200,9 +410,11 @@ pub struct Shaper { impl Shaper { /// Bind the listening socket and start forwarding. - pub async fn bind(config: Config) -> anyhow::Result { - config.up.validate().context("invalid up profile")?; - config.down.validate().context("invalid down profile")?; + pub async fn bind(setup: impl Into) -> anyhow::Result { + let setup = setup.into(); + let config = &setup.config; + setup.up.validate(&config.up).context("invalid up profile")?; + setup.down.validate(&config.down).context("invalid down profile")?; let listen = UdpSocket::bind(config.bind) .await @@ -212,11 +424,11 @@ impl Shaper { let tally = Arc::new([Tally::default(), Tally::default()]); let failed = Arc::new(OnceLock::new()); let task = tokio::spawn({ - let config = config.clone(); + let setup = setup.clone(); let tally = tally.clone(); let failed = failed.clone(); async move { - if let Err(err) = run(Arc::new(listen), config, tally).await { + if let Err(err) = run(Arc::new(listen), setup, tally).await { let _ = failed.set(format!("{err:#}")); } } @@ -224,7 +436,7 @@ impl Shaper { Ok(Self { addr, - config, + setup, tally, failed, task, @@ -238,7 +450,7 @@ impl Shaper { /// The configuration this shaper runs, including its seed. pub fn config(&self) -> &Config { - &self.config + &self.setup.config } /// What the shaper has done so far. @@ -249,18 +461,53 @@ impl Shaper { } } - /// Fail unless the shaper is still forwarding and every impairment the - /// profile configures acted on some datagram, in either direction. + /// Fail unless the shaper is still forwarding, every impairment the profile + /// configures acted on some datagram, and some datagram saw every phase of + /// a direction with steps, before the first and after each. /// /// An impairment is only held to that once the traffic makes its silence - /// implausible: a short run can see no loss, but never no delay. + /// implausible: a short run can see no loss, but never no delay. The phases + /// are held to it once their direction carried anything: traffic that all + /// came on one side of a step never saw the path change. pub fn verify(&self) -> anyhow::Result { let stats = self.stats(); if let Some(err) = self.failed.get() { anyhow::bail!("the shaper stopped forwarding: {err} ({stats})"); } - let missing = unapplied(&self.config, &stats); + let counts = [UP, DOWN].map(|dir| self.tally[dir].phases.lock().unwrap().clone()); + let phases: Vec<(Profile, u64)> = [ + (&self.setup.config.up, &self.setup.up, &counts[UP]), + (&self.setup.config.down, &self.setup.down, &counts[DOWN]), + ] + .into_iter() + .flat_map(|(profile, options, counts)| phased(profile, &options.steps, counts)) + .collect(); + let mut missing: Vec = unapplied(&phases, &stats).into_iter().map(String::from).collect(); + let batched = self + .tally + .iter() + .map(|tally| tally.batched.load(Ordering::Relaxed)) + .sum(); + if unbatched(&self.setup, &stats, batched) { + missing.push("batch".to_string()); + } + for (name, options, phases, counters) in [ + ("up", &self.setup.up, &counts[UP], &stats.up), + ("down", &self.setup.down, &counts[DOWN], &stats.down), + ] { + let Some(first) = options.steps.first().filter(|_| counters.packets > 0) else { + continue; + }; + if !phases.contains_key(&0) { + missing.push(format!("the {name} profile before its step at {:?}", first.at)); + } + for (index, step) in options.steps.iter().enumerate() { + if !phases.contains_key(&(index + 1)) { + missing.push(format!("the {name} step at {:?}", step.at)); + } + } + } anyhow::ensure!( missing.is_empty(), "the profile never applied {} ({stats}), so this run was not impaired as configured", @@ -287,6 +534,10 @@ struct Tally { throttled: AtomicU64, delayed: AtomicU64, reordered: AtomicU64, + /// Datagrams treated in each phase: 0 before the first step, n after the nth. + phases: Mutex>, + /// Datagrams a batch held past when they would otherwise have left. + batched: AtomicU64, } impl Tally { @@ -309,12 +560,22 @@ fn bump(counter: &AtomicU64) { /// Accept datagrams from clients, giving each client its own flow. /// /// A flow is a socket of its own toward the target, so the target sees one -/// address per client just as it would without the shaper in the way. -async fn run(listen: Arc, config: Config, tally: Arc<[Tally; 2]>) -> anyhow::Result<()> { - let mut flows = HashMap::::new(); +/// address per client just as it would without the shaper in the way. Each +/// flow takes a link of its own each way, unless the path is shared. +async fn run(listen: Arc, setup: Setup, tally: Arc<[Tally; 2]>) -> anyhow::Result<()> { + let config = &setup.config; + let mut flows = HashMap::::new(); let mut tasks = JoinSet::new(); let mut buf = vec![0u8; u16::MAX as usize]; + // What a step's `at` counts from. + let start = Instant::now(); + + // A shared path is one link each way, drawing the streams the first flow would. + let shared = setup + .shared + .then(|| [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, 0, start, &tally))); + loop { let (size, from) = tokio::select! { res = listen.recv_from(&mut buf) => res.context("receive from a client")?, @@ -322,31 +583,71 @@ async fn run(listen: Arc, config: Config, tally: Arc<[Tally; 2]>) -> res.context("flow task panicked")??; continue; } + }; let now = Instant::now(); - let stream = 2 * flows.len() as u64; - let link = match flows.entry(from) { + let number = flows.len() as u64; + let flow = match flows.entry(from) { hash_map::Entry::Occupied(entry) => entry.into_mut(), hash_map::Entry::Vacant(entry) => { let upstream = Arc::new(bind_toward(config.target).await?); - let (up_tx, up_rx) = mpsc::unbounded_channel(); - tasks.spawn(deliver(up_rx, upstream.clone(), config.target)); - - let (down_tx, down_rx) = mpsc::unbounded_channel(); - tasks.spawn(deliver(down_rx, listen.clone(), from)); - - let down = Link::new(&config.down, config.seed, stream + 1, now, down_tx); - tasks.spawn(reply(upstream, config.target, down, tally.clone())); - - entry.insert(Link::new(&config.up, config.seed, stream, now, up_tx)) + let [up, down] = match &shared { + Some(links) => links.clone(), + None => [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, number, start, &tally)), + }; + tasks.spawn(reply( + upstream.clone(), + config.target, + down, + listen.clone(), + from, + tally.clone(), + )); + + entry.insert(Flow { upstream, up }) } }; - link.push(now, buf[..size].to_vec(), &tally[UP]); + let datagram = buf[..size].to_vec(); + let mut up = flow.up.lock().expect("link poisoned"); + up.push(now, datagram, &flow.upstream, config.target, &tally[UP]); } } +/// One client: its socket toward the target, and the link it sends through. +struct Flow { + upstream: Arc, + up: Arc>, +} + +/// One direction of the flow numbered `flow` as a link, and the task delivering +/// what it treats. +fn link( + tasks: &mut JoinSet>, + setup: &Setup, + direction: usize, + flow: u64, + start: Instant, + tally: &Arc<[Tally; 2]>, +) -> Arc> { + let (profile, options) = match direction { + UP => (&setup.config.up, &setup.up), + _ => (&setup.config.down, &setup.down), + }; + let (queue, queued) = mpsc::unbounded_channel(); + tasks.spawn(deliver(queued, options.batch, tally.clone(), direction)); + let stream = 2 * flow + direction as u64; + Arc::new(Mutex::new(Link::new( + profile, + options, + setup.config.seed, + stream, + start, + queue, + ))) +} + /// A socket that can reach `target`: loopback for a loopback target, so the /// shaper never opens a port beyond the host when it does not need to. async fn bind_toward(target: SocketAddr) -> anyhow::Result { @@ -361,11 +662,13 @@ async fn bind_toward(target: SocketAddr) -> anyhow::Result { .with_context(|| format!("bind a socket toward {target}")) } -/// Feed what the target sends a flow into that flow's return link. +/// Feed what the target sends a flow into the link back toward its client. async fn reply( socket: Arc, target: SocketAddr, - mut link: Link, + link: Arc>, + listen: Arc, + client: SocketAddr, tally: Arc<[Tally; 2]>, ) -> anyhow::Result<()> { let mut buf = vec![0u8; u16::MAX as usize]; @@ -375,105 +678,236 @@ async fn reply( if from != target { continue; } - link.push(Instant::now(), buf[..size].to_vec(), &tally[DOWN]); + let datagram = buf[..size].to_vec(); + let mut down = link.lock().expect("link poisoned"); + down.push(Instant::now(), datagram, &listen, client, &tally[DOWN]); } } -/// Send each treated datagram at its departure time. +/// Send each treated datagram at its departure time, after its batch if the +/// link has one. async fn deliver( - mut queue: mpsc::UnboundedReceiver<(Instant, Vec)>, - socket: Arc, - dest: SocketAddr, + mut queue: mpsc::UnboundedReceiver, + batch: Option, + tally: Arc<[Tally; 2]>, + direction: usize, ) -> anyhow::Result<()> { // Ordered by departure, then by arrival, so ties keep their order. let mut pending = BinaryHeap::>::new(); - let mut datagrams = HashMap::>::new(); + let mut parcels = HashMap::::new(); let mut sequence = 0u64; + let mut held = Held::default(); + let mut ready = Vec::new(); loop { let next = pending.peek().map(|Reverse((at, _))| *at); + let wake = next.into_iter().chain(held.closes).min(); tokio::select! { item = queue.recv() => { // The link is gone, so the flow is too. - let Some((at, datagram)) = item else { return Ok(()) }; - pending.push(Reverse((at, sequence))); - datagrams.insert(sequence, datagram); - sequence += 1; + let Some(parcel) = item else { return Ok(()) }; + match batch { + Some(batch) => held.hold(batch, parcel, &mut ready, &tally[direction]), + None => ready.push(parcel), + } + } + _ = tokio::time::sleep_until(wake.unwrap_or_else(Instant::now)), if wake.is_some() => { + // The window closes a batch that never filled. + if let Some(closes) = held.closes.filter(|&closes| closes <= Instant::now()) { + held.release(closes, &mut ready, &tally[direction]); + } } - _ = tokio::time::sleep_until(next.unwrap_or_else(Instant::now)), if next.is_some() => { - let now = Instant::now(); - while let Some(&Reverse((at, id))) = pending.peek() { - if at > now { - break; - } - pending.pop(); - let datagram = datagrams.remove(&id).expect("queued datagram"); - // A send error is the path losing the datagram, the way a - // network does when the far end is gone (a killed relay, say), - // so it is not the shaper failing. - let _ = socket.send_to(&datagram, dest).await; + } + for parcel in ready.drain(..) { + pending.push(Reverse((parcel.at, sequence))); + parcels.insert(sequence, parcel); + sequence += 1; + } + + // Whatever is due leaves now. The timer rounds up to the next + // millisecond, which would delay even a datagram given no delay. + let now = Instant::now(); + while let Some(&Reverse((at, id))) = pending.peek() { + if at > now { + break; + } + pending.pop(); + let parcel = parcels.remove(&id).expect("queued datagram"); + // A send error is the path losing the datagram, the way a network + // does when the far end is gone (a killed relay, say), so it is not + // the shaper failing. + let _ = parcel.socket.send_to(&parcel.datagram, parcel.dest).await; + } + } +} + +/// The datagrams a batch is holding, and when its window closes on them. +#[derive(Default)] +struct Held { + parcels: Vec, + closes: Option, +} + +impl Held { + /// Hold `parcel`, releasing the batch into `ready` once it is full. + fn hold(&mut self, batch: Batch, parcel: Parcel, ready: &mut Vec, tally: &Tally) { + let arrived = parcel.arrived; + // The window may have closed before its timer fired; the batch it closed + // leaves without this datagram, which starts the next one. + if let Some(closes) = self.closes.filter(|&closes| closes <= arrived) { + self.release(closes, ready, tally); + } + self.closes.get_or_insert(arrived + batch.window); + self.parcels.push(parcel); + if self.parcels.len() >= batch.count { + self.release(arrived, ready, tally); + } + } + + /// Release the batch, closed at `closed`, into `ready`: every datagram leaves + /// when the latest would, and none before the batch closed. + fn release(&mut self, closed: Instant, ready: &mut Vec, tally: &Tally) { + self.closes = None; + let Some(latest) = self.parcels.iter().map(|parcel| parcel.at).max() else { + return; + }; + let at = latest.max(closed); + for mut parcel in self.parcels.drain(..) { + if at > parcel.at { + bump(&tally.batched); + // A hold is a delay, counted once whichever stage gave it. + if !parcel.delayed { + bump(&tally.delayed); } } + parcel.at = at; + ready.push(parcel); } } } -/// One direction of one flow: the decisions and the state they depend on. +/// A treated datagram: when it arrived and leaves, and the socket and address +/// it leaves by. +struct Parcel { + arrived: Instant, + at: Instant, + /// Whether the link already counted it as delayed. + delayed: bool, + datagram: Vec, + socket: Arc, + dest: SocketAddr, +} + +/// One direction of one flow, or of every flow on a shared path: the decisions +/// and the state they depend on. struct Link { profile: Profile, + jitter: Jitter, rng: Xoshiro256PlusPlus, /// When the token bucket would be full again, as in GCRA. full_at: Instant, - queue: mpsc::UnboundedSender<(Instant, Vec)>, + /// When the latest in-order datagram leaves, which the next one never beats. + floor: Instant, + /// When the run started, which a step's `at` counts from. + start: Instant, + /// Every step, earliest first. + steps: Vec, + /// How many of `steps` have applied. + stepped: usize, + queue: mpsc::UnboundedSender, } impl Link { - /// Each direction of each flow draws from its own stream of the one seed, - /// so a flow's decisions do not depend on how its datagrams interleave with - /// anyone else's. Flows are numbered in the order they first send, since a - /// client's ephemeral port is no identity across runs. + /// Each link draws from its own stream of the one seed, so a flow's + /// decisions do not depend on how its datagrams interleave with anyone + /// else's, unless the path is shared. Flows are numbered in the order they + /// first send, since a client's ephemeral port is no identity across runs. fn new( profile: &Profile, + options: &Options, seed: u64, stream: u64, - now: Instant, - queue: mpsc::UnboundedSender<(Instant, Vec)>, + start: Instant, + queue: mpsc::UnboundedSender, ) -> Self { Self { profile: profile.clone(), + jitter: options.jitter_model, rng: Xoshiro256PlusPlus::seed_from_u64(seed ^ stream.wrapping_mul(0x9E37_79B9_7F4A_7C15)), - // Not the moment the link is built, which is after `now`: that would - // make the first datagram queue behind a bucket still refilling. - full_at: now, + // Full since the run started, never since the link was built, which + // is after its first datagram arrived: that would make the datagram + // queue behind a bucket still refilling. + full_at: start, + floor: start, + start, + steps: options.steps.clone(), + stepped: 0, queue, } } - fn push(&mut self, now: Instant, datagram: Vec, tally: &Tally) { + /// Apply every step the run has reached by `now`, in order, and record the + /// phase a datagram arriving then sees. + fn step(&mut self, now: Instant, tally: &Tally) { + while let Some(step) = self.steps.get(self.stepped) { + // A step past what the clock can hold is one the run never reaches. + if self.start.checked_add(step.at).is_none_or(|at| at > now) { + break; + } + step.apply(&mut self.profile); + self.stepped += 1; + } + // Applying a step is no evidence it acted, so count the datagrams each phase treated. + *tally.phases.lock().unwrap().entry(self.stepped).or_default() += 1; + } + + /// Treat a datagram and queue it to leave by `socket` for `dest`. + fn push(&mut self, now: Instant, datagram: Vec, socket: &Arc, dest: SocketAddr, tally: &Tally) { + let Some((at, delayed)) = self.treat(now, datagram.len(), tally) else { + return; + }; + // The delivery task only ends once this link is dropped. + let _ = self.queue.send(Parcel { + arrived: now, + at, + delayed, + datagram, + socket: socket.clone(), + dest, + }); + } + + /// When a datagram of `size` bytes arriving `now` leaves, and whether it was + /// counted as delayed, or `None` when it is dropped. + fn treat(&mut self, now: Instant, size: usize, tally: &Tally) -> Option<(Instant, bool)> { + self.step(now, tally); bump(&tally.packets); // Every draw happens for every datagram, whatever the profile, so one // knob's outcome never shifts the stream another knob draws from. let lose = self.rng.random::() < self.profile.loss; let skip = self.rng.random::() < self.profile.reorder; - let spread = self.rng.random::() * 2.0 - 1.0; + let spread = match self.jitter { + Jitter::Uniform => self.rng.random::() * 2.0 - 1.0, + Jitter::Gaussian => gaussian(&mut self.rng), + }; if lose { bump(&tally.lost); - return; + return None; } let mut depart = now; if let Some(rate) = &self.profile.rate { // GCRA: a datagram may leave once the backlog, itself included, is // within the burst allowance. - let full_at = self.full_at.max(now) + rate.cost(datagram.len() as u64); + let full_at = self.full_at.max(now) + rate.cost(size as u64); let start = full_at .checked_sub(rate.cost(rate.burst)) .map_or(now, |start| start.max(now)); if start - now > rate.queue { bump(&tally.overflowed); - return; + return None; } if start > now { bump(&tally.throttled); @@ -482,19 +916,39 @@ impl Link { depart = start; } + let mut delayed = false; if skip { bump(&tally.reordered); + } else if self.jitter == Jitter::Gaussian { + // A queue: nothing leaves before the datagram in front of it. + let wait = self.profile.delay.as_secs_f64() + self.profile.jitter.as_secs_f64() * spread; + let leave = (depart + Duration::from_secs_f64(wait.max(0.0))).max(self.floor); + // Counted like the uniform model whenever a delay is configured, so + // the chance `verify` holds it to is the same. + delayed = !self.profile.delay.is_zero() || leave > depart; + self.floor = leave; + depart = leave; } else if !self.profile.delay.is_zero() { let jitter = self.profile.jitter.as_secs_f64() * spread; depart += Duration::from_secs_f64(self.profile.delay.as_secs_f64() + jitter); + delayed = true; + } + if delayed { bump(&tally.delayed); } - // The delivery task only ends once this link is dropped. - let _ = self.queue.send((depart, datagram)); + Some((depart, delayed)) } } +/// A standard normal sample by Box-Muller, which always takes exactly two draws, +/// so the jitter never shifts the stream the next datagram's knobs draw from. +fn gaussian(rng: &mut Xoshiro256PlusPlus) -> f64 { + let u1 = rng.random::().max(f64::MIN_POSITIVE); + let u2 = rng.random::(); + (-2.0 * u1.ln()).sqrt() * (std::f64::consts::TAU * u2).cos() +} + #[cfg(test)] mod tests { use super::*; @@ -552,6 +1006,22 @@ mod tests { assert_eq!(stats.down.packets, 50); } + #[tokio::test(start_paused = true)] + async fn an_undelayed_datagram_waits_for_no_timer() { + let (_shaper, client) = setup(1, Profile::default(), Profile::default()).await; + + // Off a millisecond boundary, where a timer would round up to the next one. + tokio::time::advance(Duration::from_micros(500)).await; + let start = Instant::now(); + client.send(b"ping").await.unwrap(); + let mut buf = [0u8; 4]; + client.recv(&mut buf).await.unwrap(); + + // A paused clock only moves when every task waits on a timer, so a round + // trip that took any time slept on one. + assert_eq!(start.elapsed(), Duration::ZERO); + } + #[tokio::test] async fn the_seed_reproduces_the_losses() { let lossy = Profile { @@ -652,16 +1122,11 @@ mod tests { #[test] fn silence_is_only_a_failure_once_it_is_implausible() { - let config = Config { - bind: LOCALHOST, - target: LOCALHOST, - seed: 0, - up: Profile { - loss: 0.05, - ..Default::default() - }, - down: Profile::default(), + let lossy = Profile { + loss: 0.05, + ..Default::default() }; + let both = |packets| [(lossy.clone(), packets), (Profile::default(), packets)]; let quiet = |packets| Stats { up: Counters { packets, @@ -674,24 +1139,49 @@ mod tests { }; // 0.95^20 is about a third: a short run seeing no loss proves nothing. - assert!(unapplied(&config, &quiet(20)).is_empty()); + assert!(unapplied(&both(20), &quiet(20)).is_empty()); // 0.95^1000 is about 5e-23: the loss is not in the path. - assert_eq!(unapplied(&config, &quiet(1000)), ["loss"]); + assert_eq!(unapplied(&both(1000), &quiet(1000)), ["loss"]); - let mut lossy = quiet(1000); - lossy.up.lost = 1; - assert!(unapplied(&config, &lossy).is_empty()); + let mut lost = quiet(1000); + lost.up.lost = 1; + assert!(unapplied(&both(1000), &lost).is_empty()); // A delay acts on every datagram, so even one undelayed datagram is a // shaper that is not in the path. - let config = Config { - down: Profile { - delay: Duration::from_millis(10), + let delayed = Profile { + delay: Duration::from_millis(10), + ..Default::default() + }; + assert_eq!(unapplied(&[(lossy, 1), (delayed, 1)], &quiet(1)), ["delay"]); + } + + #[test] + fn a_phase_is_charged_only_its_own_traffic() { + let ms = Duration::from_millis; + let lossy = Profile { + loss: 0.02, + ..Default::default() + }; + let steps = [Step { + at: ms(100), + loss: Some(0.0), + ..Default::default() + }]; + // One datagram survived the 2% loss, then the loss stepped off for a thousand. + let phases = phased(&lossy, &steps, &BTreeMap::from([(0, 1), (1, 1000)])); + let stats = Stats { + up: Counters { + packets: 1001, ..Default::default() }, - ..config + down: Counters::default(), }; - assert_eq!(unapplied(&config, &quiet(1)), ["delay"]); + assert!(unapplied(&phases, &stats).is_empty()); + + // The same thousand under the loss still fail. + let phases = phased(&lossy, &steps, &BTreeMap::from([(0, 1000), (1, 1)])); + assert_eq!(unapplied(&phases, &stats), ["loss"]); } #[tokio::test] @@ -724,4 +1214,806 @@ mod tests { }; refused(undelayed, "needs a delay").await; } + + /// An echo server, a shaper `setup` builds from a config aimed at it, and a + /// client dialing the shaper. + async fn shaped(setup: impl FnOnce(Config) -> Setup) -> (Shaper, UdpSocket) { + let echo = UdpSocket::bind(LOCALHOST).await.unwrap(); + let target = echo.local_addr().unwrap(); + tokio::spawn(async move { + let mut buf = vec![0u8; u16::MAX as usize]; + loop { + let (size, from) = echo.recv_from(&mut buf).await.unwrap(); + echo.send_to(&buf[..size], from).await.unwrap(); + } + }); + + let config = Config { + bind: LOCALHOST, + target, + seed: 3, + up: Profile::default(), + down: Profile::default(), + }; + let shaper = Shaper::bind(setup(config)).await.unwrap(); + + let client = UdpSocket::bind(LOCALHOST).await.unwrap(); + client.connect(shaper.addr()).await.unwrap(); + (shaper, client) + } + + /// Push `count` numbered datagrams through one link, `spacing` apart, and + /// return each id with its departure, in the order they would be sent. + fn departures( + profile: &Profile, + options: &Options, + count: u32, + spacing: Duration, + ) -> (Vec<(u32, Instant)>, Counters) { + let (queue, _) = mpsc::unbounded_channel(); + let now = Instant::now(); + let mut link = Link::new(profile, options, 7, 0, now, queue); + let tally = Tally::default(); + let mut sent: Vec<(u32, Instant)> = (0..count) + .filter_map(|id| Some((id, link.treat(now + spacing * id, 4, &tally)?.0))) + .collect(); + // Ties leave in arrival order, which is id order here. + sent.sort_by_key(|&(id, at)| (at, id)); + (sent, tally.snapshot()) + } + + const GAUSSIAN: Options = Options { + jitter_model: Jitter::Gaussian, + batch: None, + steps: Vec::new(), + }; + + #[tokio::test] + async fn only_the_gaussian_model_keeps_the_order() { + let jittery = Profile { + delay: Duration::from_millis(20), + jitter: Duration::from_millis(10), + ..Default::default() + }; + + let (_uniform, client) = shaped(|config| { + Config { + up: jittery.clone(), + ..config + } + .into() + }) + .await; + let got = round_trip(&client, 100).await; + let mut sorted = got.clone(); + sorted.sort(); + assert_eq!(sorted, (0..100).collect::>(), "jitter lost datagrams"); + assert_ne!(got, sorted, "uniform jitter never overtook, so this proves nothing"); + + let (gaussian, client) = shaped(|config| Setup { + up: GAUSSIAN, + ..Config { up: jittery, ..config }.into() + }) + .await; + let got = round_trip(&client, 100).await; + assert_eq!(got, (0..100).collect::>(), "gaussian jitter reordered"); + + let stats = gaussian.verify().unwrap(); + assert_eq!(stats.up.reordered, 0, "{stats}"); + assert_eq!(stats.up.delayed, 100, "{stats}"); + } + + #[test] + fn gaussian_jitter_never_leaves_before_it_arrived() { + // A sigma far past the delay, so most draws would be negative unclamped. + let wide = Profile { + delay: Duration::from_millis(1), + jitter: Duration::from_millis(50), + ..Default::default() + }; + let now = Instant::now(); + let (sent, counters) = departures(&wide, &GAUSSIAN, 1000, Duration::ZERO); + assert_eq!(sent.len(), 1000); + assert!(sent.iter().all(|&(_, at)| at >= now)); + assert!(counters.delayed > 0, "{counters}"); + } + + #[test] + fn gaussian_jitter_alone_never_changes_the_order() { + // Ten times the arrival spacing as sigma, so an independent draw per + // datagram would shuffle nearly all of them. + let jittery = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(50), + ..Default::default() + }; + let (sent, counters) = departures(&jittery, &GAUSSIAN, 2000, Duration::from_millis(5)); + let ids: Vec = sent.iter().map(|&(id, _)| id).collect(); + assert_eq!(ids, (0..2000).collect::>(), "gaussian jitter reordered"); + assert_eq!(counters.reordered, 0); + } + + #[test] + fn gaussian_jitter_still_varies_the_spacing() { + let spacing = Duration::from_millis(20); + let jittery = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(5), + ..Default::default() + }; + let (sent, _) = departures(&jittery, &GAUSSIAN, 2000, spacing); + + // Keeping the order is not pacing the path: a datagram that drew more + // than the one in front falls further behind, and one that drew less + // closes up against it. + let gaps: Vec = sent.windows(2).map(|pair| pair[1].1 - pair[0].1).collect(); + assert!(gaps.iter().any(|&gap| gap < spacing), "nothing closed up"); + assert!(gaps.iter().any(|&gap| gap > spacing), "nothing fell behind"); + } + + #[test] + fn a_reorder_still_overtakes_gaussian_jitter() { + let shuffled = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(50), + reorder: 0.05, + ..Default::default() + }; + let (sent, counters) = departures(&shuffled, &GAUSSIAN, 2000, Duration::from_millis(5)); + let overtaken = sent.windows(2).filter(|pair| pair[0].0 > pair[1].0).count(); + assert!(overtaken > 0, "a reorder never overtook"); + assert!(counters.reordered > 0, "{counters}"); + } + + #[test] + fn the_seed_reproduces_the_gaussian_decisions() { + let everything = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(5), + loss: 0.05, + reorder: 0.05, + ..Default::default() + }; + let (first, first_counters) = departures(&everything, &GAUSSIAN, 1000, Duration::from_millis(1)); + let (second, second_counters) = departures(&everything, &GAUSSIAN, 1000, Duration::from_millis(1)); + + // The same ids survive and leave at the same offsets from their start. + let offsets = |sent: &[(u32, Instant)]| { + let start = sent.iter().map(|&(_, at)| at).min().unwrap(); + sent.iter().map(|&(id, at)| (id, at - start)).collect::>() + }; + assert_eq!(offsets(&first), offsets(&second)); + assert_eq!(first_counters, second_counters); + assert!(first_counters.lost > 0 && first_counters.reordered > 0); + } + + #[tokio::test] + async fn a_gaussian_sigma_may_exceed_the_delay() { + // The uniform model refuses this; a gaussian clamps its draw at zero. + let wide = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(10), + ..Default::default() + }; + let (shaper, client) = shaped(|config| Setup { + up: GAUSSIAN, + ..Config { up: wide, ..config }.into() + }) + .await; + assert_eq!(round_trip(&client, 20).await, (0..20).collect::>()); + shaper.verify().unwrap(); + + // With no delay, `verify` could never tell the jitter acted. + let undelayed = Profile { + jitter: Duration::from_millis(5), + ..Default::default() + }; + let err = Shaper::bind(Setup { + up: GAUSSIAN, + ..Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: undelayed, + down: Profile::default(), + } + .into() + }) + .await + .err() + .expect("accepted gaussian jitter with no delay"); + assert!(format!("{err:#}").contains("needs a delay to vary"), "{err:#}"); + } + + /// Two clients take turns sending `count` numbered datagrams up the path + /// `setup` builds, and what reaches the target comes back, in order, with + /// how long after the first send it did. + async fn two_clients(setup: impl FnOnce(Config) -> Setup, count: u32) -> Vec<(u32, Duration)> { + let target = UdpSocket::bind(LOCALHOST).await.unwrap(); + let config = Config { + bind: LOCALHOST, + target: target.local_addr().unwrap(), + seed: 5, + up: Profile::default(), + down: Profile::default(), + }; + let shaper = Shaper::bind(setup(config)).await.unwrap(); + + let clients = [ + UdpSocket::bind(LOCALHOST).await.unwrap(), + UdpSocket::bind(LOCALHOST).await.unwrap(), + ]; + let start = Instant::now(); + for id in 0..count { + let client = &clients[id as usize % 2]; + client.send_to(&id.to_be_bytes(), shaper.addr()).await.unwrap(); + } + + let mut got = Vec::new(); + let mut buf = [0u8; 4]; + while let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(500), target.recv_from(&mut buf)).await { + got.push((u32::from_be_bytes(buf), start.elapsed())); + } + got + } + + #[tokio::test] + async fn a_shared_path_keeps_the_order_across_clients() { + let jittery = Profile { + delay: Duration::from_millis(20), + jitter: Duration::from_millis(10), + ..Default::default() + }; + let order = |shared: bool| { + let up = jittery.clone(); + async move { + let got = two_clients( + |config| Setup { + shared, + up: GAUSSIAN, + ..Config { up, ..config }.into() + }, + 200, + ) + .await; + got.into_iter().map(|(id, _)| id).collect::>() + } + }; + + // Separate links keep each client's order, but not the order between them. + let apart = order(false).await; + let mut sorted = apart.clone(); + sorted.sort(); + assert_eq!(sorted, (0..200).collect::>(), "a client lost datagrams"); + assert_ne!( + apart, sorted, + "separate links kept the order anyway, so this proves nothing" + ); + for parity in 0..2 { + assert!(apart.iter().filter(|&&id| id % 2 == parity).is_sorted()); + } + + assert_eq!( + order(true).await, + (0..200).collect::>(), + "a shared path reordered" + ); + } + + /// Send `count` numbered datagrams `spacing` apart, and collect what echoes + /// back with when it did, alongside when each was sent. + async fn paced(client: &UdpSocket, count: u32, spacing: Duration) -> (Vec, Vec<(u32, Instant)>) { + let send = async { + let mut sent = Vec::new(); + for id in 0..count { + sent.push(Instant::now()); + client.send(&id.to_be_bytes()).await.unwrap(); + tokio::time::sleep(spacing).await; + } + sent + }; + let receive = async { + let mut got = Vec::new(); + let mut buf = [0u8; 4]; + while let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(500), client.recv(&mut buf)).await { + got.push((u32::from_be_bytes(buf), Instant::now())); + } + got + }; + tokio::join!(send, receive) + } + + fn batched(count: usize, window: Duration) -> Options { + Options { + batch: Some(Batch { count, window }), + ..Default::default() + } + } + + #[tokio::test] + async fn a_batch_releases_on_count_and_on_the_window() { + let socket = Arc::new(UdpSocket::bind(LOCALHOST).await.unwrap()); + let ms = |ms| Duration::from_millis(ms); + let now = Instant::now(); + let parcel = |at: Instant| Parcel { + arrived: at, + at, + delayed: false, + datagram: Vec::new(), + socket: socket.clone(), + dest: LOCALHOST, + }; + let batch = Batch { + count: 3, + window: ms(160), + }; + let tally = Tally::default(); + let mut held = Held::default(); + let mut ready = Vec::new(); + + held.hold(batch, parcel(now), &mut ready, &tally); + held.hold(batch, parcel(now + ms(10)), &mut ready, &tally); + assert!(ready.is_empty(), "a partial batch leaked"); + assert_eq!( + held.closes, + Some(now + ms(160)), + "the window counts from the first arrival" + ); + + held.hold(batch, parcel(now + ms(20)), &mut ready, &tally); + let leaves: Vec = ready.drain(..).map(|parcel| parcel.at).collect(); + assert_eq!( + leaves, + [now + ms(20); 3], + "a full batch leaves together, as the latest would" + ); + assert_eq!( + tally.snapshot().delayed, + 2, + "the datagram that filled it waited for nothing" + ); + + // The window closes a batch that never fills, and nothing in it leaves sooner. + held.hold(batch, parcel(now + ms(30)), &mut ready, &tally); + let closes = held.closes.unwrap(); + assert_eq!(closes, now + ms(190)); + held.release(closes, &mut ready, &tally); + assert_eq!(ready.iter().map(|parcel| parcel.at).collect::>(), [closes]); + assert_eq!(tally.snapshot().delayed, 3); + ready.clear(); + + // A datagram that arrives after the window closed, before its timer fired, + // starts the next batch rather than joining the closed one. + held.hold(batch, parcel(now + ms(200)), &mut ready, &tally); + let closes = held.closes.unwrap(); + held.hold(batch, parcel(closes + ms(1)), &mut ready, &tally); + assert_eq!(ready.iter().map(|parcel| parcel.at).collect::>(), [closes]); + assert_eq!(held.parcels.len(), 1, "the late datagram joined the closed batch"); + assert_eq!(held.closes, Some(closes + ms(1) + ms(160))); + assert_eq!(tally.snapshot().delayed, 4); + assert_eq!(tally.batched.load(Ordering::Relaxed), 4); + + // Behind a delay longer than the window, a lone datagram leaves when the + // delay says: the batch held nothing, whatever `delayed` counts. + ready.clear(); + let mut held = Held::default(); + let tally = Tally::default(); + let late = Parcel { + at: now + ms(1000), + delayed: true, + ..parcel(now) + }; + held.hold(batch, late, &mut ready, &tally); + held.release(held.closes.unwrap(), &mut ready, &tally); + assert_eq!(ready[0].at, now + ms(1000)); + assert_eq!(tally.batched.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn a_batch_releases_datagrams_together() { + let (shaper, client) = shaped(|config| Setup { + up: batched(7, Duration::from_millis(160)), + ..config.into() + }) + .await; + + // Three full batches, each filled well inside its window. + let (_, got) = paced(&client, 21, Duration::from_millis(10)).await; + assert_eq!( + got.iter().map(|&(id, _)| id).collect::>(), + (0..21).collect::>() + ); + + let batches: Vec<&[(u32, Instant)]> = got.chunks(7).collect(); + for batch in &batches { + let spread = batch[6].1 - batch[0].1; + assert!( + spread < Duration::from_millis(15), + "a batch arrived spread over {spread:?}" + ); + } + for pair in batches.windows(2) { + let gap = pair[1][0].1 - pair[0][0].1; + assert!(gap > Duration::from_millis(40), "two batches arrived {gap:?} apart"); + } + + let stats = shaper.verify().unwrap(); + assert_eq!( + stats.up.delayed, 18, + "every datagram but the last of each batch waits: {stats}" + ); + } + + #[tokio::test] + async fn the_window_releases_a_batch_that_never_fills() { + let (shaper, client) = shaped(|config| Setup { + up: batched(100, Duration::from_millis(80)), + ..config.into() + }) + .await; + + let (sent, got) = paced(&client, 3, Duration::ZERO).await; + assert_eq!(got.len(), 3); + let held = got[0].1 - sent[0]; + assert!(held >= Duration::from_millis(80), "the batch left after only {held:?}"); + assert_eq!(shaper.verify().unwrap().up.delayed, 3); + } + + #[tokio::test] + async fn a_shared_batch_fills_from_every_client() { + // Three datagrams from each of two clients, into batches of six: only a + // shared path fills one, and separate links wait out the window. + let window = Duration::from_millis(300); + for shared in [true, false] { + let got = two_clients( + |config| Setup { + shared, + up: batched(6, window), + ..config.into() + }, + 6, + ) + .await; + assert_eq!(got.len(), 6); + let last = got.iter().map(|&(_, after)| after).max().unwrap(); + assert_eq!( + last < window / 2, + shared, + "shared {shared}: the batch left after {last:?}" + ); + } + } + + #[tokio::test] + async fn a_batch_needs_a_count_and_a_window() { + let refused = |options: Options, why: &'static str| async move { + let config = Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: Profile::default(), + down: Profile::default(), + }; + let err = Shaper::bind(Setup { + up: options, + ..config.into() + }) + .await + .err() + .unwrap_or_else(|| panic!("accepted a batch where {why}")); + assert!(format!("{err:#}").contains(why), "{err:#}"); + }; + refused(batched(0, Duration::from_millis(10)), "a batch of 0 never holds").await; + refused(batched(1, Duration::from_millis(10)), "a batch of 1 never holds").await; + refused(batched(7, Duration::ZERO), "no window never holds").await; + } + + #[test] + fn a_batch_that_never_held_anything_is_unapplied() { + let config = Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: Profile::default(), + down: Profile::default(), + }; + let setup = Setup { + up: batched(7, Duration::from_millis(160)), + ..config.into() + }; + let quiet = |packets| Stats { + up: Counters { + packets, + ..Default::default() + }, + ..Default::default() + }; + + // (1/7)^2 is 2%: two datagrams through batches of seven prove nothing. + assert!(!unbatched(&setup, &quiet(2), 0)); + // (1/7)^10 is 4e-9: ten datagrams a batch never held means no batch. + assert!(unbatched(&setup, &quiet(10), 0)); + assert!(!unbatched(&setup, &quiet(10), 6)); + + // A delay the profile gave is not a batch holding anything. + let mut delayed = quiet(10); + delayed.up.delayed = 10; + assert!(unbatched(&setup, &delayed, 0)); + } + + /// How long after `at` each of `sizes` leaves one link, fed at the same instant. + fn owed(link: &mut Link, at: Instant, sizes: &[usize]) -> Vec { + let tally = Tally::default(); + sizes + .iter() + .map(|&size| link.treat(at, size, &tally).expect("dropped").0 - at) + .collect() + } + + fn stepped(steps: Vec) -> Options { + Options { + steps, + ..Default::default() + } + } + + #[test] + fn each_step_applies_at_its_own_time() { + let ms = Duration::from_millis; + let profile = Profile { + delay: ms(5), + ..Default::default() + }; + let options = stepped(vec![ + Step { + at: ms(100), + delay: Some(ms(60)), + ..Default::default() + }, + Step { + at: ms(200), + delay: Some(ms(5)), + ..Default::default() + }, + ]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&profile, &options, 7, 0, start, queue); + + assert_eq!(owed(&mut link, start, &[16]), [ms(5)]); + assert_eq!(owed(&mut link, start + ms(100), &[16]), [ms(60)]); + // The step back restores what the run opened with. + assert_eq!(owed(&mut link, start + ms(200), &[16]), [ms(5)]); + } + + #[test] + fn a_loss_step_turns_the_loss_on_and_off_again() { + let secs = Duration::from_secs; + let options = stepped(vec![ + Step { + at: secs(30), + loss: Some(1.0), + ..Default::default() + }, + Step { + at: secs(60), + loss: Some(0.0), + ..Default::default() + }, + ]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&Profile::default(), &options, 7, 0, start, queue); + let tally = Tally::default(); + let mut lost = |at: Instant| (0..100).filter(|_| link.treat(at, 16, &tally).is_none()).count(); + + assert_eq!(lost(start), 0, "the profile opens clean"); + assert_eq!(lost(start + secs(30)), 100, "the step lost nothing"); + assert_eq!(lost(start + secs(60)), 0, "the step back never cleared"); + } + + #[tokio::test] + async fn a_step_gets_worse_part_way_through() { + let ms = Duration::from_millis; + let (shaper, client) = shaped(|config| Setup { + up: stepped(vec![Step { + at: ms(150), + delay: Some(ms(60)), + ..Default::default() + }]), + ..Config { + up: Profile { + delay: ms(5), + ..Default::default() + }, + ..config + } + .into() + }) + .await; + + let (sent, got) = paced(&client, 40, ms(10)).await; + assert_eq!(got.len(), 40); + let median = |range: std::ops::Range| { + let mut latency: Vec = got[range].iter().map(|&(id, at)| at - sent[id as usize]).collect(); + latency.sort(); + latency[latency.len() / 2] + }; + + // The first ten leave well inside the first 150ms, the last fifteen well after. + let (before, after) = (median(0..10), median(25..40)); + assert!( + after > before + ms(30), + "median latency went from {before:?} to {after:?}" + ); + shaper.verify().unwrap(); + } + + #[tokio::test] + async fn a_step_the_run_never_reached_is_unapplied() { + let (shaper, client) = shaped(|config| Setup { + down: stepped(vec![Step { + at: Duration::from_secs(60), + delay: Some(Duration::from_millis(60)), + ..Default::default() + }]), + ..config.into() + }) + .await; + assert_eq!(round_trip(&client, 10).await, (0..10).collect::>()); + + let err = shaper.verify().expect_err("a run that ended before its step passed"); + assert!(format!("{err:#}").contains("the down step at 60s"), "{err:#}"); + } + + // Real sockets, so the wall clock: a paused one could jump past a step while a datagram is still in the kernel. + #[tokio::test] + async fn a_step_no_datagram_saw_is_unapplied() { + let ms = Duration::from_millis; + let (shaper, client) = shaped(|config| Setup { + up: stepped(vec![ + Step { + at: ms(100), + delay: Some(ms(60)), + ..Default::default() + }, + Step { + at: ms(200), + delay: Some(ms(5)), + ..Default::default() + }, + ]), + ..Config { + up: Profile { + delay: ms(5), + ..Default::default() + }, + ..config + } + .into() + }) + .await; + + // One datagram before the first step and one after the second, so the + // 60ms phase passes with nothing in it. + let mut buf = [0u8; 4]; + client.send(&0u32.to_be_bytes()).await.unwrap(); + client.recv(&mut buf).await.unwrap(); + tokio::time::sleep(ms(250)).await; + client.send(&1u32.to_be_bytes()).await.unwrap(); + client.recv(&mut buf).await.unwrap(); + + let err = shaper.verify().expect_err("a run that skipped a step passed"); + let err = format!("{err:#}"); + assert!(err.contains("the up step at 100ms"), "{err}"); + assert!(!err.contains("the up step at 200ms"), "{err}"); + } + + #[tokio::test] + async fn a_profile_no_datagram_saw_before_its_step_is_unapplied() { + let ms = Duration::from_millis; + let (shaper, client) = shaped(|config| Setup { + up: stepped(vec![Step { + at: ms(100), + delay: Some(ms(60)), + ..Default::default() + }]), + ..Config { + up: Profile { + delay: ms(5), + ..Default::default() + }, + ..config + } + .into() + }) + .await; + + // The first datagram comes after the step, so the 5ms phase passes with nothing in it. + tokio::time::sleep(ms(250)).await; + let mut buf = [0u8; 4]; + client.send(&0u32.to_be_bytes()).await.unwrap(); + client.recv(&mut buf).await.unwrap(); + + let err = shaper.verify().expect_err("a run that started after its step passed"); + let err = format!("{err:#}"); + assert!(err.contains("the up profile before its step at 100ms"), "{err}"); + assert!(!err.contains("the up step at"), "{err}"); + } + + #[test] + fn a_step_past_the_clock_never_applies() { + let options = stepped(vec![Step { + at: Duration::MAX, + delay: Some(Duration::from_millis(60)), + ..Default::default() + }]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&Profile::default(), &options, 7, 0, start, queue); + assert_eq!(owed(&mut link, start + Duration::from_secs(1), &[16]), [Duration::ZERO]); + } + + #[tokio::test] + async fn steps_that_would_be_skipped_or_do_nothing_are_refused() { + let refused = |steps: Vec, why: &'static str| async move { + let config = Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: Profile { + delay: Duration::from_millis(20), + ..Default::default() + }, + down: Profile::default(), + }; + let err = Shaper::bind(Setup { + down: stepped(steps), + ..config.into() + }) + .await + .err() + .unwrap_or_else(|| panic!("accepted steps where {why}")); + assert!(format!("{err:#}").contains(why), "{err:#}"); + }; + let at = |secs| Step { + at: Duration::from_secs(secs), + loss: Some(0.1), + ..Default::default() + }; + + refused(vec![at(60), at(30)], "steps go in order").await; + refused(vec![at(0)], "a step at zero").await; + refused( + vec![Step { + at: Duration::from_secs(30), + ..Default::default() + }], + "changes nothing", + ) + .await; + // Restating the value already in force changes nothing either. + refused( + vec![ + at(30), + Step { + at: Duration::from_secs(60), + loss: Some(0.1), + ..Default::default() + }, + ], + "the step at 60s changes nothing", + ) + .await; + // Uniform jitter past the delay, reached by a step rather than at the start. + refused( + vec![Step { + at: Duration::from_secs(30), + jitter: Some(Duration::from_millis(10)), + ..Default::default() + }], + "exceeds delay", + ) + .await; + } } diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index 2a2841e1010..c16eb30578c 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -1,10 +1,13 @@ //! Put a seeded, impaired UDP path in front of a server. //! -//! Both directions get the same profile. The seed and profile print at start, -//! and the counters at exit, which fails if the profile never acted. +//! Both directions get the same profile from the flags, unless `--profile` +//! names a profile file, which sets each direction on its own. The seed and +//! profile print at start, and the counters at exit, which fails if the profile +//! never acted. -use std::{net::SocketAddr, time::Duration}; +use std::{net::SocketAddr, path::PathBuf, time::Duration}; +use anyhow::Context; use clap::Parser; #[derive(Parser)] @@ -16,15 +19,28 @@ struct Args { /// The address every datagram is forwarded to. #[arg(long)] target: SocketAddr, - /// Seeds every treatment decision; random when omitted. + /// Seeds every treatment decision; the profile's own, or random, when omitted. #[arg(long)] seed: Option, + /// A built-in profile's name, or a profile TOML file, in place of the flags + /// that shape the path. + #[arg( + long, + conflicts_with_all = [ + "delay", "jitter", "jitter_model", "loss", "reorder", "rate", "burst", "queue", "batch", "batch_window", + "shared", + ], + )] + profile: Option, /// The base one-way delay, e.g. `20ms`. #[arg(long, default_value = "0s", value_parser = humantime::parse_duration)] delay: Duration, - /// The most the delay varies, uniformly either way. + /// How far the delay varies: the most either way, or the sigma of a gaussian. #[arg(long, default_value = "0s", value_parser = humantime::parse_duration)] jitter: Duration, + /// How the jitter is drawn. + #[arg(long, value_enum, default_value_t = JitterModel::Uniform)] + jitter_model: JitterModel, /// The probability that a datagram is dropped. #[arg(long, default_value_t = 0.0)] loss: f64, @@ -40,12 +56,103 @@ struct Args { /// The longest a datagram waits for the rate limit before it is dropped. #[arg(long, default_value = "100ms", value_parser = humantime::parse_duration, requires = "rate")] queue: Duration, + /// Hold datagrams until this many are waiting, then release them together. + #[arg(long, requires = "batch_window")] + batch: Option, + /// The longest a batch waits to fill before it leaves anyway. + #[arg(long, value_parser = humantime::parse_duration, requires = "batch")] + batch_window: Option, + /// Every client shares one link each way, instead of each getting its own. + #[arg(long)] + shared: bool, + /// Write the profile, seed and counters to this file as JSON at exit. + #[arg(long)] + report: Option, + /// Print the same JSON as one line on stdout this often. + #[arg(long, value_parser = humantime::parse_duration)] + report_interval: Option, +} + +#[derive(Clone, Copy, clap::ValueEnum)] +enum JitterModel { + /// Uniform either way of the delay, per datagram, so datagrams can overtake each other. + Uniform, + /// A gaussian clamped at zero that never lets a datagram overtake the one in front. + Gaussian, } #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); + anyhow::ensure!( + args.report_interval.is_none_or(|period| !period.is_zero()), + "the report interval must be positive" + ); + + let (name, setup) = match &args.profile { + Some(profile) => { + let preset = moq_shaper::Preset::load(profile)?; + let mut setup = preset.setup(args.listen, args.target); + if let Some(seed) = args.seed { + setup.config.seed = seed; + } + (Some(preset.name), setup) + } + None => (None, flags(&args)), + }; + + let treatment = match &name { + Some(name) => format!("profile {name}"), + None if setup.shared || setup.up != moq_shaper::Options::default() => format!( + "profile {:?}, options {:?}, shared {}", + setup.config.up, setup.up, setup.shared + ), + None => format!("profile {:?}", setup.config.up), + }; + let shaper = moq_shaper::Shaper::bind(setup).await?; + let config = shaper.config(); + println!( + "shaper: {} -> {}, seed {}, {treatment}", + shaper.addr(), + config.target, + config.seed + ); + + let report = || Report { + profile: name.as_deref(), + seed: config.seed, + stats: shaper.stats(), + }; + + // The first tick is a full period in, so a line reports traffic, not the start. + let mut interval = args + .report_interval + .map(|period| tokio::time::interval_at(tokio::time::Instant::now() + period, period)); + let shutdown = shutdown(); + tokio::pin!(shutdown); + loop { + tokio::select! { + res = &mut shutdown => { + res?; + break; + } + _ = tick(&mut interval) => println!("{}", serde_json::to_string(&report())?), + } + } + + // Before the verdict, so a run that fails it still leaves its counters behind. + if let Some(path) = &args.report { + std::fs::write(path, serde_json::to_vec(&report())?).with_context(|| format!("write {}", path.display()))?; + } + + let stats = shaper.verify()?; + println!("shaper: {stats}"); + Ok(()) +} + +/// The setup the flags describe: one profile and one set of options, both ways. +fn flags(args: &Args) -> moq_shaper::Setup { let profile = moq_shaper::Profile { delay: args.delay, jitter: args.jitter, @@ -57,29 +164,50 @@ async fn main() -> anyhow::Result<()> { queue: args.queue, }), }; - let shaper = moq_shaper::Shaper::bind(moq_shaper::Config { + let config = moq_shaper::Config { bind: args.listen, target: args.target, seed: args.seed.unwrap_or_else(rand::random), up: profile.clone(), down: profile, - }) - .await?; - - let config = shaper.config(); - println!( - "shaper: {} -> {}, seed {}, profile {:?}", - shaper.addr(), - config.target, - config.seed, - config.up - ); + }; + let options = moq_shaper::Options { + jitter_model: match args.jitter_model { + JitterModel::Uniform => moq_shaper::Jitter::Uniform, + JitterModel::Gaussian => moq_shaper::Jitter::Gaussian, + }, + batch: args + .batch + .zip(args.batch_window) + .map(|(count, window)| moq_shaper::Batch { count, window }), + ..Default::default() + }; + moq_shaper::Setup { + shared: args.shared, + up: options.clone(), + down: options, + ..config.into() + } +} - shutdown().await?; +/// What a run did, as the JSON a harness reads back. +#[derive(serde::Serialize)] +struct Report<'a> { + /// The profile's name, or null when the flags built it. + profile: Option<&'a str>, + seed: u64, + #[serde(flatten)] + stats: moq_shaper::Stats, +} - let stats = shaper.verify()?; - println!("shaper: {stats}"); - Ok(()) +/// The next report tick, or never without an interval. +async fn tick(interval: &mut Option) { + match interval { + Some(interval) => { + interval.tick().await; + } + None => std::future::pending().await, + } } /// Wait for Ctrl-C, or SIGTERM where there is one, which is how a harness stops a child. diff --git a/rs/moq-shaper/src/preset.rs b/rs/moq-shaper/src/preset.rs new file mode 100644 index 00000000000..e8cd8095a9c --- /dev/null +++ b/rs/moq-shaper/src/preset.rs @@ -0,0 +1,248 @@ +use std::{net::SocketAddr, time::Duration}; + +use anyhow::Context; +use serde::Deserialize; + +use crate::{Batch, Config, Jitter, Options, Profile, Rate, Setup, Step}; + +/// The profiles shipped under `profiles/`, embedded so a binary needs no data files. +const BUILTIN: &[(&str, &str)] = &[ + ("near-zero", include_str!("../profiles/near-zero.toml")), + ("mild", include_str!("../profiles/mild.toml")), + ("bursty", include_str!("../profiles/bursty.toml")), + ("step", include_str!("../profiles/step.toml")), +]; + +/// A named, seeded treatment of both directions, loaded from a TOML profile. +#[derive(Clone, Debug)] +pub struct Preset { + /// The name a run records its counters under. + pub name: String, + seed: u64, + shared: bool, + up: (Profile, Options), + down: (Profile, Options), +} + +impl Preset { + /// Load a built-in profile by name, or a profile file by path. + pub fn load(name_or_path: &str) -> anyhow::Result { + let text = match BUILTIN.iter().find(|(name, _)| *name == name_or_path) { + Some((_, text)) => (*text).to_string(), + // An unknown name is almost always a typo, so say what the choices were. + None => std::fs::read_to_string(name_or_path).with_context(|| { + format!( + "no built-in profile or readable file named {name_or_path:?}; the built-ins are {}", + Self::names().collect::>().join(", ") + ) + })?, + }; + + let file: File = toml::from_str(&text).with_context(|| format!("parse profile {name_or_path:?}"))?; + file.preset() + .with_context(|| format!("invalid profile {name_or_path:?}")) + } + + /// The built-in profiles' names. + pub fn names() -> impl Iterator { + BUILTIN.iter().map(|(name, _)| *name) + } + + /// This preset listening on `bind` and forwarding to `target`, with its own seed. + pub fn setup(&self, bind: SocketAddr, target: SocketAddr) -> Setup { + Setup { + config: Config { + bind, + target, + seed: self.seed, + up: self.up.0.clone(), + down: self.down.0.clone(), + }, + shared: self.shared, + up: self.up.1.clone(), + down: self.down.1.clone(), + } + } +} + +/// A profile file. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct File { + name: String, + seed: u64, + #[serde(default)] + shared: bool, + #[serde(default)] + up: Side, + #[serde(default)] + down: Side, +} + +impl File { + fn preset(self) -> anyhow::Result { + anyhow::ensure!(!self.name.is_empty(), "the profile has no name"); + Ok(Preset { + up: self.up.split().context("up")?, + down: self.down.split().context("down")?, + name: self.name, + seed: self.seed, + shared: self.shared, + }) + } +} + +/// One direction of a profile file: its [`Profile`] and [`Options`] in one table. +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct Side { + #[serde(with = "humantime_serde")] + delay: Duration, + #[serde(with = "humantime_serde")] + jitter: Duration, + jitter_model: Option, + loss: f64, + reorder: f64, + rate: Option, + batch: Option, + steps: Vec, +} + +impl Side { + fn split(self) -> anyhow::Result<(Profile, Options)> { + // Whether datagrams can overtake each other depends on the model as soon + // as their delays differ, so a file has to say which it means rather + // than inherit whichever is the default. + let varies = !self.jitter.is_zero() + || self + .steps + .iter() + .any(|step| step.delay.is_some() || step.jitter.is_some_and(|jitter| !jitter.is_zero())); + anyhow::ensure!( + self.jitter_model.is_some() || !varies, + "the delay varies, so name a jitter_model: \"uniform\" lets a datagram overtake the one in front, \"gaussian\" never does" + ); + + let profile = Profile { + delay: self.delay, + jitter: self.jitter, + loss: self.loss, + reorder: self.reorder, + rate: self.rate, + }; + let options = Options { + jitter_model: self.jitter_model.unwrap_or_default(), + batch: self.batch, + steps: self.steps, + }; + options.validate(&profile)?; + Ok((profile, options)) + } +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr}; + + use super::*; + + const LOCALHOST: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0); + + #[test] + fn builtins_parse_and_are_named_after_their_file() { + for name in Preset::names() { + let preset = Preset::load(name).unwrap_or_else(|err| panic!("{name}: {err:#}")); + assert_eq!(preset.name, name); + } + } + + #[test] + fn builtins_treat_the_path_except_the_control() { + for name in Preset::names() { + let preset = Preset::load(name).unwrap(); + let untreated = (Profile::default(), Options::default()); + let idle = preset.up == untreated && preset.down == untreated; + assert_eq!(idle, name == "near-zero", "{name} treats the wrong amount of traffic"); + } + } + + #[test] + fn the_step_builtin_steps_both_directions_at_thirty_seconds() { + let setup = Preset::load("step").unwrap().setup(LOCALHOST, LOCALHOST); + for (which, profile, options) in [ + ("up", &setup.config.up, &setup.up), + ("down", &setup.config.down, &setup.down), + ] { + assert_eq!(profile.delay, Duration::from_millis(5), "{which}"); + assert_eq!(options.jitter_model, Jitter::Gaussian, "{which}"); + assert_eq!(options.steps.len(), 1, "{which}"); + assert_eq!(options.steps[0].at, Duration::from_secs(30), "{which}"); + assert_eq!(options.steps[0].delay, Some(Duration::from_millis(60)), "{which}"); + } + } + + #[test] + fn an_unknown_name_lists_the_builtins() { + let err = format!("{:#}", Preset::load("nope").unwrap_err()); + assert!(err.contains("near-zero"), "{err}"); + } + + #[test] + fn an_unknown_field_is_refused() { + // A field the schema does not know, like a rate in bytes, is refused rather than ignored. + let Err(err) = toml::from_str::("name = \"x\"\nseed = 1\n[up.rate]\nbytes_per_second = 1000\n") else { + panic!("a rate in bytes parsed"); + }; + assert!(err.to_string().contains("bytes_per_second"), "{err}"); + } + + #[test] + fn a_varying_delay_has_to_name_its_model() { + let parse = |text: &str| toml::from_str::(text).unwrap().preset(); + + let err = parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\njitter = \"5ms\"\n").unwrap_err(); + assert!(format!("{err:#}").contains("jitter_model"), "{err:#}"); + + let err = parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\n[[up.steps]]\nat = \"1s\"\ndelay = \"9ms\"\n") + .unwrap_err(); + assert!(format!("{err:#}").contains("jitter_model"), "{err:#}"); + + // A fixed delay cannot reorder anything, so it needs no model. + parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\n").unwrap(); + parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\njitter = \"5ms\"\njitter_model = \"uniform\"\n").unwrap(); + } + + #[test] + fn a_profile_file_loads_by_path() { + let path = std::env::temp_dir().join(format!("moq-shaper-preset-{}.toml", std::process::id())); + std::fs::write( + &path, + "name = \"file\"\nseed = 9\n[down.batch]\ncount = 3\nwindow = \"50ms\"\n", + ) + .unwrap(); + let preset = Preset::load(path.to_str().unwrap()); + std::fs::remove_file(&path).unwrap(); + + let setup = preset.unwrap().setup(LOCALHOST, LOCALHOST); + assert_eq!(setup.config.seed, 9); + assert_eq!( + setup.down.batch, + Some(Batch { + count: 3, + window: Duration::from_millis(50) + }) + ); + } + + #[tokio::test] + async fn the_builtins_all_bind() { + for name in Preset::names() { + // Binding proves the profile is usable, not just parseable. + let preset = Preset::load(name).unwrap(); + let shaper = crate::Shaper::bind(preset.setup(LOCALHOST, LOCALHOST)) + .await + .unwrap_or_else(|err| panic!("{name}: {err:#}")); + shaper.verify().unwrap(); + } + } +} diff --git a/rs/moq-shaper/tests/cli.rs b/rs/moq-shaper/tests/cli.rs new file mode 100644 index 00000000000..8c15351647c --- /dev/null +++ b/rs/moq-shaper/tests/cli.rs @@ -0,0 +1,87 @@ +//! The binary end to end: a profile by name and the +//! counters it reports while it runs and when it stops. +#![cfg(unix)] + +use std::{ + io::{BufRead, BufReader}, + net::{SocketAddr, UdpSocket}, + process::{Command, Stdio}, + time::Duration, +}; + +#[test] +fn a_profile_run_reports_its_counters() { + let target = UdpSocket::bind("127.0.0.1:0").unwrap(); + let target_addr = target.local_addr().unwrap(); + + let report = std::env::temp_dir().join(format!("moq-shaper-report-{}.json", std::process::id())); + let mut shaper = Command::new(env!("CARGO_BIN_EXE_moq-shaper")) + .args(["--listen", "127.0.0.1:0", "--target", &target_addr.to_string()]) + .args(["--profile", "mild", "--seed", "7"]) + .args(["--report-interval", "100ms", "--report", report.to_str().unwrap()]) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdout = BufReader::new(shaper.stdout.take().unwrap()); + + // The first line says where it listens, and under what. + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(line.ends_with("seed 7, profile mild\n"), "{line}"); + let addr: SocketAddr = line.split_whitespace().nth(1).unwrap().parse().unwrap(); + + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + for id in 0..50u32 { + client.send_to(&id.to_be_bytes(), addr).unwrap(); + } + target.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut buf = [0u8; 4]; + for _ in 0..50 { + target.recv_from(&mut buf).expect("a datagram never arrived"); + } + + // A line each interval, once the traffic has been counted. + let interim = loop { + line.clear(); + stdout.read_line(&mut line).unwrap(); + let json: serde_json::Value = serde_json::from_str(&line).unwrap(); + if json["up"]["packets"] == 50 { + break json; + } + }; + assert_eq!(interim["profile"], "mild"); + + let status = Command::new("kill") + .args(["-TERM", &shaper.id().to_string()]) + .status() + .unwrap(); + assert!(status.success()); + assert!(shaper.wait().unwrap().success(), "the profile never acted"); + + let written = std::fs::read(&report).expect("no report at exit"); + std::fs::remove_file(&report).unwrap(); + let json: serde_json::Value = serde_json::from_slice(&written).unwrap(); + assert_eq!(json["profile"], "mild"); + assert_eq!(json["seed"], 7); + assert_eq!(json["up"]["packets"], 50); + assert_eq!(json["up"]["delayed"], 50); + assert_eq!(json["up"]["lost"], 0); +} + +#[test] +fn a_zero_report_interval_is_refused_before_binding() { + let result = Command::new(env!("CARGO_BIN_EXE_moq-shaper")) + .args([ + "--listen", + "127.0.0.1:0", + "--target", + "127.0.0.1:9", + "--report-interval", + "0s", + ]) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(result.stdout.is_empty(), "bound before refusing the interval"); + assert!(String::from_utf8_lossy(&result.stderr).contains("report interval must be positive")); +} diff --git a/rs/moq-tokio/src/client.rs b/rs/moq-tokio/src/client.rs index 686fd6c6df7..d7c517d2582 100644 --- a/rs/moq-tokio/src/client.rs +++ b/rs/moq-tokio/src/client.rs @@ -62,6 +62,9 @@ pub struct Client { pub(crate) reconnect: bool, pub(crate) backoff: Backoff, pub(crate) goaway: Goaway, + /// Whether the TLS config pins a certificate fingerprint, which only verifies + /// the configured host, so a GOAWAY may not redirect elsewhere. + pub(crate) pinned: bool, /// The resolved Happy Eyeballs timings, used by the `tcp://` dial here; the /// QUIC backend captures its own copy from the config. #[cfg(feature = "tcp")] @@ -146,6 +149,7 @@ impl Client { reconnect: !config.once.unwrap_or(false), backoff: config.backoff, goaway: config.goaway, + pinned: !config.tls.fingerprint.is_empty(), #[cfg(feature = "tcp")] failover_delay, #[cfg(feature = "tcp")] diff --git a/rs/moq-tokio/src/connection.rs b/rs/moq-tokio/src/connection.rs index 43358538915..1e9178c819e 100644 --- a/rs/moq-tokio/src/connection.rs +++ b/rs/moq-tokio/src/connection.rs @@ -209,27 +209,42 @@ pub enum Redirect { impl Redirect { /// Resolve the URL to dial after a GOAWAY, falling back to `current` when the /// redirect is empty ("reconnect to me"), malformed, or refused by policy. + /// + /// Lenient on purpose, for a caller that only wants somewhere to dial. A + /// [`Connection`] is stricter: it ends with [`Error::RefusedRedirect`] rather + /// than redialing after a malformed or refused URI. pub fn resolve(&self, uri: &str, current: &Url) -> Url { - self.target(uri, current).unwrap_or_else(|| current.clone()) + self.target(uri, current, false) + .ok() + .flatten() + .unwrap_or_else(|| current.clone()) } - // Absence keeps the caller's address list; only an accepted URI replaces it. - fn target(&self, uri: &str, current: &Url) -> Option { - if uri.is_empty() || matches!(self, Self::Ignore) { - return None; + /// The URL a GOAWAY assigns. `Ok(None)` keeps the current address list (the + /// peer named no URI, or the policy ignores a URI it could parse), `Ok(Some)` + /// replaces it, and `Err` is an explicit URI this policy refuses. A malformed + /// URI is refused even under [`Self::Ignore`]. + /// + /// `pinned` is a certificate pin on the connection, which can only verify the + /// host it was configured for, so it refuses a host change even under + /// [`Self::Follow`]. + fn target(&self, uri: &str, current: &Url, pinned: bool) -> crate::Result> { + if uri.is_empty() { + return Ok(None); } - let Ok(target) = uri.parse::() else { - tracing::warn!(uri, "malformed GOAWAY URI; keeping the current addresses"); - return None; - }; + // The URI can carry credentials, so the error names the reason, never the URI. + // Parse before `Ignore`: a malformed redirect is terminal even when the policy + // would otherwise stay on the current address list. + let refuse = |reason: &str| Error::RefusedRedirect(reason.to_string()); + + let target = uri.parse::().map_err(|_| refuse("the GOAWAY URI is malformed"))?; + if matches!(self, Self::Ignore) { + return Ok(None); + } if scheme_tier(target.scheme()) < scheme_tier(current.scheme()) { - tracing::warn!( - uri, - "GOAWAY redirect downgrades the scheme; keeping the current addresses" - ); - return None; + return Err(refuse("the GOAWAY redirect downgrades the scheme")); } // Only as far as the URL itself says: a name is dialed, never resolved here, @@ -237,27 +252,29 @@ impl Redirect { // nothing about one that hides the same address behind a hostname. That gap // is why [`Self::SameHost`] is the default; see [`is_local`]. if is_local(&target) && !is_local(current) { - tracing::warn!( - uri, - "GOAWAY redirect widens reachability; keeping the current addresses" - ); - return None; + return Err(refuse("the GOAWAY redirect widens reachability to a local address")); } // Host only, not the full authority: the port is what a peer legitimately // moves us across when it hands off to a sibling process on the same box. - if matches!(self, Self::SameHost) && target.host_str() != current.host_str() { - tracing::warn!( - uri, - "GOAWAY redirect leaves the current host; keeping the current addresses" - ); - return None; + let same_host = target.host_str() == current.host_str(); + if matches!(self, Self::SameHost) && !same_host { + return Err(refuse("the GOAWAY redirect leaves the current host")); + } + if pinned && !same_host { + return Err(refuse("the GOAWAY redirect leaves the host a certificate pin verifies")); } - Some(target) + Ok(Some(target)) } } +/// Whether this scheme's dial installs the Rustls verifier, so a configured +/// fingerprint actually checked the peer. Plain and non-Rustls transports do not. +fn fingerprint_pins(scheme: &str) -> bool { + matches!(scheme, "https" | "wss" | "moqt" | "moql") +} + /// Rank a scheme so a peer-supplied redirect cannot silently drop encryption. /// Unknown schemes rank lowest, so a forgotten classification is refused. fn scheme_tier(scheme: &str) -> u8 { @@ -806,11 +823,20 @@ impl Connection { // ended sooner counts as a failed attempt however it ended. let healthy = connected.elapsed() >= initial; - // The connected target owns the policy, including in one-shot mode. - if let Ended::Goaway(msg) = &ended - && addr.addresses().is_some() - && goaway.redirect.target(msg.uri(), &url).is_some() - { + // The connected target owns the policy, including in one-shot mode. A + // refused redirect is terminal: the peer is leaving and named somewhere we + // won't go, so redialing the old address or a fallback would ignore it. + let assigned = match &ended { + // A fingerprint only checked the dial that installed the Rustls + // verifier. tcp, unix, iroh, and plaintext WebSocket never consult it. + Ended::Goaway(msg) => { + goaway + .redirect + .target(msg.uri(), &url, client.pinned && fingerprint_pins(url.scheme()))? + } + Ended::Closed(_) => None, + }; + if assigned.is_some() && addr.addresses().is_some() { return Err(Error::PinnedRedirect); } @@ -828,7 +854,7 @@ impl Connection { // An accepted redirect is an assignment: keep dialing it from here on, and // only it. The peer named exactly one place to go, which retires // whatever other addresses got us to this session. - let url = if let Some(target) = goaway.redirect.target(msg.uri(), &url) { + let url = if let Some(target) = assigned { addrs = Addrs::new(target.clone()); target } else { @@ -1556,8 +1582,8 @@ mod tests { assert_eq!(Redirect::Follow.resolve("https://other.example/", &plain), same); } - /// The three ways a redirect resolves to "redial what we already had": the peer - /// naming no URI, a URI we cannot parse, and a policy that ignores it outright. + /// `resolve` is the lenient form: every way a redirect can fail to assign a + /// new URL, refusals included, lands back on the current one. #[test] fn resolve_falls_back_to_the_current_url() { let current: Url = "https://relay.example/".parse().unwrap(); @@ -1567,11 +1593,7 @@ mod tests { current, "empty means 'reconnect to me'" ); - assert_eq!( - Redirect::Follow.resolve("not a url", ¤t), - current, - "a malformed URI is not a reason to stop reconnecting" - ); + assert_eq!(Redirect::Follow.resolve("not a url", ¤t), current); assert_eq!( Redirect::Ignore.resolve("https://other.example/", ¤t), current, @@ -1636,26 +1658,73 @@ mod tests { ); } + /// Only an explicit URI can assign, and one the policy will not follow is an + /// error rather than a quiet fallback: the loop ends on it instead of redialing. #[test] fn only_an_accepted_redirect_replaces_the_address_list() { let current: Url = "https://relay.example/".parse().unwrap(); + + // No URI, or a policy that ignores it: keep the current address list. + assert_eq!(Redirect::Follow.target("", ¤t, false).unwrap(), None); + assert_eq!( + Redirect::Ignore + .target("https://relay.example:5443/", ¤t, false) + .unwrap(), + None + ); + + // An explicit URI the policy will not follow is refused. for (policy, uri) in [ (Redirect::SameHost, "https://other.example/"), - (Redirect::Follow, ""), (Redirect::Follow, "not a url"), - (Redirect::Ignore, "https://relay.example:5443/"), + (Redirect::Ignore, "not a url"), (Redirect::Follow, "http://relay.example/"), (Redirect::Follow, "https://127.0.0.1/"), ] { - assert_eq!(policy.target(uri, ¤t), None, "{policy:?}: {uri}"); + assert!( + matches!(policy.target(uri, ¤t, false), Err(Error::RefusedRedirect(_))), + "{policy:?}: {uri}" + ); } + // An explicit assignment remains an assignment even if its URL is unchanged. assert_eq!( - Redirect::SameHost.target(current.as_str(), ¤t), + Redirect::SameHost.target(current.as_str(), ¤t, false).unwrap(), Some(current.clone()) ); let moved = "https://relay.example:5443/"; - assert_eq!(Redirect::SameHost.target(moved, ¤t), Some(moved.parse().unwrap())); + assert_eq!( + Redirect::SameHost.target(moved, ¤t, false).unwrap(), + Some(moved.parse().unwrap()) + ); + } + + /// A fingerprint is a Rustls check. Schemes that never install that verifier + /// must not inherit the pin. + #[test] + fn a_fingerprint_pin_only_covers_rustls_schemes() { + for scheme in ["https", "wss", "moqt", "moql"] { + assert!(fingerprint_pins(scheme), "{scheme}"); + } + for scheme in ["http", "ws", "tcp", "unix", "iroh"] { + assert!(!fingerprint_pins(scheme), "{scheme}"); + } + } + + /// A certificate pin verifies only the host it was configured for, so it + /// refuses a host change even when the policy would follow one. + #[test] + fn a_certificate_pin_holds_the_host() { + let current: Url = "https://relay.example/".parse().unwrap(); + assert!(matches!( + Redirect::Follow.target("https://other.example/", ¤t, true), + Err(Error::RefusedRedirect(_)) + )); + let moved = "https://relay.example:5443/"; + assert_eq!( + Redirect::Follow.target(moved, ¤t, true).unwrap(), + Some(moved.parse().unwrap()) + ); } /// `SameHost` lets a peer move us between ports or schemes on the endpoint we @@ -2019,4 +2088,125 @@ mod tests { assert_eq!(attempt_timeout(1, 3), Some(CONNECT_ATTEMPT), "still one more"); assert_eq!(attempt_timeout(2, 3), None, "the last of three"); } + + /// A stream-only server on a free loopback port, publishing `origin`. + /// + /// Returns its address, a receiver yielding each accepted session (so the test + /// can drain it), and the listener task. + #[cfg(feature = "tcp")] + async fn serve( + origin: moq_net::origin::Producer, + ) -> ( + std::net::SocketAddr, + tokio::sync::mpsc::UnboundedReceiver, + tokio::task::JoinHandle<()>, + ) { + let mut config = crate::listen::Config::default(); + config.tcp.bind = Some("127.0.0.1:0".parse().unwrap()); + let mut server = config.init(Default::default()).unwrap().listen().await.unwrap(); + let addr = server.tcp_local_addr().expect("tcp listener bound"); + + let (accepted, sessions) = tokio::sync::mpsc::unbounded_channel(); + let task = tokio::spawn(async move { + while let Some(request) = server.accept().await { + if let Ok(session) = request.with_publisher(&origin).ok().await { + let _ = accepted.send(session); + } + } + }); + (addr, sessions, task) + } + + /// The fleet drain: a relay withdrawn from DNS sends an empty-URI GOAWAY with a + /// deadline, and the client lands on a healthy relay by resolving the configured + /// name again. The drained relay still accepts, so a cached resolve would land + /// right back on it. The live track hands over at a group boundary, and the old + /// session closes at our handover cap, well before the peer's own deadline. + #[cfg(feature = "tcp")] + #[tokio::test] + async fn a_fleet_drain_redials_through_a_fresh_resolve() { + const WAIT: Duration = Duration::from_secs(10); + const HANDOVER: Duration = Duration::from_millis(500); + const DEADLINE: Duration = Duration::from_secs(30); + + // Two relays of one fleet, serving the same live broadcast. + let origin = crate::origin::spawn(); + let broadcast = origin.create_broadcast("cam").unwrap(); + broadcast.announce(Default::default()).unwrap(); + let track = broadcast.create_track("video", None).unwrap(); + let (addr_a, mut accepted_a, _task_a) = serve(origin.clone()).await; + let (addr_b, mut accepted_b, _task_b) = serve(origin.clone()).await; + + // Unique to this test: the table is process-wide. + const HOST: &str = "fleet-drain.test"; + crate::resolve::hosts::point(HOST, [addr_a]); + + let subscriber = crate::origin::spawn(); + let mut config = crate::connect::Config::default(); + config.goaway.handover = HANDOVER; + // A session younger than the initial delay counts as redirected immediately and + // waits out a backoff, which this test is not about. + config.backoff.initial = MIN_BACKOFF; + let client = config + .init(Default::default()) + .unwrap() + .with_subscriber(subscriber.clone()); + let url: Url = format!("tcp://{HOST}:1/").parse().unwrap(); + let _connection = client.connect(url); + + let session_a = tokio::time::timeout(WAIT, accepted_a.recv()).await.unwrap().unwrap(); + + let consumer = subscriber.consume(); + let cam = tokio::time::timeout(WAIT, consumer.routed_broadcast("cam")) + .await + .unwrap() + .unwrap(); + let mut sub = cam.track("video").unwrap().subscribe(None).await.unwrap(); + + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, b"g0".as_ref()).unwrap(); + group.finish().unwrap(); + let g0 = tokio::time::timeout(WAIT, sub.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(g0.sequence, 0); + + // Withdraw A from DNS, then drain it. + crate::resolve::hosts::point(HOST, [addr_b]); + let drained = tokio::time::Instant::now(); + session_a + .drain() + .send(moq_net::goaway::Goaway::new().with_timeout(DEADLINE)) + .unwrap(); + + let _session_b = tokio::time::timeout(WAIT, accepted_b.recv()) + .await + .expect("never redialed through the fresh resolve") + .unwrap(); + + let mut group = track.append_group().unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, b"g1".as_ref()).unwrap(); + group.finish().unwrap(); + let mut g1 = tokio::time::timeout(WAIT, sub.recv_group()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(g1.sequence, 1, "delivery resumes at the next group after the swap"); + assert_eq!(g1.read_frame().await.unwrap().unwrap().payload[..], b"g1"[..]); + + tokio::time::timeout(WAIT, session_a.closed()) + .await + .expect("the drained session never closed"); + assert!( + drained.elapsed() < DEADLINE, + "the old session outlived the handover cap and waited for the peer's deadline" + ); + assert!( + accepted_a.try_recv().is_err(), + "a cached resolve redialed the drained relay" + ); + } } diff --git a/rs/moq-tokio/src/error.rs b/rs/moq-tokio/src/error.rs index 209306d96a7..f603fc13077 100644 --- a/rs/moq-tokio/src/error.rs +++ b/rs/moq-tokio/src/error.rs @@ -58,6 +58,11 @@ pub enum Error { #[error("peer redirect refused for a connection with fixed addresses")] PinnedRedirect, + /// A peer's GOAWAY named a redirect the connection's policy refuses, or one it + /// could not parse. Terminal: the peer is leaving, so redialing is ignoring it. + #[error("GOAWAY redirect refused: {0}")] + RefusedRedirect(String), + /// Reading or writing a socket, certificate, or key file failed. #[error(transparent)] Io(Arc), diff --git a/rs/moq-tokio/src/listen.rs b/rs/moq-tokio/src/listen.rs index 2d2cea2e924..cbd88f64bdb 100644 --- a/rs/moq-tokio/src/listen.rs +++ b/rs/moq-tokio/src/listen.rs @@ -74,11 +74,14 @@ impl<'de> serde::Deserialize<'de> for Bind { } } +/// How long an accepted connection has to finish its handshake, unless overridden by `--listen-timeout`. +pub(crate) const DEFAULT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); + /// The accept side of an endpoint: what to listen on and how to be trusted. /// /// Derives [`usage::Args`], so flatten it into a binary's own parser with /// `#[usage(flatten)]`. The dial side is [`crate::connect::Config`]. -#[derive(usage::Args, Clone, Debug, Default, serde::Serialize, serde::Deserialize)] +#[derive(usage::Args, Clone, Debug, serde::Serialize, serde::Deserialize)] #[usage(unknown_flags = "error", args_override_self = false)] #[serde(deny_unknown_fields, default)] #[non_exhaustive] @@ -143,6 +146,29 @@ pub struct Config { )] pub version: Vec, + /// Maximum time for one accepted connection to finish its handshake: the + /// QUIC, WebTransport, WebSocket, or qmux one, then the MoQ SETUP, through + /// [`crate::server::Request::ok`]. Defaults to 10 seconds; set to 0 to wait forever. + /// + /// A peer that connects and then never speaks would otherwise hold its + /// connection open indefinitely, since keep-alives count as activity. The + /// connection is closed with a timeout code once this passes. See + /// [`resolved_timeout`](Self::resolved_timeout) for the value in effect. + #[usage(skip)] + #[serde(with = "crate::cli::duration::serde_duration")] + pub timeout: std::time::Duration, + + #[usage( + name = "listen-timeout", + long = "listen-timeout", + env = "MOQ_LISTEN_TIMEOUT", + default_value_t = crate::cli::Duration::fallback(DEFAULT_TIMEOUT), + default = "10s", + setting = "listen.timeout" + )] + #[serde(default, rename = "__cli_timeout", skip_serializing_if = "Option::is_none")] + pub(crate) timeout_arg: Option, + /// The certificates to serve and the roots that authenticate mTLS clients /// (`--listen-tls-*`). #[usage(flatten)] @@ -222,6 +248,30 @@ pub struct Config { pub quic: Option, } +impl Default for Config { + fn default() -> Self { + Self { + bind: None, + listen: None, + #[cfg(feature = "tcp")] + tcp: Default::default(), + #[cfg(all(feature = "uds", unix))] + unix: Default::default(), + version: Vec::new(), + timeout: DEFAULT_TIMEOUT, + timeout_arg: None, + tls: Default::default(), + preferred_v4: None, + preferred_v6: None, + lb_id: None, + lb_nonce: None, + load_balancer: None, + legacy: Default::default(), + quic: None, + } + } +} + #[cfg(feature = "noq")] pub(crate) use moq_sock::shard::Shard; /// One server's socket in a complete `SO_REUSEPORT` group. @@ -376,6 +426,13 @@ impl Config { Ok(()) } + /// How long an accepted connection has to finish its handshake, or `None` to wait + /// forever: for zero, and for a timeout too long for the clock to reach. + pub fn resolved_timeout(&self) -> Option { + let timeout = crate::cli::Duration::resolve(self.timeout_arg, self.timeout); + (!timeout.is_zero() && std::time::Instant::now().checked_add(timeout).is_some()).then_some(timeout) + } + #[cfg(feature = "noq")] /// Return the effective QUIC-LB connection-ID encoding. pub fn load_balancer(&self) -> Option { @@ -564,6 +621,30 @@ load_balancer = { id = "ab", nonce = 8 } assert!(server.local_addr().is_ok()); } + /// The handshake deadline defaults on, from the parser and in code alike, and + /// `0s` turns it off. + #[test] + fn timeout_defaults_and_disables() { + let secs = std::time::Duration::from_secs; + assert_eq!(Config::default().resolved_timeout(), Some(DEFAULT_TIMEOUT)); + assert_eq!(config_from(["test"]).resolved_timeout(), Some(DEFAULT_TIMEOUT)); + assert_eq!( + config_from(["test", "--listen-timeout", "3s"]).resolved_timeout(), + Some(secs(3)) + ); + assert_eq!(config_from(["test", "--listen-timeout", "0s"]).resolved_timeout(), None); + + let config: Config = toml::from_str(r#"timeout = "4s""#).expect("parse"); + assert_eq!(config.resolved_timeout(), Some(secs(4))); + + // Past the clock's range is forever, not a panic on the first accept. + let config = Config { + timeout: std::time::Duration::MAX, + ..Default::default() + }; + assert_eq!(config.resolved_timeout(), None); + } + /// The canonical spellings, which is what `--help` teaches. #[test] fn canonical_spellings_parse() { diff --git a/rs/moq-tokio/src/quic.rs b/rs/moq-tokio/src/quic.rs index 8d61d6f2d6a..275ba84d6e0 100644 --- a/rs/moq-tokio/src/quic.rs +++ b/rs/moq-tokio/src/quic.rs @@ -83,10 +83,17 @@ impl std::str::FromStr for CongestionControl { pub const DEFAULT_MAX_STREAMS: u64 = 10_000; /// Default idle timeout before an inactive connection is dropped. -pub(crate) const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(30); +/// +/// A peer that dies or drops off the network sends no close, so this is how long +/// its routes and subscriptions stay stuck. Short enough to fail over in seconds; +/// [`DEFAULT_KEEP_ALIVE`] keeps a live but quiet peer from reaching it. +pub(crate) const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(10); /// Default keep-alive ping interval. -pub(crate) const DEFAULT_KEEP_ALIVE: Duration = Duration::from_secs(5); +/// +/// Under a third of [`DEFAULT_IDLE_TIMEOUT`], so a lost ping is followed by +/// another (plus QUIC's probe retransmits) before the peer's timer runs out. +pub(crate) const DEFAULT_KEEP_ALIVE: Duration = Duration::from_secs(3); /// The `--quic-*` transport section, applied to dialed and accepted connections alike. #[derive(Clone, Debug, usage::Args, serde::Serialize, serde::Deserialize)] @@ -122,7 +129,9 @@ pub struct Config { )] pub gso: Option, - /// Idle timeout before an inactive connection is dropped. Defaults to 30s. + /// Idle timeout before an inactive connection is dropped. Defaults to 10s. + /// + /// QUIC uses the smaller of both ends' timeouts, so either side can shorten it. #[usage(skip)] #[serde(with = "crate::cli::duration::serde_duration")] pub idle_timeout: Duration, @@ -132,13 +141,13 @@ pub struct Config { long = "quic-idle-timeout", env = "MOQ_QUIC_IDLE_TIMEOUT", default_value_t = CliDuration::fallback(DEFAULT_IDLE_TIMEOUT), - default = "30s", + default = "10s", setting = "quic.idle_timeout" )] #[serde(default, rename = "__cli_idle_timeout", skip_serializing_if = "Option::is_none")] idle_timeout_arg: Option, - /// Keep-alive ping interval. Defaults to 5s; set `0s` to disable. + /// Keep-alive ping interval. Defaults to 3s; set `0s` to disable. /// Ignored by the iroh backend, which has no keep-alive knob. #[usage(skip)] #[serde(with = "crate::cli::duration::serde_duration")] @@ -149,7 +158,7 @@ pub struct Config { long = "quic-keep-alive", env = "MOQ_QUIC_KEEP_ALIVE", default_value_t = CliDuration::fallback(DEFAULT_KEEP_ALIVE), - default = "5s", + default = "3s", setting = "quic.keep_alive" )] #[serde(default, rename = "__cli_keep_alive", skip_serializing_if = "Option::is_none")] @@ -501,7 +510,7 @@ impl Config { /// here as the default rather than the value that was written. pub fn resolve(&self) -> Resolved { // A zero keep-alive means "disabled"; anything else (including unset) keeps - // the connection warm, defaulting to 5s. + // the connection warm, defaulting to 3s. let idle_timeout = CliDuration::resolve(self.idle_timeout_arg, self.idle_timeout); let keep_alive = CliDuration::resolve(self.keep_alive_arg, self.keep_alive); let keep_alive = (!keep_alive.is_zero()).then_some(keep_alive); diff --git a/rs/moq-tokio/src/resolve.rs b/rs/moq-tokio/src/resolve.rs index 7410cae79a1..3f58d119faf 100644 --- a/rs/moq-tokio/src/resolve.rs +++ b/rs/moq-tokio/src/resolve.rs @@ -249,6 +249,11 @@ impl Candidates { }, }; + #[cfg(test)] + if let Some(addrs) = hosts::lookup(domain) { + return Self::fixed(addrs); + } + Self { full: Query::start(domain, port, Lookup::Full), ipv4: Query::start(domain, port, Lookup::Ipv4), @@ -549,6 +554,32 @@ impl Candidates { } } +/// A name table tests can repoint, consulted before the system resolver. +/// +/// DNS is what moves a client off a drained node, so proving a redial resolves +/// afresh needs a name whose answer changes between dials. Entries carry the +/// port too, so two servers on one loopback address can stand in for two hosts. +#[cfg(test)] +pub(crate) mod hosts { + use std::collections::HashMap; + use std::net::SocketAddr; + use std::sync::Mutex; + + static HOSTS: Mutex>>> = Mutex::new(None); + + /// Answer every later lookup of `host` with `addrs`, ports included. + pub(crate) fn point(host: &str, addrs: impl IntoIterator) { + let mut hosts = HOSTS.lock().unwrap(); + hosts + .get_or_insert_with(HashMap::new) + .insert(host.to_string(), addrs.into_iter().collect()); + } + + pub(super) fn lookup(host: &str) -> Option> { + HOSTS.lock().unwrap().as_ref()?.get(host).cloned() + } +} + #[cfg(test)] impl Query { /// A lookup that answers with `addrs` after `delay`. diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index d0937f9caef..1a360703b83 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -215,6 +215,8 @@ impl Config { pub struct Server { moq: moq_net::Server, versions: moq_net::Versions, + /// From [`crate::listen::Config::resolved_timeout`]. + timeout: Option, accept: FuturesUnordered>>, #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] streams: StreamListeners, @@ -280,6 +282,7 @@ impl Server { config.validate()?; let versions = config.versions(); + let timeout = config.resolved_timeout(); // Build a QUIC backend when `--listen` is set, or when nothing else // is (the default). A stream-only server (`--listen-unix-bind` with no @@ -339,6 +342,7 @@ impl Server { let streams = StreamListeners::new( stream_binds, stream_versions(&versions), + timeout, #[cfg(all(feature = "uds", unix))] unix_allow, ); @@ -362,6 +366,7 @@ impl Server { accept: Default::default(), moq, versions, + timeout, #[cfg(any(feature = "tcp", all(feature = "uds", unix)))] streams, #[cfg(feature = "iroh")] @@ -601,15 +606,18 @@ impl Server { #[cfg(feature = "websocket")] let ws_accept = async { match ws_ref { - Some(ws) => ws.accept_with_url().await, + Some(ws) => Some(ws.accept_pending().await), None => None, } }; #[cfg(not(feature = "websocket"))] - let ws_accept = std::future::ready(None::>); + let ws_accept = std::future::ready(None::<()>); #[allow(unused_variables)] - let server = self.moq.clone(); + let setup = Setup { + server: self.moq.clone(), + timeout: self.timeout, + }; #[allow(unused_variables)] let versions = self.versions.clone(); @@ -630,14 +638,15 @@ impl Server { #[cfg(feature = "noq")] { let alpns = versions.alpns(); + let deadline = setup.deadline(); self.accept.push(async move { // Accept the transport (capturing url + mTLS identity) and exchange the // MoQ SETUP up front, so path/role are known before the caller authorizes // (like the stream bindings). - let Accepted { session, url, identity, authority, mut link } = super::noq::accept(_conn, alpns).await?; + let Accepted { session, url, identity, authority, mut link } = within(deadline, super::noq::accept(_conn, alpns)).await?; link.local = local; - let request = server.accept_request(tokio::time::Instant::now().into_std(), crate::transport::Session::new(session)).await?; - Ok(Request { transport: Transport::Quic, url, identity, authority, link, kind: RequestKind::Noq(Box::new(request)) }) + let (request, deadline) = setup.accept(session, deadline).await?; + Ok(Request { transport: Transport::Quic, url, identity, authority, link, kind: RequestKind::Noq(Box::new(request)), deadline }) }.boxed()); } } @@ -645,31 +654,30 @@ impl Server { #[cfg(feature = "iroh")] { let alpns = versions.alpns(); + let deadline = setup.deadline(); self.accept.push(async move { - let Accepted { session, url, identity, authority, link } = super::iroh::accept(_conn, alpns).await?; - let request = server.accept_request(tokio::time::Instant::now().into_std(), crate::transport::Session::new(session)).await?; - Ok(Request { transport: Transport::Iroh, url, identity, authority, link, kind: RequestKind::Iroh(Box::new(request)) }) + let Accepted { session, url, identity, authority, link } = within(deadline, super::iroh::accept(_conn, alpns)).await?; + let (request, deadline) = setup.accept(session, deadline).await?; + Ok(Request { transport: Transport::Iroh, url, identity, authority, link, kind: RequestKind::Iroh(Box::new(request)), deadline }) }.boxed()); } } - Some(_res) = ws_accept => { + Some(_pending) = ws_accept => { #[cfg(feature = "websocket")] - match _res { - Ok((session, url, accepted)) => { - // Read the SETUP off the qmux session before handing it over, so a - // slow peer doesn't stall the accept loop (spawned like the others). - let local = self.websocket_local_addr(); - self.accept.push(async move { - let request = server.accept_request(tokio::time::Instant::now().into_std(), crate::transport::Session::new(session)).await?; - let authority = url.host_str().filter(|h| !h.is_empty()).map(str::to_owned); - let link = Link { remote: Some(accepted.remote), local, alpn: accepted.protocol, ..Default::default() }; - Ok(Request { transport: Transport::WebSocket, url: Some(url), authority, identity: None, link, kind: RequestKind::Qmux(Box::new(request)) }) - }.boxed()); - } - // One connection's upgrade, not the listener's: a failed - // `accept(2)` never reaches here, having been classified, - // counted, and warned about by the listener itself. - Err(err) => tracing::debug!(%err, "WebSocket upgrade failed"), + { + // Upgrade and read the SETUP before handing the session over, so a slow + // peer doesn't stall the accept loop (spawned like the others). A failed + // `accept(2)` never reaches here, having been classified, counted, and + // warned about by the listener itself. + let local = self.websocket_local_addr(); + let deadline = setup.deadline(); + self.accept.push(async move { + let (session, url, accepted) = within(deadline, _pending).await?; + let (request, deadline) = setup.accept(session, deadline).await?; + let authority = url.host_str().filter(|h| !h.is_empty()).map(str::to_owned); + let link = Link { remote: Some(accepted.remote), local, alpn: accepted.protocol, ..Default::default() }; + Ok(Request { transport: Transport::WebSocket, url: Some(url), authority, identity: None, link, kind: RequestKind::Qmux(Box::new(request)), deadline }) + }.boxed()); } } Some(res) = self.accept.next() => { @@ -829,6 +837,138 @@ async fn serve_session(request: Request) -> crate::Result<()> { Err(session.closed().await.into()) } +/// The MoQ server an accepted connection handshakes against, and how long it has. +#[cfg(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + all(feature = "uds", unix) +))] +#[derive(Clone)] +struct Setup { + server: moq_net::Server, + timeout: Option, +} + +#[cfg(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + all(feature = "uds", unix) +))] +impl Setup { + /// When a connection accepted now must finish its handshake, or `None` to wait + /// forever, including past the clock's range. + fn deadline(&self) -> Option { + tokio::time::Instant::now().checked_add(self.timeout?) + } + + /// Read the peer's MoQ SETUP by `deadline`, closing the session with a timeout + /// code if it hasn't arrived. The deadline comes back too, since answering the + /// SETUP is part of the handshake it bounds. + async fn accept( + &self, + session: S, + deadline: Option, + ) -> crate::Result<(PendingRequest>, Option)> + where + S: web_transport_trait::Session, + crate::transport::Session: moq_net::transport::poll::Boxable, + as web_transport_trait::poll::Session>::SendStream: + web_transport_trait::MaybeSync, + as web_transport_trait::poll::Session>::RecvStream: + web_transport_trait::MaybeSync, + { + let session = crate::transport::Session::new(session); + let deadline = deadline.map(|at| Deadline::new(at, session.clone())); + let accept = async { + Ok(self + .server + .accept_request(tokio::time::Instant::now().into_std(), session) + .await?) + }; + let request = Deadline::bound(deadline.as_ref(), accept).await?; + Ok((request, deadline)) + } +} + +/// When an accepted connection's handshake must be done, and how to tell the peer +/// it wasn't. +struct Deadline { + at: tokio::time::Instant, + /// Closes the session with the timeout code. + close: Box, +} + +impl Deadline { + #[cfg(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + all(feature = "uds", unix) + ))] + fn new(at: tokio::time::Instant, session: S) -> Self + where + S: web_transport_trait::poll::Session + Clone + Send + Sync + 'static, + { + let close = move || { + let err = moq_net::Error::Timeout; + session + .clone() + .close(moq_net::SessionError::from(&err).to_code(), &err.to_string()); + }; + Self { + at, + close: Box::new(close), + } + } + + /// Finish `handshake` before the deadline, or close the session with the + /// timeout code. Without a deadline, wait forever. + async fn bound(deadline: Option<&Self>, handshake: impl Future>) -> crate::Result { + let Some(deadline) = deadline else { + return handshake.await; + }; + match tokio::time::timeout_at(deadline.at, handshake).await { + Ok(res) => res, + Err(_) => { + (deadline.close)(); + Err(moq_net::Error::Timeout.into()) + } + } + } +} + +/// Finish a transport handshake by `deadline`, or fail with a timeout. +/// +/// Nothing is established yet that could carry a close code, so an expired +/// handshake is dropped, which closes its connection. +#[cfg(any( + feature = "noq", + feature = "iroh", + feature = "websocket", + feature = "tcp", + all(feature = "uds", unix) +))] +async fn within( + deadline: Option, + handshake: impl Future>, +) -> crate::Result +where + Error: From, +{ + let Some(deadline) = deadline else { + return Ok(handshake.await?); + }; + match tokio::time::timeout_at(deadline, handshake).await { + Ok(res) => Ok(res?), + Err(_) => Err(moq_net::Error::Timeout.into()), + } +} + /// The version set offered on stream (`tcp://`/`unix://`) listeners. /// /// A URL-less transport carries the request path in the moq-lite-05+ SETUP, so @@ -891,6 +1031,7 @@ struct StreamListeners { /// at startup, long before the first `accept` binds anything. health: Vec, versions: moq_net::Versions, + timeout: Option, #[cfg(all(feature = "uds", unix))] unix_allow: Option, /// Bound sockets whose accept loops have not started. Empty once [`Self::serve`] @@ -908,6 +1049,7 @@ impl StreamListeners { fn new( binds: Vec, versions: moq_net::Versions, + timeout: Option, #[cfg(all(feature = "uds", unix))] unix_allow: Option, ) -> Self { let health = binds @@ -918,6 +1060,7 @@ impl StreamListeners { binds, health, versions, + timeout, #[cfg(all(feature = "uds", unix))] unix_allow, pending: Vec::new(), @@ -984,13 +1127,17 @@ impl StreamListeners { return; } + let setup = Setup { + server: server.clone(), + timeout: self.timeout, + }; let (tx, rx) = tokio::sync::mpsc::channel(16); for listener in self.pending.drain(..) { let task = match listener { #[cfg(feature = "tcp")] - BoundListener::Tcp(listener) => spawn_tcp_loop(listener, server.clone(), tx.clone()), + BoundListener::Tcp(listener) => spawn_tcp_loop(listener, setup.clone(), tx.clone()), #[cfg(all(feature = "uds", unix))] - BoundListener::Unix(listener) => spawn_unix_loop(listener, server.clone(), self.unix_allow.clone(), tx.clone()), + BoundListener::Unix(listener) => spawn_unix_loop(listener, setup.clone(), self.unix_allow.clone(), tx.clone()), }; self.tasks.push(task); } @@ -1033,7 +1180,7 @@ impl Drop for StreamListeners { #[cfg(feature = "tcp")] fn spawn_tcp_loop( listener: crate::tcp::Listener, - server: moq_net::Server, + setup: Setup, tx: tokio::sync::mpsc::Sender, ) -> tokio::task::JoinHandle<()> { let local = listener.local_addr().ok(); @@ -1046,17 +1193,18 @@ fn spawn_tcp_loop( pending = listener.accept_pending() => pending, Some(_) = handshakes.join_next() => continue, }; - let server = server.clone(); + let deadline = setup.deadline(); + let setup = setup.clone(); let tx = tx.clone(); handshakes.spawn(async move { - match pending.await { + match within(deadline, pending).await { Ok((session, remote)) => { let link = Link { remote: Some(remote), local, ..Default::default() }; - stream_request(session, Transport::Tcp, link, server, tx).await + stream_request(session, Transport::Tcp, link, &setup, deadline, tx).await } // Per-connection: a failed `accept(2)` is the listener's own to // classify and pace, and never surfaces here. @@ -1070,7 +1218,7 @@ fn spawn_tcp_loop( #[cfg(all(feature = "uds", unix))] fn spawn_unix_loop( listener: crate::unix::Listener, - server: moq_net::Server, + setup: Setup, allow: Option, tx: tokio::sync::mpsc::Sender, ) -> tokio::task::JoinHandle<()> { @@ -1082,11 +1230,12 @@ fn spawn_unix_loop( pending = listener.accept_pending() => pending, Some(_) = handshakes.join_next() => continue, }; - let server = server.clone(); + let deadline = setup.deadline(); + let setup = setup.clone(); let tx = tx.clone(); let allow = allow.clone(); handshakes.spawn(async move { - match pending.await { + match within(deadline, pending).await { Ok((session, cred)) => { // Enforce the allowlist (if any) before reading SETUP bytes from the peer. if let Some(allow) = &allow @@ -1095,7 +1244,7 @@ fn spawn_unix_loop( tracing::warn!(uid = cred.uid, gid = cred.gid, pid = ?cred.pid, "unix connection rejected by allow list"); return; } - stream_request(session, Transport::Unix, Link::default(), server, tx).await; + stream_request(session, Transport::Unix, Link::default(), &setup, deadline, tx).await; } // Per-connection, as in `spawn_tcp_loop`. Err(err) => tracing::warn!(%err, "unix qmux handshake failed"), @@ -1111,17 +1260,12 @@ async fn stream_request( session: qmux::Session, transport: Transport, link: Link, - server: moq_net::Server, + setup: &Setup, + deadline: Option, tx: tokio::sync::mpsc::Sender, ) { - match server - .accept_request( - tokio::time::Instant::now().into_std(), - crate::transport::Session::new(session), - ) - .await - { - Ok(request) => { + match setup.accept(session, deadline).await { + Ok((request, deadline)) => { let request = Request { transport, url: None, @@ -1129,6 +1273,7 @@ async fn stream_request( identity: None, link, kind: RequestKind::Qmux(Box::new(request)), + deadline, }; let _ = tx.send(request).await; } @@ -1240,6 +1385,8 @@ pub struct Request { /// The link facts the transport could see. link: Link, kind: RequestKind, + /// What remains of the handshake deadline, which also bounds [`Request::ok`]. + deadline: Option, } /// Why an incoming session was rejected. @@ -1310,99 +1457,47 @@ impl Request { } /// Publish the given origin to the session. - pub fn with_publisher(self, publish: impl moq_net::Consume) -> Self { - let Request { - transport, - url, - authority, - identity, - link, - kind, - } = self; - let kind = request_map!(kind, request => request.with_publisher(publish)); - Request { - transport, - url, - authority, - identity, - link, - kind, - } + pub fn with_publisher(mut self, publish: impl moq_net::Consume) -> Self { + self.kind = request_map!(self.kind, request => request.with_publisher(publish)); + self } /// Subscribe to the given origin from the session. - pub fn with_subscriber(self, subscribe: moq_net::origin::Producer) -> Self { - let Request { - transport, - url, - authority, - identity, - link, - kind, - } = self; - let kind = request_map!(kind, request => request.with_subscriber(subscribe)); - Request { - transport, - url, - authority, - identity, - link, - kind, - } + pub fn with_subscriber(mut self, subscribe: moq_net::origin::Producer) -> Self { + self.kind = request_map!(self.kind, request => request.with_subscriber(subscribe)); + self } /// Assign the identity this peer's routes are attributed to; see /// [`moq_net::server::Handshake::with_peer_hop`]. Derive it from [`Self::peer_identity`], /// never from something coarser. - pub fn with_peer_hop(self, hop: moq_net::Hop) -> Self { - let Request { - transport, - url, - authority, - identity, - link, - kind, - } = self; - let kind = request_map!(kind, request => request.with_peer_hop(hop)); - Request { - transport, - url, - authority, - identity, - link, - kind, - } + pub fn with_peer_hop(mut self, hop: moq_net::Hop) -> Self { + self.kind = request_map!(self.kind, request => request.with_peer_hop(hop)); + self } /// Attach a per-connection [`moq_net::stats::Session`] context to this session. - pub fn with_stats(self, stats: moq_net::stats::Session) -> Self { - let Request { - transport, - url, - authority, - identity, - link, - kind, - } = self; - let kind = request_map!(kind, request => request.with_stats(stats)); - Request { - transport, - url, - authority, - identity, - link, - kind, - } + pub fn with_stats(mut self, stats: moq_net::stats::Session) -> Self { + self.kind = request_map!(self.kind, request => request.with_stats(stats)); + self } /// Accept the session, starting the MoQ session loops. + /// + /// Bounded by what remains of the handshake deadline: answering SETUP can wait + /// on a peer that withholds flow-control credit, which keep-alives would + /// otherwise hold open. pub async fn ok(self) -> crate::Result { - Ok(request_into!(self.kind, request => { - let (session, driver) = request.ok().await?; - use tracing::Instrument; - tokio::spawn(moq_net::time::run(driver).instrument(tracing::Span::current())); - session - })) + let Request { kind, deadline, .. } = self; + let ok = async { + Ok(request_into!(kind, request => { + let (session, driver) = request.ok().await?; + use tracing::Instrument; + tokio::spawn(moq_net::time::run(driver).instrument(tracing::Span::current())); + session + })) + }; + Deadline::bound(deadline.as_ref(), ok).await } /// Returns the network transport carrying this session. @@ -1549,7 +1644,14 @@ mod tests { .with_protocols(["moq-lite-06"]); let addr = listener.local_addr().unwrap(); let (tx, _rx) = tokio::sync::mpsc::channel(1); - let task = spawn_tcp_loop(listener, moq_net::Server::new(), tx); + let task = spawn_tcp_loop( + listener, + Setup { + server: moq_net::Server::new(), + timeout: None, + }, + tx, + ); let mut stalled = tokio::net::TcpStream::connect(addr).await.unwrap(); stalled.read_u8().await.expect("first handshake started"); let mut remaining = Vec::new(); @@ -1576,7 +1678,15 @@ mod tests { .unwrap() .with_protocols(["moq-lite-06"]); let (tx, _rx) = tokio::sync::mpsc::channel(1); - let task = spawn_unix_loop(listener, moq_net::Server::new(), None, tx); + let task = spawn_unix_loop( + listener, + Setup { + server: moq_net::Server::new(), + timeout: None, + }, + None, + tx, + ); let mut stalled = tokio::net::UnixStream::connect(&path).await.unwrap(); stalled.read_u8().await.expect("first handshake started"); let mut remaining = Vec::new(); @@ -1607,7 +1717,14 @@ mod tests { .with_protocols(["moq-lite-06"]); let addr = listener.local_addr().unwrap(); let (tx, mut rx) = tokio::sync::mpsc::channel(1); - let task = spawn_tcp_loop(listener, moq_net::Server::new(), tx); + let task = spawn_tcp_loop( + listener, + Setup { + server: moq_net::Server::new(), + timeout: None, + }, + tx, + ); let mut stalled = tokio::net::TcpStream::connect(addr).await.unwrap(); stalled.read_u8().await.expect("handshake started"); tokio::time::pause(); @@ -1628,7 +1745,15 @@ mod tests { .unwrap() .with_protocols(["moq-lite-06"]); let (tx, mut rx) = tokio::sync::mpsc::channel(1); - let task = spawn_unix_loop(listener, moq_net::Server::new(), None, tx); + let task = spawn_unix_loop( + listener, + Setup { + server: moq_net::Server::new(), + timeout: None, + }, + None, + tx, + ); let mut stalled = tokio::net::UnixStream::connect(&path).await.unwrap(); stalled.read_u8().await.expect("handshake started"); tokio::time::pause(); @@ -1638,6 +1763,323 @@ mod tests { assert!(matches!(closed, Ok(None)), "stalled handshake outlived its accept loop"); } + /// A qmux session pair over an in-memory WebSocket, so a handshake can run on a + /// paused clock. + #[cfg(feature = "websocket")] + async fn session_pair() -> (qmux::Session, qmux::Session) { + use qmux::ws::tokio_tungstenite::{WebSocketStream, tungstenite::protocol::Role}; + + const ALPN: &str = "qmux-01.moq-lite-06"; + let (client, server) = tokio::io::duplex(64 * 1024); + let client = WebSocketStream::from_raw_socket(client, Role::Client, None).await; + let server = WebSocketStream::from_raw_socket(server, Role::Server, None).await; + ( + qmux::ws::Upgraded::new(client).with_alpn(ALPN).connect(), + qmux::ws::Upgraded::new(server).with_alpn(ALPN).accept(), + ) + } + + /// A scripted peer, recording the code it was closed with. + /// + /// With `setup` it is a moq-transport peer whose bidi SETUP stream delivers those + /// bytes and then never grants credit for the reply. Without it, it never opens + /// anything, while a bidi stream it opens itself records what is written. + #[cfg(feature = "websocket")] + #[derive(Clone)] + struct Scripted { + alpn: &'static str, + setup: Option, + written: std::sync::Arc>>, + closed: std::sync::Arc>>, + } + + #[cfg(feature = "websocket")] + impl Scripted { + fn new(alpn: &'static str) -> Self { + Self { + alpn, + setup: None, + written: Default::default(), + closed: Default::default(), + } + } + } + + /// Records every write, or accepts none at all when `stalled`. + #[cfg(feature = "websocket")] + struct ScriptedSend { + written: std::sync::Arc>>, + stalled: bool, + } + + #[cfg(feature = "websocket")] + impl web_transport_trait::SendStream for ScriptedSend { + type Error = qmux::Error; + + async fn write(&mut self, buf: &[u8]) -> Result { + if self.stalled { + std::future::pending::<()>().await; + } + self.written.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + + fn set_priority(&mut self, _order: i32) {} + + fn finish(&mut self) -> Result<(), Self::Error> { + Ok(()) + } + + fn reset(&mut self, _code: u32) {} + + async fn closed(&mut self) -> Result<(), Self::Error> { + std::future::pending().await + } + } + + /// Delivers `data` once, then nothing more. + #[cfg(feature = "websocket")] + struct ScriptedRecv { + data: Option, + } + + #[cfg(feature = "websocket")] + impl web_transport_trait::RecvStream for ScriptedRecv { + type Error = qmux::Error; + + async fn read(&mut self, dst: &mut [u8]) -> Result, Self::Error> { + let Some(mut data) = self.data.take() else { + return std::future::pending().await; + }; + let n = dst.len().min(data.len()); + dst[..n].copy_from_slice(&data.split_to(n)); + self.data = (!data.is_empty()).then_some(data); + Ok(Some(n)) + } + + fn stop(&mut self, _code: u32) {} + + async fn closed(&mut self) -> Result<(), Self::Error> { + std::future::pending().await + } + } + + #[cfg(feature = "websocket")] + impl web_transport_trait::Session for Scripted { + type SendStream = ScriptedSend; + type RecvStream = ScriptedRecv; + type Error = qmux::Error; + + async fn accept_uni(&self) -> Result { + std::future::pending().await + } + + async fn accept_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + let Some(setup) = self.setup.clone() else { + return std::future::pending().await; + }; + let send = ScriptedSend { + written: Default::default(), + stalled: true, + }; + Ok((send, ScriptedRecv { data: Some(setup) })) + } + + async fn open_bi(&self) -> Result<(Self::SendStream, Self::RecvStream), Self::Error> { + let send = ScriptedSend { + written: self.written.clone(), + stalled: false, + }; + Ok((send, ScriptedRecv { data: None })) + } + + async fn open_uni(&self) -> Result { + std::future::pending().await + } + + fn send_datagram(&self, _payload: bytes::Bytes) -> Result<(), Self::Error> { + Ok(()) + } + + async fn recv_datagram(&self) -> Result { + std::future::pending().await + } + + fn max_datagram_size(&self) -> usize { + 0 + } + + fn protocol(&self) -> Option<&str> { + Some(self.alpn) + } + + fn close(&self, code: u32, _reason: &str) { + self.closed.lock().unwrap().get_or_insert(code); + } + + async fn closed(&self) -> Self::Error { + std::future::pending().await + } + } + + /// A timeout past the clock's range waits forever instead of panicking on accept. + #[cfg(feature = "websocket")] + #[tokio::test] + async fn unreachable_timeout_has_no_deadline() { + let setup = Setup { + server: moq_net::Server::new(), + timeout: Some(std::time::Duration::MAX), + }; + assert_eq!(setup.deadline(), None); + } + + /// A peer that never sends SETUP is refused at the deadline, and told why. + #[cfg(feature = "websocket")] + #[tokio::test(start_paused = true)] + async fn stalled_setup_is_closed_at_the_deadline() { + let peer = Scripted::new("moq-lite-06"); + let timeout = std::time::Duration::from_secs(10); + let setup = Setup { + server: moq_net::Server::new(), + timeout: Some(timeout), + }; + let start = tokio::time::Instant::now(); + let Err(err) = setup.accept(peer.clone(), setup.deadline()).await else { + panic!("a peer that never sent SETUP was accepted"); + }; + assert!(matches!(err, Error::MoqNet(moq_net::Error::Timeout)), "{err}"); + assert_eq!(start.elapsed(), timeout); + assert_eq!( + *peer.closed.lock().unwrap(), + Some(moq_net::SessionError::Timeout.to_code()) + ); + } + + /// Answering SETUP is part of the handshake: a moq-transport peer that sends + /// its SETUP but never grants credit for the reply is refused at the same + /// deadline, rather than held open by keep-alives. + #[cfg(feature = "websocket")] + #[tokio::test(start_paused = true)] + async fn stalled_setup_reply_is_closed_at_the_deadline() { + // Record a real moq-transport-14 CLIENT_SETUP. The client then waits on a + // reply that never comes. + let recorder = Scripted::new("moq-00"); + let client = tokio::spawn({ + let recorder = crate::transport::Session::new(recorder.clone()); + async move { + let _ = moq_net::Client::new() + .connect(tokio::time::Instant::now().into_std(), recorder) + .await; + } + }); + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + client.abort(); + let bytes = std::mem::take(&mut *recorder.written.lock().unwrap()); + assert!(!bytes.is_empty(), "the client never wrote its SETUP"); + + let mut peer = Scripted::new("moq-00"); + peer.setup = Some(bytes.into()); + let timeout = std::time::Duration::from_secs(10); + let setup = Setup { + server: moq_net::Server::new(), + timeout: Some(timeout), + }; + let start = tokio::time::Instant::now(); + let Ok((request, deadline)) = setup.accept(peer.clone(), setup.deadline()).await else { + panic!("a readable SETUP was refused"); + }; + assert_eq!(start.elapsed(), std::time::Duration::ZERO); + + let ok = async { Ok(request.ok().await.map(drop)?) }; + let err = Deadline::bound(deadline.as_ref(), ok) + .await + .expect_err("a reply the peer never took completed"); + assert!(matches!(err, Error::MoqNet(moq_net::Error::Timeout)), "{err}"); + assert_eq!(start.elapsed(), timeout); + assert_eq!( + *peer.closed.lock().unwrap(), + Some(moq_net::SessionError::Timeout.to_code()) + ); + } + + /// The deadline bounds only the handshake: a session that finished in time + /// stays open past it. + #[cfg(feature = "websocket")] + #[tokio::test(start_paused = true)] + async fn completed_setup_outlives_the_deadline() { + use web_transport_trait::Session as _; + + let (client, server) = session_pair().await; + let timeout = std::time::Duration::from_secs(10); + let setup = Setup { + server: moq_net::Server::new(), + timeout: Some(timeout), + }; + let deadline = setup.deadline(); + + let transport = crate::transport::Session::new(client.clone()); + let connect = tokio::spawn(async move { + let (session, driver) = moq_net::Client::new() + .connect(tokio::time::Instant::now().into_std(), transport) + .await + .expect("client handshake"); + tokio::spawn(moq_net::time::run(driver)); + session + }); + + let Ok((request, _)) = setup.accept(server, deadline).await else { + panic!("a prompt SETUP was refused"); + }; + let (_session, driver) = request.ok().await.expect("server handshake"); + tokio::spawn(moq_net::time::run(driver)); + let _client = connect.await.unwrap(); + + tokio::time::sleep(timeout * 2).await; + assert!( + client.closed().now_or_never().is_none(), + "a completed handshake was closed at its deadline" + ); + } + + /// A WebSocket peer that never sends its upgrade request holds up only itself. + /// Real sockets, so this runs on the wall clock. + #[cfg(feature = "websocket")] + #[tokio::test] + async fn websocket_stalled_upgrade_does_not_block_accept() { + let ws = crate::websocket::Listener::bind("127.0.0.1:0".parse().unwrap()) + .await + .unwrap(); + let addr = ws.local_addr().unwrap(); + let config = Config { + websocket: Some(ws), + ..Default::default() + }; + let mut listener = config.init_streams().unwrap().listen().await.unwrap(); + + let _stalled = tokio::net::TcpStream::connect(addr).await.unwrap(); + let client = tokio::spawn(async move { + let session = qmux::ws::Client::new() + .with_protocol("moq-lite-06", &[qmux::Version::QMux01]) + .connect(&format!("ws://{addr}/")) + .await + .expect("upgrade"); + let (_session, driver) = moq_net::Client::new() + .connect( + tokio::time::Instant::now().into_std(), + crate::transport::Session::new(session), + ) + .await + .expect("client handshake"); + moq_net::time::run(driver).await + }); + + let request = tokio::time::timeout(std::time::Duration::from_secs(5), listener.accept()) + .await + .expect("a stalled upgrade blocked the accept loop"); + assert!(request.is_some()); + client.abort(); + } + #[test] fn version_help_lists_every_parseable_name() { #[derive(usage::Cli)] diff --git a/rs/moq-tokio/src/settings.rs b/rs/moq-tokio/src/settings.rs index 72211afdda8..bec2ac4d22e 100644 --- a/rs/moq-tokio/src/settings.rs +++ b/rs/moq-tokio/src/settings.rs @@ -17,6 +17,9 @@ pub struct Listen { #[usage(env = "MOQ_LISTEN_VERSION", cli("--listen-version"), parse = "list_by_comma")] version: Option>, + #[usage(env = "MOQ_LISTEN_TIMEOUT", cli("--listen-timeout"))] + timeout: Option, + #[usage(env = "MOQ_LISTEN_PREFERRED_V4", cli("--listen-preferred-v4"))] preferred_v4: Option, diff --git a/rs/moq-tokio/src/websocket.rs b/rs/moq-tokio/src/websocket.rs index 9ee7e63ddf0..e48d13c9619 100644 --- a/rs/moq-tokio/src/websocket.rs +++ b/rs/moq-tokio/src/websocket.rs @@ -351,7 +351,7 @@ pub(crate) async fn connect( qmux::ws::Client::new() .with_protocols(alpns.iter().map(|&a| (a, qmux_versions_for(a)))) .with_connector(connector) - .with_keep_alive(qmux::ws::KeepAlive::default()) // 5s ping / 30s deadline, parity with QUIC + .with_keep_alive(qmux::ws::KeepAlive::default()) // 5s ping / 30s deadline; TCP backs off retransmits too far for QUIC's 10s .connect(url.as_str()) .await .map_err(Error::connect)? @@ -537,6 +537,14 @@ impl Listener { /// Accept the next connection and retain the WebSocket request URL, the chosen /// sub-protocol, and the peer's address. pub(crate) async fn accept_with_url(&self) -> Option> { + Some(self.accept_pending().await.await) + } + + /// Accept a socket and return its independently driven WebSocket upgrade, so a + /// peer that stalls mid-upgrade holds up only itself. + pub(crate) async fn accept_pending( + &self, + ) -> impl Future> + Send + use<> { let (stream, addr) = self.accept_socket().await; tracing::debug!(%addr, "accepted WebSocket TCP connection"); @@ -579,11 +587,11 @@ impl Listener { Ok(response) }; - let websocket = tokio_tungstenite::accept_hdr_async_with_config(stream, callback, None) - .await - .map_err(qmux::Error::from) - .map_err(Error::accept); - Some(websocket.map(|websocket| { + async move { + let websocket = tokio_tungstenite::accept_hdr_async_with_config(stream, callback, None) + .await + .map_err(qmux::Error::from) + .map_err(Error::accept)?; let (protocol, url) = accepted .lock() .unwrap() @@ -594,8 +602,8 @@ impl Listener { Some(protocol) => upgraded.with_alpn(protocol).accept(), None => upgraded.accept(), }; - (session, url, Accepted { remote: addr, protocol }) - })) + Ok((session, url, Accepted { remote: addr, protocol })) + } } /// The `accept(2)` half: keep asking until a connection comes back. diff --git a/rs/moq-tokio/tests/backend.rs b/rs/moq-tokio/tests/backend.rs index 7ef7e52dd63..3338fbdaa57 100644 --- a/rs/moq-tokio/tests/backend.rs +++ b/rs/moq-tokio/tests/backend.rs @@ -744,11 +744,9 @@ async fn iroh_connect_test(version: Option<&str>) { #[tracing_test::traced_test] #[tokio::test] async fn noq_client_close_reaches_server() { - let quic = moq_tokio::quic::Config::default(); - assert!( - quic.idle_timeout > TIMEOUT, - "an idle timeout inside TIMEOUT would hide a lost close" - ); + // An idle timeout inside TIMEOUT would hide a lost close. + let mut quic = moq_tokio::quic::Config::default(); + quic.idle_timeout = TIMEOUT * 3; let mut server_config = moq_tokio::listen::Config::default(); server_config.bind = Some("127.0.0.1:0".parse().unwrap()); diff --git a/rs/moq-tokio/tests/reconnect.rs b/rs/moq-tokio/tests/reconnect.rs index a3532e354c3..c2ecf04bf74 100644 --- a/rs/moq-tokio/tests/reconnect.rs +++ b/rs/moq-tokio/tests/reconnect.rs @@ -133,8 +133,7 @@ async fn spawn_server() -> ( (port, sessions, handle) } -/// A client that redials fast, so a refused redirect lands back on the original -/// server inside the test's patience. +/// A client that redials fast, so a migration lands inside the test's patience. fn quick_client(redirect: moq_tokio::Redirect) -> moq_tokio::Client { let mut config = moq_tokio::connect::Config::default(); config.backoff.initial = Duration::from_millis(20); @@ -144,14 +143,15 @@ fn quick_client(redirect: moq_tokio::Redirect) -> moq_tokio::Client { config.init(Default::default()).expect("failed to init client") } -/// The default refuses peer-selected host changes and redials the configured URL. +/// The default refuses a peer-selected host change, and the refusal ends the +/// connection: the peer is leaving, so redialing the configured URL would ignore it. #[tokio::test] async fn a_redirect_to_another_host_is_refused_by_default() { let (port_a, mut sessions_a, _task_a) = spawn_server().await; let (port_b, mut sessions_b, _task_b) = spawn_server().await; let url: url::Url = format!("tcp://localhost:{port_a}/").parse().expect("parse url"); - let _connection = quick_client(Default::default()).connect(url); + let connection = quick_client(Default::default()).connect(url); let first = tokio::time::timeout(Duration::from_secs(10), sessions_a.recv()) .await @@ -163,12 +163,13 @@ async fn a_redirect_to_another_host_is_refused_by_default() { .send(moq_net::goaway::Goaway::redirect(format!("tcp://127.0.0.1:{port_b}/"))) .expect("send goaway"); - // Refused, so the redial goes back to A rather than to the host the peer named. - tokio::time::timeout(Duration::from_secs(10), sessions_a.recv()) + let err = tokio::time::timeout(Duration::from_secs(10), connection.closed()) .await - .expect("never redialed the configured URL") - .expect("server A stopped accepting"); + .expect("the refusal never ended the connection") + .expect_err("a refused redirect must end with an error"); + assert!(matches!(err, moq_tokio::Error::RefusedRedirect(_)), "ended with {err}"); + assert!(sessions_a.try_recv().is_err(), "redialed the configured URL"); assert!( sessions_b.try_recv().is_err(), "the peer moved us onto the host it named" @@ -202,21 +203,21 @@ async fn follow_still_honors_a_cross_host_redirect() { .expect("server B stopped accepting"); } -/// Refusing a peer-selected host must not discard caller-selected fallbacks. +/// A refused redirect must not fall through to a caller-selected fallback either. #[tokio::test] -async fn a_refused_redirect_preserves_configured_fallbacks() { +async fn a_refused_redirect_skips_configured_fallbacks() { let (port_a, mut sessions_a, task_a) = spawn_server().await; let (port_b, mut sessions_b, _task_b) = spawn_server().await; let primary: url::Url = format!("tcp://localhost:{port_a}/").parse().expect("primary URL"); let fallback: url::Url = format!("tcp://127.0.0.1:{port_b}/").parse().expect("fallback URL"); let addrs = moq_tokio::connect::Addrs::new(primary).or(fallback); - let _connection = quick_client(Default::default()).connect(addrs); + let connection = quick_client(Default::default()).connect(addrs); let first = tokio::time::timeout(Duration::from_secs(10), sessions_a.recv()) .await .expect("first dial timed out") .expect("server A stopped accepting"); - // Stop accepting before GOAWAY so reconnect must use the configured fallback. + // Stop accepting before GOAWAY so any redial could only land on the fallback. task_a.abort(); assert!(task_a.await.expect_err("listener was aborted").is_cancelled()); first @@ -224,6 +225,36 @@ async fn a_refused_redirect_preserves_configured_fallbacks() { .send(moq_net::goaway::Goaway::redirect("tcp://127.0.0.1:1/")) .expect("send goaway"); + let err = tokio::time::timeout(Duration::from_secs(10), connection.closed()) + .await + .expect("the refusal never ended the connection") + .expect_err("a refused redirect must end with an error"); + assert!(matches!(err, moq_tokio::Error::RefusedRedirect(_)), "ended with {err}"); + assert!( + sessions_b.try_recv().is_err(), + "fell through to the configured fallback" + ); +} + +/// An empty GOAWAY is "reconnect to me", which keeps every caller-selected fallback. +#[tokio::test] +async fn an_empty_goaway_preserves_configured_fallbacks() { + let (port_a, mut sessions_a, task_a) = spawn_server().await; + let (port_b, mut sessions_b, _task_b) = spawn_server().await; + let primary: url::Url = format!("tcp://localhost:{port_a}/").parse().expect("primary URL"); + let fallback: url::Url = format!("tcp://127.0.0.1:{port_b}/").parse().expect("fallback URL"); + let addrs = moq_tokio::connect::Addrs::new(primary).or(fallback); + let _connection = quick_client(Default::default()).connect(addrs); + let first = tokio::time::timeout(Duration::from_secs(10), sessions_a.recv()) + .await + .expect("first dial timed out") + .expect("server A stopped accepting"); + + // Stop accepting before GOAWAY so the migration must use the configured fallback. + task_a.abort(); + assert!(task_a.await.expect_err("listener was aborted").is_cancelled()); + first.drain().send(moq_net::goaway::Goaway::new()).expect("send goaway"); + tokio::time::timeout(Duration::from_secs(10), sessions_b.recv()) .await .expect("configured fallback was discarded") diff --git a/rs/moq-video/Cargo.toml b/rs/moq-video/Cargo.toml index a59422f170a..26c4d85c514 100644 --- a/rs/moq-video/Cargo.toml +++ b/rs/moq-video/Cargo.toml @@ -30,7 +30,8 @@ categories = ["multimedia", "multimedia::video", "multimedia::encoding"] default = ["mediacodec", "nvidia", "openh264"] # Native camera and screen capture. Costs the build nothing on any platform: # Linux goes through moq-v4l (bindings checked in), macOS and Windows use OS -# frameworks. Still off by default until every distribution ships it, which is +# frameworks (Windows display/window capture requires Windows 10 2004+ WGC). +# Still off by default until every distribution ships it, which is # a packaging decision rather than a build cost. moq-cli's `capture` turns it on. capture = ["dep:kio", "dep:libc", "dep:moq-v4l", "dep:x11rb", "dep:zune-jpeg"] # Android MediaCodec encode/decode and its AHardwareBuffer frame surface. Kept @@ -223,6 +224,11 @@ objc2-screen-capture-kit = { workspace = true } # vendor SDK. The Direct3D11 / Dxgi features cover the shared D3D11 device and # the `IMFDXGIBuffer` surfaces wrapping each captured texture. windows = { version = "0.62", features = [ + "Foundation", + "Foundation_Metadata", + "Graphics_Capture", + "Graphics_DirectX_Direct3D11", + "Security_Authorization_AppCapabilityAccess", "Win32_Foundation", "Win32_System_Com", "Win32_Media_MediaFoundation", @@ -232,12 +238,13 @@ windows = { version = "0.62", features = [ "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi", - # PrintWindow lives under Storage_Xps in the Win32 metadata; it is how a - # DirectComposition window renders into our capture DC. - "Win32_Storage_Xps", + "Win32_Graphics_Dwm", "Win32_System_Ole", "Win32_System_Threading", "Win32_System_Variant", + "Win32_System_WinRT", + "Win32_System_WinRT_Direct3D11", + "Win32_System_WinRT_Graphics_Capture", "Win32_UI_HiDpi", "Win32_UI_WindowsAndMessaging", ] } diff --git a/rs/moq-video/README.md b/rs/moq-video/README.md index 99d5172e717..679eacf1712 100644 --- a/rs/moq-video/README.md +++ b/rs/moq-video/README.md @@ -27,9 +27,14 @@ Per-platform, picked at compile time: feature), with native X11 monitor/window selection and capture as the X11 fallback. The Wayland picker dialog chooses the screen, and the portal's restore token is reused so demand-driven reopens don't re-prompt. -- **Windows**: native Media Foundation (camera; `IMFSourceReader`) and DXGI - Desktop Duplication (display), plus GDI single-window capture. Both convert - BGRA to CPU I420 and use the ids returned by the enumerators. +- **Windows**: native Media Foundation (camera; `IMFSourceReader`) and + Windows.Graphics.Capture (display and window; Windows 10 2004/build 19041 + or newer). Screen capture honors `Config::cursor` and converts BGRA to an + owned NV12 texture on the GPU, shared with the Media Foundation encoder; + software encoding downloads that texture. Odd screen edges are cropped to + even dimensions. Builds before 19041 return `Error::Unsupported`, with no + older capture fallback. The system capture border remains unless borderless + access is available (build 20348+) and granted. `capture::cameras()` lists AVFoundation, V4L2, or Media Foundation cameras with identifiers accepted by `capture::Source::Camera`. `capture::displays()` does @@ -37,6 +42,11 @@ the same for macOS, Windows, and X11 displays. `capture::windows()` lists macOS, Windows, and X11 windows. Wayland display selection stays in the desktop portal picker, which does not expose a stable display identifier. +Windows `display:N` selectors are enumeration indices, not persistent monitor +identities. Switching from Desktop Duplication to WGC can change which monitor +a saved selector names. Run `moq devices` again and reselect the intended display +after upgrading. + Embedded applications can consume raw capture without creating a MoQ broadcast: @@ -56,6 +66,23 @@ drops rather than latency. `read` ends with `None` when the source stopped for a benign reason, such as a window resize, so reopen to follow it. Permission denial and a source disappearing are terminal, reported as `Error::PermissionDenied` and `Error::SourceUnavailable`. +On Windows, opening an already minimized window returns `Error::SourceUnavailable`. +After capture starts, receiving no usable first frame within five seconds also +returns `Error::SourceUnavailable` and releases the capture session. +An established capture pauses while the window is minimized. Unchanged Windows +content repeats the last owned GPU texture at the configured frame rate, with +advancing presentation timestamps, so a static share remains live. + +On a Windows desktop, `just rs test -p moq-video --features capture --run-ignored only -E 'test(wgc_)'` runs the opt-in WGC hardware exercises. +Set `MOQ_WGC_WINDOW=window:HWND` to a visible, odd-sized window from +`moq devices` first. These tests capture each monitor and the selected window +with cursor capture on and off, reopen sessions, verify owned NV12 pixels after +pool reuse, and check SD/HD conversion and encoder color descriptions. They +require a GPU video processor and a hardware H.264 encoder; unsupported +hardware fails rather than silently skipping the exercise. Cursor appearance, +resize/close/minimize behavior, hybrid-GPU monitors, and border permissions +still need visual checks. The conversion workload prints submission time and +batch completion time including one final readback, not end-to-end latency. ## Encode diff --git a/rs/moq-video/src/capture/channel.rs b/rs/moq-video/src/capture/channel.rs index ded95b29f6f..a80a9d0ca7f 100644 --- a/rs/moq-video/src/capture/channel.rs +++ b/rs/moq-video/src/capture/channel.rs @@ -72,8 +72,8 @@ impl FrameChannel { /// Map a device-local timestamp into this stream's private timeline. The /// source epoch never escapes: its first sample is anchored to arrival. - /// Only the blocking-device pump feeds native timestamps, so it is gated like - /// `pump` plus `cfg(test)` for the mapping test below. + /// The blocking-device pump and WGC feed native timestamps; the mapping is + /// also compiled for its host-side tests. /// /// A device timeline that steps back or stalls (a driver restarting its clock /// at zero, or one reporting a constant) re-anchors that sample to arrival, or diff --git a/rs/moq-video/src/capture/desktopduplication.rs b/rs/moq-video/src/capture/desktopduplication.rs deleted file mode 100644 index 2f3e8d660ae..00000000000 --- a/rs/moq-video/src/capture/desktopduplication.rs +++ /dev/null @@ -1,418 +0,0 @@ -//! Native Windows display capture via DXGI Desktop Duplication. -//! -//! Duplicates a monitor's output on a Direct3D11 device and pulls each desktop -//! frame off the GPU as a BGRA texture, copies it to a CPU staging texture, and -//! converts it to packed [`I420`] for the encoder. Whole-monitor capture only -//! (the Windows analogue of macOS ScreenCaptureKit display capture); per-window -//! capture would need Windows.Graphics.Capture instead. -//! -//! Runs on the shared blocking [`pump`] thread: `AcquireNextFrame` is a blocking -//! call with no async form, and the [`IDXGIOutputDuplication`] handle is `!Send`, -//! so building and driving it on one thread is the natural fit. The read loop -//! paces itself to the target frame rate, coalescing bursts of desktop updates -//! into one frame and re-emitting the last frame while the screen is static, so a -//! still desktop still produces a steady stream. - -use std::time::{Duration, Instant}; - -use windows::Win32::Foundation::E_ACCESSDENIED; -use windows::Win32::Graphics::Direct3D11::{ - D3D11_CPU_ACCESS_READ, D3D11_MAP_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING, - ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, -}; -use windows::Win32::Graphics::Dxgi::{ - DXGI_ERROR_ACCESS_LOST, DXGI_ERROR_NOT_FOUND, DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, IDXGIAdapter, - IDXGIDevice, IDXGIOutput1, IDXGIOutputDuplication, IDXGIResource, -}; -use windows::core::Interface; - -use super::channel::FrameChannel; -use super::pump::{self, Geometry}; -use super::{Config, Stream}; -use crate::Error; -use crate::frame::{I420, Surface, d3d11}; - -const DEFAULT_FRAMERATE: u32 = 30; - -fn err(ctx: &str, e: windows::core::Error) -> Error { - Error::Codec(anyhow::anyhow!("{ctx}: {e}")) -} - -/// List the outputs on the same adapter [`open`] can capture. -pub(super) fn displays() -> Result, Error> { - let device = d3d11::create_device()?; - let adapter = adapter(&device)?; - let mut displays = Vec::new(); - - for index in 0.. { - let output = match unsafe { adapter.EnumOutputs(index) } { - Ok(output) => output, - Err(error) if error.code() == DXGI_ERROR_NOT_FOUND => break, - Err(error) => return Err(err("EnumOutputs", error)), - }; - let desc = unsafe { output.GetDesc().map_err(|error| err("GetDesc", error))? }; - if !desc.AttachedToDesktop.as_bool() { - continue; - } - - let name_end = desc - .DeviceName - .iter() - .position(|character| *character == 0) - .unwrap_or(desc.DeviceName.len()); - let name = String::from_utf16_lossy(&desc.DeviceName[..name_end]); - let width = (desc.DesktopCoordinates.right - desc.DesktopCoordinates.left).unsigned_abs(); - let height = (desc.DesktopCoordinates.bottom - desc.DesktopCoordinates.top).unsigned_abs(); - displays.push(super::Display { - id: format!("display:{index}"), - name, - width, - height, - }); - } - - Ok(displays) -} - -/// Open a display capture and stream its frames over a pump thread. -pub(super) async fn open(config: &Config, device: Option<&str>) -> Result { - let config = config.clone(); - // The device opens on the pump thread, so the selector has to be owned. - let device = device.map(str::to_string); - let chan = FrameChannel::new(); - let (geo, guard) = pump::spawn( - chan.clone(), - move || { - let cap = Duplicator::open(&config, device.as_deref())?; - let geometry = Geometry { - width: cap.width, - height: cap.height, - framerate: Some(crate::Rate::integer(cap.framerate)), - label: cap.device_name.clone(), - }; - Ok((cap, geometry)) - }, - Duplicator::read, - ) - .await?; - - Ok(Stream::new( - chan, - geo.width, - geo.height, - geo.framerate, - geo.label, - None, - Box::new(guard), - )) -} - -/// An open monitor duplication, read frame-by-frame on the pump thread. -struct Duplicator { - device: ID3D11Device, - context: ID3D11DeviceContext, - /// The duplicated output, kept so the duplication can be rebuilt after a - /// `DXGI_ERROR_ACCESS_LOST` (a mode switch or fullscreen transition). - output: IDXGIOutput1, - dupl: IDXGIOutputDuplication, - /// Reused CPU-readable copy of the desktop texture (constant size). - staging: Option, - /// Even-clamped capture size (I420 chroma is 2x2, so dimensions must be even). - width: u32, - height: u32, - framerate: u32, - interval: Duration, - /// When the next frame is due; paces `read` to `framerate`. - next_deadline: Option, - /// Most recent decoded frame, re-emitted while the screen is static. - last: Option, - device_name: String, -} - -impl Duplicator { - fn open(config: &Config, selector: Option<&str>) -> Result { - let device = d3d11::create_device()?; - let context = unsafe { - device - .GetImmediateContext() - .map_err(|e| err("GetImmediateContext", e))? - }; - - let index = select_output(selector)?; - let output = enumerate_output(&device, index)?; - let device_name = format!("display:{index}"); - let dupl = duplicate(&output, &device)?; - - let desc = unsafe { dupl.GetDesc() }; - // I420 needs even dimensions; clamp down (drop the last odd row/column). - let width = desc.ModeDesc.Width & !1; - let height = desc.ModeDesc.Height & !1; - if width == 0 || height == 0 { - return Err(Error::Codec(anyhow::anyhow!( - "display {index} reported an unusable size {}x{}", - desc.ModeDesc.Width, - desc.ModeDesc.Height - ))); - } - - let framerate = config - .framerate - .unwrap_or(crate::Rate::integer(DEFAULT_FRAMERATE)) - .rounded(); - let mut cap = Self { - device, - context, - output, - dupl, - staging: None, - width, - height, - framerate, - interval: Duration::from_micros(1_000_000 / framerate as u64), - next_deadline: None, - last: None, - device_name, - }; - - // Seed the first frame so a static screen still has something to emit, and - // so a broken duplication path fails here at open rather than mid-stream. - // The desktop image may not be ready instantly, so retry briefly. - for _ in 0..20 { - if cap.capture_once(100)? { - break; - } - } - if cap.last.is_none() { - return Err(Error::Codec(anyhow::anyhow!("no desktop frame within timeout"))); - } - - tracing::info!( - display = %cap.device_name, - width = cap.width, - height = cap.height, - framerate = cap.framerate, - "opened Desktop Duplication capture" - ); - Ok(cap) - } - - /// Rebuild the duplication after `DXGI_ERROR_ACCESS_LOST` (e.g. a resolution - /// change or a fullscreen exclusive app grabbing/releasing the output). - fn reduplicate(&mut self) -> Result<(), Error> { - self.dupl = duplicate(&self.output, &self.device).map_err(|error| match error { - Error::PermissionDenied(_) => error, - error => Error::SourceUnavailable(format!("{}: {error}", self.device_name)), - })?; - self.staging = None; - Ok(()) - } - - /// Acquire at most one desktop frame, waiting up to `timeout_ms`. Returns - /// `true` if a frame was captured into `last`, `false` on timeout (no update). - fn capture_once(&mut self, timeout_ms: u32) -> Result { - let mut info = DXGI_OUTDUPL_FRAME_INFO::default(); - let mut resource: Option = None; - match unsafe { self.dupl.AcquireNextFrame(timeout_ms, &mut info, &mut resource) } { - Ok(()) => {} - Err(e) if e.code() == DXGI_ERROR_WAIT_TIMEOUT => return Ok(false), - Err(e) if e.code() == DXGI_ERROR_ACCESS_LOST => { - self.reduplicate()?; - return Ok(false); - } - Err(e) => { - return Err(Error::SourceUnavailable(format!( - "{}: AcquireNextFrame: {e}", - self.device_name - ))); - } - } - - let resource = resource.ok_or_else(|| Error::Codec(anyhow::anyhow!("AcquireNextFrame returned no surface")))?; - let texture = resource - .cast::() - .map_err(|e| err("desktop surface is not a texture", e))?; - - // Always release the frame, even if the copy fails, or the next acquire - // deadlocks (only one frame may be held at a time). - let result = self.copy_to_last(&texture); - unsafe { - let _ = self.dupl.ReleaseFrame(); - } - result?; - Ok(true) - } - - /// Copy the desktop texture to the staging texture and convert BGRA -> I420. - fn copy_to_last(&mut self, texture: &ID3D11Texture2D) -> Result<(), Error> { - let staging = self.ensure_staging(texture)?; - unsafe { self.context.CopyResource(&staging, texture) }; - - let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); - unsafe { - self.context - .Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) - .map_err(|e| err("Map (staging)", e))?; - } - let _guard = UnmapGuard { - context: &self.context, - resource: &staging, - }; - - let pitch = mapped.RowPitch; - let len = pitch as usize * self.height as usize; - let bgra = unsafe { std::slice::from_raw_parts(mapped.pData as *const u8, len) }; - self.last = Some(I420::from_bgra(bgra, pitch, crate::Size::new(self.width, self.height))?); - Ok(()) - } - - /// Lazily create (and cache) a CPU-readable staging texture matching the - /// desktop texture's format and size. - fn ensure_staging(&mut self, texture: &ID3D11Texture2D) -> Result { - if let Some(staging) = &self.staging { - return Ok(staging.clone()); - } - let mut desc = D3D11_TEXTURE2D_DESC::default(); - unsafe { texture.GetDesc(&mut desc) }; - desc.Usage = D3D11_USAGE_STAGING; - desc.BindFlags = 0; - desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ.0 as u32; - desc.MiscFlags = 0; - desc.ArraySize = 1; - desc.MipLevels = 1; - - let mut staging: Option = None; - unsafe { - self.device - .CreateTexture2D(&desc, None, Some(&mut staging)) - .map_err(|e| err("CreateTexture2D (staging)", e))?; - } - let staging = staging.ok_or_else(|| Error::Codec(anyhow::anyhow!("CreateTexture2D returned null")))?; - self.staging = Some(staging.clone()); - Ok(staging) - } - - /// Capture the next frame, paced to the target frame rate. Coalesces a burst - /// of desktop updates into the latest frame, and re-emits the last frame when - /// the screen hasn't changed, so the output rate stays steady. - fn read(&mut self) -> Result { - let deadline = *self.next_deadline.get_or_insert_with(|| Instant::now() + self.interval); - - loop { - let now = Instant::now(); - if now >= deadline { - break; - } - let remaining = (deadline - now).as_millis().max(1) as u32; - // Blocks up to `remaining`; a new frame returns early, a static screen - // times out at the deadline. Either way we exit the loop at the deadline. - self.capture_once(remaining)?; - } - - // Schedule the next frame; if we fell badly behind (a long stall), reset to - // now so we don't then burst to catch up. - let next = deadline + self.interval; - self.next_deadline = Some(next.max(Instant::now())); - - // A screen that hasn't changed since the stream opened has no frame to hand - // back yet, which is not the duplication API going away. - Ok(self - .last - .clone() - .map_or(pump::Read::Idle, |frame| pump::Read::Frame(Surface::I420(frame)))) - } -} - -struct UnmapGuard<'a> { - context: &'a ID3D11DeviceContext, - resource: &'a ID3D11Texture2D, -} - -impl Drop for UnmapGuard<'_> { - fn drop(&mut self) { - unsafe { self.context.Unmap(self.resource, 0) }; - } -} - -/// Which monitor to capture: a bare index or the `display:{index}` form that -/// [`Stream::device`](super::Stream) reports; `None` is the first one. -fn select_output(selector: Option<&str>) -> Result { - match selector { - None => Ok(0), - Some(spec) => spec - .strip_prefix("display:") - .unwrap_or(spec) - .parse::() - .map_err(|_| Error::Codec(anyhow::anyhow!("invalid display selector {spec:?}"))), - } -} - -/// Get the `index`th output (monitor) attached to the device's adapter. -fn enumerate_output(device: &ID3D11Device, index: u32) -> Result { - let adapter = adapter(device)?; - let output = unsafe { - adapter - .EnumOutputs(index) - .map_err(|_| Error::SourceUnavailable(format!("no display at index {index}")))? - }; - output - .cast::() - .map_err(|e| err("output is not IDXGIOutput1", e)) -} - -/// The DXGI adapter backing a Direct3D device. -fn adapter(device: &ID3D11Device) -> Result { - let dxgi = device - .cast::() - .map_err(|e| err("device is not a DXGI device", e))?; - unsafe { dxgi.GetAdapter().map_err(|e| err("GetAdapter", e)) } -} - -/// Start duplicating `output` on `device`. -fn duplicate(output: &IDXGIOutput1, device: &ID3D11Device) -> Result { - unsafe { output.DuplicateOutput(device) }.map_err(|error| { - if error.code() == E_ACCESSDENIED { - Error::PermissionDenied(format!("display capture: {error}")) - } else { - err("DuplicateOutput", error) - } - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::capture::Config; - use crate::frame::Surface; - - /// Open the primary display, grab a few frames, and check geometry + frame - /// size. Ignored because Desktop Duplication needs an interactive desktop - /// session with a GPU output; skips cleanly when that isn't available. - #[test] - #[ignore] - fn duplicates_primary_display() { - let mut cap = match Duplicator::open(&Config::default(), None) { - Ok(cap) => cap, - Err(e) => { - eprintln!("skipping: no Desktop Duplication available: {e}"); - return; - } - }; - - assert!(cap.width >= 2 && cap.width % 2 == 0, "bad width {}", cap.width); - assert!(cap.height >= 2 && cap.height % 2 == 0, "bad height {}", cap.height); - - for i in 0..5 { - let frame = cap.read().expect("read frame"); - let pump::Read::Frame(Surface::I420(i420)) = frame else { - panic!("frame {i} was not I420"); - }; - assert_eq!(i420.width, cap.width); - assert_eq!(i420.height, cap.height); - assert_eq!( - i420.data.len(), - I420::len(crate::Size::new(cap.width, cap.height)).unwrap() - ); - } - eprintln!("captured 5 frames at {}x{}", cap.width, cap.height); - } -} diff --git a/rs/moq-video/src/capture/mod.rs b/rs/moq-video/src/capture/mod.rs index 3f08347d53d..c6105efcf07 100644 --- a/rs/moq-video/src/capture/mod.rs +++ b/rs/moq-video/src/capture/mod.rs @@ -5,8 +5,8 @@ //! - Linux camera -> native V4L2 (YUYV / MJPEG -> CPU I420), or a PipeWire //! camera node (`pipewire` feature), X11 display and window -> X11, Wayland //! display -> xdg-desktop-portal + PipeWire (`pipewire` feature). -//! - Windows camera -> native Media Foundation (`IMFSourceReader`), screen -> -//! DXGI Desktop Duplication, window -> GDI (BGRA -> CPU I420). +//! - Windows camera -> native Media Foundation (`IMFSourceReader`), display and +//! window -> Windows.Graphics.Capture (GPU NV12, Windows 10 2004 or newer). //! //! [`encode::publish_capture`](crate::encode::publish_capture) consumes [`Config`]. @@ -54,12 +54,9 @@ mod pipewire; #[cfg(target_os = "windows")] mod mediafoundation; -// DXGI Desktop Duplication screen capture on Windows. -#[cfg(target_os = "windows")] -mod desktopduplication; -// Native GDI window enumeration and capture on Windows. -#[cfg(target_os = "windows")] -mod window; +// The WGC notification state and selector tests also run on headless hosts. +#[cfg(any(target_os = "windows", test))] +mod wgc; // Blocking-device -> async-channel bridge used by V4L2 / Media Foundation. #[cfg(any(target_os = "linux", target_os = "windows"))] @@ -121,18 +118,18 @@ impl Source { /// A short human-readable name for the source, used in logs and as the /// captured device label. /// - /// macOS-only: one ScreenCaptureKit backend serves display, window, and app, - /// so it names the source from the config. The other backends label a stream + /// ScreenCaptureKit and WGC serve multiple source kinds, so they name the + /// source from the config. The other backends label a stream /// with the device they resolved (`/dev/video0`, a Media Foundation friendly /// name), which the config doesn't know. - #[cfg(target_os = "macos")] + #[cfg(any(target_os = "macos", target_os = "windows", test))] pub(crate) fn label(&self) -> String { match self { Self::Camera(None) => "camera".to_string(), Self::Camera(Some(id)) => format!("camera:{id}"), Self::Display(None) => "display".to_string(), - Self::Display(Some(id)) => format!("display:{id}"), - Self::Window(id) => format!("window:{id}"), + Self::Display(Some(id)) => format!("display:{}", id.strip_prefix("display:").unwrap_or(id)), + Self::Window(id) => format!("window:{}", id.strip_prefix("window:").unwrap_or(id)), Self::App(id) => format!("app:{id}"), } } @@ -442,7 +439,7 @@ pub async fn open(config: &Config) -> Result { } #[cfg(target_os = "windows")] { - desktopduplication::open(config, device.as_deref()).await + wgc::open(config).await } #[cfg(all(target_os = "linux", feature = "pipewire"))] { @@ -473,7 +470,7 @@ pub async fn open(config: &Config) -> Result { } #[cfg(target_os = "windows")] { - window::open(config, id).await + wgc::open(config).await } #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] { @@ -576,7 +573,7 @@ pub async fn displays() -> Result, Error> { } #[cfg(target_os = "windows")] { - blocking(desktopduplication::displays).await + blocking(wgc::displays).await } #[cfg(target_os = "linux")] { @@ -600,7 +597,7 @@ pub async fn windows() -> Result, Error> { } #[cfg(target_os = "windows")] { - blocking(window::windows).await + blocking(wgc::windows).await } #[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))] { @@ -682,6 +679,26 @@ where mod tests { use super::*; + #[test] + fn source_labels_add_the_source_kind() { + assert_eq!(Source::Camera(None).label(), "camera"); + assert_eq!(Source::Camera(Some("camera-id".into())).label(), "camera:camera-id"); + assert_eq!(Source::Display(None).label(), "display"); + assert_eq!(Source::Display(Some("0".into())).label(), "display:0"); + assert_eq!(Source::Window("123".into()).label(), "window:123"); + assert_eq!(Source::App("com.example.app".into()).label(), "app:com.example.app"); + } + + #[test] + fn source_labels_preserve_prefixed_display_ids() { + assert_eq!(Source::Display(Some("display:0".into())).label(), "display:0"); + } + + #[test] + fn source_labels_preserve_prefixed_window_ids() { + assert_eq!(Source::Window("window:123".into()).label(), "window:123"); + } + #[test] fn camera_selectors_name_their_backend() { assert_eq!(LinuxCamera::select(None, false), LinuxCamera::V4l2(None)); diff --git a/rs/moq-video/src/capture/pipewire.rs b/rs/moq-video/src/capture/pipewire.rs index 1ceb8a0f28e..2a7a9f975cd 100644 --- a/rs/moq-video/src/capture/pipewire.rs +++ b/rs/moq-video/src/capture/pipewire.rs @@ -19,7 +19,7 @@ //! grant is asked for again rather than silently resumed. //! - Compositors only deliver frames on damage, so a static screen would starve //! the encoder. A loop timer re-emits the last frame whenever a frame interval -//! passes without a fresh one, mirroring the Windows Desktop Duplication pacing. +//! passes without a fresh one. //! Cameras deliver every interval, so their streams have no such timer. use std::borrow::Cow; diff --git a/rs/moq-video/src/capture/settle.rs b/rs/moq-video/src/capture/settle.rs index 8efff70a558..9cbcb049b9a 100644 --- a/rs/moq-video/src/capture/settle.rs +++ b/rs/moq-video/src/capture/settle.rs @@ -45,6 +45,12 @@ impl Settle { Self { opened, pending: None } } + /// Wake an event-driven source even if its final resized frame was static. + #[cfg(any(target_os = "windows", test))] + pub fn deadline(&self) -> Option { + self.pending.as_ref().map(|(_, since)| *since + HOLD) + } + /// Feed the geometry observed at `now`. pub fn observe(&mut self, current: &T, now: Instant) -> Settled { if *current == self.opened { @@ -74,6 +80,19 @@ impl Settle { mod tests { use super::*; + #[test] + fn a_static_resize_has_a_deadline_without_another_frame() { + let now = Instant::now(); + let mut settle = Settle::new((800, 600)); + assert_eq!(settle.deadline(), None); + assert_eq!(settle.observe(&(900, 600), now), Settled::Waiting); + assert_eq!(settle.deadline(), Some(now + HOLD)); + assert_eq!(settle.observe(&(902, 600), now + HOLD / 2), Settled::Waiting); + assert_eq!(settle.deadline(), Some(now + HOLD + HOLD / 2)); + assert_eq!(settle.observe(&(800, 600), now + HOLD), Settled::Open); + assert_eq!(settle.deadline(), None); + } + #[test] fn the_open_geometry_captures() { let start = Instant::now(); diff --git a/rs/moq-video/src/capture/wgc.rs b/rs/moq-video/src/capture/wgc.rs new file mode 100644 index 00000000000..ce0c7ede8c9 --- /dev/null +++ b/rs/moq-video/src/capture/wgc.rs @@ -0,0 +1,388 @@ +//! Windows.Graphics.Capture notification ownership and selector validation. +//! Native API calls live separately so the stop/wakeup contract runs in host CI. + +use std::sync::{Condvar, Mutex}; +use std::time::{Duration, Instant}; + +use moq_net::Timestamp; + +use crate::Error; + +#[cfg(target_os = "windows")] +mod native; +#[cfg(target_os = "windows")] +pub(super) use native::{displays, open, windows}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Event { + Stop, + Closed, + Borderless, + Frame, + Deadline, +} + +#[derive(Default)] +struct Events { + stop: bool, + closed: bool, + borderless: bool, + frame: bool, +} + +impl Events { + fn next(&mut self) -> Option { + if self.stop { + Some(Event::Stop) + } else if self.closed { + Some(Event::Closed) + } else if std::mem::take(&mut self.borderless) { + Some(Event::Borderless) + } else if std::mem::take(&mut self.frame) { + Some(Event::Frame) + } else { + None + } + } +} + +#[derive(Default)] +struct Signal { + state: Mutex, + wake: Condvar, +} + +impl Signal { + fn notify(&self, event: Event) { + let mut state = self.state.lock().unwrap(); + match event { + Event::Stop => state.stop = true, + Event::Closed => state.closed = true, + Event::Borderless => state.borderless = true, + Event::Frame => state.frame = true, + Event::Deadline => unreachable!("deadlines are derived from the clock"), + } + drop(state); + self.wake.notify_one(); + } + + fn wait(&self, deadline: Option) -> Event { + let mut state = self.state.lock().unwrap(); + loop { + if let Some(event) = state.next() { + return event; + } + state = match deadline { + Some(deadline) => { + let now = Instant::now(); + if now >= deadline { + return Event::Deadline; + } + self.wake.wait_timeout(state, deadline - now).unwrap().0 + } + None => self.wake.wait(state).unwrap(), + }; + } + } +} + +// A source can stop producing events before its first frame, for example if +// a window is minimized just after capture starts. Startup must still finish. +struct Startup { + deadline: Option, +} + +impl Startup { + fn new(now: Instant) -> Self { + Self { + deadline: Some(now + Duration::from_secs(5)), + } + } + + fn check(&self, now: Instant) -> Result<(), Error> { + if self.deadline.is_some_and(|deadline| now >= deadline) { + return Err(Error::SourceUnavailable( + "WGC did not deliver its first frame within 5 seconds".into(), + )); + } + Ok(()) + } + + fn delivered(&mut self) { + self.deadline = None; + } +} + +// WGC can omit unchanged frames. Retain only the owned output, keeping the +// capture stream paced without holding a frame-pool slot or converting again. +struct Delivery { + frame: Option, + clock: Option<(Timestamp, Instant)>, + interval: Duration, + next: Instant, +} + +impl Delivery { + fn new(interval: Duration, now: Instant) -> Self { + Self { + frame: None, + clock: None, + interval, + next: now, + } + } + + fn replace(&mut self, frame: T, timestamp: Timestamp, now: Instant) { + self.frame = Some(frame); + // Changed frames can arrive after a repeated frame was already delivered. + // Keep one presentation clock instead of rewinding to their acquisition time. + self.clock.get_or_insert((timestamp, now)); + } + + fn clear(&mut self) { + self.frame = None; + } + + fn deadline(&self) -> Option { + self.frame.as_ref().map(|_| self.next) + } + + fn next(&mut self, now: Instant) -> Result, Error> { + if now < self.next { + return Ok(None); + } + let Some(frame) = self.frame.as_ref() else { + return Ok(None); + }; + let (timestamp, started) = self.clock.unwrap(); + let elapsed = + u64::try_from(now.saturating_duration_since(started).as_micros()).map_err(|_| moq_net::TimeOverflow)?; + let timestamp = timestamp.checked_add(Timestamp::from_micros(elapsed)?)?; + self.next = now + self.interval; + Ok(Some((frame, timestamp))) + } +} + +fn display_index(selector: Option<&str>) -> Result { + selector.map_or(Ok(0), |selector| { + selector + .strip_prefix("display:") + .unwrap_or(selector) + .parse() + .map_err(|_| Error::SourceUnavailable(format!("invalid Windows display selector {selector:?}"))) + }) +} + +fn window_handle(selector: &str) -> Result { + let value = selector + .strip_prefix("window:") + .unwrap_or(selector) + .parse::() + .map_err(|_| Error::SourceUnavailable(format!("invalid Windows window selector {selector:?}")))?; + if value == 0 { + return Err(Error::SourceUnavailable("a window handle cannot be null".into())); + } + Ok(value) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + #[test] + fn startup_without_frame_events_expires() { + let now = Instant::now(); + let startup = Startup::new(now); + let deadline = now + Duration::from_secs(5); + assert_eq!(startup.deadline, Some(deadline)); + assert!(startup.check(deadline - Duration::from_nanos(1)).is_ok()); + // Model WGC silence after minimizing between StartCapture and its first + // frame. No callback is required to reach the failure boundary. + assert!(matches!(startup.check(deadline), Err(Error::SourceUnavailable(_)))); + } + + #[test] + fn startup_callbacks_without_usable_frames_do_not_extend_the_deadline() { + let now = Instant::now(); + let startup = Startup::new(now); + let signal = Signal::default(); + for second in 1..=5 { + signal.notify(Event::Frame); + signal.notify(Event::Borderless); + assert_eq!(signal.wait(startup.deadline), Event::Borderless); + assert_eq!(signal.wait(startup.deadline), Event::Frame); + // Empty pools and invalid content sizes do not deliver a frame. + assert_eq!(startup.deadline, Some(now + Duration::from_secs(5))); + assert_eq!(startup.check(now + Duration::from_secs(second)).is_err(), second == 5); + } + } + + #[test] + fn first_delivery_removes_the_startup_deadline() { + let now = Instant::now(); + let mut startup = Startup::new(now); + assert!(startup.check(now + Duration::from_secs(1)).is_ok()); + startup.delivered(); + assert_eq!(startup.deadline, None); + assert!(startup.check(now + Duration::from_secs(10)).is_ok()); + } + + #[test] + fn terminal_events_take_precedence_over_expired_startup() { + let startup = Startup::new(Instant::now() - Duration::from_secs(5)); + let signal = Signal::default(); + assert_eq!(signal.wait(startup.deadline), Event::Deadline); + signal.notify(Event::Frame); + signal.notify(Event::Borderless); + signal.notify(Event::Closed); + assert_eq!(signal.wait(startup.deadline), Event::Closed); + signal.notify(Event::Stop); + assert_eq!(signal.wait(startup.deadline), Event::Stop); + } + + #[test] + fn unchanged_frames_keep_the_delivery_deadline_and_advance_timestamps() { + let now = Instant::now(); + let interval = Duration::from_millis(20); + let mut delivery = Delivery::new(interval, now); + let pixels = Arc::new([1, 2, 3]); + delivery.replace(pixels.clone(), Timestamp::from_micros(1_000_000).unwrap(), now); + + for step in 0..10 { + let now = now + interval * step; + assert_eq!(delivery.deadline(), Some(now)); + let (frame, timestamp) = delivery.next(now).unwrap().unwrap(); + assert!( + Arc::ptr_eq(frame, &pixels), + "reuse the owned output without a conversion" + ); + assert_eq!(timestamp.as_micros(), 1_000_000 + u128::from(step) * 20_000); + assert!(delivery.next(now + interval / 2).unwrap().is_none()); + } + } + + #[test] + fn new_frames_and_resume_keep_the_same_presentation_clock() { + let now = Instant::now(); + let interval = Duration::from_millis(20); + let mut delivery = Delivery::new(interval, now); + delivery.replace(1, Timestamp::from_micros(1_000_000).unwrap(), now); + delivery.next(now).unwrap().unwrap(); + let repeated = now + interval * 5; + assert_eq!(delivery.next(repeated).unwrap().unwrap().1.as_micros(), 1_100_000); + // This changed frame was acquired before the most recent repeated delivery. + delivery.replace(2, Timestamp::from_micros(1_090_000).unwrap(), repeated); + let (frame, timestamp) = delivery.next(repeated + interval).unwrap().unwrap(); + assert_eq!(*frame, 2); + assert_eq!(timestamp.as_micros(), 1_120_000); + + // Minimized, empty, or resizing sources stop repeating until new content arrives. + delivery.clear(); + assert_eq!(delivery.deadline(), None); + let resumed = now + Duration::from_secs(1); + assert!(delivery.next(resumed).unwrap().is_none()); + delivery.replace(3, Timestamp::from_micros(1_990_000).unwrap(), resumed); + let (frame, timestamp) = delivery.next(resumed).unwrap().unwrap(); + assert_eq!(*frame, 3); + assert_eq!(timestamp.as_micros(), 2_000_000); + assert_eq!(delivery.deadline(), Some(resumed + interval)); + } + + #[tokio::test] + #[cfg(feature = "openh264")] + async fn unchanged_frames_reach_an_active_subscriber() { + use crate::encode::{Config, Encoder, Kind, Producer}; + + let now = Instant::now(); + let interval = Duration::from_millis(20); + let mut delivery = Delivery::new(interval, now); + let surface = crate::frame::I420 { + width: 320, + height: 240, + data: vec![0x80; 320 * 240 * 3 / 2], + color: None, + }; + delivery.replace(surface, Timestamp::from_micros(0).unwrap(), now); + let mut config = Config::new(320, 240, crate::Rate::integer(50)); + config.kind = Kind::Software; + let mut encoder = Encoder::new(&config).unwrap(); + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let track = broadcast + .create_track("video", catalog.track_info(hang::catalog::PRIORITY.video)) + .unwrap(); + let subscriber = track.subscribe(None); + let rendition = config.probe().await.unwrap(); + let container = moq_mux::catalog::hang::Container::try_from(&rendition).unwrap(); + let mut subscriber = moq_mux::container::Consumer::new(subscriber, container); + let mut producer = Producer::with_track(track, catalog.clone(), rendition).unwrap(); + assert!(producer.demand().is_used()); + + // Only one captured picture arrives; more than three intervals of paced + // output must still reach a real video subscriber with advancing timestamps. + for step in 0..10 { + let now = now + interval * step; + assert_eq!(delivery.deadline(), Some(now)); + let (surface, timestamp) = delivery.next(now).unwrap().unwrap(); + let frame = crate::Frame::new(crate::Surface::I420(surface.clone()), timestamp); + producer.publish(&encoder.encode(&frame).unwrap()).unwrap(); + let received = subscriber.read().await.unwrap().unwrap(); + assert_eq!(received.timestamp.as_micros(), u128::from(step) * 20_000); + } + } + + #[test] + fn selectors_preserve_native_ids_and_refuse_malformed_input() { + assert_eq!(display_index(None).unwrap(), 0); + assert_eq!(display_index(Some("display:2")).unwrap(), 2); + assert_eq!(display_index(Some("2")).unwrap(), 2); + assert_eq!(window_handle("window:1234").unwrap(), 1234); + assert_eq!(window_handle("1234").unwrap(), 1234); + for invalid in ["", "-1", "window:2", "display:"] { + assert!(display_index(Some(invalid)).is_err()); + } + for invalid in ["0", "window:0", "-1", "display:1"] { + assert!(window_handle(invalid).is_err()); + } + } + + #[test] + fn stop_wakes_without_a_frame_or_polling_timeout() { + let signal = Arc::new(Signal::default()); + let barrier = Arc::new(std::sync::Barrier::new(2)); + let worker = std::thread::spawn({ + let signal = signal.clone(); + let barrier = barrier.clone(); + move || { + barrier.wait(); + signal.wait(None) + } + }); + barrier.wait(); + signal.notify(Event::Stop); + assert_eq!(worker.join().unwrap(), Event::Stop); + } + + #[test] + fn terminal_events_win_over_queued_frames_and_access_callbacks() { + let signal = Signal::default(); + signal.notify(Event::Frame); + signal.notify(Event::Borderless); + signal.notify(Event::Closed); + assert_eq!(signal.wait(None), Event::Closed); + signal.notify(Event::Stop); + assert_eq!(signal.wait(None), Event::Stop); + } + + #[test] + fn notifications_are_retained_and_frames_coalesce() { + let signal = Signal::default(); + signal.notify(Event::Frame); + signal.notify(Event::Frame); + signal.notify(Event::Borderless); + assert_eq!(signal.wait(None), Event::Borderless); + assert_eq!(signal.wait(None), Event::Frame); + assert_eq!(signal.wait(Some(Instant::now())), Event::Deadline); + } +} diff --git a/rs/moq-video/src/capture/wgc/native.rs b/rs/moq-video/src/capture/wgc/native.rs new file mode 100644 index 00000000000..19843fa674d --- /dev/null +++ b/rs/moq-video/src/capture/wgc/native.rs @@ -0,0 +1,679 @@ +//! One WGC session for either a monitor or a window, owned by its MTA thread. + +use std::ffi::c_void; +use std::sync::Arc; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +use windows::Foundation::{Metadata::ApiInformation, TypedEventHandler}; +use windows::Graphics::Capture::{ + Direct3D11CaptureFrame, Direct3D11CaptureFramePool, GraphicsCaptureAccess, GraphicsCaptureAccessKind, + GraphicsCaptureItem, GraphicsCaptureSession, +}; +use windows::Graphics::DirectX::{Direct3D11::IDirect3DDevice, DirectXPixelFormat}; +use windows::Security::Authorization::AppCapabilityAccess::AppCapabilityAccessStatus; +use windows::Win32::Foundation::{CloseHandle, E_ACCESSDENIED, HANDLE, HWND, LPARAM, RECT}; +use windows::Win32::Graphics::Direct3D11::{ID3D11Device, ID3D11Texture2D}; +use windows::Win32::Graphics::Dwm::{DWMWA_CLOAKED, DWMWA_EXTENDED_FRAME_BOUNDS, DwmGetWindowAttribute}; +use windows::Win32::Graphics::Dxgi::IDXGIDevice; +use windows::Win32::Graphics::Gdi::{EnumDisplayMonitors, GetMonitorInfoW, HDC, HMONITOR, MONITORINFOEXW}; +use windows::Win32::System::Threading::{ + OpenProcess, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION, QueryFullProcessImageNameW, +}; +use windows::Win32::System::WinRT::Direct3D11::{CreateDirect3D11DeviceFromDXGIDevice, IDirect3DDxgiInterfaceAccess}; +use windows::Win32::System::WinRT::Graphics::Capture::IGraphicsCaptureItemInterop; +use windows::Win32::System::WinRT::{RO_INIT_MULTITHREADED, RoInitialize, RoUninitialize}; +use windows::Win32::UI::HiDpi::{ + DPI_AWARENESS_CONTEXT, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2, SetThreadDpiAwarenessContext, +}; +use windows::Win32::UI::WindowsAndMessaging::{ + EnumWindows, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, IsIconic, IsWindowVisible, +}; +use windows::core::{IInspectable, Interface, PWSTR, factory, h}; + +use super::super::settle::{Settle, Settled}; +use super::super::{Config, Display, FrameChannel, Source, Stream, Window}; +use super::{Delivery, Event, Signal, Startup, display_index, window_handle}; +use crate::frame::{Surface, d3d11}; +use crate::{Error, Rate, Size}; + +const BUFFERS: i32 = 2; + +pub(in crate::capture) async fn open(config: &Config) -> Result { + let chan = FrameChannel::new(); + let signal = Arc::new(Signal::default()); + let label = config.source.label(); + let rate = config.framerate.unwrap_or(Rate::integer(30)); + let handle = std::thread::Builder::new() + .name("moq-wgc".into()) + .spawn({ + let chan = chan.clone(); + let signal = signal.clone(); + let config = config.clone(); + move || { + // A panic must also release a reader awaiting its first frame. + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&config, &chan, &signal))); + match result { + Ok(Ok(())) => chan.close(), + Ok(Err(error)) => chan.fail(error), + Err(_) => chan.fail(Error::Codec(anyhow::anyhow!("WGC capture thread panicked"))), + } + } + }) + .map_err(|error| Error::Codec(anyhow::anyhow!("start WGC thread: {error}")))?; + let guard = Guard { + signal, + handle: Some(handle), + }; + let first = chan + .recv() + .await? + .ok_or_else(|| Error::SourceUnavailable(format!("{label} ended before its first frame")))?; + let Size { width, height } = first.size(); + tracing::info!(source = %label, width, height, "opened screen capture (WGC)"); + Ok(Stream::new( + chan, + width, + height, + Some(rate), + label, + Some(first), + Box::new(guard), + )) +} + +struct Guard { + signal: Arc, + handle: Option>, +} + +impl Drop for Guard { + fn drop(&mut self) { + self.signal.notify(Event::Stop); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +struct Apartment; + +impl Drop for Apartment { + fn drop(&mut self) { + unsafe { RoUninitialize() }; + } +} + +fn run(config: &Config, chan: &Arc, signal: &Arc) -> Result<(), Error> { + unsafe { RoInitialize(RO_INIT_MULTITHREADED) }.map_err(|e| error("initialize WinRT", e))?; + let _apartment = Apartment; + if !ApiInformation::IsPropertyPresent( + h!("Windows.Graphics.Capture.GraphicsCaptureSession"), + h!("IsCursorCaptureEnabled"), + ) + .map_err(|e| error("query WGC cursor support", e))? + || !GraphicsCaptureSession::IsSupported().map_err(|e| error("query WGC support", e))? + { + return Err(Error::Unsupported( + "Windows.Graphics.Capture requires Windows 10 2004 (build 19041) or newer".into(), + )); + } + let mut capture = Capture::new(config, chan, signal)?; + // Never block this owner thread on a consent dialog: stop must remain usable. + // The session keeps its border until access completes successfully. + let access = if ApiInformation::IsPropertyPresent( + h!("Windows.Graphics.Capture.GraphicsCaptureSession"), + h!("IsBorderRequired"), + ) + .map_err(|e| error("query WGC border support", e))? + { + match GraphicsCaptureAccess::RequestAccessAsync(GraphicsCaptureAccessKind::Borderless) { + Ok(access) => { + let signal = signal.clone(); + if let Err(err) = access.when(move |result| match result { + Ok(AppCapabilityAccessStatus::Allowed) => signal.notify(Event::Borderless), + Ok(status) => tracing::debug!(?status, "WGC borderless access not granted; keeping border"), + Err(err) => tracing::debug!(%err, "WGC borderless request failed; keeping border"), + }) { + let _ = access.Cancel(); + tracing::debug!(%err, "WGC borderless callback unavailable; keeping border"); + } + Some(access) + } + Err(err) => { + tracing::debug!(%err, "WGC borderless unavailable; keeping border"); + None + } + } + } else { + None + }; + let result = capture.run(config, chan, signal); + if let Some(access) = access { + let _ = access.Cancel(); + } + result +} + +struct Capture { + item: GraphicsCaptureItem, + pool: Direct3D11CaptureFramePool, + session: Option, + device: ID3D11Device, + frame_token: Option, + closed_token: Option, + window: Option, +} + +impl Capture { + fn new(config: &Config, chan: &Arc, signal: &Arc) -> Result { + let _dpi = DpiContext::enter()?; + let interop = + factory::().map_err(|e| error("WGC item factory", e))?; + let (item, window): (GraphicsCaptureItem, _) = match &config.source { + Source::Display(selector) => { + let index = display_index(selector.as_deref())?; + let monitors = monitors()?; + let monitor = monitors + .get(index) + .ok_or_else(|| Error::SourceUnavailable(format!("no display at index {index}")))?; + ( + unsafe { interop.CreateForMonitor(monitor.handle) } + .map_err(|e| error("create WGC monitor item", e))?, + None, + ) + } + Source::Window(selector) => { + let handle = HWND(window_handle(selector)? as *mut c_void); + if unsafe { IsIconic(handle) }.as_bool() { + return Err(Error::SourceUnavailable( + "cannot start capture from a minimized window".into(), + )); + } + ( + unsafe { interop.CreateForWindow(handle) }.map_err(|e| error("create WGC window item", e))?, + Some(handle), + ) + } + _ => return Err(Error::Unsupported("WGC captures one display or window".into())), + }; + let size = item.Size().map_err(|e| error("get WGC item size", e))?; + if size.Width < 2 || size.Height < 2 { + return Err(Error::SourceUnavailable("WGC item has no capturable area".into())); + } + let device = d3d11::create_device()?; + let dxgi: IDXGIDevice = device.cast().map_err(|e| error("query capture DXGI device", e))?; + let winrt: IDirect3DDevice = unsafe { CreateDirect3D11DeviceFromDXGIDevice(&dxgi) } + .and_then(|device| device.cast()) + .map_err(|e| error("create WinRT D3D device", e))?; + let pool = Direct3D11CaptureFramePool::CreateFreeThreaded( + &winrt, + DirectXPixelFormat::B8G8R8A8UIntNormalized, + BUFFERS, + size, + ) + .map_err(|e| error("create WGC frame pool", e))?; + let mut capture = Self { + item, + pool, + session: None, + device, + frame_token: None, + closed_token: None, + window, + }; + capture.closed_token = Some( + capture + .item + .Closed(&TypedEventHandler::::new({ + let signal = signal.clone(); + let chan = chan.clone(); + let label = config.source.label(); + move |_, _| { + chan.fail(Error::SourceUnavailable(format!("{label} closed"))); + signal.notify(Event::Closed); + Ok(()) + } + })) + .map_err(|e| error("register WGC item closed", e))?, + ); + capture.frame_token = Some( + capture + .pool + .FrameArrived(&TypedEventHandler::::new({ + let signal = signal.clone(); + move |_, _| { + signal.notify(Event::Frame); + Ok(()) + } + })) + .map_err(|e| error("register WGC frames", e))?, + ); + let session = capture + .pool + .CreateCaptureSession(&capture.item) + .map_err(|e| error("create WGC session", e))?; + capture.session = Some(session); + capture + .session + .as_ref() + .unwrap() + .SetIsCursorCaptureEnabled(config.cursor) + .map_err(|e| error("configure WGC cursor", e))?; + Ok(capture) + } + + fn run(&mut self, config: &Config, chan: &FrameChannel, signal: &Signal) -> Result<(), Error> { + self.session + .as_ref() + .unwrap() + .StartCapture() + .map_err(|e| error("start WGC", e))?; + let mut startup = Startup::new(Instant::now()); + let rate = config.framerate.unwrap_or(Rate::integer(30)); + let interval = + Duration::from_nanos(1_000_000_000 * u64::from(rate.denominator()) / u64::from(rate.numerator())); + let mut opened = None; + let mut settle = None; + let mut pending: Option = None; + let mut delivery = Delivery::new(interval, Instant::now()); + loop { + let resize_deadline = settle.as_ref().and_then(Settle::deadline); + let deadline = resize_deadline + .into_iter() + .chain(startup.deadline) + .chain(delivery.deadline()) + .chain(pending.as_ref().map(|_| delivery.next)) + .min(); + match signal.wait(deadline) { + Event::Stop => return Ok(()), + Event::Closed => return Err(Error::SourceUnavailable(format!("{} closed", config.source.label()))), + Event::Borderless => { + if let Err(err) = self.session.as_ref().unwrap().SetIsBorderRequired(false) { + tracing::debug!(%err, "WGC borderless denied; keeping border"); + } + } + Event::Frame => { + // Bound each drain so an active desktop cannot starve shutdown. + for _ in 0..BUFFERS { + let Some(frame) = self.next()? else { break }; + pending = Some(frame); + } + } + Event::Deadline => {} + } + let now = Instant::now(); + startup.check(now)?; + if self.window.is_some_and(|window| unsafe { IsIconic(window) }.as_bool()) { + if opened.is_none() { + return Err(Error::SourceUnavailable( + "cannot start capture from a minimized window".into(), + )); + } + pending = None; + delivery.clear(); + settle = opened.map(Settle::new); + continue; + } + if let Some(frame) = pending.as_ref() { + let native = frame.0.ContentSize().map_err(|e| error("read WGC content size", e))?; + if native.Width < 2 || native.Height < 2 { + pending = None; + delivery.clear(); + settle = opened.map(Settle::new); + continue; + } + let size = Size::new(native.Width as u32, native.Height as u32); + if opened.is_none() { + opened = Some(size); + settle = Some(Settle::new(size)); + } + match settle.as_mut().unwrap().observe(&size, now) { + Settled::Open => {} + Settled::Waiting => { + pending = None; + delivery.clear(); + } + Settled::Changed => return Ok(()), + } + } + if settle + .as_ref() + .and_then(Settle::deadline) + .is_some_and(|deadline| now >= deadline) + { + tracing::info!(source = %config.source.label(), "WGC source resized; ending capture for reopen"); + return Ok(()); + } + if now >= delivery.next + && let Some(frame) = pending.take() + { + let ticks = frame + .0 + .SystemRelativeTime() + .map_err(|e| error("read WGC timestamp", e))? + .Duration; + let micros = + u64::try_from(ticks).map_err(|_| Error::Codec(anyhow::anyhow!("negative WGC timestamp")))? / 10; + let timestamp = moq_net::Timestamp::from_micros(micros)?; + let surface = frame.0.Surface().map_err(|e| error("read WGC surface", e))?; + let access: IDirect3DDxgiInterfaceAccess = surface.cast().map_err(|e| error("query WGC surface", e))?; + let texture: ID3D11Texture2D = + unsafe { access.GetInterface() }.map_err(|e| error("read WGC texture", e))?; + let texture = d3d11::Texture::capture(&self.device, &texture, opened.unwrap())?; + delivery.replace(texture, timestamp, now); + } + if let Some((texture, timestamp)) = delivery.next(now)? { + // Only AddRef the owned NV12 output. The WGC pool frame was released + // after conversion, and the cached texture is never written again. + let texture = d3d11::Texture { + device: texture.device.clone(), + texture: texture.texture.clone(), + width: texture.width, + height: texture.height, + color: texture.color, + }; + chan.push_native(Surface::Texture(texture), timestamp); + startup.delivered(); + } + } + } + + fn next(&self) -> Result, Error> { + let mut frame = std::ptr::null_mut(); + // The projection rejects S_OK + null, but WGC uses it to mean an empty pool. + unsafe { + (self.pool.vtable().TryGetNextFrame)(self.pool.as_raw(), &mut frame) + .ok() + .map_err(|e| error("read WGC frame", e))?; + } + Ok((!frame.is_null()).then(|| Captured(unsafe { Direct3D11CaptureFrame::from_raw(frame) }))) + } +} + +impl Drop for Capture { + fn drop(&mut self) { + // No notification mutex is held while calling into callbacks/COM teardown. + if let Some(token) = self.frame_token.take() { + let _ = self.pool.RemoveFrameArrived(token); + } + if let Some(token) = self.closed_token.take() { + let _ = self.item.RemoveClosed(token); + } + if let Some(session) = self.session.take() { + let _ = session.Close(); + } + let _ = self.pool.Close(); + } +} + +struct Captured(Direct3D11CaptureFrame); +impl Drop for Captured { + fn drop(&mut self) { + let _ = self.0.Close(); + } +} + +struct Monitor { + handle: HMONITOR, + info: MONITORINFOEXW, +} + +fn monitors() -> Result, Error> { + let mut handles = Vec::new(); + unsafe { + EnumDisplayMonitors( + None, + None, + Some(collect_monitor), + LPARAM((&mut handles as *mut Vec) as isize), + ) + } + .ok() + .map_err(|e| error("enumerate monitors", e))?; + let mut result = Vec::new(); + for handle in handles { + let mut info = MONITORINFOEXW::default(); + info.monitorInfo.cbSize = std::mem::size_of::() as u32; + unsafe { GetMonitorInfoW(handle, &mut info.monitorInfo) } + .ok() + .map_err(|e| error("get monitor info", e))?; + result.push(Monitor { handle, info }); + } + // MONITORINFOF_PRIMARY = 1. Keep the default display first on hybrid systems. + result.sort_by_key(|monitor| monitor.info.monitorInfo.dwFlags & 1 == 0); + Ok(result) +} + +unsafe extern "system" fn collect_monitor( + monitor: HMONITOR, + _: HDC, + _: *mut RECT, + data: LPARAM, +) -> windows::core::BOOL { + unsafe { &mut *(data.0 as *mut Vec) }.push(monitor); + true.into() +} + +pub(in crate::capture) fn displays() -> Result, Error> { + let _dpi = DpiContext::enter()?; + Ok(monitors()? + .into_iter() + .enumerate() + .map(|(index, monitor)| { + let rect = monitor.info.monitorInfo.rcMonitor; + let name = &monitor.info.szDevice; + Display { + id: format!("display:{index}"), + name: String::from_utf16_lossy(&name[..name.iter().position(|v| *v == 0).unwrap_or(name.len())]), + width: (rect.right - rect.left).max(0) as u32, + height: (rect.bottom - rect.top).max(0) as u32, + } + }) + .collect()) +} + +pub(in crate::capture) fn windows() -> Result, Error> { + let _dpi = DpiContext::enter()?; + let mut handles = Vec::new(); + unsafe { EnumWindows(Some(collect_window), LPARAM((&mut handles as *mut Vec) as isize)) } + .map_err(|e| error("enumerate windows", e))?; + let mut result = Vec::new(); + for handle in handles { + let mut cloaked = 0u32; + let mut rect = RECT::default(); + unsafe { + if DwmGetWindowAttribute( + handle, + DWMWA_CLOAKED, + (&mut cloaked as *mut u32).cast(), + std::mem::size_of_val(&cloaked) as u32, + ) + .is_err() || cloaked != 0 + || DwmGetWindowAttribute( + handle, + DWMWA_EXTENDED_FRAME_BOUNDS, + (&mut rect as *mut RECT).cast(), + std::mem::size_of_val(&rect) as u32, + ) + .is_err() + { + continue; + } + } + let width = (rect.right - rect.left).max(0) as u32; + let height = (rect.bottom - rect.top).max(0) as u32; + let length = unsafe { GetWindowTextLengthW(handle) }; + if width < 2 || height < 2 || length <= 0 { + continue; + } + let mut title = vec![0u16; length as usize + 1]; + let copied = unsafe { GetWindowTextW(handle, &mut title) }.max(0) as usize; + if copied == 0 { + continue; + } + result.push(Window { + id: format!("window:{}", handle.0 as usize), + title: String::from_utf16_lossy(&title[..copied]), + app: app_name(handle), + width, + height, + }); + } + Ok(result) +} + +unsafe extern "system" fn collect_window(handle: HWND, data: LPARAM) -> windows::core::BOOL { + if unsafe { IsWindowVisible(handle) }.as_bool() { + unsafe { &mut *(data.0 as *mut Vec) }.push(handle); + } + true.into() +} + +fn app_name(handle: HWND) -> String { + let mut id = 0; + unsafe { GetWindowThreadProcessId(handle, Some(&mut id)) }; + let Ok(process) = (unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, id) }) else { + return String::new(); + }; + let process = Process(process); + let mut path = vec![0u16; 32_768]; + let mut length = path.len() as u32; + if unsafe { QueryFullProcessImageNameW(process.0, PROCESS_NAME_WIN32, PWSTR(path.as_mut_ptr()), &mut length) } + .is_err() + { + return String::new(); + } + let path = String::from_utf16_lossy(&path[..length as usize]); + std::path::Path::new(&path) + .file_stem() + .and_then(|name| name.to_str()) + .unwrap_or_default() + .to_string() +} + +struct Process(HANDLE); +impl Drop for Process { + fn drop(&mut self) { + let _ = unsafe { CloseHandle(self.0) }; + } +} + +struct DpiContext(DPI_AWARENESS_CONTEXT); +impl DpiContext { + fn enter() -> Result { + let previous = unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2) }; + if previous.0.is_null() { + return Err(error( + "enable per-monitor DPI awareness", + windows::core::Error::from_thread(), + )); + } + Ok(Self(previous)) + } +} +impl Drop for DpiContext { + fn drop(&mut self) { + unsafe { SetThreadDpiAwarenessContext(self.0) }; + } +} + +fn error(context: &str, error: windows::core::Error) -> Error { + if error.code() == E_ACCESSDENIED { + Error::PermissionDenied(format!("{context}: {error}")) + } else { + Error::Codec(anyhow::anyhow!("{context}: {error}")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + #[ignore = "requires a Windows desktop; creates a minimized test window"] + async fn wgc_minimized_window_is_rejected_before_the_first_frame() { + use windows::Win32::UI::WindowsAndMessaging::{ + CreateWindowExW, DestroyWindow, WS_MINIMIZE, WS_OVERLAPPEDWINDOW, + }; + use windows::core::w; + + struct TestWindow(HWND); + impl Drop for TestWindow { + fn drop(&mut self) { + let _ = unsafe { DestroyWindow(self.0) }; + } + } + let window = TestWindow(unsafe { + CreateWindowExW( + Default::default(), + w!("STATIC"), + w!("WGC minimized-window test"), + WS_OVERLAPPEDWINDOW | WS_MINIMIZE, + 0, + 0, + 640, + 480, + None, + None, + None, + None, + ) + .expect("create test window") + }); + assert!(unsafe { IsIconic(window.0) }.as_bool()); + let config = Config { + source: Source::Window(format!("window:{}", window.0.0 as usize)), + ..Default::default() + }; + assert!(matches!( + open(&config).await, + Err(Error::SourceUnavailable(message)) if message.contains("minimized") + )); + } + + #[tokio::test] + #[ignore = "requires an interactive Windows desktop; captures each enumerated monitor"] + async fn wgc_displays_reopen_with_cursor_on_and_off() { + let displays = displays().expect("enumerate monitors"); + assert!(!displays.is_empty(), "requires at least one attached monitor"); + for display in displays { + capture_and_release(display.source()).await; + } + } + + #[tokio::test] + #[ignore = "set MOQ_WGC_WINDOW to an open window:HWND; use an odd-sized visible window"] + async fn wgc_window_reopens_with_cursor_on_and_off() { + let selector = std::env::var("MOQ_WGC_WINDOW").expect("set MOQ_WGC_WINDOW=window:HWND from moq devices"); + capture_and_release(Source::Window(selector)).await; + } + + async fn capture_and_release(source: Source) { + for cursor in [true, false] { + let config = Config { + source: source.clone(), + cursor, + ..Default::default() + }; + let mut stream = open(&config).await.expect("open WGC"); + let frame = stream.read().await.unwrap().expect("first WGC frame"); + assert_eq!(frame.size().width % 2, 0); + assert_eq!(frame.size().height % 2, 0); + assert!( + matches!(&frame.surface, Surface::Texture(_)), + "capture must stay on the GPU" + ); + let color = stream.color(); + assert_eq!(color, Some(crate::Color::infer(frame.size()))); + // Stop before another frame arrives; the returned pixels must outlive the pool. + drop(stream); + let pixels = frame + .surface + .to_i420() + .expect("read owned texture after closing capture"); + assert_eq!(pixels.size(), frame.size()); + assert_eq!(pixels.color(), color); + } + } +} diff --git a/rs/moq-video/src/capture/window.rs b/rs/moq-video/src/capture/window.rs deleted file mode 100644 index 1c22c6be79c..00000000000 --- a/rs/moq-video/src/capture/window.rs +++ /dev/null @@ -1,599 +0,0 @@ -//! Native Windows single-window capture. -//! -//! Desktop Duplication is monitor-scoped, so selected windows use GDI. The -//! backend enumerates visible top-level windows, renders the selected one into -//! a memory DC at the requested frame rate, and converts BGRA to I420. The -//! shared latest-frame channel keeps the GDI loop independent from encoder -//! latency. - -use std::ffi::c_void; -use std::mem::size_of; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; - -use windows::Win32::Foundation::{CloseHandle, E_ACCESSDENIED, FILETIME, HANDLE, HWND, LPARAM, RECT, WPARAM}; -use windows::Win32::Graphics::Gdi::{ - BI_RGB, BITMAPINFO, BITMAPINFOHEADER, BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DIB_RGB_COLORS, DeleteDC, - DeleteObject, GetDIBits, GetWindowDC, HBITMAP, HDC, HGDIOBJ, ReleaseDC, SRCCOPY, SelectObject, -}; -use windows::Win32::Storage::Xps::{PRINT_WINDOW_FLAGS, PrintWindow}; -use windows::Win32::System::Threading::{ - GetProcessTimes, OpenProcess, PROCESS_NAME_WIN32, PROCESS_QUERY_LIMITED_INFORMATION, QueryFullProcessImageNameW, -}; -use windows::Win32::UI::HiDpi::{ - DPI_AWARENESS_CONTEXT, DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE, SetThreadDpiAwarenessContext, -}; -use windows::Win32::UI::WindowsAndMessaging::{ - CURSOR_SHOWING, CURSORINFO, DI_NORMAL, DrawIconEx, EnumWindows, GetCursorInfo, GetIconInfo, GetPropW, - GetWindowRect, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, HICON, ICONINFO, IsIconic, IsWindow, - IsWindowVisible, PW_RENDERFULLCONTENT, RemovePropW, SMTO_ABORTIFHUNG, SMTO_BLOCK, SendMessageTimeoutW, SetPropW, - WM_NULL, -}; -use windows::core::{BOOL, PCWSTR, PWSTR}; - -use super::channel::FrameChannel; -use super::pump::{self, Geometry}; -use super::settle::{Settle, Settled}; -use super::{Config, Stream, Window}; -use crate::Error; -use crate::frame::{I420, Surface}; - -const DEFAULT_FRAMERATE: u32 = 30; -const PROBE_TIMEOUT_MS: u32 = 50; -static NEXT_WINDOW_IDENTITY: AtomicUsize = AtomicUsize::new(1); - -/// List visible, titled top-level windows by native HWND. -pub(super) fn windows() -> Result, Error> { - let _dpi = DpiContext::enter()?; - let mut handles = Vec::::new(); - unsafe { - EnumWindows( - Some(collect_window), - LPARAM((&mut handles as *mut Vec).cast::() as isize), - ) - .map_err(|error| Error::Codec(anyhow::anyhow!("enumerate Windows windows: {error}")))?; - } - - let mut result = Vec::new(); - for handle in handles { - let mut rect = RECT::default(); - if unsafe { GetWindowRect(handle, &mut rect) }.is_err() { - continue; - } - let width = (rect.right - rect.left).max(0) as u32; - let height = (rect.bottom - rect.top).max(0) as u32; - if width < 2 || height < 2 { - continue; - } - let title = title(handle); - if title.is_empty() { - continue; - } - result.push(Window { - id: format!("window:{}", handle.0 as usize), - title, - app: app_name(handle), - width, - height, - }); - } - Ok(result) -} - -unsafe extern "system" fn collect_window(handle: HWND, data: LPARAM) -> BOOL { - if unsafe { IsWindowVisible(handle) }.as_bool() && unsafe { GetWindowTextLengthW(handle) } > 0 { - let handles = unsafe { &mut *(data.0 as *mut Vec) }; - handles.push(handle); - } - true.into() -} - -pub(super) async fn open(config: &Config, selector: &str) -> Result { - let handle = parse(selector)?; - let handle = handle.0 as usize; - let config = config.clone(); - let chan = FrameChannel::new(); - let (geometry, guard) = pump::spawn( - chan.clone(), - move || { - let handle = HWND(handle as *mut c_void); - let capture = Capture::open(&config, handle)?; - let geometry = Geometry { - width: capture.width, - height: capture.height, - framerate: Some(crate::Rate::integer(capture.framerate)), - label: capture.name.clone(), - }; - Ok((capture, geometry)) - }, - Capture::read, - ) - .await?; - - Ok(Stream::new( - chan, - geometry.width, - geometry.height, - geometry.framerate, - geometry.label, - None, - Box::new(guard), - )) -} - -struct Capture { - handle: HWND, - identity: WindowIdentity, - width: u32, - height: u32, - framerate: u32, - interval: Duration, - next: Instant, - name: String, - cursor: bool, - settle: Settle<(u32, u32)>, - /// Whether the window was minimized on the previous read, so the hold is - /// logged when it starts and ends rather than once per frame. - iconic: bool, - _dpi: DpiContext, -} - -impl Capture { - fn open(config: &Config, handle: HWND) -> Result { - let dpi = DpiContext::enter()?; - if !unsafe { IsWindow(Some(handle)) }.as_bool() { - return Err(Error::SourceUnavailable(format!( - "window {} no longer exists", - handle.0 as usize - ))); - } - let mut rect = RECT::default(); - unsafe { GetWindowRect(handle, &mut rect) } - .map_err(|error| Error::SourceUnavailable(format!("read window geometry: {error}")))?; - let width = ((rect.right - rect.left).max(0) as u32) & !1; - let height = ((rect.bottom - rect.top).max(0) as u32) & !1; - if width == 0 || height == 0 { - return Err(Error::SourceUnavailable("window has no capturable area".to_string())); - } - let identity = WindowIdentity::new(handle); - let framerate = config - .framerate - .unwrap_or(crate::Rate::integer(DEFAULT_FRAMERATE)) - .rounded(); - Ok(Self { - handle, - identity, - width, - height, - framerate, - interval: Duration::from_micros(1_000_000 / u64::from(framerate)), - next: Instant::now(), - name: format!("window:{}", handle.0 as usize), - cursor: config.cursor, - settle: Settle::new((width, height)), - iconic: false, - _dpi: dpi, - }) - } - - fn read(&mut self) -> Result { - let now = Instant::now(); - if self.next > now { - std::thread::sleep(self.next - now); - } - self.next = Instant::now() + self.interval; - - if !self.identity.matches(self.handle) { - return Err(Error::SourceUnavailable(format!( - "{} was closed or replaced", - self.name - ))); - } - // A minimized window reports an off-screen rect and has no capturable - // area, so hold the capture instead of ending it: reopening would fail - // outright, killing the broadcast for what is only a taskbar click. - if unsafe { IsIconic(self.handle) }.as_bool() { - if !self.iconic { - self.iconic = true; - tracing::info!(source = %self.name, "window minimized; holding capture"); - } - return Ok(pump::Read::Idle); - } - if self.iconic { - self.iconic = false; - tracing::info!(source = %self.name, "window restored; resuming capture"); - } - - let mut rect = RECT::default(); - unsafe { GetWindowRect(self.handle, &mut rect) } - .map_err(|error| Error::SourceUnavailable(format!("read window geometry: {error}")))?; - let width = ((rect.right - rect.left).max(0) as u32) & !1; - let height = ((rect.bottom - rect.top).max(0) as u32) & !1; - match self.settle.observe(&(width, height), Instant::now()) { - Settled::Open => {} - // Mid-drag: the window is a size the encoder can't take, but one that - // is still moving, so skip the frame and look again next interval. - Settled::Waiting => return Ok(pump::Read::Idle), - // The encoder's geometry is fixed at open, so end the stream and let - // the caller reopen against the size the drag came to rest at. - Settled::Changed => { - tracing::info!( - source = %self.name, - from = %format_args!("{}x{}", self.width, self.height), - to = %format_args!("{width}x{height}"), - "window resized; ending capture" - ); - return Ok(pump::Read::Done); - } - } - let bgra = snapshot(self.handle, self.width, self.height, self.cursor)?; - if !self.identity.matches(self.handle) { - return Err(Error::SourceUnavailable(format!( - "{} was closed or replaced", - self.name - ))); - } - Ok(pump::Read::Frame(Surface::I420(I420::from_bgra( - &bgra, - self.width * 4, - crate::Size::new(self.width, self.height), - )?))) - } -} - -struct DpiContext(DPI_AWARENESS_CONTEXT); - -impl DpiContext { - fn enter() -> Result { - let previous = unsafe { SetThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE) }; - if previous.0.is_null() { - return Err(last_capture_error("enable per-monitor DPI awareness")); - } - Ok(Self(previous)) - } -} - -impl Drop for DpiContext { - fn drop(&mut self) { - unsafe { SetThreadDpiAwarenessContext(self.0) }; - } -} - -/// Identity for a window we do not own, so a recycled `HWND` is not mistaken -/// for the one that was selected. -/// -/// A unique property gives exact identity when the target accepts it. UIPI -/// refuses that write to a higher-integrity window, so elevated targets fall -/// back to the owning process id and creation time. -struct WindowIdentity { - marker: Option, - /// Owning process, 0 when it could not be read. - process: u32, - /// Owning process's creation time, `None` when it could not be read. - created: Option, -} - -impl WindowIdentity { - fn new(handle: HWND) -> Self { - let value = NEXT_WINDOW_IDENTITY.fetch_add(1, Ordering::Relaxed).max(1); - let token = HANDLE(value as *mut c_void); - let name = format!("moq.capture.identity.{}.{value}\0", std::process::id()) - .encode_utf16() - .collect::>(); - let marker = unsafe { SetPropW(handle, PCWSTR(name.as_ptr()), Some(token)) } - .is_ok() - .then_some(WindowMarker { handle, name, token }); - let process = process_id(handle); - Self { - marker, - process, - created: process_created(process), - } - } - - fn matches(&self, handle: HWND) -> bool { - if !unsafe { IsWindow(Some(handle)) }.as_bool() { - return false; - } - if let Some(marker) = &self.marker { - return marker.matches(handle); - } - let process = process_id(handle); - if process == 0 || process != self.process { - return false; - } - // A creation time we failed to read is not evidence the process changed, - // so only compare when both reads succeeded. - match (self.created, process_created(process)) { - (Some(before), Some(after)) => before == after, - _ => true, - } - } -} - -struct WindowMarker { - handle: HWND, - name: Vec, - token: HANDLE, -} - -impl WindowMarker { - fn matches(&self, handle: HWND) -> bool { - unsafe { GetPropW(handle, PCWSTR(self.name.as_ptr())).0 == self.token.0 } - } -} - -impl Drop for WindowMarker { - fn drop(&mut self) { - if self.matches(self.handle) { - let _ = unsafe { RemovePropW(self.handle, PCWSTR(self.name.as_ptr())) }; - } - } -} - -fn snapshot(handle: HWND, width: u32, height: u32, cursor: bool) -> Result, Error> { - let source = unsafe { GetWindowDC(Some(handle)) }; - if source.is_invalid() { - return Err(last_capture_error("get window device context")); - } - let source = WindowDc(handle, source); - let memory = unsafe { CreateCompatibleDC(Some(source.1)) }; - if memory.is_invalid() { - return Err(last_capture_error("create memory device context")); - } - let memory = MemoryDc(memory); - let bitmap = unsafe { CreateCompatibleBitmap(source.1, width as i32, height as i32) }; - if bitmap.is_invalid() { - return Err(last_capture_error("create window bitmap")); - } - let bitmap = Bitmap(bitmap); - let previous = unsafe { SelectObject(memory.0, HGDIOBJ(bitmap.0.0)) }; - if previous.is_invalid() { - return Err(last_capture_error("select window bitmap")); - } - let selected = Selected { dc: memory.0, previous }; - unsafe { - BitBlt( - memory.0, - 0, - 0, - width as i32, - height as i32, - Some(source.1), - 0, - 0, - SRCCOPY, - ) - .map_err(|error| capture_error("copy window pixels", error))?; - } - // A window composed by DirectComposition (a browser, Electron, any UWP app) - // puts nothing in its window DC, so the copy above is black on its own. - // PrintWindow asks the window to render over it, and PW_RENDERFULLCONTENT is - // what makes a GPU-composited window honor that; a window that declines - // leaves the copy in place. - print_window(handle, memory.0); - if cursor { - draw_cursor(handle, memory.0); - } - drop(selected); - - let mut info = BITMAPINFO { - bmiHeader: BITMAPINFOHEADER { - biSize: size_of::() as u32, - biWidth: width as i32, - biHeight: -(height as i32), - biPlanes: 1, - biBitCount: 32, - biCompression: BI_RGB.0, - ..Default::default() - }, - ..Default::default() - }; - let mut pixels = vec![0u8; width as usize * height as usize * 4]; - let rows = unsafe { - GetDIBits( - memory.0, - bitmap.0, - 0, - height, - Some(pixels.as_mut_ptr().cast()), - &mut info, - DIB_RGB_COLORS, - ) - }; - if rows != height as i32 { - return Err(last_capture_error("read window bitmap")); - } - Ok(pixels) -} - -fn draw_cursor(handle: HWND, target: HDC) { - let mut cursor = CURSORINFO { - cbSize: size_of::() as u32, - ..Default::default() - }; - if unsafe { GetCursorInfo(&mut cursor) }.is_err() || cursor.flags != CURSOR_SHOWING { - return; - } - - let icon = HICON(cursor.hCursor.0); - let mut info = ICONINFO::default(); - if unsafe { GetIconInfo(icon, &mut info) }.is_err() { - return; - } - let _mask = (!info.hbmMask.is_invalid()).then(|| Bitmap(info.hbmMask)); - let _color = (!info.hbmColor.is_invalid()).then(|| Bitmap(info.hbmColor)); - - let mut rect = RECT::default(); - if unsafe { GetWindowRect(handle, &mut rect) }.is_err() { - return; - } - let x = cursor.ptScreenPos.x - rect.left - info.xHotspot as i32; - let y = cursor.ptScreenPos.y - rect.top - info.yHotspot as i32; - let _ = unsafe { DrawIconEx(target, x, y, icon, 0, 0, 0, None, DI_NORMAL) }; -} - -/// Ask the window to render itself over `target`, best effort. -/// -/// Deliberately not latched off after a failure: a window that misses one frame -/// is not a window that can never be printed, and giving up permanently is what -/// would leave a composited window black for the rest of the capture. -fn print_window(handle: HWND, target: HDC) { - // PrintWindow runs on the target's UI thread and has no timeout of its own, - // while the pump thread joins on this call at shutdown, so a hung window - // would wedge the capture. A bounded probe first also covers UIPI, which - // refuses a cross-integrity send outright. Either way the copied window DC - // still yields a frame. - if !responding(handle) { - return; - } - let _ = unsafe { PrintWindow(handle, target, PRINT_WINDOW_FLAGS(PW_RENDERFULLCONTENT)) }; -} - -/// Bounded check that the window's thread is still pumping messages. -fn responding(handle: HWND) -> bool { - unsafe { - SendMessageTimeoutW( - handle, - WM_NULL, - WPARAM(0), - LPARAM(0), - SMTO_BLOCK | SMTO_ABORTIFHUNG, - PROBE_TIMEOUT_MS, - None, - ) - .0 != 0 - } -} - -fn title(handle: HWND) -> String { - let length = unsafe { GetWindowTextLengthW(handle) }; - if length <= 0 { - return String::new(); - } - let mut buffer = vec![0u16; length as usize + 1]; - let copied = unsafe { GetWindowTextW(handle, &mut buffer) }.max(0) as usize; - String::from_utf16_lossy(&buffer[..copied]) -} - -fn app_name(handle: HWND) -> String { - let process_id = process_id(handle); - if process_id == 0 { - return String::new(); - } - let Ok(process) = (unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, process_id) }) else { - return String::new(); - }; - let process = ProcessHandle(process); - let mut path = vec![0u16; 32_768]; - let mut length = path.len() as u32; - if unsafe { QueryFullProcessImageNameW(process.0, PROCESS_NAME_WIN32, PWSTR(path.as_mut_ptr()), &mut length) } - .is_err() - { - return String::new(); - } - let path = String::from_utf16_lossy(&path[..length as usize]); - std::path::Path::new(&path) - .file_stem() - .and_then(|name| name.to_str()) - .unwrap_or_default() - .to_string() -} - -/// The process that owns `handle`, or 0 when it cannot be read. -fn process_id(handle: HWND) -> u32 { - let mut process = 0; - unsafe { GetWindowThreadProcessId(handle, Some(&mut process)) }; - process -} - -/// A process's creation time as a raw FILETIME, which pins one process instance -/// across pid reuse. -fn process_created(process: u32) -> Option { - if process == 0 { - return None; - } - // PROCESS_QUERY_LIMITED_INFORMATION is the access right that a medium - // integrity process is granted against an elevated one, which is the whole - // point of identifying the window this way. - let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, false, process) }.ok()?; - let handle = ProcessHandle(handle); - let mut created = FILETIME::default(); - let mut exit = FILETIME::default(); - let mut kernel = FILETIME::default(); - let mut user = FILETIME::default(); - unsafe { GetProcessTimes(handle.0, &mut created, &mut exit, &mut kernel, &mut user) }.ok()?; - Some((u64::from(created.dwHighDateTime) << 32) | u64::from(created.dwLowDateTime)) -} - -struct ProcessHandle(HANDLE); - -impl Drop for ProcessHandle { - fn drop(&mut self) { - let _ = unsafe { CloseHandle(self.0) }; - } -} - -fn parse(selector: &str) -> Result { - let value = selector - .strip_prefix("window:") - .unwrap_or(selector) - .parse::() - .map_err(|_| Error::SourceUnavailable(format!("invalid Windows window selector {selector:?}")))?; - Ok(HWND(value as *mut c_void)) -} - -fn last_capture_error(context: &str) -> Error { - capture_error(context, windows::core::Error::from_thread()) -} - -fn capture_error(context: &str, error: windows::core::Error) -> Error { - if error.code() == E_ACCESSDENIED { - Error::PermissionDenied(format!("{context}: {error}")) - } else { - Error::SourceUnavailable(format!("{context}: {error}")) - } -} - -struct WindowDc(HWND, HDC); - -impl Drop for WindowDc { - fn drop(&mut self) { - unsafe { - ReleaseDC(Some(self.0), self.1); - } - } -} - -struct MemoryDc(HDC); - -impl Drop for MemoryDc { - fn drop(&mut self) { - unsafe { - let _ = DeleteDC(self.0); - } - } -} - -struct Bitmap(HBITMAP); - -impl Drop for Bitmap { - fn drop(&mut self) { - unsafe { - let _ = DeleteObject(HGDIOBJ(self.0.0)); - } - } -} - -struct Selected { - dc: HDC, - previous: HGDIOBJ, -} - -impl Drop for Selected { - fn drop(&mut self) { - unsafe { - SelectObject(self.dc, self.previous); - } - } -} diff --git a/rs/moq-video/src/encode/encoder.rs b/rs/moq-video/src/encode/encoder.rs index f62ad5972ca..f228594d763 100644 --- a/rs/moq-video/src/encode/encoder.rs +++ b/rs/moq-video/src/encode/encoder.rs @@ -1436,4 +1436,41 @@ mod tests { let keyframe = frames.first().expect("a keyframe"); assert_eq!(declared_color(&keyframe.payload), Some(BT709_DESCRIBED)); } + + #[cfg(all(target_os = "windows", feature = "capture", feature = "openh264"))] + #[test] + #[ignore = "requires Windows WGC video processing and a Media Foundation hardware H.264 encoder"] + fn wgc_nv12_encodes_with_matching_color_on_gpu_and_software() { + use super::backend::test_util::{BT601_DESCRIBED, BT709_DESCRIBED, declared_color}; + use crate::frame::d3d11; + + let device = d3d11::create_device().expect("D3D11 hardware device"); + for (size, declared) in [ + (Size::new(640, 480), BT601_DESCRIBED), + (Size::new(1280, 720), BT709_DESCRIBED), + ] { + let pixels = [0, 0, 255, 255].repeat(size.pixels() as usize); + let source = d3d11::upload_bgra(&device, size, &pixels); + for backend in ["mediafoundation", "openh264"] { + let texture = d3d11::Texture::capture(&device, &source, size).expect("BGRA to NV12"); + let frame = Frame::new(Surface::Texture(texture), moq_net::Timestamp::from_micros(0).unwrap()); + let config = Config { + kind: Kind::Named(backend.into()), + color: frame.surface.color(), + ..Config::new(size.width, size.height, crate::Rate::new(30, 1).unwrap()) + }; + let mut encoder = Encoder::new(&config).expect("requested encoder must be available"); + assert_eq!(encoder.name(), backend); + let mut encoded = encoder.encode(&frame).expect("encode WGC texture"); + encoded.extend(encoder.finish().unwrap()); + assert!(!encoded.is_empty(), "{backend} must produce a frame"); + assert!( + encoded + .iter() + .any(|frame| declared_color(&frame.payload).as_ref() == Some(&declared)), + "{backend} {size} SPS color" + ); + } + } + } } diff --git a/rs/moq-video/src/frame.rs b/rs/moq-video/src/frame.rs index 1fe39ddaf74..2f82cdffaa3 100644 --- a/rs/moq-video/src/frame.rs +++ b/rs/moq-video/src/frame.rs @@ -5,7 +5,7 @@ //! Capture and the VideoToolbox decoder both produce it, and the VideoToolbox //! encoder consumes it directly, no copy and no color conversion. //! - `Surface::Texture` is a Windows Direct3D11 NV12 texture, produced by Media -//! Foundation capture and decode one GPU blit removed from their own pools +//! Foundation capture/decode and Windows.Graphics.Capture, one GPU blit removed from their pools //! (which they recycle, so a frame has to be lifted out of them), and consumed //! by the hardware encoder MFT on the same device with no copy at all, so a //! camera or a decoder reaches an encoder without touching the CPU. Drawing one @@ -43,6 +43,9 @@ use yuv::{YuvChromaSubsampling, YuvConversionMode, YuvPlanarImageMut, rgba_to_yu use crate::{Color, Error, Size}; +#[cfg(any(target_os = "windows", test))] +mod processor; + /// One raw (uncompressed) video frame: the pixels plus when they are shown. /// /// The currency of the crate's raw side: capture sources and @@ -634,7 +637,7 @@ impl Surface { #[cfg(apple)] Surface::PixelBuffer(s) => s.color(), #[cfg(target_os = "windows")] - Surface::Texture(_) => None, + Surface::Texture(texture) => texture.color, #[cfg(all(target_os = "linux", feature = "nvidia"))] Surface::Cuda(c) => c.color(), #[cfg(all(target_os = "linux", feature = "nvidia"))] @@ -1957,6 +1960,7 @@ pub mod d3d11 { D3D11_FORMAT_SUPPORT_VIDEO_ENCODER, D3D11_MAP_READ, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEX2D_VPIV, D3D11_TEX2D_VPOV, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE, D3D11_VIDEO_PROCESSOR_COLOR_SPACE, D3D11_VIDEO_PROCESSOR_CONTENT_DESC, + D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT, D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0, D3D11_VIDEO_PROCESSOR_STREAM, D3D11_VIDEO_USAGE_PLAYBACK_NORMAL, D3D11_VPIV_DIMENSION_TEXTURE2D, D3D11_VPOV_DIMENSION_TEXTURE2D, @@ -1964,23 +1968,23 @@ pub mod d3d11 { ID3D11VideoProcessor, ID3D11VideoProcessorEnumerator, ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView, }; - #[cfg(test)] - use windows::Win32::Graphics::Dxgi::Common::DXGI_FORMAT_NV12; use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT, DXGI_RATIONAL, DXGI_SAMPLE_DESC}; + use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12}; use windows::Win32::Media::MediaFoundation::{IMFDXGIBuffer, IMFSample}; use windows::core::Interface; + use super::processor::Plan; use super::{Cache, I420}; - use crate::{Error, Size}; + use crate::{Color, Error, Size}; fn err(ctx: &str, e: windows::core::Error) -> Error { Error::Codec(anyhow::anyhow!("{ctx}: {e}")) } /// Create a hardware Direct3D11 device, multithread-protected (Media - /// Foundation's internal threads or DXGI duplication and our capture thread + /// Foundation's internal threads or WGC and our capture thread /// both touch it). The shared low-level constructor behind the Media - /// Foundation device manager and the Desktop Duplication capture path. + /// Foundation device manager and Windows.Graphics.Capture. pub(crate) fn create_device() -> Result { let mut device: Option = None; unsafe { @@ -2008,8 +2012,8 @@ pub mod d3d11 { Ok(device) } - /// A GPU texture (NV12) on the Direct3D11 device of whichever Media Foundation - /// object produced it: the capture source reader, or the DXVA decoder. Holds + /// A GPU texture (NV12) on its producer's Direct3D11 device: a Media Foundation + /// source reader, DXVA decoder, or Windows.Graphics.Capture session. Holds /// that device so the download fallback and the hardware encoder run on the /// device that owns the texture. Cloning the COM handles is a cheap `AddRef`, /// which is what keeps capture -> encode and decode -> encode zero-copy. @@ -2018,6 +2022,7 @@ pub mod d3d11 { pub(crate) texture: ID3D11Texture2D, pub(crate) width: u32, pub(crate) height: u32, + pub(crate) color: Option, } impl Texture { @@ -2076,6 +2081,27 @@ pub mod d3d11 { texture, width, height, + color: None, + }) + } + + /// Convert a recycled WGC BGRA surface to an owned, even-sized NV12 texture. + #[cfg(feature = "capture")] + pub(crate) fn capture(device: &ID3D11Device, source: &ID3D11Texture2D, size: Size) -> Result { + let plan = Plan::capture(size)?; + let mut desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { source.GetDesc(&mut desc) }; + if desc.Format != DXGI_FORMAT_B8G8R8A8_UNORM || desc.Width < size.width || desc.Height < size.height { + return Err(Error::Codec(anyhow::anyhow!( + "invalid WGC BGRA surface geometry or format" + ))); + } + Ok(Self { + device: device.clone(), + texture: process(device, source, plan)?, + width: plan.target.width, + height: plan.target.height, + color: Some(Color::infer(plan.target)), }) } @@ -2183,9 +2209,7 @@ pub mod d3d11 { width: self.width, height: self.height, data, - // A deinterleave, not a color conversion, and nothing here names - // the space these samples are in. Left unknown to be inferred. - color: None, + color: self.color, }) } @@ -2201,53 +2225,60 @@ pub mod d3d11 { pub(crate) fn resize(&self, width: u32, height: u32) -> Result { let source = Size::new(self.width, self.height); let target = Size::new(width, height); - let key = ScalerKey::new(&self.device, source, target); - - let scaler = { - let mut scalers = SCALERS - .lock() - .map_err(|_| Error::Codec(anyhow::anyhow!("video-processor cache lock poisoned")))?; - scalers - .get_or_insert_with(key, || { - Ok::<_, std::convert::Infallible>(ScalerState::discover(&self.device, source, target)) - }) - .expect("scaler discovery is infallible") - }; - let mut state = scaler - .lock() - .map_err(|_| Error::Codec(anyhow::anyhow!("video processor lock poisoned")))?; - let result = match &*state { - ScalerState::Ready(scaler) => scaler.scale(&self.texture), - ScalerState::Unsupported { reason, .. } => { - return Err(Error::Codec(anyhow::anyhow!("GPU resize is unsupported: {reason}"))); - } - }; - let texture = match result { - Ok(texture) => texture, - Err(ScaleError::Unsupported(err)) => { - *state = ScalerState::Unsupported { - _device: self.device.clone(), - reason: err.to_string(), - }; - return Err(err); - } - Err(ScaleError::Transient(err)) => return Err(err), - }; - drop(state); - drop(scaler); - if let Ok(mut scalers) = SCALERS.lock() { - scalers.prune(); - } - + let texture = process(&self.device, &self.texture, Plan::resize(source, target, self.color))?; Ok(Self { device: self.device.clone(), texture, width, height, + color: self.color, }) } } + fn process(device: &ID3D11Device, source: &ID3D11Texture2D, plan: Plan) -> Result { + let key = ScalerKey { + device: device.as_raw() as usize, + plan, + }; + let scaler = { + let mut scalers = SCALERS + .lock() + .map_err(|_| Error::Codec(anyhow::anyhow!("video-processor cache lock poisoned")))?; + scalers + .get_or_insert_with(key, || { + Ok::<_, std::convert::Infallible>(ScalerState::discover(device, plan)) + }) + .expect("processor discovery is infallible") + }; + let mut state = scaler + .lock() + .map_err(|_| Error::Codec(anyhow::anyhow!("video processor lock poisoned")))?; + let result = match &*state { + ScalerState::Ready(scaler) => scaler.scale(source), + ScalerState::Unsupported { reason, .. } => { + return Err(Error::Codec(anyhow::anyhow!("GPU processing is unsupported: {reason}"))); + } + }; + let texture = match result { + Ok(texture) => texture, + Err(ScaleError::Unsupported(err)) => { + *state = ScalerState::Unsupported { + _device: device.clone(), + reason: err.to_string(), + }; + return Err(err); + } + Err(ScaleError::Transient(err)) => return Err(err), + }; + drop(state); + drop(scaler); + if let Ok(mut scalers) = SCALERS.lock() { + scalers.prune(); + } + Ok(texture) + } + /// Enough reusable video processors for a large rendition ladder without /// retaining every device and scale a long-lived process has ever seen. const SCALER_CACHE_CAPACITY: usize = 16; @@ -2270,8 +2301,8 @@ pub mod d3d11 { } impl ScalerState { - fn discover(device: &ID3D11Device, source: Size, target: Size) -> Self { - match Scaler::new(device, source, target) { + fn discover(device: &ID3D11Device, plan: Plan) -> Self { + match Scaler::new(device, plan) { Ok(scaler) => Self::Ready(scaler), Err(err) => Self::Unsupported { _device: device.clone(), @@ -2290,22 +2321,11 @@ pub mod d3d11 { #[derive(Clone, PartialEq, Eq, Hash)] struct ScalerKey { device: usize, - source: Size, - target: Size, - } - - impl ScalerKey { - fn new(device: &ID3D11Device, source: Size, target: Size) -> Self { - Self { - device: device.as_raw() as usize, - source, - target, - } - } + plan: Plan, } /// One Direct3D11 video processor, configured for a single source and target - /// size. The GPU scaler behind [`Texture::resize`]. + /// size, crop, format and color conversion. Shared by resize and WGC capture. struct Scaler { /// Keeps the device keying this entry alive, so its address stays unique. device: ID3D11Device, @@ -2314,6 +2334,7 @@ pub mod d3d11 { enumerator: ID3D11VideoProcessorEnumerator, processor: ID3D11VideoProcessor, target: Size, + input_format: DXGI_FORMAT, } /// Whether a failed scale proves this key unsupported or can succeed later. @@ -2323,7 +2344,8 @@ pub mod d3d11 { } impl Scaler { - fn new(device: &ID3D11Device, source: Size, target: Size) -> Result { + fn new(device: &ID3D11Device, plan: Plan) -> Result { + let (source, target) = (plan.source, plan.target); let video = device .cast::() .map_err(|e| err("query ID3D11VideoDevice", e))?; @@ -2351,14 +2373,31 @@ pub mod d3d11 { let enumerator = unsafe { video.CreateVideoProcessorEnumerator(&desc) } .map_err(|e| err("CreateVideoProcessorEnumerator", e))?; + let input_format = if plan.bgra { + DXGI_FORMAT_B8G8R8A8_UNORM + } else { + DXGI_FORMAT_NV12 + }; + for (format, required) in [ + (input_format, D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT), + (DXGI_FORMAT_NV12, D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT), + ] { + let support = unsafe { enumerator.CheckVideoProcessorFormat(format) } + .map_err(|e| err("CheckVideoProcessorFormat", e))?; + if support & required.0 as u32 == 0 { + return Err(Error::Codec(anyhow::anyhow!( + "video processor does not support {format:?} as {required:?}" + ))); + } + } let processor = unsafe { video.CreateVideoProcessor(&enumerator, 0) }.map_err(|e| err("CreateVideoProcessor", e))?; let full = RECT { left: 0, top: 0, - right: source.width as i32, - bottom: source.height as i32, + right: plan.picture.width as i32, + bottom: plan.picture.height as i32, }; let scaled = RECT { left: 0, @@ -2368,19 +2407,20 @@ pub mod d3d11 { }; unsafe { context.VideoProcessorSetStreamFrameFormat(&processor, 0, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE); - // The whole picture into the whole destination: the scale itself. + // Capture crops odd edges; resize maps the entire input picture. context.VideoProcessorSetStreamSourceRect(&processor, 0, true, Some(&full)); context.VideoProcessorSetStreamDestRect(&processor, 0, true, Some(&scaled)); // Drivers ship denoise and edge enhancement on by default here. // This is a resize, not a filter chain, so a rung must not come out // looking different from the frame it was scaled from. context.VideoProcessorSetStreamAutoProcessingMode(&processor, 0, false); - // One space in, the same space out. Resampling moves samples - // around, it must not reinterpret them, and a processor left to - // its own devices will happily convert between ranges. - let space = D3D11_VIDEO_PROCESSOR_COLOR_SPACE::default(); - context.VideoProcessorSetStreamColorSpace(&processor, 0, &space); - context.VideoProcessorSetOutputColorSpace(&processor, &space); + let input = D3D11_VIDEO_PROCESSOR_COLOR_SPACE { + _bitfield: plan.input_space(), + }; + let output = D3D11_VIDEO_PROCESSOR_COLOR_SPACE { _bitfield: plan.space }; + context.VideoProcessorSetStreamColorSpace(&processor, 0, &input); + context.VideoProcessorSetOutputColorSpace(&processor, &output); + context.VideoProcessorSetOutputTargetRect(&processor, true, Some(&scaled)); } Ok(Self { @@ -2390,6 +2430,7 @@ pub mod d3d11 { enumerator, processor, target, + input_format, }) } @@ -2397,7 +2438,12 @@ pub mod d3d11 { fn scale(&self, source: &ID3D11Texture2D) -> Result { let mut desc = D3D11_TEXTURE2D_DESC::default(); unsafe { source.GetDesc(&mut desc) }; - let output = alloc(&self.device, self.target.width, self.target.height, desc.Format) + if desc.Format != self.input_format { + return Err(ScaleError::Transient(Error::Codec(anyhow::anyhow!( + "video processor input format changed" + )))); + } + let output = alloc(&self.device, self.target.width, self.target.height, DXGI_FORMAT_NV12) .map_err(ScaleError::Transient)?; let input_desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { @@ -2518,9 +2564,29 @@ pub mod d3d11 { texture, width, height, + color: frame.color, }) } + /// A synthetic pool surface for the opt-in WGC conversion/encoding tests. + #[cfg(all(test, feature = "capture"))] + pub(crate) fn upload_bgra(device: &ID3D11Device, size: Size, pixels: &[u8]) -> ID3D11Texture2D { + assert_eq!(pixels.len(), size.pixels() as usize * 4); + let texture = alloc(device, size.width, size.height, DXGI_FORMAT_B8G8R8A8_UNORM).unwrap(); + let context = unsafe { device.GetImmediateContext() }.unwrap(); + unsafe { + context.UpdateSubresource( + &texture, + 0, + None, + pixels.as_ptr().cast(), + size.width * 4, + pixels.len() as u32, + ); + } + texture + } + /// The Direct3D11 texture behind a Media Foundation sample, and which slice of /// it this sample is. Errors if the sample is system-memory backed. fn resolve(sample: &IMFSample) -> Result<(ID3D11Texture2D, u32), Error> { @@ -2583,6 +2649,94 @@ pub mod d3d11 { unsafe { self.context.Unmap(self.resource, 0) }; } } + + #[cfg(all(test, feature = "capture"))] + mod tests { + use super::*; + + #[test] + #[ignore = "requires a Windows GPU with BGRA-to-NV12 video processing"] + fn wgc_conversion_crops_odd_edges_and_preserves_color() { + let device = create_device().expect("D3D11 hardware device"); + for size in [Size::new(641, 481), Size::new(1281, 721)] { + let mut pixels = Vec::with_capacity(size.pixels() as usize * 4); + for y in 0..size.height { + for x in 0..size.width { + // A green odd edge must be cropped, not scaled into the red picture. + pixels.extend_from_slice(if x + 1 == size.width || y + 1 == size.height { + &[0, 255, 0, 255] + } else { + &[0, 0, 255, 255] + }); + } + } + let source = upload_bgra(&device, size, &pixels); + let output = Texture::capture(&device, &source, size).expect("BGRA to NV12"); + let expected_size = Size::new(size.width & !1, size.height & !1); + let color = Color::infer(expected_size); + assert_eq!(Size::new(output.width, output.height), expected_size); + assert_eq!(output.color, Some(color)); + assert_eq!(output.device.as_raw(), device.as_raw()); + assert_ne!(output.texture.as_raw(), source.as_raw(), "must own the output"); + + // Simulate immediate reuse of a WGC pool frame after conversion. + pixels.fill(0); + unsafe { + device.GetImmediateContext().unwrap().UpdateSubresource( + &source, + 0, + None, + pixels.as_ptr().cast(), + size.width * 4, + pixels.len() as u32, + ); + } + drop(source); + assert_red(&output.download_i420().unwrap(), color); + // HD -> SD scaling must keep 709, not infer 601 from the new height. + let scaled = output.resize(320, 240).expect("NV12 GPU resize"); + assert_eq!(scaled.color, Some(color)); + assert_red(&scaled.download_i420().unwrap(), color); + } + } + + fn assert_red(frame: &I420, color: Color) { + assert_eq!(frame.color(), Some(color)); + let expected = color.coefficients().apply([255, 0, 0]); + for (plane, value) in [frame.y(), frame.u(), frame.v()].into_iter().zip(expected) { + assert!( + plane.iter().all(|sample| sample.abs_diff(value) <= 3), + "{color:?}: expected {value}" + ); + } + } + + #[test] + #[ignore = "Windows GPU workload; reports submission and batch completion, not capture latency"] + fn wgc_conversion_workload() { + let device = create_device().expect("D3D11 hardware device"); + for size in [Size::new(640, 480), Size::new(1280, 720), Size::new(1920, 1080)] { + let pixels = [0, 0, 255, 255].repeat(size.pixels() as usize); + let source = upload_bgra(&device, size, &pixels); + Texture::capture(&device, &source, size) + .unwrap() + .download_i420() + .unwrap(); + let start = std::time::Instant::now(); + let mut output = None; + for _ in 0..32 { + output = Some(Texture::capture(&device, &source, size).unwrap()); + } + let submitted = start.elapsed(); + // Mapping the final result fences the preceding ordered GPU blits. + output.unwrap().download_i420().unwrap(); + eprintln!( + "WGC {size}: 32 frames, submit={submitted:?}, complete_with_one_readback={:?}", + start.elapsed() + ); + } + } + } } #[cfg(test)] diff --git a/rs/moq-video/src/frame/processor.rs b/rs/moq-video/src/frame/processor.rs new file mode 100644 index 00000000000..1966020d8d6 --- /dev/null +++ b/rs/moq-video/src/frame/processor.rs @@ -0,0 +1,111 @@ +//! D3D11 processor configuration, testable without a GPU or Windows host. + +#[cfg(any(feature = "capture", test))] +use crate::Error; +use crate::{Color, Size}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub(super) struct Plan { + pub source: Size, + pub picture: Size, + pub target: Size, + pub bgra: bool, + pub space: u32, +} + +impl Plan { + pub fn resize(source: Size, target: Size, color: Option) -> Self { + Self { + source, + picture: source, + target, + bgra: false, + space: color.map(space).unwrap_or(0), + } + } + + #[cfg(any(feature = "capture", test))] + pub fn capture(source: Size) -> Result { + let target = Size::new(source.width & !1, source.height & !1); + if target.width == 0 || target.height == 0 || source.width > i32::MAX as u32 || source.height > i32::MAX as u32 + { + return Err(Error::SourceUnavailable("screen has no capturable area".into())); + } + Ok(Self { + source, + picture: target, + target, + bgra: true, + space: space(Color::infer(target)), + }) + } + + pub fn input_space(self) -> u32 { + if self.bgra { RGB_FULL } else { self.space } + } +} + +// D3D11_VIDEO_PROCESSOR_COLOR_SPACE: RGB_Range is bit 1 (0 = full), +// YCbCr_Matrix bit 2 (1 = 709), Nominal_Range bits 4..6 (1 = limited, 2 = full). +const RGB_FULL: u32 = 2 << 4; + +fn space(color: Color) -> u32 { + let matrix = u32::from(matches!(color, Color::Bt709Limited | Color::Bt709Full)); + let range = if color.limited() { 1 } else { 2 }; + (matrix << 2) | (range << 4) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn odd_capture_crops_instead_of_scaling() { + let plan = Plan::capture(Size::new(801, 601)).unwrap(); + assert_eq!(plan.source, Size::new(801, 601)); + assert_eq!(plan.picture, Size::new(800, 600)); + assert_eq!(plan.picture, plan.target); + } + + #[test] + fn capture_refuses_empty_or_unrepresentable_rectangles() { + for size in [Size::new(0, 600), Size::new(800, 1), Size::new(u32::MAX, 600)] { + assert!(Plan::capture(size).is_err()); + } + } + + #[test] + fn capture_converts_full_rgb_to_the_declared_matrix_and_range() { + for (height, matrix) in [(480, 0), (576, 0), (578, 1), (1080, 1)] { + let plan = Plan::capture(Size::new(1920, height)).unwrap(); + assert_eq!(plan.input_space(), 2 << 4); + assert_eq!(plan.space, (matrix << 2) | (1 << 4)); + } + } + + #[test] + fn resizing_preserves_color_across_the_sd_boundary() { + for color in [ + Color::Bt601Limited, + Color::Bt601Full, + Color::Bt709Limited, + Color::Bt709Full, + ] { + let plan = Plan::resize(Size::new(1920, 1080), Size::new(640, 360), Some(color)); + assert_eq!(plan.input_space(), space(color)); + assert_eq!(plan.space, space(color)); + } + assert_eq!(Plan::resize(Size::new(640, 480), Size::new(320, 240), None).space, 0); + } + + #[test] + fn the_cache_distinguishes_conversion_resize_and_color() { + let size = Size::new(1280, 720); + let capture = Plan::capture(size).unwrap(); + let resize = Plan::resize(size, size, Some(Color::Bt709Limited)); + let full = Plan::resize(size, size, Some(Color::Bt709Full)); + let unknown = Plan::resize(size, size, None); + let keys = std::collections::HashSet::from([capture, resize, full, unknown]); + assert_eq!(keys.len(), 4); + } +} diff --git a/rs/moq-video/src/lib.rs b/rs/moq-video/src/lib.rs index 272c3717737..43f23fb1562 100644 --- a/rs/moq-video/src/lib.rs +++ b/rs/moq-video/src/lib.rs @@ -15,8 +15,8 @@ //! //! - `capture` describes a frame source and grabs frames per platform: //! AVFoundation/ScreenCaptureKit on macOS, native V4L2 on Linux, native Media -//! Foundation (camera), DXGI Desktop Duplication (screen), and GDI (window) on -//! Windows, plus portal/PipeWire on Wayland and X11 capture on Linux. Use +//! Foundation (camera) and Windows.Graphics.Capture (display/window) on +//! Windows 10 2004+, plus portal/PipeWire on Wayland and X11 capture on Linux. Use //! `capture::open` for an embeddable raw-frame stream or //! `encode::publish_capture` for turnkey publication. It requires the opt-in //! `capture` feature, which costs the build host nothing on any platform. diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 956d21a7b15..551391837b9 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -53,7 +53,7 @@ fi # The impact map: a module is in scope when a changed path matches its # pattern. An empty pattern is a repository-wide lint that runs on every diff. declare -A scope=( - [js]='^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|sh/js/|sh/rs/stats-docs\.py$|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' + [js]='^(js/|doc/|drafts/|demo/(boy|web)/|test/(audio-quality|interop)/clients/js|test/wasm/|sh/js/|sh/rs/stats-docs\.py$|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' # Workers with lockfiles outside the Bun workspace. [workers]='^(infra/apt/|infra/rpm/|demo/pub/|sh/js/workers\.sh$)' # sh/rs/select.sh widens to the whole workspace for inputs every crate shares. diff --git a/test/README.md b/test/README.md index 968b286777b..c45654b9a3c 100644 --- a/test/README.md +++ b/test/README.md @@ -8,8 +8,10 @@ tests live in each language's own justfile. | [interop](interop/README.md) | `just test interop` | every client built from this checkout interoperates | | [wasm](wasm/README.md) | `just test wasm` | the `@moq/wasm` bindings work in a real browser | | [ts](ts/README.md) | `just test ts` | the subscriber's `export ts` output is IRD-compliant | +| [audio-quality](audio-quality/README.md) | `just test audio-quality` | browser audio playout stays within budget over an impaired path | +| [drain](drain/README.md) | `just test drain` | a JS viewer migrates off a draining relay without a dropped group | -All three stand up a `moq-relay` and clients, so two of them running at once, or +All five stand up a `moq-relay` and clients, so two of them running at once, or the same one running from two worktrees, would otherwise collide. `lib/harness.sh` is what keeps them apart. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md new file mode 100644 index 00000000000..b79baa83b1c --- /dev/null +++ b/test/audio-quality/README.md @@ -0,0 +1,235 @@ +# Audio quality + +Plays a tone in headless Chromium over a seeded, impaired UDP path and grades what the listener +would have heard: how often the ring ran dry, how much audio a re-anchor threw away, how much of the +run was silent, and where the playout delay settled. A regression in any of them fails a row +against [`budgets.json`](budgets.json) instead of arriving as a bug report. + +```bash +just test audio-quality # the whole matrix, 60s a row +just test audio-quality --list # what it would run +just test audio-quality --profiles mild --codecs opus --duration 20 +just test audio-quality --rings plain # the production path only +just test audio-quality --out ~/runs/after # keep the run directory +just test audio-quality --enforce # fail on the budgets +just test audio-quality --replays # the replays only, in seconds +``` + +The matrix is codec x profile x ring: 24 browser rows, about 30 minutes, plus 6 [replay +rows](#traces) of recorded arrivals. `--profiles`, `--rings`, and `--codecs` take comma-separated +lists; `--seed` replays a given impairment. The table prints either way, and only `--enforce` (which +the nightly job passes) turns a breach into a failed run. + +## Rows + +| Axis | Values | +| --- | --- | +| codec | `opus` at 48 kHz over fMP4, `aac` at 44.1 kHz over MPEG-TS | +| profile | see below | +| ring | `isolated` (SharedArrayBuffer, cross-origin isolated page) and `plain` (postMessage) | + +`plain` is the production path: most pages are not cross-origin isolated. One build is served under +`/isolated/` with COOP and COEP and under `/plain/` without, and the page's `crossOriginIsolated` +decides which ring ran. + +| Profile | Path | Delay | +| --- | --- | --- | +| `near-zero` | through the shaper, untreated | auto | +| `mild` | 5 ms one way, 5 ms Gaussian sigma, in order | auto | +| `wide` | 40 ms one way, 40 ms Gaussian sigma, in order | auto | +| `bursty` | batches of seven datagrams, released within 160 ms | auto | +| `step` | 5 ms one way, stepping to 60 ms at 30 s | auto | +| `fixed-250` | through the shaper, untreated | a fixed 250 ms | + +`fixed-250` is the control: every other row adapts, so without it the whole matrix could pass while a +fixed preset regressed, and a fixed preset is what many viewers land on. Loss, reorder, and rate +limits stay off: the buffer's job is absorbing arrival spread, and congestion response would make a +failure hard to attribute. The `mild` and `wide` profiles preserve datagram order and share one path across the page's sessions. +`bursty` and `step` use the shaper's named profiles on that same shared path. The step is timed from +shaper startup, before the page connects; the driver gives up on a row with no audio 20 s in, so the +measured window starts before the step, and `--duration` must exceed 30 s so playback spans it. + +The AAC arm keeps ffmpeg's default PES packing, whose multi-frame bursts are the flush-span shape +the reporter measured on the public relay (#3477). The source is a sine tone rather than a film, so a +quiet quantum at the output is always a gap or a stall and never the soundtrack. + +## The path + +```text + page ──UDP──▶ moq-shaper ──▶ moq-relay WebTransport, the impaired path + moq (opus, aac) ─────────────▶ moq-relay the publishers, on the clean path +``` + +Every port is reserved for the run ([the harness contract](../README.md)), and the shaper is the +same `moq-shaper` binary the [transport drills](../drill/README.md) run under, so the two share one +impairment implementation. It exits nonzero when a configured impairment never acted, and that +voids the row. + +The shaper carries UDP and nothing else. The page fetches `/certificate.sha256` from the address it +dials, so the driver answers that fetch with the relay's own hash (the certificate is pinned by +hash, so a different port needs nothing more). With nothing listening on TCP there, the WebSocket +fallback cannot connect: the session is WebTransport through the shaper, or the row is void. + +## Traces + +The shaper's profiles are synthetic. [`traces/`](traces) holds real arrivals, each graded as a +`replay` row whose profile is the file's name: + +| Trace | Recorded from | +| --- | --- | +| `relay-bbb` | the public relay's `bbb.hang`: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured | +| `local-aac` | this harness's AAC publisher through a local relay: the shallow control | +| `relay-mic` | `` in headless Chromium, publishing its fake capture device through the public relay | + +A trace is a [`Trace`](clients/js/src/schema.ts): the catalog's audio rendition, the smallest +PROBE round trip the session saw, how long it observed, and every frame's `[at, timestamp, group]`, +with `at` on the viewer's clock. `record.ts` stamps a frame as it comes off its group's stream, the way +`Container.Consumer` receives it, using only public `@moq/net` and `@moq/hang` in the same Chromium +over WebTransport. It stamps before the consumer's in-order delivery, whose waits and skips depend +on the delay, so the replay decides those again. + +`replay.ts` plays a trace through the player's own `Container.Consumer` and rings on a simulated +clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a +real `Sync` resolves for the recorded catalog and round trip, and reads each quantum through the +tap's classifier. The consumer makes the player's group ordering, max age skips, and discontinuity +resets again at that delay; a group's stream is taken to finish with its last recorded frame. Every +frame is the trace's median spacing long, so a frame that never arrived stays missing audio, and +rendering runs to the end of the observation, so an outage after the last arrival is heard. It +writes the same samples a page does, so `analyze.ts` reduces both alike, and the shaper, transport, +and clock checks have nothing to void. Decoding is taken as instant. A replay is deterministic, so +its budgets are exactly what it measured. + +```bash +just test audio-quality-record # all three, 35 s each, needs the network +just test audio-quality-record --lanes relay-bbb +``` + +Recording is by hand: the public relay and a browser publisher are not something a nightly run +should depend on. Re-measure the replay budgets after re-recording. The recorder asks the relay for +ten seconds of backlog so a late group is kept, which makes a trace's tune-in heavier than a +player's; the warmup excludes it from the grade. + +## The metric contract + +[`clients/js/src/schema.ts`](clients/js/src/schema.ts) is the contract. The page emits `Sample`s, +[`src/analyze.ts`](clients/js/src/analyze.ts) reduces a row to a `Summary`, and `grade.ts` reads that +against the budgets. A native lane or the latency ledger that emits the same `Summary` is comparable +by construction. + +**Units.** Every duration is milliseconds as a float, every count an integer, every share a fraction +of 1. Samples missing at the device rate are converted to ms before they reach a summary. + +**Clocks.** Every timestamp is taken on a named clock and reduced to `viewer`, the page's monotonic +`performance.now()`, before any two are subtracted: + +| Clock | What | Reduced to `viewer` by | +| --- | --- | --- | +| `viewer` | `performance.now()` | nothing: it is the reference | +| `render` | the `AudioContext` frame clock | offset and rate fitted from paired readings every 250 ms | +| `media` | the publisher's timestamps, through the stream | rate fitted; the offset is unknowable from the viewer | +| `publisher`, `relay` | other processes | an offset measured NTP style over the session, uncertain by half a round trip | + +Nothing in this lane carries a publisher or relay timestamp yet, so the stages that need one are +reported as unmeasured rather than guessed. A render clock more than 1% off the viewer's voids the +row: it is a throttled or stalled context, and everything counted on it would be wrong. + +**Aggregation.** A budget key is `_`: `total` over the graded window, +`per_min` (the total over the window's minutes), `p50`/`p95` (nearest rank over the window's +samples), `max`, `share` (of the window), or `last`. So "underruns: 3" and "underruns: 3/min" are +different keys and are never graded against each other. + +**Counters.** The player publishes no count of its own underruns, so the page fans a listening +AudioWorklet out from the player's output node. The player's render worklet copies what the ring +holds into the front of each 128-sample quantum and leaves the rest zero, so the tap reads every +quantum's fill level from the output itself: + +- `underruns`: quanta the ring filled only partly, or not at all, while playing. +- `short_quanta`: the underruns that were partly filled: the ring ran dry part-way through. +- `underrun_episodes`: maximal runs of consecutive underruns, one audible gap each. +- `underrun_ms`: the missing audio, summed, and the longest episode as `max`. +- `skip_aheads`: re-anchors that discarded buffered audio, seen as a forward step of more than 40 ms + in the lag between the viewer clock and the playhead (judged on the median of four samples either + side, after removing the media clock's drift). +- `discarded_ms`: the media those skip-aheads jumped over. Audio dropped before it reached the ring + moves no playhead, so it is not counted. +- `stalled`: the share of the window the ring spent stalled, refilling rather than playing. A gap + that starts within 60 ms of a stall is that stall, and is not an underrun. +- `silence`: the share of rendered quanta under -60 dBFS at the output, whatever the cause. +- `target_ms`: the resolved playout delay (`sync.out.delay`). +- `converge_ms`: from first audio to the last time the target moved more than one 20 ms bucket from + its final value. +- `render_load`: Chromium's render capacity, the share of each quantum's budget the graph used. + +Every row discards the first 5 s after first audio, which is the tune-in rather than the steady +state; `converge_ms` grades the tune-in instead. + +**Stages.** An end-to-end delay splits into exclusive spans, each defined by its two boundaries: +`capture`, `encode`, `publish_flush`, `network`, `jitter_buffer`, `decode`, `render`, `device`, and +the named remainder `unaccounted`. The identity is `sum == end-to-end` within the larger of 2 ms or +2%. This lane measures `jitter_buffer` (the resolved target) and `device` (`outputLatency` plus +`baseLatency`) on the viewer, and reports the publisher's declared `publish_flush` without summing it +with them. No clock spans the whole path here, so `end-to-end` is null rather than an identity that +holds by construction. + +## Void rules + +A row that cannot be trusted is void, reported rather than graded, and fails an enforced run. A +replay row crosses no shaper, so the `shaper` void does not apply to it. + +| Void | Why | +| --- | --- | +| `shaper` | It exited nonzero (a configured impairment never acted), left no report, or forwarded nothing to the page. | +| `transport` | The session negotiated something other than WebTransport. | +| `ring` | `crossOriginIsolated` does not match the row's ring, so the other ring ran. | +| `clock` | The AudioContext clock was unreadable or ran more than 1% off the viewer's. | +| `window` | No audio after the warmup. | +| `driver` | The page never reached a session and first audio within 20 s, or the driver threw. | + +## Budgets + +`budgets.json` holds a ceiling per graded key per row, keyed by the whole row: runtime, codec, rate, +profile, and ring. Under `--enforce` a run fails on a breach, a budgeted key the run left +unmeasured, a void row, or a row with no budget. Tightening a ceiling is a visible diff; loosening +one needs a reason in review. The file's `note` says how the current ceilings were measured. + +The nightly `Audio quality` job runs the matrix under `--enforce` and keeps a failing run directory +for a week: each process's log, the shaper's counters, every row's raw samples and summary. The +replays also run under `--enforce` on every PR that touches the player's packages (`js/watch`, +`js/hang`, `js/net`, `js/signals`) or the harness (`.github/workflows/audio-quality.yml`), so a +change that moves them updates `budgets.json` in the same PR. + +## Layout + +```text +run.sh builds, starts the relay and publishers, runs each row behind the shaper +record.sh records traces/, by hand +relay.toml anonymous relay, self-signed cert +budgets.json a ceiling per metric per row +traces/ recorded arrivals, one replay profile each +clients/js/ + index.html src/page.ts the watch element, driven from the query string + src/probe.ts samples its public signals every 250 ms + src/tap.ts the listening worklet (tap-worklet.ts, quantum.ts) + src/schema.ts the metric contract + src/analyze.ts samples to a summary, unit tested + driver.ts one row in headless Chromium, and its void checks + recorder.html mic.html the recorder and the microphone publisher (src/recorder.ts, src/mic.ts) + record.ts one trace in headless Chromium + replay.ts the replay rows, written as a page's samples + analyze.ts grade.ts the summary per row, then the table and the verdict +``` + +## Attribution + +The harness is the reporter's, from #3477: fperex built it on their fork `fperex/moq` (branches +`debug/rt-audio` and `debug-findings-solution`), with the black-box probe, the analyzer's skip-ahead +rule, the Playwright matrix over `moq-shaper`, the budget file graded under `--enforce`, and the +nightly job. The harness and opt-in shaper features preserve that author's credit. The player and estimator +changes remain separate: the underrun count comes from a tap on the output rather than a counter in the player, and +the profiles now include the fork's batch, step, and order-preserving jitter treatments. + +## Not covered + +Safari and Firefox, video, and perceptual scoring: the grade is glitches and latency, not an opinion +about how it sounds. The `relay-mic` trace captures Chromium's fake device rather than a real +microphone, so it carries a browser publisher's encode and send cadence but not a sound card's. diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json new file mode 100644 index 00000000000..3682697b25a --- /dev/null +++ b/test/audio-quality/budgets.json @@ -0,0 +1,425 @@ +{ + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. Updated 2026-09-29: mild and wide now use shared order-preserving Gaussian jitter; bursty and step are new. Those 16 rows were re-recorded for 60 s each, seed 7, on the same shared Linux workstation with concurrent bounded builds. Their ceilings use twice the measurement rounded upward to two significant figures and the same floors above; target_ms_p95 adds one 20 ms bucket. Other rows retain their previous measurements. These remain current-player regression ceilings, not target audio quality; re-record on the nightly runner before tightening them. The replay rows are deterministic, so their ceilings are exactly what they measured on 2026-10-01, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in.", + "rows": [ + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "bursty", + "ring": "isolated", + "underruns_per_min": 1100.0, + "underrun_ms_per_min": 3000.0, + "skip_aheads_per_min": 6.6, + "discarded_ms_per_min": 500.0, + "stalled_share": 0.01, + "silence_share": 0.05, + "target_ms_p95": 404.8 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "bursty", + "ring": "plain", + "underruns_per_min": 740.0, + "underrun_ms_per_min": 2200.0, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 350.0, + "stalled_share": 0.01, + "silence_share": 0.034, + "target_ms_p95": 368.5 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "fixed-250", + "ring": "isolated", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 440 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "fixed-250", + "ring": "plain", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 440 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "mild", + "ring": "isolated", + "underruns_per_min": 320.0, + "underrun_ms_per_min": 890.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.014, + "target_ms_p95": 211 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "mild", + "ring": "plain", + "underruns_per_min": 340.0, + "underrun_ms_per_min": 970.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.014, + "target_ms_p95": 206 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "near-zero", + "ring": "isolated", + "underruns_per_min": 760, + "underrun_ms_per_min": 2200, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.028, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "near-zero", + "ring": "plain", + "underruns_per_min": 1500, + "underrun_ms_per_min": 4400, + "skip_aheads_per_min": 17, + "discarded_ms_per_min": 690, + "stalled_share": 0.01, + "silence_share": 0.066, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "step", + "ring": "isolated", + "underruns_per_min": 970.0, + "underrun_ms_per_min": 2800.0, + "skip_aheads_per_min": 6.6, + "discarded_ms_per_min": 380.0, + "stalled_share": 0.01, + "silence_share": 0.042, + "target_ms_p95": 207.3 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "step", + "ring": "plain", + "underruns_per_min": 450.0, + "underrun_ms_per_min": 1200.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 120.0, + "stalled_share": 0.01, + "silence_share": 0.02, + "target_ms_p95": 212.3 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "wide", + "ring": "isolated", + "underruns_per_min": 140.0, + "underrun_ms_per_min": 360.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 140.0, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 319.8 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "wide", + "ring": "plain", + "underruns_per_min": 330.0, + "underrun_ms_per_min": 940.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.018, + "target_ms_p95": 308.5 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "bursty", + "ring": "isolated", + "underruns_per_min": 4800.0, + "underrun_ms_per_min": 13000.0, + "skip_aheads_per_min": 8.8, + "discarded_ms_per_min": 490.0, + "stalled_share": 0.018, + "silence_share": 0.2, + "target_ms_p95": 128 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "bursty", + "ring": "plain", + "underruns_per_min": 2900.0, + "underrun_ms_per_min": 7100.0, + "skip_aheads_per_min": 6.6, + "discarded_ms_per_min": 470.0, + "stalled_share": 0.028, + "silence_share": 0.13, + "target_ms_p95": 139.3 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "fixed-250", + "ring": "isolated", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 300 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "fixed-250", + "ring": "plain", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 300 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "mild", + "ring": "isolated", + "underruns_per_min": 1300.0, + "underrun_ms_per_min": 2600.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.026, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "mild", + "ring": "plain", + "underruns_per_min": 1400.0, + "underrun_ms_per_min": 2900.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.032, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "near-zero", + "ring": "isolated", + "underruns_per_min": 6100, + "underrun_ms_per_min": 16000, + "skip_aheads_per_min": 8.0, + "discarded_ms_per_min": 340, + "stalled_share": 0.034, + "silence_share": 0.25, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "near-zero", + "ring": "plain", + "underruns_per_min": 4600, + "underrun_ms_per_min": 12000, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 290, + "stalled_share": 0.01, + "silence_share": 0.19, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "step", + "ring": "isolated", + "underruns_per_min": 1300.0, + "underrun_ms_per_min": 2700.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.034, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "step", + "ring": "plain", + "underruns_per_min": 2500.0, + "underrun_ms_per_min": 5700.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.084, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "wide", + "ring": "isolated", + "underruns_per_min": 2200.0, + "underrun_ms_per_min": 5600.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.092, + "target_ms_p95": 150.5 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "wide", + "ring": "plain", + "underruns_per_min": 2300.0, + "underrun_ms_per_min": 5700.0, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 200.0, + "stalled_share": 0.01, + "silence_share": 0.09, + "target_ms_p95": 154.3 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "local-aac", + "ring": "isolated", + "underruns_per_min": 102.9, + "underrun_ms_per_min": 290.8, + "skip_aheads_per_min": 2, + "discarded_ms_per_min": 93.6, + "stalled_share": 0, + "silence_share": 0.005, + "target_ms_p95": 186 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "local-aac", + "ring": "plain", + "underruns_per_min": 102.9, + "underrun_ms_per_min": 290.8, + "skip_aheads_per_min": 2, + "discarded_ms_per_min": 93.6, + "stalled_share": 0, + "silence_share": 0.005, + "target_ms_p95": 186 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "relay-bbb", + "ring": "isolated", + "underruns_per_min": 0, + "underrun_ms_per_min": 0, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0, + "target_ms_p95": 289 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "relay-bbb", + "ring": "plain", + "underruns_per_min": 0, + "underrun_ms_per_min": 0, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0, + "target_ms_p95": 289 + }, + { + "runtime": "replay", + "codec": "opus", + "rate": 48000, + "profile": "relay-mic", + "ring": "isolated", + "underruns_per_min": 225.9, + "underrun_ms_per_min": 588.2, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0.01, + "target_ms_p95": 123 + }, + { + "runtime": "replay", + "codec": "opus", + "rate": 48000, + "profile": "relay-mic", + "ring": "plain", + "underruns_per_min": 225.9, + "underrun_ms_per_min": 588.2, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0.01, + "target_ms_p95": 123 + } + ] +} diff --git a/test/audio-quality/clients/js/analyze.ts b/test/audio-quality/clients/js/analyze.ts new file mode 100644 index 00000000000..7a0eabaa1ee --- /dev/null +++ b/test/audio-quality/clients/js/analyze.ts @@ -0,0 +1,116 @@ +/** + * Reduces one row of a run directory to `.summary.json`, and prints it. + * + * Reads what the row left: `.ndjson` and `.page.json` from the driver, and the shaper's log + * and exit status from run.sh. + * + * bun analyze.ts --run --row [--warmup 5] + * + * @module + */ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { analyze } from "./src/analyze.ts"; +import { + type Environment, + METRICS, + parseRow, + type Sample, + type Shaper, + type ShaperCounters, + STAGES, + type Void, +} from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + run: { type: "string" }, + row: { type: "string" }, + warmup: { type: "string", default: "5" }, + }, +}); +if (!values.run || !values.row) { + console.error("usage: analyze.ts --run --row [--warmup 5]"); + process.exit(2); +} +const run = values.run; +const tag = values.row; +const read = (file: string) => { + const path = join(run, file); + return existsSync(path) ? readFileSync(path, "utf8") : undefined; +}; + +const samples: Sample[] = (read(`${tag}.ndjson`) ?? "") + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as Sample); + +const page = JSON.parse(read(`${tag}.page.json`) ?? "null") as { + environment?: Environment; + notes: string[]; + voids: Void[]; +} | null; + +/** The shaper prints its seed at start and its counters at exit; run.sh records the exit status. */ +function shaperOf(): Shaper | null { + const log = read(`shaper-${tag}.log`); + const status = read(`shaper-${tag}.status`); + if (log === undefined || status === undefined) return null; + const counters = (direction: "up" | "down"): ShaperCounters | null => { + const match = log.match( + new RegExp( + `${direction}: (\\d+) packets, (\\d+) lost, (\\d+) overflowed, (\\d+) throttled, (\\d+) delayed, (\\d+) reordered`, + ), + ); + if (!match) return null; + const [packets, lost, overflowed, throttled, delayed, reordered] = match.slice(1).map(Number) as number[]; + return { packets, lost, overflowed, throttled, delayed, reordered } as ShaperCounters; + }; + const seed = log.match(/seed (\d+)/)?.[1]; + return { + seed: seed === undefined ? null : Number(seed), + status: Number.parseInt(status, 10), + up: counters("up"), + down: counters("down"), + }; +} + +const summary = analyze({ + row: parseRow(tag), + samples, + environment: page?.environment, + voids: page?.voids ?? [{ assertion: "driver", detail: "left no page report" }], + notes: page?.notes ?? [], + shaper: shaperOf(), + warmupMs: Number.parseFloat(values.warmup) * 1000, +}); +await Bun.write(join(run, `${tag}.summary.json`), JSON.stringify(summary, null, 1)); + +const lines = [`## ${tag}`, "", `window ${summary.windowMs} ms after a ${summary.warmupMs} ms warmup`, ""]; +if (summary.voids.length > 0) { + lines.push(`**void**: ${summary.voids.map((v) => `${v.assertion} (${v.detail})`).join("; ")}`, ""); +} +lines.push("| metric | unit | clock | aggregation | value |", "|---|---|---|---|---|"); +for (const [name, spec] of Object.entries(METRICS)) { + for (const aggregation of spec.aggregations) { + lines.push( + `| ${name} | ${spec.unit} | ${spec.clock} | ${aggregation} | ${summary.metrics[`${name}_${aggregation}`] ?? "n/a"} |`, + ); + } +} +lines.push( + "", + "| stage | ms | source |", + "|---|---|---|", + ...STAGES.map((s) => `| ${s} | ${summary.stages[s].ms ?? "n/a"} | ${summary.stages[s].source} |`), +); +const shaper = summary.shaper; +if (shaper) { + lines.push( + "", + `shaper seed ${shaper.seed} exit ${shaper.status}: up ${JSON.stringify(shaper.up)} down ${JSON.stringify(shaper.down)}`, + ); +} +if (summary.notes.length > 0) lines.push("", "notes:", ...summary.notes.slice(0, 20).map((n) => `- ${n}`)); +console.log(lines.join("\n")); diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts new file mode 100644 index 00000000000..4d18ea295c8 --- /dev/null +++ b/test/audio-quality/clients/js/driver.ts @@ -0,0 +1,166 @@ +/** + * Plays one matrix row in headless Chromium, and refuses a row whose numbers would mean something + * other than what its key says. + * + * The page dials the shaper, which forwards UDP and nothing else. So the page's fetch of the + * certificate hash is answered here with the relay's own (the certificate is pinned by hash, so + * dialing another port needs nothing more), and a WebSocket fallback finds nobody listening: the + * session is WebTransport through the shaper, or nothing. + * + * bun driver.ts --url http://127.0.0.1:4501 --fingerprint http://127.0.0.1:4500/certificate.sha256 \ + * --broadcast tone-opus.hang --page dist --ring plain --delay auto --duration 60 \ + * --tag chromium-opus-48000-mild-plain --out + * + * Writes `.ndjson` (the samples, appended as they are drained) and `.page.json` (the + * environment, console notes, and voids). Exits nonzero when the row could not be played at all. + * + * @module + */ +import { appendFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { launch, sleep } from "../../../interop/clients/js/harness.ts"; +import type { Environment, Ring, Void } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + url: { type: "string" }, + fingerprint: { type: "string" }, + broadcast: { type: "string" }, + page: { type: "string" }, + ring: { type: "string" }, + delay: { type: "string", default: "auto" }, + duration: { type: "string", default: "60" }, + tag: { type: "string" }, + out: { type: "string" }, + }, +}); + +const { url, fingerprint, broadcast, page: pageDir, tag, out } = values; +const durationMs = Number.parseFloat(values.duration) * 1000; +const ring = values.ring as Ring; +if ( + !url || + !fingerprint || + !broadcast || + !pageDir || + !tag || + !out || + !Number.isFinite(durationMs) || + durationMs <= 0 || + (ring !== "plain" && ring !== "isolated") +) { + console.error( + "usage: driver.ts --url U --fingerprint F --broadcast B --page DIR --ring plain|isolated --tag T --out DIR [--delay auto] [--duration S>0]", + ); + process.exit(2); +} + +/** + * How long the session and the first audio get, together, before the row is abandoned. It stays short + * of the step profile's change at 30s, so a step row always measures audio from before the change. + */ +const STARTUP_MS = 20_000; +const startupDeadline = Date.now() + STARTUP_MS; +/** How often the probe is drained into the ndjson. */ +const DRAIN_MS = 1000; + +const hash = await fetch(fingerprint).then((r) => { + if (!r.ok) throw new Error(`${fingerprint}: ${r.status}`); + return r.text(); +}); + +// Cross-origin isolation is a property of the document, not the bundle, so one build served under +// two prefixes runs both rings. `/plain` is the production path: most viewers are not isolated, and +// get the postMessage ring. +const root = resolve(pageDir); +const isolation = { "cross-origin-opener-policy": "same-origin", "cross-origin-embedder-policy": "require-corp" }; +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const path = new URL(req.url).pathname; + const match = path.match(/^\/(isolated|plain)\/(.*)$/); + if (!match) return new Response("not found", { status: 404 }); + const file = Bun.file(join(root, match[2] || "index.html")); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file, { headers: match[1] === "isolated" ? isolation : undefined }); + }, +}); + +const query = new URLSearchParams({ url, broadcast, delay: values.delay }); +const pageUrl = `http://127.0.0.1:${server.port}/${ring}/?${query}`; +console.log(`page: ${pageUrl}`); + +const samplesFile = join(out, `${tag}.ndjson`); +const voids: Void[] = []; +const refuse = (assertion: string, detail: string) => { + console.error(`void: ${assertion}: ${detail}`); + voids.push({ assertion, detail }); +}; + +/** Poll the page until `ready` returns a value, or throw naming what never happened. */ +async function waitFor(page: Page, what: string, ready: () => T | undefined): Promise { + while (Date.now() < startupDeadline) { + const value = await page.evaluate(ready); + if (value !== undefined && value !== null) return value; + await sleep(200); + } + throw new Error(`${what} not within ${STARTUP_MS / 1000}s`); +} + +const drain = async (page: Page) => { + const samples = await page.evaluate(() => globalThis.audioQuality.drain()); + if (samples.length > 0) appendFileSync(samplesFile, `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); +}; + +// No fake devices: nothing here captures. The autoplay flag stands in for the click a viewer makes. +const browser = await launch(["--autoplay-policy=no-user-gesture-required"]); +let environment: Environment | undefined; +let notes: string[] = []; +let failed = false; +try { + const page = await browser.newPage(); + page.on("pageerror", (error) => console.error(`[page error] ${error.message}`)); + await page.route(new URL("/certificate.sha256", url).href, (route) => + route.fulfill({ body: hash, headers: { "access-control-allow-origin": "*" } }), + ); + await page.goto(pageUrl, { waitUntil: "load" }); + + environment = await waitFor(page, "session", () => { + const env = globalThis.audioQuality?.environment(); + return env?.transport ? env : undefined; + }); + if (environment.transport !== "webtransport") { + refuse("transport", `negotiated ${environment.transport}, which never crosses the UDP shaper`); + } + if (environment.crossOriginIsolated !== (ring === "isolated")) { + refuse("ring", `asked for ${ring}, but crossOriginIsolated is ${environment.crossOriginIsolated}`); + } + + // The row's duration is audio played, not startup: the window starts once the ring does. + await waitFor(page, "first audio", () => globalThis.audioQuality.playing() || undefined); + + const end = Date.now() + durationMs; + while (Date.now() < end) { + await sleep(Math.min(DRAIN_MS, end - Date.now())); + await drain(page); + } + // A run that ends inside a gap still counts it. + await page.evaluate(() => globalThis.audioQuality.finish()); + await drain(page); + // The context rate is only known once the graph exists, which is after the catalog. + environment = (await page.evaluate(() => globalThis.audioQuality.environment())) ?? environment; + notes = await page.evaluate(() => globalThis.audioQuality.notes()); + await page.close(); +} catch (err) { + failed = true; + refuse("driver", err instanceof Error ? err.message : String(err)); +} finally { + await browser.close().catch(() => {}); + server.stop(true); +} + +await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes, voids }, null, 1)); +process.exit(failed ? 1 : 0); diff --git a/test/audio-quality/clients/js/grade.ts b/test/audio-quality/clients/js/grade.ts new file mode 100644 index 00000000000..819b7be29fe --- /dev/null +++ b/test/audio-quality/clients/js/grade.ts @@ -0,0 +1,102 @@ +/** + * Grades a run's summaries against `budgets.json`, prints the table, and writes `grade.md`. + * + * Every budget value is a ceiling on one `_` key. Under `--enforce` a run fails + * on any of: a value over its ceiling, a budgeted value the run never measured, a void row, or a row + * with no budget at all. Each of those would otherwise read as a pass while grading nothing. + * + * bun grade.ts --run --budgets [--enforce] + * + * @module + */ +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { type Budgets, METRIC_KEYS, type Row, rowKey, type Summary } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + run: { type: "string" }, + budgets: { type: "string" }, + enforce: { type: "boolean", default: false }, + }, +}); +if (!values.run || !values.budgets) { + console.error("usage: grade.ts --run --budgets [--enforce]"); + process.exit(2); +} +const run = values.run; + +const budgets = JSON.parse(readFileSync(values.budgets, "utf8")) as Budgets; +const summaries = readdirSync(run) + .filter((f) => f.endsWith(".summary.json")) + .sort() + .map((f) => JSON.parse(readFileSync(join(run, f), "utf8")) as Summary); +if (summaries.length === 0) { + console.error(`error: no summaries in ${run}`); + process.exit(1); +} + +/** A budget applies to exactly one matrix cell, matched on the whole row. */ +const budgetFor = (row: Row) => + budgets.rows.find( + (b) => + b.runtime === row.runtime && + b.codec === row.codec && + b.rate === row.rate && + b.profile === row.profile && + b.ring === row.ring, + ); + +const failures: string[] = []; +for (const summary of summaries) { + const key = rowKey(summary.row); + const budget = budgetFor(summary.row); + if (!budget) { + failures.push(`${key}: no budget`); + continue; + } + for (const v of summary.voids) failures.push(`${key}: void, ${v.assertion} (${v.detail})`); + if (summary.voids.length > 0) continue; + for (const metric of METRIC_KEYS) { + const ceiling = budget[metric]; + if (typeof ceiling !== "number") continue; + const value = summary.metrics[metric]; + if (value === null || value === undefined) failures.push(`${key}: ${metric} unmeasured, ceiling ${ceiling}`); + else if (value > ceiling) failures.push(`${key}: ${metric} ${value} > ${ceiling}`); + } +} + +/** The columns a reader scans first: what the listener would have noticed, and the delay paid for it. */ +const COLUMNS: [string, string][] = [ + ["underruns/min", "underruns_per_min"], + ["short/min", "short_quanta_per_min"], + ["episodes/min", "underrun_episodes_per_min"], + ["gap ms/min", "underrun_ms_per_min"], + ["skips/min", "skip_aheads_per_min"], + ["discarded ms/min", "discarded_ms_per_min"], + ["stalled", "stalled_share"], + ["silence", "silence_share"], + ["target p95", "target_ms_p95"], + ["converge ms", "converge_ms_last"], +]; + +const lines = [ + `| row | ${COLUMNS.map(([label]) => label).join(" | ")} | void |`, + `|---|${COLUMNS.map(() => "---|").join("")}---|`, + ...summaries.map( + (s) => + `| ${rowKey(s.row)} | ${COLUMNS.map(([, k]) => s.metrics[k] ?? "n/a").join(" | ")} | ${s.voids.map((v) => v.assertion).join(",") || "-"} |`, + ), + "", + ...(failures.length > 0 ? ["failures:", ...failures.map((f) => `- ${f}`)] : ["within budget"]), +]; +const report = lines.join("\n"); +console.log(report); +await Bun.write(join(run, "grade.md"), `${report}\n`); + +if (!values.enforce) { + if (failures.length > 0) console.log("\nreporting only: pass --enforce to fail on these"); + process.exit(0); +} +process.exit(failures.length > 0 ? 1 : 0); diff --git a/test/audio-quality/clients/js/index.html b/test/audio-quality/clients/js/index.html new file mode 100644 index 00000000000..8b151b202c7 --- /dev/null +++ b/test/audio-quality/clients/js/index.html @@ -0,0 +1,10 @@ + + + + + moq audio quality + + + + + diff --git a/test/audio-quality/clients/js/mic.html b/test/audio-quality/clients/js/mic.html new file mode 100644 index 00000000000..43c0cb2d816 --- /dev/null +++ b/test/audio-quality/clients/js/mic.html @@ -0,0 +1,10 @@ + + + + + moq audio quality microphone + + + + + diff --git a/test/audio-quality/clients/js/package.json b/test/audio-quality/clients/js/package.json new file mode 100644 index 00000000000..dd2918b1669 --- /dev/null +++ b/test/audio-quality/clients/js/package.json @@ -0,0 +1,22 @@ +{ + "name": "@moq/audio-quality", + "private": true, + "type": "module", + "scripts": { + "check": "tsc --noEmit", + "test": "bun test --only-failures" + }, + "dependencies": { + "@moq/hang": "workspace:*", + "@moq/net": "workspace:*", + "@moq/publish": "workspace:*", + "@moq/signals": "workspace:*", + "@moq/watch": "workspace:*" + }, + "devDependencies": { + "@types/audioworklet": "^0.0.100", + "esbuild": "^0.28.2", + "playwright": "^1.63.0", + "vite": "^8.3.0" + } +} diff --git a/test/audio-quality/clients/js/record.ts b/test/audio-quality/clients/js/record.ts new file mode 100644 index 00000000000..7c710de081a --- /dev/null +++ b/test/audio-quality/clients/js/record.ts @@ -0,0 +1,144 @@ +/** + * Records one arrival trace in headless Chromium, and writes it as a checked-in `Trace`. + * + * The recorder page subscribes to the broadcast's audio and stamps every frame the container consumer + * hands over. With `--mic`, a second page first publishes Chromium's fake capture device through + * `` to the same relay and broadcast, so the trace is a browser publisher's cadence. + * + * bun record.ts --page dist --url https://cdn.moq.dev/demo --broadcast bbb.hang --duration 35 \ + * --source "..." --description "..." --out ../../traces/relay-bbb.json + * + * The recording starts at the first arrival, tune-in burst included, since a player tunes in too. + * + * @module + */ +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { launch, sleep } from "../../../interop/clients/js/harness.ts"; +import type { Arrival, Trace } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + page: { type: "string" }, + url: { type: "string" }, + broadcast: { type: "string" }, + duration: { type: "string", default: "35" }, + source: { type: "string" }, + description: { type: "string" }, + mic: { type: "boolean", default: false }, + out: { type: "string" }, + }, +}); + +const { page: pageDir, url, broadcast, source, description, out } = values; +const durationMs = Number(values.duration) * 1000; +if (!pageDir || !url || !broadcast || !source || !description || !out || !Number.isFinite(durationMs) || durationMs <= 0) { + console.error( + "usage: record.ts --page DIR --url U --broadcast B --source S --description D --out FILE [--duration S] [--mic]", + ); + process.exit(2); +} + +/** How long the session and the first arrival get before the recording is abandoned. */ +const STARTUP_MS = 30_000; + +// Cross-origin isolated, for the 5 us `performance.now()` a non-isolated page coarsens to 100 us. +const root = resolve(pageDir); +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const file = Bun.file(join(root, new URL(req.url).pathname)); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file, { + headers: { "cross-origin-opener-policy": "same-origin", "cross-origin-embedder-policy": "require-corp" }, + }); + }, +}); +const pageUrl = (file: string) => `http://127.0.0.1:${server.port}/${file}?${new URLSearchParams({ url, broadcast })}`; + +/** Poll the page until `ready` returns a value, or throw naming what never happened. */ +async function waitFor(page: Page, what: string, ready: () => T | undefined): Promise { + const deadline = Date.now() + STARTUP_MS; + while (Date.now() < deadline) { + const error = await page.evaluate(() => globalThis.recorder?.error()); + if (error) throw new Error(`recorder: ${error}`); + const value = await page.evaluate(ready); + if (value !== undefined && value !== null) return value; + await sleep(200); + } + throw new Error(`${what} not within ${STARTUP_MS / 1000}s`); +} + +const browser = await launch(["--use-fake-device-for-media-stream", "--use-fake-ui-for-media-stream"]); +let failed = false; +try { + const context = await browser.newContext({ permissions: ["microphone", "camera"] }); + if (values.mic) { + const publisher = await context.newPage(); + publisher.on("pageerror", (error) => console.error(`[mic error] ${error.message}`)); + await publisher.goto(pageUrl("mic.html"), { waitUntil: "load" }); + // Subscribing before the element has an audio rendition finds no catalog to read. + const deadline = Date.now() + STARTUP_MS; + while (!(await publisher.evaluate(() => document.querySelector("moq-publish")?.audio.out.catalog.peek()))) { + if (Date.now() > deadline) throw new Error(`the microphone never published within ${STARTUP_MS / 1000}s`); + await sleep(200); + } + await sleep(2000); + } + + const page = await context.newPage(); + page.on("pageerror", (error) => console.error(`[recorder error] ${error.message}`)); + page.on("console", (msg) => { + if (msg.type() === "error" || msg.type() === "warning") console.error(`[recorder ${msg.type()}] ${msg.text()}`); + }); + await page.goto(pageUrl("recorder.html"), { waitUntil: "load" }); + + const info = await waitFor(page, "an audio subscription", () => globalThis.recorder.info()); + const arrivals: Arrival[] = []; + await waitFor(page, "a first arrival", () => { + const drained = globalThis.recorder.drain(); + return drained.length > 0 ? drained : undefined; + }).then((first) => arrivals.push(...first)); + + // Ends on the page's clock, not on the next arrival, so a publisher that goes quiet is recorded + // as the silence it is instead of holding the recording open. + const start = arrivals[0][0]; + while ((await page.evaluate(() => performance.now())) < start + durationMs) { + await sleep(1000); + const error = await page.evaluate(() => globalThis.recorder.error()); + if (error) throw new Error(`recorder: ${error}`); + arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain()))); + } + arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain()))); + // The RTT is read last: the PROBE minimum only falls as the session runs. + const rtt = await page.evaluate(() => globalThis.recorder.info()?.rtt ?? null); + + const trimmed = arrivals.filter(([at]) => at - start <= durationMs); + const round = (x: number, places: number) => Math.round(x * 10 ** places) / 10 ** places; + const trace: Trace = { + version: 2, + source, + description, + transport: info.transport, + rtt: rtt === null ? null : round(rtt, 1), + config: info.config as Trace["config"], + duration: durationMs, + arrivals: trimmed.map(([at, timestamp, group]) => [round(at - start, 2), round(timestamp, 3), group]), + }; + // One arrival per line, so the file reads as a table. + const { arrivals: rows, ...header } = trace; + const head = JSON.stringify(header, null, "\t").slice(0, -2); + await Bun.write(out, `${head},\n\t"arrivals": [\n${rows.map((r) => `\t\t${JSON.stringify(r)}`).join(",\n")}\n\t]\n}\n`); + console.log( + `${out}: ${trace.arrivals.length} arrivals over ${round(durationMs / 1000, 1)} s, ${trace.config.codec} at ${trace.config.sampleRate} Hz, ${trace.transport}, rtt ${trace.rtt} ms`, + ); +} catch (err) { + failed = true; + console.error(err instanceof Error ? err.message : String(err)); +} finally { + await browser.close().catch(() => {}); + server.stop(true); +} +process.exit(failed ? 1 : 0); diff --git a/test/audio-quality/clients/js/recorder.html b/test/audio-quality/clients/js/recorder.html new file mode 100644 index 00000000000..f3185f8b369 --- /dev/null +++ b/test/audio-quality/clients/js/recorder.html @@ -0,0 +1,10 @@ + + + + + moq audio quality recorder + + + + + diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts new file mode 100644 index 00000000000..9b072658625 --- /dev/null +++ b/test/audio-quality/clients/js/replay.ts @@ -0,0 +1,171 @@ +/** + * Replays the recorded traces through the player's consumer and rings, and writes each row the way + * `driver.ts` writes a browser row, so `analyze.ts` reduces both alike. + * + * The player half is `js/watch/src/audio/replay.ts`: the real container consumer and rings on a + * simulated clock, at the delay a real `Sync` resolves for the recorded catalog and RTT. This half + * reads its quanta through the same classifier the page's output tap runs, and samples the ring + * every 250 ms of simulated time, as the probe samples a page. Every profile is the trace's name at + * the "auto" delay a viewer gets by default. + * + * bun replay.ts --traces ../../traces --profiles relay-bbb --rings plain --codecs aac --list + * bun replay.ts --traces ../../traces --out + * + * `--list` prints the row tags; otherwise it writes `.ndjson` and `.page.json` per row. + * + * @module + */ +import { readdirSync, readFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { parseArgs } from "node:util"; +import type * as Catalog from "@moq/hang/catalog"; +import { classify, Ledger } from "./src/quantum.ts"; +import { + type Environment, + type Ring, + type Row, + rowKey, + SAMPLE_INTERVAL_MS, + type Sample, + SILENCE_RMS, + type Stall, + type Trace, + traceCodec, +} from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + traces: { type: "string" }, + profiles: { type: "string" }, + rings: { type: "string", default: "isolated,plain" }, + codecs: { type: "string", default: "opus,aac" }, + out: { type: "string" }, + list: { type: "boolean", default: false }, + }, +}); +if (!values.traces || (!values.list && !values.out)) { + console.error("usage: replay.ts --traces DIR [--profiles a,b] [--rings r] [--codecs c] (--list | --out DIR)"); + process.exit(2); +} +const dir = values.traces; +const split = (list: string) => list.split(",").filter((s) => s !== ""); + +const traces = readdirSync(dir) + .filter((f) => f.endsWith(".json")) + .sort() + .map((f) => ({ profile: basename(f, ".json"), trace: JSON.parse(readFileSync(join(dir, f), "utf8")) as Trace })); +const profiles = values.profiles === undefined ? undefined : split(values.profiles); +const rings = split(values.rings); +const codecs = split(values.codecs); +const selectors: [string, string[], string[]][] = [ + ["profile", profiles ?? [], traces.map((t) => t.profile)], + ["ring", rings, ["isolated", "plain"]], + ["codec", codecs, ["opus", "aac"]], +]; +for (const [name, list, known] of selectors) { + const unknown = list.find((v) => !known.includes(v)); + if (unknown !== undefined) { + console.error(`unknown ${name} '${unknown}' (known: ${known.join(", ")})`); + process.exit(2); + } +} + +const rows: { row: Row; trace: Trace }[] = []; +for (const { profile, trace } of traces) { + if (trace.version !== 2) throw new Error(`${profile}: trace version ${trace.version}, expected 2`); + const codec = traceCodec(trace); + if (profiles && !profiles.includes(profile)) continue; + if (!codecs.includes(codec)) continue; + for (const ring of rings as Ring[]) { + rows.push({ row: { runtime: "replay", codec, rate: trace.config.sampleRate, profile, ring }, trace }); + } +} + +if (values.list) { + for (const { row } of rows) console.log(rowKey(row)); + process.exit(0); +} +const out = values.out as string; + +// Loaded only to play: listing runs before `run.sh` has installed the workspace the player needs. +const { replay, target } = await import("../../../../js/watch/src/audio/replay.ts"); + +for (const { row, trace } of rows) { + const tag = rowKey(row); + const rate = row.rate; + const delay = await target({ + delay: "auto", + config: trace.config as unknown as Catalog.AudioConfig, + rtt: trace.rtt ?? undefined, + }); + + const ledger = new Ledger(rate, SILENCE_RMS); + const samples: Sample[] = []; + let stalls: Stall[] = []; + let stalled = true; + let frame = 0; + let next = SAMPLE_INTERVAL_MS; + + // The consumer's warnings, as the page's probe keeps a browser row's console. + const notes: string[] = []; + const warn = console.warn; + console.warn = (...args: unknown[]) => { + if (notes.length < 200) notes.push(args.map(String).join(" ").slice(0, 200)); + }; + + const arrivals = trace.arrivals.map(([at, timestamp, group]) => ({ at, timestamp, group })); + const sample = (quantum: { at: number; timestamp: number; stalled: boolean }) => { + const gaps = ledger.take(); + samples.push({ + at: quantum.at, + render: quantum.at, + timestamp: quantum.timestamp, + stalled: quantum.stalled, + delay, + quanta: ledger.counts.quanta, + quiet: ledger.counts.quiet, + gaps: gaps.length > 0 ? gaps : undefined, + stalls: stalls.length > 0 ? stalls : undefined, + }); + stalls = []; + }; + + const quanta = replay(arrivals, { + ring: row.ring === "isolated" ? "shared" : "post", + rate, + delay, + duration: trace.duration, + }); + let last: { at: number; timestamp: number; stalled: boolean } | undefined; + for await (const quantum of quanta) { + // The render clock is the simulated one: the trace's `at`, which starts at zero. + ledger.add(frame, quantum.output.length, classify([quantum.output])); + frame += quantum.output.length; + if (quantum.stalled !== stalled) { + stalled = quantum.stalled; + stalls.push({ at: quantum.at, stalled }); + } + last = { at: quantum.at, timestamp: quantum.timestamp, stalled: quantum.stalled }; + + if (quantum.at >= next) { + next += SAMPLE_INTERVAL_MS; + sample(last); + } + } + // A trace that ends inside a gap still counts it. + ledger.finish(); + if (last) sample(last); + console.warn = warn; + + const environment: Environment = { + crossOriginIsolated: row.ring === "isolated", + transport: trace.transport, + codec: trace.config.codec, + rate, + jitter: trace.config.jitter, + contextRate: rate, + }; + await Bun.write(join(out, `${tag}.ndjson`), `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); + await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes, voids: [] }, null, 1)); + console.log(`${tag}: ${samples.length} samples at a ${delay} ms target`); +} diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts new file mode 100644 index 00000000000..b6dba0ae501 --- /dev/null +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, test } from "bun:test"; +import { analyze, type Input, percentile, SKIP_MS, skipAheads } from "./analyze.ts"; +import type { Sample, Shaper } from "./schema.ts"; + +const row = { runtime: "chromium", codec: "opus", rate: 48000, profile: "mild", ring: "plain" } as const; +const counters = { packets: 100, lost: 0, overflowed: 0, throttled: 0, delayed: 100, reordered: 0 }; +const shaper: Shaper = { seed: 7, status: 0, up: counters, down: counters }; + +/** A clean run: 40 s of samples, the render clock 1000 ms behind the viewer's, the playhead 200 ms behind. */ +function clean(): Sample[] { + return Array.from({ length: 160 }, (_, i) => { + const at = 1000 + i * 250; + return { + at, + render: at - 1000, + timestamp: at - 200, + stalled: false, + delay: 120, + quanta: i * 94, + quiet: 0, + }; + }); +} + +const input = (samples: Sample[], extra: Partial = {}): Input => ({ + row, + samples, + environment: { crossOriginIsolated: false, transport: "webtransport" }, + voids: [], + notes: [], + shaper, + warmupMs: 5000, + ...extra, +}); + +describe("skipAheads", () => { + test("the quantization band is not a skip", () => { + const lags = Array.from({ length: 40 }, (_, i) => 200 + (i % 2 === 0 ? 25 : -25)); + expect(skipAheads(lags).count).toBe(0); + }); + + test("a step in the lag is one skip, of the step's size", () => { + const lags = Array.from({ length: 40 }, (_, i) => (i < 20 ? 300 : 200)); + expect(skipAheads(lags)).toEqual({ count: 1, ms: 100 }); + }); + + test("a step at the threshold is not one", () => { + const lags = Array.from({ length: 40 }, (_, i) => (i < 20 ? 200 + SKIP_MS : 200)); + expect(skipAheads(lags).count).toBe(0); + }); +}); + +test("percentile is nearest rank", () => { + expect(percentile([5, 1, 4, 2, 3], 50)).toBe(3); + expect(percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 95)).toBe(10); + expect(percentile([], 50)).toBeNull(); +}); + +describe("analyze", () => { + test("a clean run has nothing to report and nothing void", () => { + const summary = analyze(input(clean())); + expect(summary.voids).toEqual([]); + expect(summary.metrics).toMatchObject({ + underruns_total: 0, + skip_aheads_total: 0, + stalled_share: 0, + silence_share: 0, + target_ms_p95: 120, + converge_ms_last: 0, + }); + // Every key in the schema is present, measured or null. + expect(summary.metrics.render_load_max).toBeNull(); + expect(summary.drift.find((d) => d.clock === "render")).toEqual({ clock: "render", rate: 0, offset: 1000 }); + }); + + test("a gap on the render clock is counted where it lands on the viewer's", () => { + const samples = clean(); + // 20 s into the viewer clock is 19 s on the render clock. + samples[100] = { ...samples[100], gaps: [{ at: 19_000, ms: 8, quanta: 3, short: 1 }] } as Sample; + // Inside the warmup, so not graded. + samples[10] = { ...samples[10], gaps: [{ at: 1_000, ms: 4, quanta: 2, short: 1 }] } as Sample; + const summary = analyze(input(samples)); + expect(summary.metrics).toMatchObject({ + underruns_total: 3, + short_quanta_total: 1, + underrun_episodes_total: 1, + underrun_ms_total: 8, + underrun_ms_max: 8, + }); + }); + + test("a gap that is the ring stalling is not an underrun", () => { + const samples = clean(); + samples[100] = { + ...samples[100], + gaps: [{ at: 19_000, ms: 300, quanta: 110, short: 0 }], + stalls: [ + { at: 19_020, stalled: true }, + { at: 19_300, stalled: false }, + ], + } as Sample; + expect(analyze(input(samples)).metrics.underruns_total).toBe(0); + }); + + test("a re-anchor that discards audio is a skip-ahead", () => { + const samples = clean().map((s, i) => (i >= 80 ? { ...s, timestamp: s.at - 100 } : s)); + expect(analyze(input(samples)).metrics).toMatchObject({ skip_aheads_total: 1, discarded_ms_total: 100 }); + }); + + test("a shaper that failed or never carried the page voids the row", () => { + const failed = analyze(input(clean(), { shaper: { ...shaper, status: 1 } })); + expect(failed.voids.map((v) => v.assertion)).toEqual(["shaper"]); + const idle = analyze(input(clean(), { shaper: { ...shaper, down: { ...counters, packets: 0 } } })); + expect(idle.voids.map((v) => v.assertion)).toEqual(["shaper"]); + }); + + test("a render clock that is not running at wall rate voids the row", () => { + const samples = clean().map((s) => ({ ...s, render: (s.at - 1000) * 0.5 })); + expect(analyze(input(samples)).voids.map((v) => v.assertion)).toEqual(["clock"]); + }); + + test("no audio at all voids the window", () => { + const samples = clean().map((s) => ({ ...s, timestamp: undefined, stalled: true })); + expect(analyze(input(samples)).voids.map((v) => v.assertion)).toContain("window"); + }); + + test("convergence is the last move, not the first plateau", () => { + const samples = clean().map((s, i) => ({ ...s, delay: i < 40 ? 100 : i < 60 ? 300 : 100 })); + // First audio at sample 0, the target back on its final value from sample 60. + expect(analyze(input(samples)).metrics.converge_ms_last).toBe(60 * 250); + }); +}); diff --git a/test/audio-quality/clients/js/src/analyze.ts b/test/audio-quality/clients/js/src/analyze.ts new file mode 100644 index 00000000000..84d398b6bd0 --- /dev/null +++ b/test/audio-quality/clients/js/src/analyze.ts @@ -0,0 +1,308 @@ +/** + * Reduces one row's samples to the {@link Summary} the grader reads. + * + * Pure, so every derivation below is unit tested against synthetic samples rather than trusted. + * `../analyze.ts` is the command that feeds it a run directory. + * + * Adapted from `debug-findings/analysis/analyze.mjs` on the reporter's fork (`fperex/moq`, branch + * `debug/rt-audio`), whose skip-ahead rule and per-minute normalisation this keeps. The event stream + * it read came from probes patched into the player; this reads public signals and the output tap. + * + * @module + */ +import { + BUCKET_MS, + type Drift, + type Environment, + type Gap, + MAX_RENDER_DRIFT, + METRIC_KEYS, + type Ms, + type Row, + type Sample, + type Shaper, + STAGES, + type Stage, + type StageSpan, + type Summary, + type Void, +} from "./schema.ts"; + +/** Everything a row left behind. */ +export type Input = { + row: Row; + samples: Sample[]; + environment: Environment | undefined; + voids: Void[]; + notes: string[]; + shaper: Shaper | null; + /** Discarded after first audio: every profile spends its first seconds converging on purpose. */ + warmupMs: Ms; +}; + +/** + * A gap that starts this close to a stall is the stall, not an underrun. + * + * The page learns of a stall after the ring does: up to five quanta late on the postMessage ring, + * one poll late on the isolated one. + */ +export const STALL_SLACK_MS = 60; + +/** A step in the de-trended playhead lag larger than this is a skip-ahead. See {@link skipAheads}. */ +export const SKIP_MS = 40; + +/** Samples either side of a candidate skip reduced to a median before it is judged. */ +export const SKIP_WINDOW = 4; + +const median = (values: number[]): number | undefined => { + if (values.length === 0) return undefined; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +}; + +/** Nearest-rank percentile. Null for an empty series. */ +export function percentile(values: number[], p: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)] ?? null; +} + +/** Least-squares slope of `ys` over `xs`. Null with nothing to fit. */ +export function slope(xs: number[], ys: number[]): number | null { + const n = xs.length; + if (n < 2) return null; + const mx = xs.reduce((a, b) => a + b, 0) / n; + const my = ys.reduce((a, b) => a + b, 0) / n; + let num = 0; + let den = 0; + for (let i = 0; i < n; i++) { + num += ((xs[i] ?? 0) - mx) * ((ys[i] ?? 0) - my); + den += ((xs[i] ?? 0) - mx) ** 2; + } + return den === 0 ? null : num / den; +} + +/** + * Forward steps in the lag between the viewer clock and the playhead, and the media they skipped. + * + * `lags` is the de-trended `at - timestamp` per sample, undefined where there was no playhead or the + * ring was stalled. The playhead a page reads is quantized (it moves 240 to 296 ms per 250 ms sample + * on a clean path), so one sample's excess is noise: a skip is a step between the medians either side + * of a sample, larger than {@link SKIP_MS}, which is more than that band and more than one estimator + * bucket plus a render quantum. + */ +export function skipAheads(lags: (Ms | undefined)[]): { count: number; ms: Ms } { + let count = 0; + let ms = 0; + const defined = (slice: (Ms | undefined)[]) => slice.filter((l): l is Ms => l !== undefined); + for (let i = SKIP_WINDOW; i < lags.length - SKIP_WINDOW; i++) { + if (lags[i] === undefined) continue; + const before = median(defined(lags.slice(i - SKIP_WINDOW, i))); + const after = median(defined(lags.slice(i + 1, i + 1 + SKIP_WINDOW))); + if (before === undefined || after === undefined) continue; + const jumped = before - after; + if (jumped > SKIP_MS) { + count++; + ms += jumped; + // One step is one skip, and the trailing window straddles it for several samples. + i += SKIP_WINDOW; + } + } + return { count, ms }; +} + +/** + * Spans in which the ring was stalled, on the render clock. + * + * The ring starts stalled, before the page has a context to stamp that on, so a first change to + * unstalled closes a span that opened before anything was recorded. + */ +export function stallSpans(samples: Sample[]): { start: Ms; end: Ms }[] { + const changes = samples.flatMap((s) => s.stalls ?? []); + const spans: { start: Ms; end: Ms }[] = []; + let open: Ms | undefined = changes[0]?.stalled === false ? Number.NEGATIVE_INFINITY : undefined; + for (const change of changes) { + if (change.stalled && open === undefined) open = change.at; + else if (!change.stalled && open !== undefined) { + spans.push({ start: open, end: change.at }); + open = undefined; + } + } + if (open !== undefined) spans.push({ start: open, end: Number.POSITIVE_INFINITY }); + return spans; +} + +const round = (x: number | null | undefined, places = 1): number | null => + x === null || x === undefined || !Number.isFinite(x) ? null : Math.round(x * 10 ** places) / 10 ** places; + +/** Reduce one row. */ +export function analyze(input: Input): Summary { + const { row, samples, environment, warmupMs } = input; + const voids = [...input.voids]; + const notes = [...input.notes]; + + // ── the shaper's verdict ──────────────────────────────────────────────── + // A replay row never crosses one: its path is the recording. + const shaper = input.shaper; + if (row.runtime === "replay") { + // Nothing to check. + } else if (!shaper) { + voids.push({ assertion: "shaper", detail: "left no report, so nothing says the path was shaped" }); + } else if (shaper.status !== 0) { + voids.push({ assertion: "shaper", detail: `exited ${shaper.status}: the profile never acted as configured` }); + } else if ((shaper.down?.packets ?? 0) === 0) { + voids.push({ assertion: "shaper", detail: "forwarded nothing to the page, so the session never crossed it" }); + } + + // ── the window ────────────────────────────────────────────────────────── + const first = samples.find((s) => s.timestamp !== undefined && !s.stalled); + const t0 = (first?.at ?? 0) + warmupMs; + const t1 = samples.at(-1)?.at ?? t0; + const window = samples.filter((s) => s.at >= t0 && s.at <= t1); + const minutes = Math.max(1e-9, (t1 - t0) / 60_000); + if (!first || window.length < 2) { + voids.push({ assertion: "window", detail: `no audio after the warmup (${samples.length} samples in all)` }); + } + + // ── the render clock, reduced to the viewer's ─────────────────────────── + // Each sample reads both clocks back to back, so the pairs fit the render clock's rate and offset. + const paired = samples.filter((s) => s.render !== undefined); + const renderRate = slope( + paired.map((s) => s.at), + paired.map((s) => s.render as Ms), + ); + const renderOffset = median(paired.map((s) => s.at - (s.render as Ms))) ?? null; + if (renderRate === null || renderOffset === null) { + voids.push({ assertion: "clock", detail: "the AudioContext clock was never readable" }); + } else if (Math.abs(renderRate - 1) > MAX_RENDER_DRIFT) { + voids.push({ + assertion: "clock", + detail: `the AudioContext clock ran at ${round(renderRate, 3)}x the viewer's: throttled or stalled`, + }); + } + const toViewer = (render: Ms) => render + (renderOffset ?? 0); + + // ── underruns, from the tap ───────────────────────────────────────────── + const stalls = stallSpans(samples); + const chosen = (gap: Gap) => + stalls.some((s) => gap.at >= s.start - STALL_SLACK_MS && gap.at <= s.end + STALL_SLACK_MS); + const gaps = samples + .flatMap((s) => s.gaps ?? []) + .filter((g) => toViewer(g.at) >= t0 && toViewer(g.at) <= t1 && !chosen(g)); + const underruns = gaps.reduce((a, g) => a + g.quanta, 0); + const short = gaps.reduce((a, g) => a + g.short, 0); + const underrunMs = gaps.reduce((a, g) => a + g.ms, 0); + + const tapped = window.filter((s) => s.quanta !== undefined && s.quiet !== undefined); + const firstTap = tapped[0]; + const lastTap = tapped.at(-1); + const quanta = (lastTap?.quanta ?? 0) - (firstTap?.quanta ?? 0); + const silence = quanta > 0 ? ((lastTap?.quiet ?? 0) - (firstTap?.quiet ?? 0)) / quanta : null; + if (!lastTap) notes.push("tap: never reported, so underruns and silence are unmeasured"); + + // ── the playhead ──────────────────────────────────────────────────────── + // The lag drifts when the media timeline does not run at wall rate, a property of the source + // rather than the player. Fitted and removed first, or it reads as a skip every few seconds. + // A least-squares fit would read a skip itself as drift and hide part of it, so the rate is the + // median slope across spans of two skip windows, which a step contaminates only a few of. + const playing = window.filter((s) => s.timestamp !== undefined && !s.stalled); + const reach = 2 * SKIP_WINDOW; + const mediaRate = + median( + playing.slice(reach).map((s, i) => { + const from = playing[i] as Sample; + return (s.at - (s.timestamp as Ms) - (from.at - (from.timestamp as Ms))) / (s.at - from.at); + }), + ) ?? null; + const base = playing[0]?.at ?? 0; + const lags = window.map((s) => + s.timestamp === undefined || s.stalled ? undefined : s.at - s.timestamp - (mediaRate ?? 0) * (s.at - base), + ); + const skips = skipAheads(lags); + const stalledShare = window.length > 0 ? window.filter((s) => s.stalled).length / window.length : null; + + // ── the target ────────────────────────────────────────────────────────── + const targets = window.flatMap((s) => (s.delay === undefined ? [] : [s.delay])); + // Measured backwards from the end: a target that settles and moves again has not converged. + const series = samples.filter((s) => first && s.at >= first.at && s.delay !== undefined); + const final = series.at(-1)?.delay; + let settled = series[0]; + for (let i = series.length - 1; i >= 0; i--) { + if (Math.abs((series[i]?.delay ?? 0) - (final ?? 0)) > BUCKET_MS) { + settled = series[i + 1]; + break; + } + } + const converge = first && settled ? settled.at - first.at : null; + + const loads = window.flatMap((s) => (s.renderLoad === undefined ? [] : [s.renderLoad])); + + const metrics: Record = Object.fromEntries(METRIC_KEYS.map((k) => [k, null])); + const perMin = (total: number) => round(total / minutes); + Object.assign(metrics, { + underruns_total: lastTap ? underruns : null, + underruns_per_min: lastTap ? perMin(underruns) : null, + short_quanta_total: lastTap ? short : null, + short_quanta_per_min: lastTap ? perMin(short) : null, + underrun_episodes_total: lastTap ? gaps.length : null, + underrun_episodes_per_min: lastTap ? perMin(gaps.length) : null, + underrun_ms_total: lastTap ? round(underrunMs) : null, + underrun_ms_per_min: lastTap ? perMin(underrunMs) : null, + underrun_ms_max: lastTap ? round(Math.max(0, ...gaps.map((g) => g.ms))) : null, + skip_aheads_total: skips.count, + skip_aheads_per_min: perMin(skips.count), + discarded_ms_total: round(skips.ms), + discarded_ms_per_min: perMin(skips.ms), + stalled_share: round(stalledShare, 3), + silence_share: round(silence, 3), + target_ms_p50: round(percentile(targets, 50)), + target_ms_p95: round(percentile(targets, 95)), + target_ms_max: round(percentile(targets, 100)), + converge_ms_last: round(converge), + render_load_max: round(percentile(loads, 100), 3), + }); + + // ── stages ────────────────────────────────────────────────────────────── + // Only the viewer's clocks are readable here, so only the receiver's stages are measured, and no + // end-to-end exists to sum them to. `publish_flush` is what the publisher declares, reported but + // never summed with viewer-clock spans. + const last = window.at(-1); + const span = (ms: Ms | undefined, source: StageSpan["source"]): StageSpan => + ms === undefined ? { ms: null, source: "unmeasured" } : { ms: round(ms), source }; + const known: Partial> = { + publish_flush: span(environment?.jitter, "declared"), + jitter_buffer: span(last?.delay, "measured"), + device: + last?.outputLatency === undefined + ? span(undefined, "unmeasured") + : span(last.outputLatency + (last.baseLatency ?? 0), "measured"), + }; + const stages = Object.fromEntries(STAGES.map((s) => [s, known[s] ?? span(undefined, "unmeasured")])) as Record< + Stage, + StageSpan + >; + + const drift: Drift[] = [ + { clock: "viewer", rate: 0, offset: 0 }, + { clock: "render", rate: round(renderRate === null ? null : renderRate - 1, 6), offset: round(renderOffset) }, + // The lag's slope is the media clock's rate against the viewer's, negated. + { clock: "media", rate: round(mediaRate === null ? null : -mediaRate, 6), offset: null }, + { clock: "publisher", rate: null, offset: null }, + { clock: "relay", rate: null, offset: null }, + ]; + + return { + version: 1, + row, + windowMs: round(t1 - t0) ?? 0, + warmupMs, + environment: environment ?? null, + voids, + metrics, + stages, + endToEnd: null, + drift, + shaper, + notes: notes.slice(0, 50), + }; +} diff --git a/test/audio-quality/clients/js/src/mic.ts b/test/audio-quality/clients/js/src/mic.ts new file mode 100644 index 00000000000..ae536caa9c6 --- /dev/null +++ b/test/audio-quality/clients/js/src/mic.ts @@ -0,0 +1,27 @@ +/** + * The publisher page for the microphone trace: the demo's `` capturing audio only. + * + * Whatever device Chromium offers is captured through the element's real path (getUserMedia, the + * capture worklet, WebCodecs Opus), so the trace carries a browser publisher's own cadence. Under + * `record.ts` that device is Chromium's fake one, a tone. + * + * ?url=https://cdn.moq.dev/anon&broadcast=aq-mic-1234.hang + * + * @module + */ +import "@moq/publish/element"; + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const publish = document.createElement("moq-publish"); +publish.setAttribute("url", required("url")); +publish.setAttribute("name", required("broadcast")); +publish.setAttribute("source", "camera"); +publish.setAttribute("invisible", ""); +publish.setAttribute("preview", "none"); +document.body.appendChild(publish); diff --git a/test/audio-quality/clients/js/src/page.ts b/test/audio-quality/clients/js/src/page.ts new file mode 100644 index 00000000000..ee742da8ccd --- /dev/null +++ b/test/audio-quality/clients/js/src/page.ts @@ -0,0 +1,43 @@ +/** + * The page under measurement: the demo's player and nothing else. + * + * The element and its registration imports are the ones `demo/web` uses, so a regression in the + * element's attribute handling or the worklet reaches these numbers the way it reaches a viewer. + * Everything comes from the query string, so one build serves every row: + * + * ?url=http://127.0.0.1:4499&broadcast=tone-opus.hang&delay=auto + * + * The driver drains the probe through `globalThis.audioQuality`. + * + * @module + */ +import "@moq/watch/element"; +import type MoqWatch from "@moq/watch/element"; +import { type Probe, probe } from "./probe.ts"; + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const watch = document.createElement("moq-watch") as MoqWatch; +watch.setAttribute("url", required("url")); +watch.setAttribute("name", required("broadcast")); +// "auto" adapts, and is what every profile but the control runs. A fixed delay carries its unit: +// the element rejects `delay="250"`. +watch.setAttribute("delay", params.get("delay") ?? "auto"); +watch.appendChild(document.createElement("canvas")); +document.body.appendChild(watch); + +// Audio is the measurement: muted, nothing would reach the ring. Volume first, because un-muting +// restores the stashed volume. +watch.volume = 1; +watch.muted = false; +watch.paused = false; + +declare global { + var audioQuality: Probe; +} +globalThis.audioQuality = probe(watch); diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts new file mode 100644 index 00000000000..195f3b852a1 --- /dev/null +++ b/test/audio-quality/clients/js/src/probe.ts @@ -0,0 +1,153 @@ +/** + * Samples a `` every 250 ms, from its public signals and a tap on its output. + * + * Nothing here patches the player or reaches past its `out` surface, so a number it reports is one + * any consumer could read. What the player does not publish (how full each render quantum was) is + * read off its output by the tap in `tap.ts`, every quantum rather than a sample of them. + * + * Adapted from the black-box probe in `debug-findings/analysis/blackbox.js` on the reporter's fork + * (`fperex/moq`, branch `debug/rt-audio`). See ../../../README.md. + * + * @module + */ +import type MoqWatch from "@moq/watch/element"; +import { type Environment, SAMPLE_INTERVAL_MS, type Sample, SILENCE_RMS, type Stall } from "./schema.ts"; +import { type Tap, tap } from "./tap.ts"; + +/** Chromium's render capacity surface, which is not in lib.dom. */ +type RenderCapacity = { + start(options: { updateInterval: number }): void; + addEventListener(type: "update", listener: (event: { averageLoad: number }) => void): void; + removeEventListener(type: "update", listener: (event: { averageLoad: number }) => void): void; +}; + +/** What the probe has collected, until someone drains it. */ +export type Probe = { + /** Samples taken since the last drain. */ + drain(): Sample[]; + /** End the measurement: take one last sample, with any gap still open closed into it. */ + finish(): Promise; + /** What the page is, once the catalog says the session is up. */ + environment(): Environment | undefined; + /** Console warnings and errors so far. */ + notes(): string[]; + /** Whether the ring has a playhead and is not stalled. */ + playing(): boolean; +}; + +/** Start sampling `watch`, from before it has produced anything, so startup is on the record too. */ +export function probe(watch: MoqWatch): Probe { + const samples: Sample[] = []; + const notes: string[] = []; + const note = (line: string) => { + if (notes.length < 200) notes.push(line.slice(0, 200)); + }; + + const { audio, sync, broadcast } = watch.player; + + // Stall changes are taken as they happen: a re-stall shorter than the sample grid is exactly the + // one a 250 ms read would miss, and it is what tells a stall from an underrun. + let stalls: Stall[] = []; + audio.out.stalled.subscribe((stalled) => { + const context = audio.out.context.peek(); + if (context) stalls.push({ at: context.currentTime * 1000, stalled }); + }); + + // The tap follows the output node, which the player rebuilds with its graph. A tap left on the old + // node would report a perfect run as one long silence. + let tapped: AudioNode | undefined; + let current: Tap | undefined; + let renderLoad: number | undefined; + let unlisten: (() => void) | undefined; + const attach = () => { + const root = audio.out.root.peek(); + if (!root || root === tapped) return; + tapped = root; + current?.close(); + current = undefined; + + tap(root, { floor: SILENCE_RMS }).then( + (t) => { + if (tapped === root) current = t; + else t.close(); + }, + (err: unknown) => note(`tap: ${err instanceof Error ? err.message : String(err)}`), + ); + + // The previous root's context stops reporting into this one's reading. + unlisten?.(); + unlisten = undefined; + const capacity = (root.context as unknown as { renderCapacity?: RenderCapacity }).renderCapacity; + if (!capacity) { + note("renderCapacity: unavailable, render_load will be null"); + return; + } + const update = (event: { averageLoad: number }) => { + renderLoad = event.averageLoad; + }; + capacity.addEventListener("update", update); + unlisten = () => capacity.removeEventListener("update", update); + // A whole second: shorter intervals are refused by some builds, and load is smooth anyway. + capacity.start({ updateInterval: 1 }); + }; + + const sample = (): Sample => { + attach(); + const context = audio.out.context.peek(); + const counts = current?.counts(); + const gaps = current?.take() ?? []; + const taken = stalls; + stalls = []; + return { + // Read back to back, so the pair calibrates the render clock against the viewer's. + at: performance.now(), + render: context ? context.currentTime * 1000 : undefined, + timestamp: audio.out.timestamp.peek(), + stalled: audio.out.stalled.peek(), + delay: sync.out.delay.peek(), + outputLatency: context ? context.outputLatency * 1000 : undefined, + baseLatency: context ? context.baseLatency * 1000 : undefined, + renderLoad, + quanta: counts?.quanta, + quiet: counts?.quiet, + gaps: gaps.length > 0 ? gaps : undefined, + stalls: taken.length > 0 ? taken : undefined, + }; + }; + + // Console output is evidence: a decoder reset shows up here before it shows up in a count. + for (const level of ["warn", "error"] as const) { + const original = console[level]; + console[level] = (...args: unknown[]) => { + note(`${level}: ${args.map(String).join(" ")}`); + original(...args); + }; + } + + setInterval(() => samples.push(sample()), SAMPLE_INTERVAL_MS); + + return { + drain: () => samples.splice(0, samples.length), + async finish() { + attach(); + await current?.finish(); + samples.push(sample()); + }, + environment() { + const catalog = broadcast.out.catalog.peek(); + if (!catalog) return undefined; + // One rendition is published per broadcast here, so the first is the one playing. + const config = Object.values(catalog.audio?.renditions ?? {})[0]; + return { + crossOriginIsolated: globalThis.crossOriginIsolated === true, + transport: watch.connection.transport.peek(), + codec: config?.codec, + rate: config?.sampleRate, + jitter: config?.jitter, + contextRate: audio.out.context.peek()?.sampleRate, + }; + }, + notes: () => notes.slice(), + playing: () => audio.out.timestamp.peek() !== undefined && !audio.out.stalled.peek(), + }; +} diff --git a/test/audio-quality/clients/js/src/quantum.test.ts b/test/audio-quality/clients/js/src/quantum.test.ts new file mode 100644 index 00000000000..7fb5444889b --- /dev/null +++ b/test/audio-quality/clients/js/src/quantum.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, test } from "bun:test"; +import { classify, Ledger } from "./quantum.ts"; + +const quantum = (values: number[], channels = 2) => Array.from({ length: channels }, () => Float32Array.from(values)); +const tone = (length: number) => Array.from({ length }, (_, i) => 0.5 * Math.sin(i / 3 + 0.1)); + +describe("classify", () => { + test("a quantum the ring filled is full", () => { + expect(classify(quantum(tone(128))).fill).toBe("full"); + }); + + test("trailing zeros in every channel are the samples the ring did not have", () => { + const values = [...tone(100), ...new Array(28).fill(0)]; + expect(classify(quantum(values))).toMatchObject({ fill: "short", missing: 28 }); + }); + + test("one channel still playing means the ring supplied that sample", () => { + const left = Float32Array.from([...tone(100), ...new Array(28).fill(0)]); + const right = Float32Array.from(tone(128)); + expect(classify([left, right])).toMatchObject({ fill: "full", missing: 0 }); + }); + + test("all zeros is silent, however it came about", () => { + expect(classify(quantum(new Array(128).fill(0)))).toMatchObject({ fill: "silent", missing: 128 }); + expect(classify([])).toMatchObject({ fill: "silent", missing: 0 }); + }); +}); + +describe("Ledger", () => { + const RATE = 48000; + const full = classify(quantum(tone(128))); + const short = classify(quantum([...tone(64), ...new Array(64).fill(0)])); + const silent = classify(quantum(new Array(128).fill(0))); + + test("the tune-in before the first audible quantum is not a gap", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, silent); + ledger.add(128, 128, silent); + ledger.add(256, 128, full); + expect(ledger.counts).toEqual({ quanta: 1, quiet: 0 }); + expect(ledger.take()).toEqual([]); + }); + + test("consecutive unfilled quanta are one gap, closed by the next full one", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, full); + ledger.add(128, 128, short); + ledger.add(256, 128, silent); + ledger.add(384, 128, short); + expect(ledger.take()).toEqual([]); + + ledger.add(512, 128, full); + const [gap, ...rest] = ledger.take(); + expect(rest).toEqual([]); + // Starts at the first missing sample, half-way through the first short quantum. + expect(gap?.at).toBeCloseTo((192 / RATE) * 1000); + expect(gap?.ms).toBeCloseTo(((64 + 128 + 64) / RATE) * 1000); + expect(gap).toMatchObject({ quanta: 3, short: 2 }); + expect(ledger.counts).toEqual({ quanta: 5, quiet: 1 }); + }); + + test("a run that ends inside a gap still reports it once finished", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, full); + ledger.add(128, 128, silent); + ledger.add(256, 128, silent); + expect(ledger.take()).toEqual([]); + + ledger.finish(); + const [gap, ...rest] = ledger.take(); + expect(rest).toEqual([]); + expect(gap?.ms).toBeCloseTo((256 / RATE) * 1000); + expect(gap).toMatchObject({ quanta: 2, short: 0 }); + }); +}); diff --git a/test/audio-quality/clients/js/src/quantum.ts b/test/audio-quality/clients/js/src/quantum.ts new file mode 100644 index 00000000000..a3cb38d3b8a --- /dev/null +++ b/test/audio-quality/clients/js/src/quantum.ts @@ -0,0 +1,117 @@ +/** + * What one render quantum at the player's output says about the ring behind it. + * + * The player's render worklet copies what its ring holds into the front of each output quantum and + * leaves the rest as the zeros Web Audio hands it. So a ring that ran dry shows up at the output as + * exact zeros at the end of a quantum across every channel, and an empty or stalled ring as a quantum + * of nothing but zeros. The published tone never lands on exact zero in every channel at once, which + * is what makes the output readable as the ring's fill level without a counter in the player. + * + * Free of worklet globals so the classification is unit tested rather than trusted. + * + * @module + */ +import type { Gap } from "./schema.ts"; + +/** How the ring filled one quantum. */ +export type Fill = "full" | "short" | "silent"; + +/** One quantum, classified. */ +export type Quantum = { + fill: Fill; + /** Samples per channel the ring did not supply: the trailing zeros, or the whole quantum. */ + missing: number; + /** Mean square over every sample of every channel. */ + power: number; +}; + +/** Classify one quantum of planar audio. No channels at all is a quantum that played nothing. */ +export function classify(channels: Float32Array[]): Quantum { + const length = channels[0]?.length ?? 0; + if (length === 0) return { fill: "silent", missing: 0, power: 0 }; + + let last = -1; + let sum = 0; + for (const channel of channels) { + for (let i = 0; i < channel.length; i++) { + const value = channel[i] ?? 0; + if (value !== 0 && i > last) last = i; + sum += value * value; + } + } + const power = sum / (length * channels.length); + if (last < 0) return { fill: "silent", missing: length, power }; + + const missing = length - 1 - last; + return { fill: missing > 0 ? "short" : "full", missing, power }; +} + +/** Cumulative counts since the first audible quantum. */ +export type Counts = { + /** Quanta rendered. */ + quanta: number; + /** Quanta under the silence floor, whatever the reason. */ + quiet: number; +}; + +/** + * Folds quanta into {@link Counts}, and closes a {@link Gap} each time the ring fills a quantum again. + * + * Nothing before the first non-silent quantum counts: the ring is filling for the first time, which + * is the tune-in, not a gap. + */ +export class Ledger { + readonly counts: Counts = { quanta: 0, quiet: 0 }; + readonly #rate: number; + readonly #floor: number; + #started = false; + #open: { frame: number; missing: number; quanta: number; short: number } | undefined; + #closed: Gap[] = []; + + /** `rate` is the context's sample rate; `floor` is the silence floor as an RMS level. */ + constructor(rate: number, floor: number) { + this.#rate = rate; + this.#floor = floor; + } + + /** Account for one quantum of `length` samples that started at `frame` on the render clock. */ + add(frame: number, length: number, quantum: Quantum): void { + if (!this.#started) { + if (quantum.fill === "silent") return; + this.#started = true; + } + + this.counts.quanta++; + if (Math.sqrt(quantum.power) < this.#floor) this.counts.quiet++; + if (quantum.fill === "full") { + this.#close(); + return; + } + + // A short quantum inside an open gap is a ring that got a little and ran out again: still one + // gap, since only a full quantum ends it. + this.#open ??= { frame: frame + length - quantum.missing, missing: 0, quanta: 0, short: 0 }; + const open = this.#open; + open.missing += quantum.missing; + open.quanta++; + if (quantum.fill === "short") open.short++; + } + + /** Gaps closed since the last call. A gap still open is reported once a full quantum ends it. */ + take(): Gap[] { + return this.#closed.splice(0, this.#closed.length); + } + + /** Close a gap still open, for the next {@link take}: the measurement ended inside it. */ + finish(): void { + this.#close(); + } + + #close(): void { + if (!this.#open) return; + const ms = (samples: number) => (samples / this.#rate) * 1000; + const { frame, missing, quanta, short } = this.#open; + this.#closed.push({ at: ms(frame), ms: ms(missing), quanta, short }); + this.#open = undefined; + } +} diff --git a/test/audio-quality/clients/js/src/recorder.ts b/test/audio-quality/clients/js/src/recorder.ts new file mode 100644 index 00000000000..c0494c5cc19 --- /dev/null +++ b/test/audio-quality/clients/js/src/recorder.ts @@ -0,0 +1,130 @@ +/** + * The recorder page: subscribes to a broadcast's audio and stamps each frame as it comes off its + * group's stream, which is where the player's container consumer receives it. + * + * Built only on the public `@moq/net` and `@moq/hang` surface, so it records the path a viewer's + * player takes (a WebTransport session in Chromium, every group on its own stream) without a debug + * hook in the player. It reads groups the way `Container.Consumer` does, concurrently and decoded + * with the same format, but stamps them before the consumer's in-order delivery: a group the + * consumer holds back waiting for a missing one, or skips for age, is a decision the replay makes + * again at its own delay, and baking one in would censor the trace. + * + * ?url=https://cdn.moq.dev/demo&broadcast=bbb.hang + * + * `record.ts` drains it through `globalThis.recorder`. + * + * @module + */ +import * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; +import * as Moq from "@moq/net"; +import type { Arrival } from "./schema.ts"; + +/** How far back the relay may serve a group. Long, so a late group is recorded rather than dropped. */ +const MAX_AGE = Moq.Time.Milli(10_000); + +/** What the recorder has, until the driver drains it. */ +export type Recorder = { + /** Arrivals since the last drain, `at` on this page's `performance.now()`. */ + drain(): Arrival[]; + /** The session and rendition, once subscribed. */ + info(): { transport: string; rtt: number | null; config: Catalog.AudioConfig } | undefined; + /** The failure that stopped recording, if one did. */ + error(): string | undefined; +}; + +declare global { + var recorder: Recorder; +} + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const arrivals: Arrival[] = []; +let info: ReturnType; +let failure: string | undefined; +const fail = (err: unknown) => { + failure ??= err instanceof Error ? err.message : String(err); + console.error("recorder:", failure); +}; + +globalThis.recorder = { + drain: () => arrivals.splice(0, arrivals.length), + info: () => info, + error: () => failure, +}; + +/** The container format the player's decoder would pick for `config`. */ +function format(config: Catalog.AudioConfig): Container.Format { + const container = config.container; + if (container.kind === "legacy") return new Container.Legacy.Format(config); + if (container.kind === "loc") return new Container.Loc.Format("audio"); + if (container.kind === "cmaf") { + const init = Uint8Array.from(atob(container.init), (c) => c.charCodeAt(0)); + return new Container.Cmaf.Format(Container.Cmaf.decodeInitSegment(init)); + } + throw new Error(`unsupported container: ${JSON.stringify(container)}`); +} + +/** Stamp every frame of one group as it is read. Markers carry no media, so they are not arrivals. */ +async function readGroup(group: Moq.Group.Consumer, decoder: Container.Format): Promise { + for (;;) { + const next = await group.readFrame(); + const at = performance.now(); + if (!next) return; + for (const frame of decoder.decode(next.payload)) { + if (decoder.end?.(frame) !== undefined) continue; + arrivals.push([at, frame.timestamp / 1000, group.sequence]); + } + } +} + +async function record(): Promise { + const origin = new Moq.Origin.Producer(); + const connection = await Moq.Connection.connect({ url: new URL(required("url")), consume: origin }); + + let rtt: number | null = null; + connection.probe.subscribe((probe) => { + if (probe.rtt !== undefined) rtt = rtt === null ? probe.rtt : Math.min(rtt, probe.rtt); + }); + + const request = origin.request(Moq.Path.from(required("broadcast"))); + let broadcast = request.active.peek(); + while (!broadcast) { + await request.active.changed(); + broadcast = request.active.peek(); + } + + let root: Catalog.Root | undefined; + for await (const update of Catalog.watch(broadcast)) { + if (Object.keys(update.audio?.renditions ?? {}).length > 0) { + root = update; + break; + } + } + // Every recorded broadcast carries one audio rendition, so the first is the one a player picks. + const [name, config] = Object.entries(root?.audio?.renditions ?? {})[0] ?? []; + if (!name || !config) throw new Error("the catalog ended without an audio rendition"); + + const decoder = format(config); + const track = broadcast.track(name).subscribe({ priority: Catalog.PRIORITY.audio, maxAge: MAX_AGE }); + info = { + transport: connection.transport, + get rtt() { + return rtt; + }, + config, + }; + + for (;;) { + const group = await track.recvGroup(); + if (!group) throw new Error("the track ended"); + readGroup(group, decoder).catch(fail); + } +} + +record().catch(fail); diff --git a/test/audio-quality/clients/js/src/run.test.ts b/test/audio-quality/clients/js/src/run.test.ts new file mode 100644 index 00000000000..13bda94a13f --- /dev/null +++ b/test/audio-quality/clients/js/src/run.test.ts @@ -0,0 +1,34 @@ +import { expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; + +const script = fileURLToPath(new URL("../../../run.sh", import.meta.url)); + +test("a step row must play beyond the profile change", () => { + for (const duration of ["20", "30"]) { + const result = Bun.spawnSync(["bash", script, "--profiles", "step", "--duration", duration, "--list"]); + expect(result.exitCode).toBe(2); + expect(result.stderr.toString()).toContain("must exceed the step at 30s"); + } + const result = Bun.spawnSync(["bash", script, "--profiles", "step", "--duration", "31", "--list"]); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString()).toContain("chromium-opus-48000-step-plain"); +}); + +test("short steady rows remain supported", () => { + const result = Bun.spawnSync(["bash", script, "--profiles", "mild", "--duration", "20", "--list"]); + expect(result.exitCode).toBe(0); +}); + +test("the nightly matrix includes both new profiles", () => { + const result = Bun.spawnSync(["bash", script, "--list"]); + expect(result.exitCode).toBe(0); + const rows = result.stdout + .toString() + .trim() + .split("\n") + .filter((row) => row.startsWith("chromium-")); + expect(rows).toHaveLength(24); + for (const profile of ["bursty", "step"]) { + expect(rows.filter((row) => row.includes(`-${profile}-`))).toHaveLength(4); + } +}); diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts new file mode 100644 index 00000000000..6f65d35b60c --- /dev/null +++ b/test/audio-quality/clients/js/src/schema.ts @@ -0,0 +1,457 @@ +/** + * The metric contract: what the page samples, what a row reduces to, and what a budget grades. + * + * This file is the schema, not a description of one. The page emits {@link Sample}s, `analyze.ts` + * reduces them to a {@link Summary}, and `grade.ts` reads that against `budgets.json`. {@link METRICS} + * says for every graded number what it counts, which clock it sits on, and how a series became one + * value. Another lane (native playout, the latency ledger) that emits the same {@link Summary} is + * comparable to this one by construction. + * + * Conventions, stated once for everything below: + * + * - Every duration is milliseconds as a float. A sample count is converted at the rate it was + * counted at before it reaches a {@link Summary}. + * - Every count is a non-negative integer. Every share is a fraction of 1. + * - Every timestamp sits on a named {@link Clock} and is reduced to `viewer` before any two are + * subtracted. {@link Drift} records what each reduction took. + * + * @module + */ + +/** Milliseconds, as a float. The unit of every duration in this schema. */ +export type Ms = number; + +/** + * The clocks a timestamp can be taken on. + * + * - `viewer`: the page's `performance.now()`, monotonic. The reference every other clock is reduced + * to, so a stage span is always a difference of two `viewer` values. + * - `render`: the page's `AudioContext` frame clock, the audio device's oscillator. Same host, so it + * is reduced by an offset and a drift fitted over the run from paired readings (see + * {@link MAX_RENDER_DRIFT}). + * - `media`: the publisher's presentation timestamps, seen through the stream. Not a clock the viewer + * can read directly: its rate against `viewer` is fitted, its offset is unknowable from here. + * - `publisher`, `relay`: other processes, often other machines. A timestamp taken there is reduced + * to `viewer` by an offset measured over the same session, NTP style: the offset at the midpoint of + * a round trip, with half the round trip as its uncertainty. A stage whose uncertainty exceeds + * {@link identityTolerance} is reported unmeasured rather than guessed. The browser lane carries no + * such timestamp yet, so every stage that needs one is null here. + */ +export type Clock = "viewer" | "render" | "media" | "publisher" | "relay"; + +/** + * How a series became the one number a budget grades. + * + * - `total`: summed over the graded window. + * - `per_min`: `total` divided by the graded window's length in minutes. + * - `p50`, `p95`: nearest-rank percentile of the window's samples. + * - `max`: the largest value in the window. + * - `share`: the fraction of the window for which the condition held. + * - `last`: the value at the end of the window. + */ +export type Aggregation = "total" | "per_min" | "p50" | "p95" | "max" | "share" | "last"; + +/** What one metric is: enough to read a number off a summary without guessing. */ +export type MetricSpec = { + /** What the raw measurement is counted in, before aggregation. */ + unit: "ms" | "count" | "share"; + /** The clock its timestamps were taken on, before reduction to `viewer`. */ + clock: Clock; + /** The aggregations reported for it, and therefore the budget keys that can grade it. */ + aggregations: Aggregation[]; + /** One plain line saying what it measures. */ + description: string; +}; + +/** + * Every graded metric, keyed by its base name. + * + * A budget key is `_`: `underruns` graded per minute is `underruns_per_min`. Naming + * the aggregation in the key is what keeps "underruns: 3" and "underruns: 3/min" from being graded + * against each other. + */ +export const METRICS = { + underruns: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: + "Render quanta the ring could only partly fill, or not fill at all, while it was playing. A quantum rendered while the ring is stalled is not one.", + }, + short_quanta: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: "Of the underruns, the quanta the ring filled the front of and then ran dry part-way through.", + }, + underrun_episodes: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: "Maximal runs of consecutive underruns: one audible gap each, however many quanta it spanned.", + }, + underrun_ms: { + unit: "ms", + clock: "render", + aggregations: ["total", "per_min", "max"], + description: + "Audio the ring failed to supply while playing: the missing samples at the device rate. `max` is the longest episode.", + }, + skip_aheads: { + unit: "count", + clock: "viewer", + aggregations: ["total", "per_min"], + description: + "Re-anchors that discarded buffered audio: a forward step in the lag between the viewer clock and the playhead.", + }, + discarded_ms: { + unit: "ms", + clock: "media", + aggregations: ["total", "per_min"], + description: + "Media time the skip-aheads jumped over without playing. Audio dropped before it reached the ring moves no playhead and is not counted.", + }, + stalled: { + unit: "share", + clock: "viewer", + aggregations: ["share"], + description: + "Share of the window the ring spent stalled, refilling rather than playing: silence the player chose.", + }, + silence: { + unit: "share", + clock: "render", + aggregations: ["share"], + description: + "Share of rendered quanta under the silence floor at the player's output, whatever the cause. The published tone is never quiet, so any of it is a gap or a stall.", + }, + target_ms: { + unit: "ms", + clock: "viewer", + aggregations: ["p50", "p95", "max"], + description: "The playout delay the receiver resolved (`sync.out.delay`), sampled through the window.", + }, + converge_ms: { + unit: "ms", + clock: "viewer", + aggregations: ["last"], + description: + "From first audio to the last time the resolved target moved more than one bucket from its final value.", + }, + render_load: { + unit: "share", + clock: "render", + aggregations: ["max"], + description: + "Chromium's render capacity: the share of each quantum's budget the graph used. High says the runner, not the player, was short of time.", + }, +} as const satisfies Record; + +/** One of the {@link METRICS} names. */ +export type Metric = keyof typeof METRICS; + +/** Every `_` key, in schema order, so tables read the same every run. */ +export const METRIC_KEYS: string[] = Object.entries(METRICS).flatMap(([name, spec]) => + spec.aggregations.map((aggregation) => `${name}_${aggregation}`), +); + +/** A quantum whose RMS at the player's output is under this is quiet. About -60 dBFS. */ +export const SILENCE_RMS = 0.001; + +/** How often the page samples. */ +export const SAMPLE_INTERVAL_MS = 250; + +/** A resolved target within this of its final value has settled. One estimator bucket. */ +export const BUCKET_MS = 20; + +/** + * The render clock may run this far from the viewer's, as a fraction, before the row is void. + * + * Both oscillators are on one host, so real drift is parts per million. Past 1% the device is not + * running at wall rate at all, which is a throttled or stalled context, and every render-clock count + * would be read off a clock that was not moving. + */ +export const MAX_RENDER_DRIFT = 0.01; + +/** + * The stages an end-to-end delay splits into, in order, as exclusive spans. + * + * Exclusive is what makes the sum mean anything: each stage is defined by the two boundaries that + * end the previous stage and start the next, so no two can claim the same millisecond. + * + * - `capture`: sound at the publisher's input to its samples being readable. + * - `encode`: samples readable to the encoded frame existing. + * - `publish_flush`: frame existing to it being written to the transport, where a publisher that + * batches frames holds them. + * - `network`: written by the publisher to delivered to the viewer's container consumer, relay + * included. + * - `jitter_buffer`: delivered to its decoded samples leaving the ring for a render quantum. The + * decoder runs inside this span in a live pipeline, so it is not also `decode`. + * - `decode`: decode time that did not overlap the jitter buffer, which is zero while the ring holds + * anything. + * - `render`: leaving the ring to leaving the audio graph. + * - `device`: leaving the graph to the speaker (`outputLatency` plus `baseLatency`). + * - `unaccounted`: end-to-end minus the sum of the rest. Named because a large one is the finding. + * + * The identity is `sum(stages) == end-to-end`, within {@link identityTolerance}. + */ +export const STAGES = [ + "capture", + "encode", + "publish_flush", + "network", + "jitter_buffer", + "decode", + "render", + "device", + "unaccounted", +] as const; + +/** One of {@link STAGES}. */ +export type Stage = (typeof STAGES)[number]; + +/** A stage span and where its number came from. */ +export type StageSpan = { + /** The span, or null when this lane cannot measure it. */ + ms: Ms | null; + /** `measured` on the viewer clock, `declared` by the publisher, or `unmeasured`. */ + source: "measured" | "declared" | "unmeasured"; +}; + +/** The tolerance on the sum-to-end-to-end identity: the larger of 2 ms or 2% of the total. */ +export const identityTolerance = (endToEnd: Ms): Ms => Math.max(2, endToEnd * 0.02); + +/** What reducing one clock to `viewer` took. */ +export type Drift = { + /** Which clock. */ + clock: Clock; + /** Its rate against `viewer`, minus one: 0.001 runs 1 ms fast per second. Null when unmeasured. */ + rate: number | null; + /** What was added to put it on the viewer epoch. Null when unmeasured or unknowable. */ + offset: Ms | null; +}; + +// ── the row ───────────────────────────────────────────────────────────────── + +/** Which ring the page ran, decided by whether the document is cross-origin isolated. */ +export type Ring = "isolated" | "plain"; + +/** The audio codecs the matrix publishes. */ +export type Codec = "opus" | "aac"; + +/** + * Where a row played: a headless browser over the shaper, or a {@link Trace} replayed through the + * player's rings on a simulated clock, whose profile is the trace's name. + */ +export type Runtime = "chromium" | "replay"; + +/** + * One matrix cell. + * + * A budget is keyed by the whole thing. The codec, its rate, and the ring each move the expected + * floor as much as the profile does, so a profile-only key would grade one cell against another's + * threshold. + */ +export type Row = { + runtime: Runtime; + codec: Codec; + /** Sample rate in Hz. */ + rate: number; + profile: string; + ring: Ring; +}; + +/** The row as a flat string: a file name and a table label. */ +export const rowKey = (row: Row): string => `${row.runtime}-${row.codec}-${row.rate}-${row.profile}-${row.ring}`; + +/** Parse {@link rowKey}. The profile is the only field that may contain a dash. */ +export function parseRow(key: string): Row { + const parts = key.split("-"); + const [runtime, codec, rate] = parts; + const ring = parts.at(-1); + const profile = parts.slice(3, -1).join("-"); + const hz = Number.parseInt(rate ?? "", 10); + if ( + (runtime !== "chromium" && runtime !== "replay") || + (codec !== "opus" && codec !== "aac") || + (ring !== "isolated" && ring !== "plain") || + !Number.isFinite(hz) || + profile === "" + ) { + throw new Error(`not a row key: ${key}`); + } + return { runtime, codec, rate: hz, profile, ring }; +} + +// ── what the recorder keeps ───────────────────────────────────────────────── + +/** + * One audio frame reaching the viewer's container consumer: when, on the `viewer` clock; its + * timestamp, on the `media` clock; and the group that carried it. + */ +export type Arrival = [at: Ms, timestamp: Ms, group: number]; + +/** A recorded arrival trace, as checked in under `traces/`. The file name is its profile. */ +export type Trace = { + /** Bumped when a field's meaning changes. */ + version: 2; + /** Where it was recorded: the relay, the broadcast, its publisher, the date. */ + source: string; + /** The shape it carries, and why it is kept. */ + description: string; + /** The transport the session negotiated. */ + transport: string; + /** The smallest round trip the connection's PROBE reported, which "auto" sizes from. */ + rtt: Ms | null; + /** The audio rendition, as the catalog advertised it. */ + config: { + codec: string; + sampleRate: number; + numberOfChannels: number; + container: { kind: string }; + jitter?: number; + delay?: number; + [key: string]: unknown; + }; + /** How long the recording observed, from the first arrival: silence after the last one is an outage. */ + duration: Ms; + /** Every arrival in order, `at` counted from the first. */ + arrivals: Arrival[]; +}; + +/** The codec a trace's rendition is, as a row names it. */ +export function traceCodec(trace: Trace): Codec { + const codec = trace.config.codec; + if (codec === "opus") return "opus"; + if (codec.startsWith("mp4a")) return "aac"; + throw new Error(`no row codec for ${codec}`); +} + +// ── what the page emits ───────────────────────────────────────────────────── + +/** A run of consecutive quanta the ring did not fill, as the output tap saw it. */ +export type Gap = { + /** Where the first missing sample would have played, on the `render` clock. */ + at: Ms; + /** The missing audio, at the device rate. */ + ms: Ms; + /** Quanta in it the ring did not fill, short and silent alike. */ + quanta: number; + /** Of those, the quanta it filled the front of. */ + short: number; +}; + +/** A change in whether the ring is stalled, stamped on the `render` clock when the page saw it. */ +export type Stall = { + at: Ms; + stalled: boolean; +}; + +/** One probe sample: what the page could read at that instant, plus what the tap reported since. */ +export type Sample = { + /** `performance.now()`: the `viewer` clock. */ + at: Ms; + /** `AudioContext.currentTime`, read at the same instant: the `render` clock. */ + render?: Ms; + + /** `audio.out.timestamp`: the playhead on the `media` clock. Undefined while there is none. */ + timestamp?: Ms; + /** `audio.out.stalled`: the ring is refilling rather than playing. */ + stalled?: boolean; + /** `sync.out.delay`: the resolved playout target. */ + delay?: Ms; + + /** `AudioContext.outputLatency`. */ + outputLatency?: Ms; + /** `AudioContext.baseLatency`. */ + baseLatency?: Ms; + /** `AudioContext.renderCapacity` average load, Chromium only. */ + renderLoad?: number; + + /** The tap's cumulative quanta rendered since the first audible one. */ + quanta?: number; + /** The tap's cumulative quanta under {@link SILENCE_RMS}. */ + quiet?: number; + /** Gaps the tap closed since the previous sample. */ + gaps?: Gap[]; + /** Stall changes since the previous sample. */ + stalls?: Stall[]; +}; + +/** What the page reports once the session is up. */ +export type Environment = { + /** Whether the document is cross-origin isolated, and therefore which ring runs. */ + crossOriginIsolated: boolean; + /** The transport the session negotiated. Anything but WebTransport never crossed the UDP shaper. */ + transport?: string; + /** The audio rendition's codec string. */ + codec?: string; + /** The audio rendition's sample rate, Hz. */ + rate?: number; + /** The publisher's declared flush span: the `publish_flush` stage. */ + jitter?: Ms; + /** The AudioContext's rate, Hz, which the render clock counts in. */ + contextRate?: number; +}; + +// ── what a row reduces to ─────────────────────────────────────────────────── + +/** The shaper's counters for one direction, as its exit line prints them. */ +export type ShaperCounters = { + packets: number; + lost: number; + overflowed: number; + throttled: number; + delayed: number; + reordered: number; +}; + +/** The shaper's run: its seed, how it exited, and what it did. */ +export type Shaper = { + seed: number | null; + /** Its exit status. Nonzero means the profile never acted, or forwarding stopped. */ + status: number | null; + up: ShaperCounters | null; + down: ShaperCounters | null; +}; + +/** Why a row cannot be graded. A void row is reported, and fails an enforced run. */ +export type Void = { + /** The assertion that failed. */ + assertion: string; + /** What was seen instead. */ + detail: string; +}; + +/** Everything one row produced: the graded numbers and what makes them trustworthy. */ +export type Summary = { + /** Bumped when a metric's meaning changes, not when one is added. */ + version: 1; + row: Row; + /** Graded window, after the warmup. */ + windowMs: Ms; + /** Discarded after first audio, before grading. */ + warmupMs: Ms; + environment: Environment | null; + /** Empty means the row is gradeable. */ + voids: Void[]; + /** Every {@link METRIC_KEYS} entry. Null is unmeasured, never zero. */ + metrics: Record; + stages: Record; + /** The delay the stages sum to, or null when no clock spans the whole path. */ + endToEnd: Ms | null; + drift: Drift[]; + shaper: Shaper | null; + /** Console warnings and errors, truncated. */ + notes: string[]; +}; + +/** One budget: the whole row it applies to, and a ceiling per graded key. */ +export type Budget = Row & { [key: string]: number | string }; + +/** The checked-in budget file. */ +export type Budgets = { + /** How the ceilings were arrived at. */ + note: string; + rows: Budget[]; +}; diff --git a/test/audio-quality/clients/js/src/tap-worklet.ts b/test/audio-quality/clients/js/src/tap-worklet.ts new file mode 100644 index 00000000000..81e1a9d18b0 --- /dev/null +++ b/test/audio-quality/clients/js/src/tap-worklet.ts @@ -0,0 +1,46 @@ +/** + * The harness's own AudioWorklet, fanned out from the player's output: it listens and never plays. + * + * It sees every render quantum rather than a 250 ms sample of them, which is what lets a gap of a few + * milliseconds between two probe samples be counted at all. See `quantum.ts` for how a quantum is + * read. + * + * @module + */ +import { classify, Ledger } from "./quantum.ts"; +import type { TapInit, TapReport } from "./tap.ts"; + +/** Quanta between reports: about 170 ms at 48 kHz, inside the probe's 250 ms. */ +const REPORT_EVERY = 64; + +class Tap extends AudioWorkletProcessor { + #ledger: Ledger; + #since = 0; + + constructor(options: AudioWorkletNodeOptions) { + super(); + const init = options.processorOptions as TapInit; + this.#ledger = new Ledger(sampleRate, init.floor); + // The run is over: close a gap still open and report at once, rather than lose it. + this.port.onmessage = () => { + this.#ledger.finish(); + this.#report(true); + }; + } + + #report(finished = false): void { + this.#since = 0; + const report: TapReport = { counts: { ...this.#ledger.counts }, gaps: this.#ledger.take(), finished }; + this.port.postMessage(report); + } + + process(inputs: Float32Array[][]): boolean { + const input = inputs[0] ?? []; + this.#ledger.add(currentFrame, input[0]?.length ?? 128, classify(input)); + + if (++this.#since >= REPORT_EVERY) this.#report(); + return true; + } +} + +registerProcessor("audio-quality-tap", Tap); diff --git a/test/audio-quality/clients/js/src/tap.ts b/test/audio-quality/clients/js/src/tap.ts new file mode 100644 index 00000000000..ef0d3e5d316 --- /dev/null +++ b/test/audio-quality/clients/js/src/tap.ts @@ -0,0 +1,84 @@ +/** + * Fan the tap worklet out from a player's output node, and collect what it reports. + * + * The tap reaches the destination through a zero gain, because Web Audio only renders what the + * destination pulls: left dangling it would never be asked for a quantum. The zero gain keeps it out + * of what the listener hears. + * + * @module + */ +import type { Counts } from "./quantum.ts"; +import type { Gap } from "./schema.ts"; +import TapWorklet from "./tap-worklet.ts?worklet"; + +/** What the worklet is constructed with. */ +export type TapInit = { + /** The silence floor, as an RMS level. */ + floor: number; +}; + +/** What the worklet posts every few dozen quanta. */ +export type TapReport = { + /** Cumulative since the first audible quantum. */ + counts: Counts; + /** Gaps closed since the last report. */ + gaps: Gap[]; + /** Whether this answers a finish, so no gap is still open. */ + finished: boolean; +}; + +/** A running tap. */ +export type Tap = { + /** The latest cumulative counts, or undefined before the first report. */ + counts(): Counts | undefined; + /** Gaps closed since the last call. */ + take(): Gap[]; + /** Close a gap still open, and resolve once its report is in for {@link take}. */ + finish(): Promise; + close(): void; +}; + +/** Attach a tap to `root`. */ +export async function tap(root: AudioNode, init: TapInit): Promise { + const context = root.context as AudioContext; + await context.audioWorklet.addModule(TapWorklet); + + const node = new AudioWorkletNode(context, "audio-quality-tap", { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: init, + }); + const mute = new GainNode(context, { gain: 0 }); + root.connect(node); + node.connect(mute); + mute.connect(context.destination); + + let counts: Counts | undefined; + const gaps: Gap[] = []; + let reported: (() => void) | undefined; + node.port.onmessage = (event: MessageEvent) => { + counts = event.data.counts; + gaps.push(...event.data.gaps); + // A periodic report already in flight is not the answer, and may still miss the open gap. + if (event.data.finished) reported?.(); + }; + + return { + counts: () => counts, + take: () => gaps.splice(0, gaps.length), + finish() { + const done = new Promise((resolve) => { + reported = resolve; + }); + node.port.postMessage("finish"); + return done; + }, + close() { + node.port.onmessage = null; + root.disconnect(node); + node.disconnect(); + mute.disconnect(); + }, + }; +} diff --git a/test/audio-quality/clients/js/tsconfig.json b/test/audio-quality/clients/js/tsconfig.json new file mode 100644 index 00000000000..b61d3dce467 --- /dev/null +++ b/test/audio-quality/clients/js/tsconfig.json @@ -0,0 +1,16 @@ +{ + "extends": "../../../../js/tsconfig.json", + "compilerOptions": { + "types": ["audioworklet", "bun", "vite/client"] + }, + "include": [ + "analyze.ts", + "driver.ts", + "grade.ts", + "record.ts", + "replay.ts", + "src", + "vite.config.ts", + "../../../../js/common/worklet.d.ts" + ] +} diff --git a/test/audio-quality/clients/js/vite.config.ts b/test/audio-quality/clients/js/vite.config.ts new file mode 100644 index 00000000000..68040577ab7 --- /dev/null +++ b/test/audio-quality/clients/js/vite.config.ts @@ -0,0 +1,26 @@ +/** + * Vite config for the audio quality page. `@moq/watch` is consumed as workspace source, so its render + * worklet is inlined as a blob URL by the same plugin its own build uses, along with the harness's tap + * worklet: a worklet fetched over the network would be one more thing that can stall inside the + * measurement. + * + * `base: "./"` because one build is served under both `/isolated/` and `/plain/`. + * + * @module + */ +import { defineConfig } from "vite"; +import { workletInline } from "../../../../js/common/vite-plugin-worklet"; + +/** + * esnext keeps WebCodecs / WebTransport syntax intact for headless Chromium. Three pages: the player + * under measurement, and the recorder and microphone publisher that `record.ts` drives. + */ +export default defineConfig({ + base: "./", + plugins: [workletInline()], + build: { + target: "esnext", + outDir: "dist", + rollupOptions: { input: ["index.html", "recorder.html", "mic.html"] }, + }, +}); diff --git a/test/audio-quality/record.sh b/test/audio-quality/record.sh new file mode 100755 index 00000000000..22114369693 --- /dev/null +++ b/test/audio-quality/record.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Records the arrival traces under traces/, which the replay rows grade. See README.md. +# +# Three lanes, each --duration seconds from the first arrival: +# relay-bbb the public relay's Big Buck Bunny, the AAC flush-span shape #3477 measured +# local-aac an AAC tone through a local relay, the shallow control +# relay-mic a browser publishing Chromium's fake capture device through the public relay +# +# Not part of a run: re-record by hand when the path or the publishers change, and re-measure the +# replay budgets after. +set -euo pipefail + +AQ_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$AQ_DIR/../.." && pwd) +CLIENT="$AQ_DIR/clients/js" + +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$AQ_DIR/../lib/harness.sh" + +RERUN="just test audio-quality-record$(harness_argv "$@")" + +ALL_LANES=(relay-bbb local-aac relay-mic) +LANES=("${ALL_LANES[@]}") +DURATION=35 +PUBLIC="https://cdn.moq.dev" + +while [[ $# -gt 0 ]]; do + case "$1" in + --lanes) + IFS=, read -r -a LANES <<<"${2:?--lanes requires a value}" + shift 2 + ;; + --duration) + DURATION="${2:?--duration requires a value}" + shift 2 + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done +for lane in "${LANES[@]}"; do + [[ " ${ALL_LANES[*]} " == *" $lane "* ]] || { + echo "error: unknown lane '$lane' (known: ${ALL_LANES[*]})" >&2 + exit 2 + } +done + +harness_begin audio-quality-record "$RERUN" + +echo "building the pages..." +( + cd "$CLIENT" + bun install --frozen-lockfile + bunx playwright install chromium + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir +) + +DATE=$(date -u +%Y-%m-%d) +record() { + local lane="$1" + shift + echo "" + echo "── $lane ──" + bun "$CLIENT/record.ts" --page "$HARNESS_RUN/page" --duration "$DURATION" \ + --out "$AQ_DIR/traces/$lane.json" "$@" +} + +for lane in "${LANES[@]}"; do + case "$lane" in + relay-bbb) + record "$lane" --url "$PUBLIC/demo" --broadcast bbb.hang \ + --source "$PUBLIC/demo bbb.hang, the public demo's MPEG-TS import, recorded $DATE" \ + --description "The public relay's Big Buck Bunny: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured." + ;; + local-aac) + (cd "$WORKSPACE" && cargo build --locked -p moq-relay -p moq-cli) + target="${CARGO_TARGET_DIR:-$WORKSPACE/target}/debug" + harness_port relay + url="http://127.0.0.1:$HARNESS_PORT" + sed "s/4443/$HARNESS_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" + harness_spawn relay "$HARNESS_RUN/relay.log" "$target/moq-relay" "$HARNESS_RUN/relay.toml" + harness_ready "$url/certificate.sha256" 30 "$HARNESS_PID" || { + echo "relay never became ready" >&2 + exit 1 + } + # The harness's own AAC publisher, so the control is the shape its rows play. + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$target/moq" --connect "$url" --broadcast tone-aac.hang import ts + } + harness_spawn pub-aac "$HARNESS_RUN/pub-aac.log" publish + record "$lane" --url "$url" --broadcast tone-aac.hang \ + --source "a local moq-relay on 127.0.0.1, ffmpeg's AAC sine over MPEG-TS through moq import ts, recorded $DATE" \ + --description "The shallow control: the audio quality harness's AAC publisher with nothing but loopback between it and the viewer." + ;; + relay-mic) + broadcast="aq-mic-$RANDOM$RANDOM.hang" + record "$lane" --mic --url "$PUBLIC/anon" --broadcast "$broadcast" \ + --source "$PUBLIC/anon, in headless Chromium capturing its fake audio device, recorded $DATE" \ + --description "A browser publisher through the public relay: getUserMedia, the capture worklet, and WebCodecs Opus, with the relay's round trip in between." + ;; + esac +done diff --git a/test/audio-quality/relay.toml b/test/audio-quality/relay.toml new file mode 100644 index 00000000000..5b8ed37b1d4 --- /dev/null +++ b/test/audio-quality/relay.toml @@ -0,0 +1,22 @@ +# Relay config for the audio quality harness. +# Anonymous access, self-signed localhost cert, QUIC + HTTP on 127.0.0.1:4443. +# +# run.sh rewrites the port into a copy in the run directory. The page reaches QUIC through the +# shaper on another port; the driver answers the page's /certificate.sha256 fetch with this relay's +# hash, so the shaper only carries UDP. + +[log] +level = "info" + +[listen] +# QUIC on UDP. 127.0.0.1 avoids IPv6 flakiness on CI runners. +bind = "127.0.0.1:4443" +tls.generate = ["localhost", "127.0.0.1"] + +[web.http] +# HTTP on TCP, serving /certificate.sha256 to the publishers and the driver. +listen = "127.0.0.1:4443" + +[auth] +# Allow anonymous access to everything. +public = "**" diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh new file mode 100755 index 00000000000..7a71b3368a8 --- /dev/null +++ b/test/audio-quality/run.sh @@ -0,0 +1,357 @@ +#!/usr/bin/env bash +# Plays a tone in headless Chromium over a seeded, impaired UDP path, and grades what it heard. +# +# One row is one page playing one codec through one shaper profile on one ring, for --duration +# seconds of audio. See README.md for the profiles, the metric contract, and the void rules. +set -euo pipefail + +AQ_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$AQ_DIR/../.." && pwd) +CLIENT="$AQ_DIR/clients/js" + +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$AQ_DIR/../lib/harness.sh" + +# Captured before the parse consumes it, so the rerun command carries every flag. +RERUN="$(harness_env AQ_PROFILE RELAY_BIN)just test audio-quality$(harness_argv "$@")" + +ALL_PROFILES=(near-zero mild wide bursty step fixed-250) +ALL_RINGS=(isolated plain) +ALL_CODECS=(opus aac) + +# Every recorded trace is a profile too, replayed through the rings instead of played in a browser. +ALL_TRACES=() +for trace in "$AQ_DIR"/traces/*.json; do + ALL_TRACES+=("$(basename "$trace" .json)") +done + +PROFILES=("${ALL_PROFILES[@]}" "${ALL_TRACES[@]}") +RINGS=("${ALL_RINGS[@]}") +CODECS=("${ALL_CODECS[@]}") +DURATION=60 +SEED=7 +OUT="" +LIST=0 +ENFORCE=0 +PROFILE="${AQ_PROFILE:-debug}" + +split() { + local IFS=, + # shellcheck disable=SC2206 # splitting on commas is the point + SPLIT=($1) +} + +need() { + if [[ $2 -lt 2 || -z "${3:-}" || "${3:-}" == -* ]]; then + echo "error: $1 requires a value" >&2 + exit 2 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --profiles) + need "$1" $# "${2:-}" + split "$2" + PROFILES=("${SPLIT[@]}") + shift 2 + ;; + --replays) + # Only the recorded traces: seconds, and no browser, relay, or shaper. + PROFILES=("${ALL_TRACES[@]}") + shift + ;; + --rings) + need "$1" $# "${2:-}" + split "$2" + RINGS=("${SPLIT[@]}") + shift 2 + ;; + --codecs) + need "$1" $# "${2:-}" + split "$2" + CODECS=("${SPLIT[@]}") + shift 2 + ;; + --duration) + need "$1" $# "${2:-}" + DURATION="$2" + shift 2 + ;; + --seed) + need "$1" $# "${2:-}" + SEED="$2" + shift 2 + ;; + --out) + need "$1" $# "${2:-}" + OUT="$2" + shift 2 + ;; + --list) + LIST=1 + shift + ;; + --enforce) + ENFORCE=1 + shift + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +if [[ ! "$DURATION" =~ ^[0-9]+$ ]] || ((DURATION <= 5)); then + echo "error: --duration must be whole seconds, more than the 5s warmup (got '$DURATION')" >&2 + exit 2 +fi +if [[ ! "$SEED" =~ ^[0-9]+$ ]]; then + echo "error: --seed must be a non-negative integer (got '$SEED')" >&2 + exit 2 +fi + +valid() { + local want="$1" name="$2" + shift 2 + local known + for known in "$@"; do + [[ "$known" == "$want" ]] && return 0 + done + echo "error: unknown $name '$want' (known: $*)" >&2 + exit 2 +} +for p in "${PROFILES[@]}"; do + valid "$p" profile "${ALL_PROFILES[@]}" "${ALL_TRACES[@]}" + if [[ "$p" == step ]] && ((DURATION <= 30)); then + echo "error: --duration must exceed the step at 30s" >&2 + exit 2 + fi +done +for r in "${RINGS[@]}"; do valid "$r" ring "${ALL_RINGS[@]}"; done +for c in "${CODECS[@]}"; do valid "$c" codec "${ALL_CODECS[@]}"; done + +if [[ $LIST -eq 0 && -n "$OUT" && -d "$OUT" ]] && [[ -n "$(ls -A "$OUT" 2>/dev/null)" ]]; then + echo "error: --out $OUT is not empty; remove it or name a new directory" >&2 + exit 2 +fi + +# The rate is part of the row key: a 44.1 kHz stream's frames do not land on a 48 kHz quantum. +rate_of() { + case "$1" in + opus) echo 48000 ;; + aac) echo 44100 ;; + esac +} + +# Every profile but the control adapts. The element requires a unit on a fixed delay. +delay_of() { + case "$1" in + fixed-250) echo "250ms" ;; + *) echo "auto" ;; + esac +} + +# The path treatment, as moq-shaper flags. Loss, reorder, and rate limits stay off: the buffer's job +# is absorbing arrival spread, and congestion response would make a failure hard to attribute. +shaper_of() { + case "$1" in + near-zero | fixed-250) ;; + mild) echo "--delay 5ms --jitter 5ms --jitter-model gaussian --shared" ;; + wide) echo "--delay 40ms --jitter 40ms --jitter-model gaussian --shared" ;; + bursty | step) echo "--profile $1" ;; + esac +} + +ROWS=() +TRACES=() +for profile in "${PROFILES[@]}"; do + if [[ " ${ALL_TRACES[*]} " == *" $profile "* ]]; then + TRACES+=("$profile") + continue + fi + for codec in "${CODECS[@]}"; do + for ring in "${RINGS[@]}"; do + ROWS+=("chromium-$codec-$(rate_of "$codec")-$profile-$ring") + done + done +done + +# A trace's codec and rate are its recording's, so replay.ts names those rows. +joined() { + local IFS=, + echo "$*" +} +replay=(bun "$CLIENT/replay.ts" --traces "$AQ_DIR/traces" --profiles "$(joined "${TRACES[@]}")") +replay+=(--rings "$(joined "${RINGS[@]}")" --codecs "$(joined "${CODECS[@]}")") +listed=$("${replay[@]}" --list) +mapfile -t REPLAYS <<<"$listed" +[[ -n "$listed" ]] || REPLAYS=() + +if [[ $LIST -eq 1 ]]; then + printf '%s\n' "${ROWS[@]}" "${REPLAYS[@]}" + echo "${#ROWS[@]} rows at ${DURATION}s each, seed $SEED; ${#REPLAYS[@]} replays of their recorded trace" >&2 + exit 0 +fi + +tools=(bun) +[[ ${#ROWS[@]} -eq 0 ]] || tools+=(cargo ffmpeg) +for tool in "${tools[@]}"; do + command -v "$tool" >/dev/null 2>&1 || { + echo "error: $tool not found; run inside 'nix develop'" >&2 + exit 1 + } +done + +harness_begin audio-quality "$RERUN" + +echo "installing the workspace..." +(cd "$CLIENT" && bun install --frozen-lockfile) + +failed=0 + +# The browser rows need the relay, the publishers, the shaper, and a page; replays need none of it. +if [[ ${#ROWS[@]} -gt 0 ]]; then + # ── build ─────────────────────────────────────────────────────────────── + flag=() + [[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") + echo "building moq-relay, moq, moq-shaper ($PROFILE)..." + (cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli -p moq-shaper) + TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" + RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" + MOQ="$TARGET_BASE/$PROFILE/moq" + SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" + + # Into the run directory: vite empties its output first, so a shared dist/ would vanish under a + # concurrent run's page loads. + echo "building the page..." + ( + cd "$CLIENT" + bunx playwright install chromium + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir + ) + + # ── relay ─────────────────────────────────────────────────────────────── + harness_port relay + RELAY_PORT="$HARNESS_PORT" + RELAY_URL="http://127.0.0.1:$RELAY_PORT" + sed "s/4443/$RELAY_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" + harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" + if ! harness_ready "$RELAY_URL/certificate.sha256" 30 "$HARNESS_PID"; then + echo "relay never became ready" >&2 + sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true + exit 1 + fi + harness_endpoint relay "$RELAY_URL" + + # ── publishers ────────────────────────────────────────────────────────── + # A sine tone, never silent, so any quiet quantum at the player's output is a gap or a stall. + # The publishers talk to the relay directly: impairing the ingest would grade the receiver on a + # stream already damaged before it was published. The AAC arm keeps ffmpeg's default PES + # packing, whose multi-frame bursts are the flush-span shape the reporter measured on the public + # relay (#3477). + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish_opus() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -ac 2 -c:a libopus -b:a 128k \ + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-opus.hang" import fmp4 + } + + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish_aac() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-aac.hang" import ts + } + + for codec in "${CODECS[@]}"; do + harness_spawn "pub-$codec" "$HARNESS_RUN/pub-$codec.log" "publish_$codec" + done + + # ── rows ──────────────────────────────────────────────────────────────── + # One shaper port for the run: rows play one at a time. + harness_port shaper + SHAPER_PORT="$HARNESS_PORT" + + echo "" + echo "running ${#ROWS[@]} rows at ${DURATION}s each (seed $SEED)" + for tag in "${ROWS[@]}"; do + IFS=- read -r _ codec _ rest <<<"$tag" + ring="${rest##*-}" + profile="${rest%-*}" + echo "" + echo "── $tag ──" + + # shellcheck disable=SC2046 # the profile's flags are meant to split + harness_spawn "shaper-$tag" "$HARNESS_RUN/shaper-$tag.log" "$SHAPER" \ + --listen "127.0.0.1:$SHAPER_PORT" --target "127.0.0.1:$RELAY_PORT" --seed "$SEED" $(shaper_of "$profile") + shaper_pid="$HARNESS_PID" + # It prints its seed and profile once bound; a page dialing before that would lose its handshake. + until grep -q '^shaper:' "$HARNESS_RUN/shaper-$tag.log" 2>/dev/null; do + if harness_exited "$shaper_pid"; then + echo "shaper exited before binding for $tag" >&2 + sed 's/^/ shaper: /' "$HARNESS_RUN/shaper-$tag.log" >&2 || true + exit 1 + fi + sleep 0.05 + done + + status=0 + harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ + --url "http://127.0.0.1:$SHAPER_PORT" \ + --fingerprint "$RELAY_URL/certificate.sha256" \ + --broadcast "tone-$codec.hang" \ + --page "$HARNESS_RUN/page" \ + --ring "$ring" \ + --delay "$(delay_of "$profile")" \ + --duration "$DURATION" \ + --tag "$tag" \ + --out "$HARNESS_RUN" + harness_wait "$HARNESS_PID" || status=$? + if [[ $status -ne 0 ]]; then + failed=1 + sed 's/^/ driver: /' "$HARNESS_RUN/driver-$tag.log" >&2 || true + fi + + # SIGTERM, not the harness's SIGKILL: the shaper prints its counters and checks the profile + # acted on the way out, and exits nonzero if it did not. + kill -TERM -- -"$shaper_pid" 2>/dev/null || true + shaper_status=0 + harness_wait "$shaper_pid" || shaper_status=$? + echo "$shaper_status" >"$HARNESS_RUN/shaper-$tag.status" + + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } + done +fi + +# ── replays ───────────────────────────────────────────────────────────────── +# Simulated time, so the whole set takes seconds, and deterministic, so the budgets are exact. +if [[ ${#REPLAYS[@]} -gt 0 ]]; then + echo "" + echo "replaying ${#REPLAYS[@]} rows" + "${replay[@]}" --out "$HARNESS_RUN" || failed=1 + for tag in "${REPLAYS[@]}"; do + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } + done +fi + +# ── grade ─────────────────────────────────────────────────────────────────── +echo "" +grade=(bun "$CLIENT/grade.ts" --run "$HARNESS_RUN" --budgets "$AQ_DIR/budgets.json") +[[ $ENFORCE -eq 0 ]] || grade+=(--enforce) +"${grade[@]}" || failed=1 + +if [[ -n "$OUT" ]]; then + mkdir -p "$OUT" + cp -R "$HARNESS_RUN/." "$OUT/" + echo "saved: $OUT" +fi +exit "$failed" diff --git a/test/audio-quality/traces/local-aac.json b/test/audio-quality/traces/local-aac.json new file mode 100644 index 00000000000..13e0221589e --- /dev/null +++ b/test/audio-quality/traces/local-aac.json @@ -0,0 +1,1567 @@ +{ + "version": 2, + "source": "a local moq-relay on 127.0.0.1, ffmpeg's AAC sine over MPEG-TS through moq import ts, recorded 2026-09-29", + "description": "The shallow control: the audio quality harness's AAC publisher with nothing but loopback between it and the viewer.", + "transport": "webtransport", + "rtt": 1, + "config": { + "codec": "mp4a.40.2", + "container": { + "kind": "legacy" + }, + "description": "1210", + "sampleRate": 44100, + "numberOfChannels": 2, + "jitter": 163 + }, + "duration": 35000, + "arrivals": [ + [0,10016.133,0], + [2.55,10039.344,1], + [5.71,10062.555,2], + [5.98,10085.766,3], + [6.4,10108.977,4], + [6.61,10132.188,5], + [6.84,10155.4,6], + [7.09,10178.677,7], + [7.59,10201.888,8], + [7.73,10225.1,9], + [7.87,10248.311,10], + [8.02,10271.522,11], + [8.17,10294.733,12], + [8.34,10317.944,13], + [9.43,10341.211,14], + [9.59,10364.422,15], + [9.75,10387.633,16], + [10.38,10410.844,17], + [10.53,10434.055,18], + [10.69,10457.266,19], + [10.84,10480.477,20], + [10.96,10503.755,21], + [11.11,10526.966,22], + [11.26,10550.177,23], + [11.4,10573.388,24], + [11.55,10596.6,25], + [11.72,10619.811,26], + [11.9,10643.022,27], + [12.08,10666.288,28], + [12.26,10689.5,29], + [12.45,10712.711,30], + [12.66,10735.922,31], + [12.83,10759.133,32], + [13.02,10782.344,33], + [13.2,10805.555,34], + [14.59,10828.833,35], + [14.84,10852.044,36], + [15.16,10875.255,37], + [15.72,10968.1,41], + [15.88,10944.888,40], + [16.06,10921.677,39], + [16.2,10898.466,38], + [123.89,11014.588,43], + [124.13,10991.377,42], + [124.91,11130.644,48], + [125.08,11107.433,47], + [125.19,11084.222,46], + [125.34,11061.011,45], + [125.43,11037.8,44], + [287.9,11200.333,51], + [288.18,11177.122,50], + [288.3,11153.911,49], + [288.42,11293.177,55], + [288.64,11269.966,54], + [288.75,11246.755,53], + [288.89,11223.544,52], + [450.27,11432.511,61], + [450.48,11409.3,60], + [450.59,11386.088,59], + [450.71,11362.877,58], + [450.93,11339.666,57], + [451.07,11316.455,56], + [451.23,11455.722,62], + [613.15,11595.044,68], + [613.37,11571.833,67], + [613.47,11548.622,66], + [613.56,11525.411,65], + [613.67,11502.2,64], + [613.87,11478.988,63], + [613.97,11618.255,69], + [774.29,11757.588,75], + [774.5,11734.377,74], + [774.59,11711.166,73], + [774.69,11687.955,72], + [774.88,11664.744,71], + [774.97,11641.533,70], + [775.08,11780.8,76], + [937.57,11920.122,82], + [937.76,11896.911,81], + [937.87,11873.7,80], + [937.98,11850.488,79], + [938.17,11827.277,78], + [938.29,11804.066,77], + [938.4,11943.333,83], + [1100.28,12036.244,87], + [1100.49,12013.033,86], + [1100.59,11989.822,85], + [1100.7,11966.611,84], + [1100.87,12105.877,90], + [1100.98,12082.666,89], + [1101.08,12059.455,88], + [1262.9,12152.366,92], + [1263.13,12129.155,91], + [1263.3,12268.422,97], + [1263.43,12245.211,96], + [1263.64,12222,95], + [1263.74,12198.788,94], + [1263.88,12175.577,93], + [1426.26,12407.744,103], + [1426.47,12384.533,102], + [1426.59,12361.322,101], + [1426.73,12338.111,100], + [1426.95,12314.9,99], + [1427.06,12291.688,98], + [1427.2,12430.955,104], + [1587.95,12570.288,110], + [1588.2,12547.077,109], + [1588.31,12523.866,108], + [1588.43,12500.655,107], + [1588.64,12477.444,106], + [1588.73,12454.233,105], + [1588.84,12593.5,111], + [1751.58,12732.822,117], + [1751.77,12709.611,116], + [1751.88,12686.4,115], + [1751.99,12663.188,114], + [1752.17,12639.977,113], + [1752.3,12616.766,112], + [1752.43,12756.033,118], + [1916.67,12895.366,124], + [1917,12872.155,123], + [1917.16,12848.944,122], + [1917.41,12825.733,121], + [1917.55,12802.522,120], + [1917.67,12779.311,119], + [1917.81,12918.577,125], + [2077.07,13057.9,131], + [2077.3,13034.688,130], + [2077.46,13011.477,129], + [2077.61,12988.266,128], + [2077.91,12965.055,127], + [2078.04,12941.844,126], + [2078.14,13081.111,132], + [2238.87,13220.444,138], + [2239.24,13197.233,137], + [2239.38,13174.022,136], + [2239.53,13150.811,135], + [2239.66,13127.6,134], + [2239.93,13104.388,133], + [2240.06,13243.655,139], + [2400.81,13382.988,145], + [2401.13,13359.777,144], + [2401.28,13336.566,143], + [2401.42,13313.355,142], + [2401.56,13290.144,141], + [2401.9,13266.933,140], + [2402.04,13406.2,146], + [2564.49,13545.522,152], + [2564.8,13522.311,151], + [2564.92,13499.1,150], + [2565.06,13475.888,149], + [2565.16,13452.677,148], + [2565.24,13429.466,147], + [2565.32,13568.733,153], + [2726.35,13708.066,159], + [2726.66,13684.855,158], + [2726.82,13661.644,157], + [2727.09,13638.433,156], + [2727.2,13615.222,155], + [2727.32,13592.011,154], + [2727.44,13731.277,160], + [2888.53,13870.6,166], + [2888.76,13847.388,165], + [2888.87,13824.177,164], + [2889.11,13800.966,163], + [2889.22,13777.755,162], + [2889.43,13754.544,161], + [2889.61,13893.811,167], + [3051.61,14033.144,173], + [3051.87,14009.933,172], + [3052.02,13986.722,171], + [3052.18,13963.511,170], + [3052.33,13940.3,169], + [3052.68,13917.088,168], + [3052.82,14056.355,174], + [3213.91,14079.622,175], + [3214.18,14218.888,181], + [3215.8,14172.466,179], + [3216.03,14149.255,178], + [3216.22,14126.044,177], + [3216.36,14102.833,176], + [3216.53,14195.677,180], + [3375.85,14335.011,186], + [3376.06,14311.8,185], + [3376.16,14288.588,184], + [3376.38,14265.377,183], + [3376.49,14242.166,182], + [3376.59,14381.433,188], + [3376.72,14358.222,187], + [3538.87,14520.766,194], + [3539.21,14497.555,193], + [3539.38,14474.344,192], + [3539.52,14451.133,191], + [3539.74,14427.922,190], + [3539.85,14404.711,189], + [3539.98,14543.977,195], + [3711.34,14706.511,202], + [3711.56,14683.3,201], + [3711.88,14660.088,200], + [3712.04,14636.877,199], + [3712.79,14613.666,198], + [3713,14590.455,197], + [3713.21,14567.244,196], + [3868.88,14845.844,208], + [3869.2,14822.633,207], + [3869.32,14799.422,206], + [3869.43,14776.211,205], + [3869.61,14753,204], + [3869.75,14729.788,203], + [3869.95,14869.055,209], + [4026.26,15008.377,215], + [4026.49,14985.166,214], + [4026.61,14961.955,213], + [4026.74,14938.744,212], + [4026.94,14915.533,211], + [4027.06,14892.322,210], + [4027.19,15031.588,216], + [4188.73,15170.922,222], + [4188.97,15147.711,221], + [4189.08,15124.5,220], + [4189.25,15101.288,219], + [4189.35,15078.077,218], + [4189.48,15054.866,217], + [4189.62,15194.133,223], + [4351.39,15333.455,229], + [4351.65,15310.244,228], + [4351.78,15287.033,227], + [4351.9,15263.822,226], + [4352.15,15240.611,225], + [4352.27,15217.4,224], + [4352.42,15356.666,230], + [4514.23,15496,236], + [4514.45,15472.788,235], + [4514.59,15449.577,234], + [4514.71,15426.366,233], + [4514.9,15403.155,232], + [4515,15379.944,231], + [4515.14,15519.211,237], + [4686.54,15658.544,243], + [4686.87,15635.333,242], + [4688.84,15588.911,240], + [4689.05,15565.7,239], + [4689.37,15542.488,238], + [4689.54,15681.755,244], + [4689.68,15612.122,241], + [4839.79,15821.077,250], + [4840.04,15797.866,249], + [4840.2,15774.655,248], + [4840.39,15751.444,247], + [4840.51,15728.233,246], + [4840.62,15705.022,245], + [4840.78,15844.288,251], + [5001.26,15983.622,257], + [5001.71,15960.411,256], + [5002.02,15937.2,255], + [5002.21,15913.988,254], + [5002.41,15890.777,253], + [5002.74,15867.566,252], + [5002.88,16006.833,258], + [5165.08,16146.155,264], + [5165.3,16122.944,263], + [5165.42,16099.733,262], + [5165.52,16076.522,261], + [5165.75,16053.311,260], + [5165.85,16030.1,259], + [5166.02,16169.366,265], + [5333.68,16192.644,266], + [5333.97,16308.7,271], + [5334.26,16285.488,270], + [5334.39,16262.277,269], + [5334.52,16239.066,268], + [5334.66,16215.855,267], + [5334.84,16331.911,272], + [5493.92,16471.233,278], + [5494.18,16448.022,277], + [5494.28,16424.811,276], + [5494.38,16401.6,275], + [5494.57,16378.388,274], + [5494.67,16355.177,273], + [5494.78,16494.444,279], + [5657.09,16633.777,285], + [5657.29,16610.566,284], + [5657.39,16587.355,283], + [5657.47,16564.144,282], + [5657.56,16540.933,281], + [5657.77,16517.722,280], + [5657.87,16656.988,286], + [5813.98,16796.322,292], + [5814.21,16773.111,291], + [5814.34,16749.9,290], + [5814.44,16726.688,289], + [5814.59,16703.477,288], + [5814.69,16680.266,287], + [5814.92,16819.533,293], + [5977.89,16958.855,299], + [5978.19,16935.644,298], + [5978.3,16912.433,297], + [5978.47,16889.222,296], + [5978.57,16866.011,295], + [5978.66,16842.8,294], + [5978.87,16982.066,300], + [6139.22,17121.4,306], + [6139.41,17098.188,305], + [6139.52,17074.977,304], + [6139.68,17051.766,303], + [6139.77,17028.555,302], + [6139.87,17005.344,301], + [6139.99,17144.611,307], + [6311.7,17283.933,313], + [6312.03,17260.722,312], + [6312.19,17237.511,311], + [6312.34,17214.3,310], + [6312.49,17191.088,309], + [6312.84,17167.877,308], + [6312.99,17307.144,314], + [6464.69,17446.477,320], + [6464.96,17423.266,319], + [6465.11,17400.055,318], + [6465.21,17376.844,317], + [6465.29,17353.633,316], + [6465.38,17330.422,315], + [6465.6,17469.688,321], + [6627.69,17609.011,327], + [6628.22,17585.8,326], + [6628.41,17562.588,325], + [6628.55,17539.377,324], + [6628.95,17516.166,323], + [6629.06,17492.955,322], + [6629.37,17632.222,328], + [6788.63,17771.555,334], + [6788.84,17748.344,333], + [6788.94,17725.133,332], + [6789.06,17701.922,331], + [6789.26,17678.711,330], + [6789.34,17655.5,329], + [6789.45,17794.766,335], + [6952.27,17934.1,341], + [6952.49,17910.888,340], + [6952.62,17887.677,339], + [6952.79,17864.466,338], + [6952.89,17841.255,337], + [6953,17818.044,336], + [6953.21,17957.311,342], + [7114.68,18096.633,348], + [7114.92,18073.422,347], + [7115.09,18050.211,346], + [7115.21,18027,345], + [7115.46,18003.788,344], + [7115.57,17980.577,343], + [7115.69,18119.844,349], + [7277.22,18259.177,355], + [7277.54,18235.966,354], + [7277.66,18212.755,353], + [7277.77,18189.544,352], + [7277.89,18166.333,351], + [7278.14,18143.122,350], + [7278.24,18282.388,356], + [7438.99,18421.711,362], + [7439.21,18398.5,361], + [7439.32,18375.288,360], + [7439.45,18352.077,359], + [7439.68,18328.866,358], + [7439.79,18305.655,357], + [7439.89,18444.922,363], + [7601.84,18584.255,369], + [7602.06,18561.044,368], + [7602.18,18537.833,367], + [7602.29,18514.622,366], + [7602.56,18491.411,365], + [7602.71,18468.2,364], + [7602.88,18607.466,370], + [7764.61,18746.788,376], + [7764.84,18723.577,375], + [7764.95,18700.366,374], + [7765.13,18677.155,373], + [7765.24,18653.944,372], + [7765.34,18630.733,371], + [7765.49,18770,377], + [7926.66,18909.333,383], + [7926.85,18886.122,382], + [7926.96,18862.911,381], + [7927.13,18839.7,380], + [7927.21,18816.488,379], + [7927.31,18793.277,378], + [7927.41,18932.544,384], + [8089.77,19071.877,390], + [8090.01,19048.666,389], + [8090.12,19025.455,388], + [8090.34,19002.244,387], + [8090.45,18979.033,386], + [8090.59,18955.822,385], + [8090.78,19095.088,391], + [8251.55,19234.411,397], + [8251.75,19211.2,396], + [8251.92,19187.988,395], + [8252.09,19164.777,394], + [8252.38,19141.566,393], + [8252.48,19118.355,392], + [8252.59,19257.622,398], + [8414.65,19396.955,404], + [8414.87,19373.744,403], + [8414.99,19350.533,402], + [8415.09,19327.322,401], + [8415.28,19304.111,400], + [8415.36,19280.9,399], + [8415.47,19420.166,405], + [8577.91,19559.488,411], + [8578.23,19536.277,410], + [8578.38,19513.066,409], + [8578.75,19489.855,408], + [8578.89,19466.644,407], + [8579.06,19443.433,406], + [8579.18,19582.7,412], + [8739.86,19722.033,418], + [8740.09,19698.822,417], + [8740.21,19675.611,416], + [8740.31,19652.4,415], + [8740.57,19629.188,414], + [8740.67,19605.977,413], + [8740.8,19745.244,419], + [8904.17,19884.566,425], + [8904.5,19861.355,424], + [8904.68,19838.144,423], + [8905,19814.933,422], + [8905.15,19791.722,421], + [8905.31,19768.511,420], + [8905.49,19907.777,426], + [9064.8,20047.111,432], + [9065.11,20023.9,431], + [9065.23,20000.688,430], + [9065.39,19977.477,429], + [9065.62,19954.266,428], + [9065.72,19931.055,427], + [9065.83,20070.322,433], + [9228,20209.655,439], + [9228.27,20186.444,438], + [9228.38,20163.233,437], + [9228.48,20140.022,436], + [9228.59,20116.811,435], + [9228.87,20093.6,434], + [9229.3,20232.866,440], + [9390.18,20372.188,446], + [9390.56,20348.977,445], + [9390.71,20325.766,444], + [9390.84,20302.555,443], + [9390.99,20279.344,442], + [9391.12,20256.133,441], + [9391.31,20395.4,447], + [9570.51,20511.522,452], + [9570.64,20534.733,453], + [9570.9,20488.311,451], + [9571.07,20465.1,450], + [9571.23,20441.888,449], + [9571.41,20418.677,448], + [9572.07,20557.944,454], + [9728.41,20697.266,460], + [9733.17,20674.055,459], + [9733.53,20650.844,458], + [9733.84,20627.633,457], + [9736.38,20604.422,456], + [9736.55,20581.211,455], + [9736.67,20720.477,461], + [9877.29,20859.811,467], + [9877.49,20836.6,466], + [9877.58,20813.388,465], + [9877.76,20790.177,464], + [9877.84,20766.966,463], + [9877.94,20743.755,462], + [9878.05,20883.022,468], + [10039.92,21022.344,474], + [10040.41,20999.133,473], + [10040.62,20975.922,472], + [10040.83,20952.711,471], + [10040.94,20929.5,470], + [10041.05,20906.288,469], + [10041.14,21045.555,475], + [10202.67,21184.888,481], + [10202.87,21161.677,480], + [10203.01,21138.466,479], + [10203.15,21115.255,478], + [10203.37,21092.044,477], + [10203.48,21068.833,476], + [10203.59,21208.1,482], + [10364.37,21347.433,488], + [10364.59,21324.222,487], + [10364.69,21301.011,486], + [10364.84,21277.8,485], + [10364.92,21254.588,484], + [10365.06,21231.377,483], + [10365.15,21370.644,489], + [10536.68,21509.966,495], + [10536.93,21486.755,494], + [10537.08,21463.544,493], + [10537.21,21440.333,492], + [10537.47,21393.911,490], + [10537.66,21533.177,496], + [10537.77,21417.122,491], + [10718.08,21579.666,498], + [10718.25,21672.511,502], + [10720.29,21649.3,501], + [10720.46,21626.088,500], + [10720.62,21602.877,499], + [10720.88,21556.455,497], + [10721.45,21695.722,503], + [10852.5,21835.044,509], + [10852.69,21811.833,508], + [10852.81,21788.622,507], + [10852.96,21765.411,506], + [10853.17,21742.2,505], + [10853.52,21718.988,504], + [10853.64,21858.255,510], + [11014.78,21997.588,516], + [11015.06,21974.377,515], + [11015.16,21951.166,514], + [11015.26,21927.955,513], + [11015.44,21904.744,512], + [11015.53,21881.533,511], + [11015.63,22020.8,517], + [11180.89,22136.911,522], + [11181.07,22160.122,523], + [11181.74,22113.7,521], + [11182.13,22090.488,520], + [11182.3,22067.277,519], + [11182.46,22044.066,518], + [11182.58,22183.333,524], + [11341.58,22322.666,530], + [11341.82,22299.455,529], + [11341.94,22276.244,528], + [11342.07,22253.033,527], + [11342.34,22229.822,526], + [11342.45,22206.611,525], + [11342.57,22345.877,531], + [11504.42,22485.211,537], + [11504.62,22462,536], + [11504.7,22438.788,535], + [11504.85,22415.577,534], + [11504.94,22392.366,533], + [11505.06,22369.155,532], + [11505.24,22508.422,538], + [11665.44,22647.744,544], + [11665.6,22624.533,543], + [11665.68,22601.322,542], + [11665.83,22578.111,541], + [11665.91,22554.9,540], + [11666.02,22531.688,539], + [11666.23,22670.955,545], + [11828.72,22810.288,551], + [11829.11,22787.077,550], + [11829.23,22763.866,549], + [11829.3,22740.655,548], + [11829.71,22717.444,547], + [11829.81,22694.233,546], + [11829.91,22833.5,552], + [11990.61,22972.822,558], + [11990.8,22949.611,557], + [11990.89,22926.4,556], + [11991.04,22903.188,555], + [11991.11,22879.977,554], + [11991.2,22856.766,553], + [11991.28,22996.033,559], + [12155.63,23135.366,565], + [12155.83,23112.155,564], + [12155.94,23088.944,563], + [12156.07,23065.733,562], + [12156.3,23042.522,561], + [12156.39,23019.311,560], + [12156.55,23158.577,566], + [12317.01,23297.9,572], + [12317.17,23274.688,571], + [12317.28,23251.477,570], + [12317.41,23228.266,569], + [12317.46,23205.055,568], + [12317.53,23181.844,567], + [12317.64,23321.111,573], + [12477.48,23460.444,579], + [12477.66,23437.233,578], + [12477.78,23414.022,577], + [12477.87,23390.811,576], + [12477.99,23367.6,575], + [12478.07,23344.388,574], + [12478.2,23483.655,580], + [12641.84,23622.988,586], + [12642.02,23599.777,585], + [12642.11,23576.566,584], + [12642.24,23553.355,583], + [12642.31,23530.144,582], + [12642.39,23506.933,581], + [12642.57,23646.2,587], + [12803.44,23785.522,593], + [12803.62,23762.311,592], + [12803.71,23739.1,591], + [12803.8,23715.888,590], + [12803.95,23692.677,589], + [12804.04,23669.466,588], + [12804.16,23808.733,594], + [12968.53,23878.433,597], + [12968.66,23948.066,600], + [12968.89,23924.855,599], + [12969.03,23901.644,598], + [12969.2,23855.222,596], + [12969.39,23832.011,595], + [12969.71,23971.277,601], + [13128.76,24110.6,607], + [13128.92,24087.388,606], + [13129.01,24064.177,605], + [13129.19,24040.966,604], + [13129.28,24017.755,603], + [13129.36,23994.544,602], + [13129.53,24133.811,608], + [13291.69,24273.144,614], + [13291.9,24249.933,613], + [13292.02,24226.722,612], + [13292.14,24203.511,611], + [13292.34,24180.3,610], + [13292.46,24157.088,609], + [13292.57,24296.355,615], + [13456.24,24435.677,621], + [13456.44,24412.466,620], + [13456.56,24389.255,619], + [13456.67,24366.044,618], + [13456.78,24342.833,617], + [13456.87,24319.622,616], + [13456.99,24458.888,622], + [13622.09,24505.377,624], + [13622.38,24482.166,623], + [13623.46,24598.222,628], + [13624.14,24575.011,627], + [13625.59,24551.8,626], + [13625.77,24528.588,625], + [13627.41,24621.433,629], + [13870.43,24760.766,635], + [13873.91,24737.555,634], + [13876.49,24714.344,633], + [13883.19,24691.133,632], + [13889,24667.922,631], + [13891.52,24644.711,630], + [13892.02,24783.977,636], + [13992.79,24923.3,642], + [13993.03,24900.088,641], + [13993.18,24876.877,640], + [13993.26,24853.666,639], + [13993.39,24830.455,638], + [13993.73,24807.244,637], + [13994.1,24946.511,643], + [14174.97,25016.211,646], + [14175.28,25085.844,649], + [14175.79,25062.633,648], + [14175.91,25039.422,647], + [14176.04,24993,645], + [14176.17,24969.788,644], + [14176.27,25109.055,650], + [14286.82,25248.377,656], + [14292.13,25225.166,655], + [14294,25178.744,653], + [14294.68,25155.533,652], + [14295.73,25132.322,651], + [14296.09,25271.588,657], + [14296.46,25201.955,654], + [14472.09,25318.077,659], + [14472.22,25294.866,658], + [14472.45,25434.133,664], + [14472.66,25410.922,663], + [14472.85,25387.711,662], + [14472.94,25364.5,661], + [14473.1,25341.288,660], + [14674.91,25573.455,670], + [14675.71,25550.244,669], + [14676.58,25503.822,667], + [14680.92,25480.611,666], + [14681.54,25457.4,665], + [14687.88,25596.666,671], + [14688.29,25527.033,668], + [14776.93,25736,677], + [14777.53,25712.788,676], + [14777.66,25689.577,675], + [14777.8,25666.366,674], + [14777.91,25643.155,673], + [14778.21,25619.944,672], + [14778.88,25759.211,678], + [14942.41,25898.544,684], + [14942.61,25875.333,683], + [14942.82,25852.122,682], + [14943,25828.911,681], + [14943.36,25805.7,680], + [14943.5,25782.488,679], + [14944.51,25921.755,685], + [15084.12,26061.077,691], + [15084.39,26037.866,690], + [15085.33,26014.655,689], + [15086.9,25991.444,688], + [15087.67,25968.233,687], + [15088.27,25945.022,686], + [15089.31,26084.288,692], + [15247.24,26223.622,698], + [15247.94,26200.411,697], + [15248.07,26177.2,696], + [15248.27,26153.988,695], + [15248.36,26130.777,694], + [15248.45,26107.566,693], + [15248.55,26246.833,699], + [15411.58,26386.155,705], + [15412.06,26362.944,704], + [15412.23,26339.733,703], + [15412.52,26316.522,702], + [15412.7,26293.311,701], + [15412.89,26270.1,700], + [15413.07,26409.366,706], + [15567.84,26548.7,712], + [15568.09,26525.488,711], + [15568.2,26502.277,710], + [15568.37,26479.066,709], + [15568.47,26455.855,708], + [15568.61,26432.644,707], + [15568.72,26571.911,713], + [15735.29,26711.233,719], + [15735.6,26688.022,718], + [15735.74,26664.811,717], + [15735.88,26641.6,716], + [15736.18,26618.388,715], + [15736.31,26595.177,714], + [15736.46,26734.444,720], + [15895.29,26873.777,726], + [15895.63,26850.566,725], + [15895.79,26827.355,724], + [15895.94,26804.144,723], + [15896.26,26780.933,722], + [15896.38,26757.722,721], + [15896.51,26896.988,727], + [16059.43,27036.322,733], + [16059.7,27013.111,732], + [16059.83,26989.9,731], + [16059.97,26966.688,730], + [16060.11,26943.477,729], + [16060.26,26920.266,728], + [16060.4,27059.533,734], + [16219.19,27198.855,740], + [16219.51,27175.644,739], + [16219.69,27152.433,738], + [16219.97,27129.222,737], + [16220.06,27106.011,736], + [16220.15,27082.8,735], + [16220.24,27222.066,741], + [16382.51,27361.4,747], + [16382.77,27338.188,746], + [16382.9,27314.977,745], + [16383.09,27291.766,744], + [16383.17,27268.555,743], + [16383.25,27245.344,742], + [16383.33,27384.611,748], + [16545.33,27523.933,754], + [16545.69,27500.722,753], + [16545.83,27477.511,752], + [16546.09,27454.3,751], + [16546.22,27431.088,750], + [16546.35,27407.877,749], + [16546.46,27547.144,755], + [16707.81,27686.477,761], + [16708.14,27663.266,760], + [16708.28,27640.055,759], + [16708.45,27616.844,758], + [16708.53,27593.633,757], + [16708.59,27570.422,756], + [16708.71,27709.688,762], + [16868,27849.011,768], + [16868.21,27825.8,767], + [16868.3,27802.588,766], + [16868.5,27779.377,765], + [16868.61,27756.166,764], + [16868.73,27732.955,763], + [16868.87,27872.222,769], + [17029.84,28011.555,775], + [17030.07,27988.344,774], + [17030.18,27965.133,773], + [17030.28,27941.922,772], + [17030.5,27918.711,771], + [17030.61,27895.5,770], + [17030.73,28034.766,776], + [17200.69,28174.1,782], + [17200.89,28150.888,781], + [17200.97,28127.677,780], + [17201.05,28104.466,779], + [17201.2,28081.255,778], + [17201.28,28058.044,777], + [17201.36,28197.311,783], + [17356.81,28336.633,789], + [17357.26,28313.422,788], + [17357.44,28290.211,787], + [17357.6,28267,786], + [17357.78,28243.788,785], + [17358.17,28220.577,784], + [17358.29,28359.844,790], + [17519.52,28499.177,796], + [17519.78,28475.966,795], + [17519.89,28452.755,794], + [17520.02,28429.544,793], + [17520.41,28406.333,792], + [17520.69,28383.122,791], + [17520.87,28522.388,797], + [17684,28661.711,803], + [17684.66,28638.5,802], + [17685.2,28615.288,801], + [17685.51,28592.077,800], + [17685.77,28568.866,799], + [17686,28545.655,798], + [17686.4,28684.922,804], + [17842.95,28824.255,810], + [17843.22,28801.044,809], + [17843.34,28777.833,808], + [17843.52,28754.622,807], + [17843.62,28731.411,806], + [17843.7,28708.2,805], + [17843.86,28847.466,811], + [18012.8,28986.788,817], + [18012.99,28963.577,816], + [18013.08,28940.366,815], + [18013.21,28917.155,814], + [18013.28,28893.944,813], + [18013.35,28870.733,812], + [18013.58,29010,818], + [18167.89,29149.333,824], + [18168.08,29126.122,823], + [18168.16,29102.911,822], + [18168.27,29079.7,821], + [18168.34,29056.488,820], + [18168.41,29033.277,819], + [18168.55,29172.544,825], + [18331.13,29311.877,831], + [18331.34,29288.666,830], + [18331.46,29265.455,829], + [18331.57,29242.244,828], + [18331.64,29219.033,827], + [18331.8,29195.822,826], + [18331.88,29335.088,832], + [18497.56,29474.411,838], + [18497.9,29451.2,837], + [18498.08,29427.988,836], + [18498.38,29404.777,835], + [18498.5,29381.566,834], + [18498.63,29358.355,833], + [18498.85,29497.622,839], + [18655.23,29636.955,845], + [18655.43,29613.744,844], + [18655.53,29590.533,843], + [18655.62,29567.322,842], + [18655.79,29544.111,841], + [18655.88,29520.9,840], + [18655.99,29660.166,846], + [18822.2,29799.488,852], + [18822.63,29776.277,851], + [18822.8,29753.066,850], + [18823.05,29729.855,849], + [18823.24,29706.644,848], + [18823.83,29683.433,847], + [18824,29822.7,853], + [18988.77,29962.033,859], + [18989.25,29938.822,858], + [18989.48,29915.611,857], + [18989.64,29892.4,856], + [18989.97,29869.188,855], + [18990.09,29845.977,854], + [18990.24,29985.244,860], + [19152.68,30124.566,866], + [19152.82,30101.355,865], + [19152.89,30078.144,864], + [19152.96,30054.933,863], + [19153.05,30031.722,862], + [19153.33,30008.511,861], + [19153.46,30147.777,867], + [19334.36,30171.055,868], + [19337.72,30310.322,874], + [19352.35,30287.111,873], + [19352.91,30263.9,872], + [19353.25,30240.688,871], + [19357.88,30217.477,870], + [19364.11,30194.266,869], + [19469.63,30426.444,879], + [19469.95,30403.233,878], + [19470.11,30380.022,877], + [19470.37,30356.811,876], + [19470.47,30333.6,875], + [19470.69,30472.866,881], + [19470.87,30449.655,880], + [19631.34,30612.188,887], + [19631.51,30588.977,886], + [19631.59,30565.766,885], + [19631.71,30542.555,884], + [19631.79,30519.344,883], + [19631.84,30496.133,882], + [19631.93,30635.4,888], + [19792.62,30774.733,894], + [19792.8,30751.522,893], + [19792.9,30728.311,892], + [19792.99,30705.1,891], + [19793.06,30681.888,890], + [19793.22,30658.677,889], + [19793.3,30797.944,895], + [19954.65,30937.266,901], + [19954.84,30914.055,900], + [19954.93,30890.844,899], + [19955.09,30867.633,898], + [19955.17,30844.422,897], + [19955.26,30821.211,896], + [19955.44,30960.477,902], + [20118.29,31099.811,908], + [20118.48,31076.6,907], + [20118.57,31053.388,906], + [20118.74,31030.177,905], + [20118.81,31006.966,904], + [20118.89,30983.755,903], + [20119,31123.022,909], + [20280.22,31262.344,915], + [20280.41,31239.133,914], + [20280.5,31215.922,913], + [20280.66,31192.711,912], + [20280.74,31169.5,911], + [20280.81,31146.288,910], + [20280.91,31285.555,916], + [20442.9,31424.888,922], + [20443.1,31401.677,921], + [20443.2,31378.466,920], + [20443.33,31355.255,919], + [20443.41,31332.044,918], + [20443.48,31308.833,917], + [20443.57,31448.1,923], + [20605.07,31587.433,929], + [20605.24,31564.222,928], + [20605.32,31541.011,927], + [20605.45,31517.8,926], + [20605.52,31494.588,925], + [20605.59,31471.377,924], + [20605.69,31610.644,930], + [20767.74,31749.966,936], + [20767.92,31726.755,935], + [20768.01,31703.544,934], + [20768.16,31680.333,933], + [20768.24,31657.122,932], + [20768.32,31633.911,931], + [20768.41,31773.177,937], + [20930.12,31912.511,943], + [20930.3,31889.3,942], + [20930.38,31866.088,941], + [20930.53,31842.877,940], + [20930.6,31819.666,939], + [20930.68,31796.455,938], + [20930.77,31935.722,944], + [21093.23,32075.044,950], + [21093.4,32051.833,949], + [21093.49,32028.622,948], + [21093.64,32005.411,947], + [21093.72,31982.2,946], + [21093.8,31958.988,945], + [21093.95,32098.255,951], + [21254.99,32237.588,957], + [21255.18,32214.377,956], + [21255.27,32191.166,955], + [21255.41,32167.955,954], + [21255.5,32144.744,953], + [21255.58,32121.533,952], + [21255.67,32260.8,958], + [21417.68,32400.122,964], + [21417.87,32376.911,963], + [21418,32353.7,962], + [21418.09,32330.488,961], + [21418.23,32307.277,960], + [21418.31,32284.066,959], + [21418.41,32423.333,965], + [21580.61,32562.666,971], + [21580.8,32539.455,970], + [21580.94,32516.244,969], + [21581.04,32493.033,968], + [21581.13,32469.822,967], + [21581.31,32446.611,966], + [21581.39,32585.877,972], + [21743.16,32725.211,978], + [21743.34,32702,977], + [21743.43,32678.788,976], + [21743.59,32655.577,975], + [21743.67,32632.366,974], + [21743.74,32609.155,973], + [21743.84,32748.422,979], + [21905.34,32887.744,985], + [21905.53,32864.533,984], + [21905.63,32841.322,983], + [21905.77,32818.111,982], + [21905.87,32794.9,981], + [21905.98,32771.688,980], + [21906.08,32910.955,986], + [22067.64,33050.288,992], + [22067.83,33027.077,991], + [22067.91,33003.866,990], + [22068.01,32980.655,989], + [22068.18,32957.444,988], + [22068.3,32934.233,987], + [22068.39,33073.5,993], + [22231.3,33212.822,999], + [22231.52,33189.611,998], + [22231.76,33166.4,997], + [22231.98,33143.188,996], + [22232.07,33119.977,995], + [22232.17,33096.766,994], + [22232.28,33236.033,1000], + [22395.19,33375.366,1006], + [22395.45,33352.155,1005], + [22395.63,33328.944,1004], + [22395.81,33305.733,1003], + [22395.98,33282.522,1002], + [22396.12,33259.311,1001], + [22396.27,33398.577,1007], + [22556.88,33537.9,1013], + [22557.18,33514.688,1012], + [22557.32,33491.477,1011], + [22557.45,33468.266,1010], + [22557.73,33445.055,1009], + [22557.9,33421.844,1008], + [22558.04,33561.111,1014], + [22720.22,33700.444,1020], + [22720.47,33677.233,1019], + [22720.67,33654.022,1018], + [22720.77,33630.811,1017], + [22720.98,33607.6,1016], + [22721.08,33584.388,1015], + [22721.19,33723.655,1021], + [22882.47,33862.988,1027], + [22882.74,33839.777,1026], + [22882.95,33816.566,1025], + [22883.05,33793.355,1024], + [22883.15,33770.144,1023], + [22883.34,33746.933,1022], + [22883.45,33886.2,1028], + [23044.69,34025.522,1034], + [23044.92,34002.311,1033], + [23045.09,33979.1,1032], + [23045.19,33955.888,1031], + [23045.34,33932.677,1030], + [23045.43,33909.466,1029], + [23045.53,34048.733,1035], + [23207.07,34188.066,1041], + [23207.29,34164.855,1040], + [23207.4,34141.644,1039], + [23207.55,34118.433,1038], + [23207.79,34095.222,1037], + [23207.9,34072.011,1036], + [23208.06,34211.277,1042], + [23370,34350.6,1048], + [23370.25,34327.388,1047], + [23370.36,34304.177,1046], + [23370.46,34280.966,1045], + [23370.7,34257.755,1044], + [23370.79,34234.544,1043], + [23370.91,34373.811,1049], + [23532.84,34513.144,1055], + [23533.06,34489.933,1054], + [23533.17,34466.722,1053], + [23533.29,34443.511,1052], + [23533.51,34420.3,1051], + [23533.6,34397.088,1050], + [23533.71,34536.355,1056], + [23696.22,34675.677,1062], + [23696.48,34652.466,1061], + [23696.58,34629.255,1060], + [23696.73,34606.044,1059], + [23696.83,34582.833,1058], + [23696.91,34559.622,1057], + [23697.07,34698.888,1063], + [23858.15,34838.222,1069], + [23858.39,34815.011,1068], + [23858.64,34791.8,1067], + [23858.84,34768.588,1066], + [23858.97,34745.377,1065], + [23859.07,34722.166,1064], + [23859.18,34861.433,1070], + [24019.44,35000.766,1076], + [24019.67,34977.555,1075], + [24019.8,34954.344,1074], + [24019.87,34931.133,1073], + [24019.95,34907.922,1072], + [24020.12,34884.711,1071], + [24020.19,35023.977,1077], + [24181.11,35163.3,1083], + [24181.28,35140.088,1082], + [24181.35,35116.877,1081], + [24181.49,35093.666,1080], + [24181.56,35070.455,1079], + [24181.63,35047.244,1078], + [24181.71,35186.511,1084], + [24345.36,35325.844,1090], + [24345.66,35302.633,1089], + [24345.77,35279.422,1088], + [24345.95,35256.211,1087], + [24346.05,35233,1086], + [24346.14,35209.788,1085], + [24346.26,35349.055,1091], + [24509.21,35488.377,1097], + [24509.52,35465.166,1096], + [24509.64,35441.955,1095], + [24509.77,35418.744,1094], + [24510.03,35395.533,1093], + [24510.14,35372.322,1092], + [24510.27,35511.588,1098], + [24673.12,35650.922,1104], + [24673.44,35627.711,1103], + [24673.58,35604.5,1102], + [24673.71,35581.288,1101], + [24673.96,35558.077,1100], + [24674.08,35534.866,1099], + [24674.22,35674.133,1105], + [24833.74,35813.455,1111], + [24833.99,35790.244,1110], + [24834.12,35767.033,1109], + [24834.21,35743.822,1108], + [24834.37,35720.611,1107], + [24834.46,35697.4,1106], + [24834.61,35836.666,1112], + [24997.09,35976,1118], + [24997.33,35952.788,1117], + [24997.47,35929.577,1116], + [24997.56,35906.366,1115], + [24997.68,35883.155,1114], + [24997.86,35859.944,1113], + [24997.99,35999.211,1119], + [25159.71,36138.544,1125], + [25159.88,36115.333,1124], + [25160,36092.122,1123], + [25160.08,36068.911,1122], + [25160.2,36045.7,1121], + [25160.27,36022.488,1120], + [25160.4,36161.755,1126], + [25322.74,36301.077,1132], + [25322.97,36277.866,1131], + [25323.07,36254.655,1130], + [25323.18,36231.444,1129], + [25323.3,36208.233,1128], + [25323.42,36185.022,1127], + [25323.55,36324.288,1133], + [25486.23,36463.622,1139], + [25486.51,36440.411,1138], + [25486.61,36417.2,1137], + [25486.78,36393.988,1136], + [25486.88,36370.777,1135], + [25487,36347.566,1134], + [25487.21,36486.833,1140], + [25645.47,36626.155,1146], + [25645.64,36602.944,1145], + [25645.75,36579.733,1144], + [25645.88,36556.522,1143], + [25646.01,36533.311,1142], + [25646.13,36510.1,1141], + [25646.51,36649.366,1147], + [25817.49,36695.855,1149], + [25817.7,36672.644,1148], + [25818.43,36742.277,1151], + [25818.57,36811.911,1154], + [25818.83,36788.7,1153], + [25819.08,36765.488,1152], + [25819.39,36719.066,1150], + [25981.76,36974.444,1161], + [25982.05,36951.233,1160], + [25982.15,36904.811,1158], + [25982.3,36881.6,1157], + [25982.38,36858.388,1156], + [25982.46,36835.177,1155], + [25982.62,36928.022,1159], + [26132.89,37113.777,1167], + [26133.16,37090.566,1166], + [26133.27,37067.355,1165], + [26133.47,37044.144,1164], + [26133.58,37020.933,1163], + [26133.69,36997.722,1162], + [26133.81,37136.988,1168], + [26296.13,37276.322,1174], + [26296.4,37253.111,1173], + [26296.51,37229.9,1172], + [26296.62,37206.688,1171], + [26296.87,37183.477,1170], + [26296.99,37160.266,1169], + [26297.11,37299.533,1175], + [26457.47,37438.855,1181], + [26457.68,37415.644,1180], + [26457.78,37392.433,1179], + [26457.98,37369.222,1178], + [26458.08,37346.011,1177], + [26458.21,37322.8,1176], + [26458.33,37462.066,1182], + [26620.12,37601.4,1188], + [26620.36,37578.188,1187], + [26620.46,37554.977,1186], + [26620.62,37531.766,1185], + [26620.72,37508.555,1184], + [26620.82,37485.344,1183], + [26620.94,37624.611,1189], + [26783.36,37763.933,1195], + [26783.56,37740.722,1194], + [26783.66,37717.511,1193], + [26783.84,37694.3,1192], + [26783.94,37671.088,1191], + [26784.03,37647.877,1190], + [26784.21,37787.144,1196], + [26945.3,37926.477,1202], + [26945.52,37903.266,1201], + [26945.62,37880.055,1200], + [26945.81,37856.844,1199], + [26945.91,37833.633,1198], + [26946.07,37810.422,1197], + [26946.2,37949.688,1203], + [27109.08,38089.011,1209], + [27109.31,38065.8,1208], + [27109.42,38042.588,1207], + [27109.61,38019.377,1206], + [27109.72,37996.166,1205], + [27109.84,37972.955,1204], + [27109.98,38112.222,1210], + [27271.98,38251.555,1216], + [27272.19,38228.344,1215], + [27272.29,38205.133,1214], + [27272.42,38181.922,1213], + [27272.63,38158.711,1212], + [27272.74,38135.5,1211], + [27272.86,38274.766,1217], + [27435.38,38414.1,1223], + [27435.63,38390.888,1222], + [27435.74,38367.677,1221], + [27435.93,38344.466,1220], + [27436.03,38321.255,1219], + [27436.12,38298.044,1218], + [27436.39,38437.311,1224], + [27596.35,38576.633,1230], + [27596.62,38553.422,1229], + [27596.73,38530.211,1228], + [27596.93,38507,1227], + [27597.08,38483.788,1226], + [27597.31,38460.577,1225], + [27597.45,38599.844,1231], + [27757.81,38739.177,1237], + [27758.02,38715.966,1236], + [27758.12,38692.755,1235], + [27758.21,38669.544,1234], + [27758.39,38646.333,1233], + [27758.48,38623.122,1232], + [27758.59,38762.388,1238], + [27922.07,38901.711,1244], + [27922.25,38878.5,1243], + [27922.34,38855.288,1242], + [27922.42,38832.077,1241], + [27922.54,38808.866,1240], + [27922.63,38785.655,1239], + [27922.72,38924.922,1245], + [28084.02,39064.255,1251], + [28084.23,39041.044,1250], + [28084.3,39017.833,1249], + [28084.42,38994.622,1248], + [28084.49,38971.411,1247], + [28084.59,38948.2,1246], + [28084.84,39087.466,1252], + [28247.27,39226.788,1258], + [28247.5,39203.577,1257], + [28247.61,39180.366,1256], + [28247.8,39157.155,1255], + [28247.91,39133.944,1254], + [28248.03,39110.733,1253], + [28248.24,39250,1259], + [28409.79,39389.333,1265], + [28410.01,39366.122,1264], + [28410.16,39342.911,1263], + [28410.26,39319.7,1262], + [28410.4,39296.488,1261], + [28410.73,39273.277,1260], + [28410.86,39412.544,1266], + [28573.39,39551.877,1272], + [28573.67,39528.666,1271], + [28573.79,39505.455,1270], + [28573.94,39482.244,1269], + [28574.06,39459.033,1268], + [28574.15,39435.822,1267], + [28574.31,39575.088,1273], + [28734.46,39714.411,1279], + [28734.79,39691.2,1278], + [28734.94,39667.988,1277], + [28735.17,39644.777,1276], + [28735.27,39621.566,1275], + [28735.38,39598.355,1274], + [28735.52,39737.622,1280], + [28897.75,39876.955,1286], + [28898.07,39853.744,1285], + [28898.43,39830.533,1284], + [28898.65,39807.322,1283], + [28898.75,39784.111,1282], + [28898.86,39760.9,1281], + [28898.99,39900.166,1287], + [29060.62,40039.488,1293], + [29060.87,40016.277,1292], + [29061.02,39993.066,1291], + [29061.29,39969.855,1290], + [29061.41,39946.644,1289], + [29061.57,39923.433,1288], + [29061.84,40062.7,1294], + [29221.44,40202.033,1300], + [29221.7,40178.822,1299], + [29221.82,40155.611,1298], + [29221.94,40132.4,1297], + [29222.23,40109.188,1296], + [29222.37,40085.977,1295], + [29222.51,40225.244,1301], + [29383.83,40364.566,1307], + [29385.58,40318.144,1305], + [29385.79,40294.933,1304], + [29385.92,40271.722,1303], + [29386.06,40248.511,1302], + [29386.19,40387.777,1308], + [29386.32,40341.355,1306], + [29546.27,40527.111,1314], + [29546.56,40503.9,1313], + [29547.11,40480.688,1312], + [29547.25,40457.477,1311], + [29547.36,40434.266,1310], + [29547.47,40411.055,1309], + [29547.58,40550.322,1315], + [29712.46,40689.655,1321], + [29712.74,40666.444,1320], + [29712.88,40643.233,1319], + [29713.11,40620.022,1318], + [29713.22,40596.811,1317], + [29713.32,40573.6,1316], + [29713.46,40712.866,1322], + [29872.6,40852.188,1328], + [29872.84,40828.977,1327], + [29872.94,40805.766,1326], + [29873.14,40782.555,1325], + [29873.22,40759.344,1324], + [29873.3,40736.133,1323], + [29873.45,40875.4,1329], + [30037.37,41014.733,1335], + [30037.59,40991.522,1334], + [30037.72,40968.311,1333], + [30037.91,40945.1,1332], + [30038.05,40921.888,1331], + [30038.17,40898.677,1330], + [30038.29,41037.944,1336], + [30209.54,41177.266,1342], + [30211.82,41154.055,1341], + [30212.55,41130.844,1340], + [30212.65,41107.633,1339], + [30217.43,41061.211,1337], + [30217.86,41200.477,1343], + [30218.12,41084.422,1338], + [30367.11,41339.811,1349], + [30367.3,41316.6,1348], + [30367.41,41293.388,1347], + [30367.54,41270.177,1346], + [30367.74,41246.966,1345], + [30367.82,41223.755,1344], + [30367.91,41363.022,1350], + [30524.24,41502.344,1356], + [30524.54,41479.133,1355], + [30524.66,41455.922,1354], + [30524.8,41432.711,1353], + [30525.17,41409.5,1352], + [30525.33,41386.288,1351], + [30525.48,41525.555,1357], + [30695.69,41664.888,1363], + [30696.02,41641.677,1362], + [30696.24,41618.466,1361], + [30696.45,41595.255,1360], + [30696.62,41572.044,1359], + [30696.88,41548.833,1358], + [30700.84,41688.1,1364], + [30850.04,41827.433,1370], + [30851.28,41804.222,1369], + [30851.44,41781.011,1368], + [30851.76,41757.8,1367], + [30851.93,41734.588,1366], + [30852.12,41711.377,1365], + [30852.3,41850.644,1371], + [31012.69,41989.966,1377], + [31013.13,42013.177,1378], + [31013.35,41966.755,1376], + [31013.53,41943.544,1375], + [31013.7,41920.333,1374], + [31013.87,41897.122,1373], + [31014.05,41873.911,1372], + [31176.13,42152.511,1384], + [31176.5,42129.3,1383], + [31176.63,42106.088,1382], + [31176.81,42082.877,1381], + [31176.94,42059.666,1380], + [31177.1,42036.455,1379], + [31177.27,42175.722,1385], + [31338.66,42315.044,1391], + [31339.11,42291.833,1390], + [31339.29,42268.622,1389], + [31339.54,42245.411,1388], + [31339.72,42222.2,1387], + [31339.91,42198.988,1386], + [31340.18,42338.255,1392], + [31497.75,42477.588,1398], + [31498,42454.377,1397], + [31498.13,42431.166,1396], + [31498.34,42407.955,1395], + [31498.45,42384.744,1394], + [31498.58,42361.533,1393], + [31498.7,42500.8,1399], + [31665.88,42640.122,1405], + [31666.22,42616.911,1404], + [31666.42,42593.7,1403], + [31666.61,42570.488,1402], + [31666.78,42547.277,1401], + [31666.99,42524.066,1400], + [31667.24,42663.333,1406], + [31826.52,42733.033,1409], + [31826.67,42802.666,1412], + [31826.91,42779.455,1411], + [31827.05,42756.244,1410], + [31827.19,42709.822,1408], + [31827.29,42686.611,1407], + [31827.42,42825.877,1413], + [31989.84,42965.211,1419], + [31991.38,42942,1418], + [31991.5,42918.788,1417], + [31991.62,42895.577,1416], + [31991.71,42872.366,1415], + [31992.28,42849.155,1414], + [31992.43,42988.422,1420], + [32146.34,43127.744,1426], + [32146.57,43104.533,1425], + [32146.67,43081.322,1424], + [32146.78,43058.111,1423], + [32146.91,43034.9,1422], + [32147.16,43011.688,1421], + [32147.26,43150.955,1427], + [32309.4,43290.288,1433], + [32309.62,43174.233,1428], + [32309.74,43313.5,1434], + [32309.84,43267.077,1432], + [32309.95,43243.866,1431], + [32310.09,43220.655,1430], + [32310.22,43197.444,1429], + [32472.45,43452.822,1440], + [32472.7,43429.611,1439], + [32472.81,43406.4,1438], + [32472.99,43383.188,1437], + [32473.14,43359.977,1436], + [32473.24,43336.766,1435], + [32473.36,43476.033,1441], + [32635.04,43615.366,1447], + [32635.3,43592.155,1446], + [32635.5,43568.944,1445], + [32635.63,43545.733,1444], + [32635.9,43522.522,1443], + [32636.04,43499.311,1442], + [32636.19,43638.577,1448], + [32798.25,43777.9,1454], + [32798.51,43754.688,1453], + [32798.62,43731.477,1452], + [32798.74,43708.266,1451], + [32798.98,43685.055,1450], + [32799.09,43661.844,1449], + [32799.22,43801.111,1455], + [32962.33,43940.444,1461], + [32963.18,43917.233,1460], + [32963.36,43894.022,1459], + [32963.49,43870.811,1458], + [32963.61,43847.6,1457], + [32963.74,43824.388,1456], + [32963.87,43963.655,1462], + [33125.55,43986.933,1463], + [33125.63,44102.988,1468], + [33125.81,44079.777,1467], + [33125.88,44056.566,1466], + [33125.97,44033.355,1465], + [33126.06,44010.144,1464], + [33126.15,44126.2,1469], + [33287.84,44265.522,1475], + [33288.12,44242.311,1474], + [33288.22,44219.1,1473], + [33288.32,44195.888,1472], + [33288.51,44172.677,1471], + [33288.6,44149.466,1470], + [33288.75,44288.733,1476], + [33453.51,44428.066,1482], + [33454.39,44404.855,1481], + [33454.56,44381.644,1480], + [33454.68,44358.433,1479], + [33454.88,44335.222,1478], + [33454.98,44312.011,1477], + [33455.11,44451.277,1483], + [33613.18,44590.6,1489], + [33614.42,44567.388,1488], + [33614.71,44544.177,1487], + [33614.88,44520.966,1486], + [33615.02,44497.755,1485], + [33615.18,44474.544,1484], + [33615.34,44613.811,1490], + [33775.06,44776.355,1497], + [33775.2,44753.144,1496], + [33775.44,44729.933,1495], + [33775.57,44706.722,1494], + [33775.69,44683.511,1493], + [33775.8,44660.3,1492], + [33775.94,44637.088,1491], + [33938.08,44915.677,1503], + [33938.3,44892.466,1502], + [33938.39,44869.255,1501], + [33938.55,44846.044,1500], + [33938.64,44822.833,1499], + [33938.79,44799.622,1498], + [33938.94,44938.888,1504], + [34098.71,45078.222,1510], + [34098.93,45055.011,1509], + [34099.04,45031.8,1508], + [34099.24,45008.588,1507], + [34099.34,44985.377,1506], + [34099.45,44962.166,1505], + [34099.57,45101.433,1511], + [34260.46,45240.766,1517], + [34260.69,45217.555,1516], + [34260.81,45194.344,1515], + [34260.93,45171.133,1514], + [34261.16,45147.922,1513], + [34261.26,45124.711,1512], + [34261.39,45263.977,1518], + [34425.51,45287.244,1519], + [34425.7,45403.3,1524], + [34426.04,45380.088,1523], + [34426.2,45356.877,1522], + [34426.35,45333.666,1521], + [34426.5,45310.455,1520], + [34426.71,45426.511,1525], + [34589.27,45565.844,1531], + [34589.57,45542.633,1530], + [34591.72,45519.422,1529], + [34591.97,45496.211,1528], + [34592.09,45473,1527], + [34592.22,45449.788,1526], + [34592.38,45589.055,1532], + [34747.99,45728.377,1538], + [34748.24,45705.166,1537], + [34748.34,45681.955,1536], + [34748.51,45658.744,1535], + [34748.6,45635.533,1534], + [34748.7,45612.322,1533], + [34748.89,45751.588,1539], + [34910.41,45890.922,1545], + [34910.63,45867.711,1544], + [34910.74,45844.5,1543], + [34910.93,45821.288,1542], + [34911.05,45798.077,1541], + [34911.16,45774.866,1540], + [34911.31,45914.133,1546] + ] +} diff --git a/test/audio-quality/traces/relay-bbb.json b/test/audio-quality/traces/relay-bbb.json new file mode 100644 index 00000000000..fd605e36afe --- /dev/null +++ b/test/audio-quality/traces/relay-bbb.json @@ -0,0 +1,1955 @@ +{ + "version": 2, + "source": "https://cdn.moq.dev/demo bbb.hang, the public demo's MPEG-TS import, recorded 2026-09-29", + "description": "The public relay's Big Buck Bunny: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured.", + "transport": "webtransport", + "rtt": null, + "config": { + "codec": "mp4a.40.2", + "container": { + "kind": "legacy" + }, + "description": "1210", + "sampleRate": 44100, + "numberOfChannels": 2, + "bitrate": 139613, + "jitter": 186 + }, + "duration": 35000, + "arrivals": [ + [0,234535,9642], + [2.98,240014.922,9878], + [9.78,240038.133,9879], + [42.28,240061.344,9880], + [42.65,234558.211,9643], + [42.87,240084.555,9881], + [43.17,240107.766,9882], + [43.41,240131.044,9883], + [43.69,240154.255,9884], + [43.97,240177.466,9885], + [44.19,240200.677,9886], + [44.4,234581.422,9644], + [44.65,240223.888,9887], + [44.85,240247.1,9888], + [45.08,240270.311,9889], + [46.07,240293.588,9890], + [46.25,240316.8,9891], + [46.46,240340.011,9892], + [46.65,240363.222,9893], + [47.12,240386.433,9894], + [47.32,234604.633,9645], + [47.78,240409.644,9895], + [47.99,240432.855,9896], + [48.33,234627.911,9646], + [48.59,240456.122,9897], + [48.81,240479.333,9898], + [49,240502.544,9899], + [49.2,234651.122,9647], + [49.41,240525.755,9900], + [49.6,240548.966,9901], + [49.81,234674.333,9648], + [50.03,240572.177,9902], + [50.27,240595.388,9903], + [50.55,234697.544,9649], + [51.48,240618.666,9904], + [51.71,235394.133,9679], + [51.88,234720.755,9650], + [52.03,234883.3,9657], + [52.18,234743.966,9651], + [52.38,234767.177,9652], + [52.59,234326.033,9633], + [52.85,234790.455,9653], + [53.07,234349.244,9634], + [53.31,234372.455,9635], + [53.54,234813.666,9654], + [53.88,234395.666,9636], + [54.16,234418.877,9637], + [57.75,234442.155,9638], + [58.49,234465.366,9639], + [58.78,238900.355,9830], + [58.93,234860.088,9656], + [59.06,234488.577,9640], + [59.42,234511.788,9641], + [59.59,234836.877,9655], + [59.74,238969.988,9833], + [59.91,238993.266,9834], + [60.05,239039.688,9836], + [60.19,239062.9,9837], + [60.34,239086.111,9838], + [60.47,239109.322,9839], + [60.63,239132.533,9840], + [60.83,239155.811,9841], + [60.97,239179.022,9842], + [61.11,239202.233,9843], + [61.25,239225.444,9844], + [61.38,239248.655,9845], + [61.52,239271.866,9846], + [61.66,239504.1,9856], + [61.81,239527.311,9857], + [61.97,239550.522,9858], + [62.11,239573.733,9859], + [62.34,239596.944,9860], + [62.52,239620.155,9861], + [62.71,239295.077,9847], + [63.2,239318.344,9848], + [63.37,239341.555,9849], + [63.53,239643.422,9862], + [63.68,239666.633,9863], + [63.9,239689.844,9864], + [64.05,239364.766,9850], + [64.75,239387.977,9851], + [64.98,239411.188,9852], + [65.18,239713.055,9865], + [65.37,239736.266,9866], + [65.57,239759.477,9867], + [65.76,239782.688,9868], + [66.32,239805.966,9869], + [66.56,239829.177,9870], + [66.77,239434.4,9853], + [67,239457.611,9854], + [67.21,239480.888,9855], + [67.43,239852.388,9871], + [67.65,239875.6,9872], + [67.88,239898.811,9873], + [68.09,239922.022,9874], + [68.34,239945.233,9875], + [68.76,239968.5,9876], + [68.99,239991.711,9877], + [69.18,238761.033,9824], + [69.35,238714.611,9822], + [69.53,238737.822,9823], + [69.72,238784.244,9825], + [69.92,238807.455,9826], + [70.13,238830.722,9827], + [70.5,238853.933,9828], + [70.71,238877.144,9829], + [77.44,238923.566,9831], + [77.63,238946.777,9832], + [77.8,239016.477,9835], + [77.98,238644.911,9819], + [78.14,238668.188,9820], + [78.3,238691.4,9821], + [78.45,238598.488,9817], + [78.6,238621.7,9818], + [78.76,237971.544,9790], + [78.94,237994.755,9791], + [79.12,238018.033,9792], + [79.31,238041.244,9793], + [79.52,238064.455,9794], + [79.71,238087.666,9795], + [79.88,238110.877,9796], + [80.04,238134.088,9797], + [80.19,238157.3,9798], + [80.34,238180.566,9799], + [80.52,238203.777,9800], + [80.69,238226.988,9801], + [80.88,238250.2,9802], + [81.05,238273.411,9803], + [81.25,238296.622,9804], + [81.41,238319.833,9805], + [81.63,238343.111,9806], + [81.81,238366.322,9807], + [81.98,238389.533,9808], + [82.22,238412.744,9809], + [82.38,238435.955,9810], + [82.54,238459.166,9811], + [82.77,238482.377,9812], + [82.99,238505.644,9813], + [83.16,238528.855,9814], + [83.37,238552.066,9815], + [83.56,238575.277,9816], + [83.72,237483.933,9769], + [83.96,237182.066,9756], + [84.13,237205.277,9757], + [84.63,237228.555,9758], + [84.8,237251.766,9759], + [85.95,237274.977,9760], + [86.24,237298.188,9761], + [86.44,237321.4,9762], + [86.63,237344.611,9763], + [86.9,237367.822,9764], + [87.11,237391.088,9765], + [87.3,237414.3,9766], + [87.47,236044.288,9707], + [87.69,236067.5,9708], + [87.88,237437.511,9767], + [88.07,237507.144,9770], + [88.27,237530.355,9771], + [88.46,237553.633,9772], + [88.63,237576.844,9773], + [88.85,236090.777,9709], + [89.02,236113.988,9710], + [89.19,236137.2,9711], + [89.37,236160.411,9712], + [89.55,237600.055,9774], + [89.72,236183.622,9713], + [89.9,236206.833,9714], + [90.1,236230.044,9715], + [90.5,236276.522,9717], + [90.72,236299.733,9718], + [91.03,236346.155,9720], + [91.19,237623.266,9775], + [91.42,236392.577,9722], + [91.57,236415.855,9723], + [91.8,236439.066,9724], + [91.95,236462.277,9725], + [92.09,236485.488,9726], + [92.32,237692.944,9778], + [92.5,236508.7,9727], + [92.82,235231.588,9672], + [93.02,235812.111,9697], + [93.29,236555.122,9729], + [93.46,236578.388,9730], + [93.78,236624.811,9732], + [94,236648.022,9733], + [94.2,236671.233,9734], + [94.49,237739.366,9780], + [94.66,237762.577,9781], + [95.03,236694.444,9735], + [95.28,237809,9783], + [95.5,236717.655,9736], + [95.75,237832.211,9784], + [95.97,236740.933,9737], + [96.32,236787.355,9739], + [96.57,236810.566,9740], + [96.97,236856.988,9742], + [97.19,237855.488,9785], + [97.4,236880.2,9743], + [97.57,236903.466,9744], + [97.82,236926.677,9745], + [98.03,236949.888,9746], + [98.27,236973.1,9747], + [98.5,236996.311,9748], + [98.82,237019.522,9749], + [99.15,237042.733,9750], + [99.31,237066.011,9751], + [99.5,237089.222,9752], + [99.68,237112.433,9753], + [99.86,237135.644,9754], + [100.03,237158.855,9755], + [100.2,237878.7,9786], + [100.39,237901.911,9787], + [102.17,237925.122,9788], + [102.47,237948.333,9789], + [102.69,237460.722,9768], + [102.9,235254.8,9673], + [103.09,235278.077,9674], + [103.92,235301.288,9675], + [104.1,235324.5,9676], + [104.32,235347.711,9677], + [104.5,235370.922,9678], + [104.65,235161.955,9669], + [104.8,235185.166,9670], + [104.97,235208.377,9671], + [105.12,235858.533,9699], + [105.29,235881.744,9700], + [105.46,235904.955,9701], + [105.61,235928.233,9702], + [105.79,235951.444,9703], + [105.97,235974.655,9704], + [106.14,235997.866,9705], + [106.34,236021.077,9706], + [106.56,234906.511,9658], + [106.76,234929.722,9659], + [106.95,234953,9660], + [107.12,234976.211,9661], + [107.34,234999.422,9662], + [107.52,235022.633,9663], + [107.71,235045.844,9664], + [107.93,235069.055,9665], + [108.14,235092.266,9666], + [108.32,235115.533,9667], + [108.54,235138.744,9668], + [109.17,235417.344,9680], + [109.4,235440.611,9681], + [109.59,235463.822,9682], + [109.8,240897.255,9916], + [110.02,240920.466,9917], + [110.24,240943.744,9918], + [110.45,240966.955,9919], + [110.68,240990.166,9920], + [110.9,241013.377,9921], + [111.12,241036.588,9922], + [111.34,241059.8,9923], + [111.58,241083.011,9924], + [111.85,241106.277,9925], + [112.1,241129.488,9926], + [112.36,241152.7,9927], + [115.86,235487.033,9683], + [116.5,235510.244,9684], + [116.87,235533.455,9685], + [117.19,235556.666,9686], + [117.55,235579.877,9687], + [117.84,235603.155,9688], + [118.08,235626.366,9689], + [118.32,235649.577,9690], + [118.58,235672.788,9691], + [118.82,235696,9692], + [119.11,235719.211,9693], + [119.44,235742.422,9694], + [119.7,235765.688,9695], + [120.07,235788.9,9696], + [120.42,241175.911,9928], + [120.69,243474.677,10027], + [121.3,242824.522,9999], + [121.55,243242.511,10017], + [121.81,242847.733,10000], + [122.07,242801.311,9998], + [122.36,242870.944,10001], + [122.63,243265.722,10018], + [122.96,243219.3,10016], + [123.26,243288.933,10019], + [123.53,242894.222,10002], + [123.88,243312.144,10020], + [124.18,243335.355,10021], + [124.49,242917.433,10003], + [125.22,243358.566,10022], + [125.51,242522.655,9986], + [125.78,242940.644,10004], + [126.09,242545.866,9987], + [126.36,242569.144,9988], + [126.65,242592.355,9989], + [127.34,242615.566,9990], + [128.12,242638.777,9991], + [128.82,242661.988,9992], + [129.38,243381.833,10023], + [129.97,242685.2,9993], + [131.38,242708.411,9994], + [132.19,242731.677,9995], + [132.6,242754.888,9996], + [132.97,242778.1,9997], + [133.36,242499.444,9985], + [133.72,242476.233,9984], + [134.14,242453.022,9983], + [134.47,242429.811,9982], + [134.77,242406.6,9981], + [135.22,242963.855,10005], + [135.52,242987.066,10006], + [136.13,243405.044,10024], + [136.43,243428.255,10025], + [136.72,243010.277,10007], + [137.05,243033.488,10008], + [137.41,243451.466,10026], + [137.75,243079.966,10010], + [138.07,243103.177,10011], + [138.36,243497.888,10028], + [138.66,243521.1,10029], + [139,243126.388,10012], + [139.59,243149.6,10013], + [139.93,243544.377,10030], + [140.28,243172.811,10014], + [140.62,240641.877,9905], + [140.94,240665.088,9906], + [141.25,240688.3,9907], + [141.59,240711.511,9908], + [141.92,240734.722,9909], + [142.25,240757.933,9910], + [142.38,237785.788,9782], + [143.09,237716.155,9779], + [143.4,237669.688,9777], + [143.68,237646.477,9776], + [143.97,236833.777,9741], + [144.26,236764.144,9738], + [144.55,236601.6,9731], + [144.83,236531.911,9728], + [145.11,236369.366,9721], + [145.84,236322.944,9719], + [146.35,236253.311,9716], + [146.74,235835.322,9698], + [147.43,240781.2,9911], + [147.84,240804.411,9912], + [148.72,240827.622,9913], + [149.34,240850.833,9914], + [150.47,243056.755,10009], + [151.16,242127.944,9969], + [151.94,242151.155,9970], + [152.75,242174.366,9971], + [153.65,242197.577,9972], + [154.65,242220.788,9973], + [155.68,242244.055,9974], + [156.55,242267.266,9975], + [157.66,242290.477,9976], + [158.09,242313.688,9977], + [158.54,242336.9,9978], + [159.02,242360.111,9979], + [159.44,242383.322,9980], + [159.85,241918.977,9960], + [160.5,241942.188,9961], + [161.22,241965.4,9962], + [161.93,241988.611,9963], + [162.73,242011.822,9964], + [163.45,242035.033,9965], + [164.22,242058.244,9966], + [164.95,242081.522,9967], + [165.68,242104.733,9968], + [166.67,241779.655,9954], + [167.23,241802.866,9955], + [167.68,241826.077,9956], + [168.12,241849.288,9957], + [168.65,241872.5,9958], + [169.09,241895.711,9959], + [169.49,244171.266,10057], + [169.9,241570.633,9945], + [170.39,241593.9,9946], + [171.01,241617.111,9947], + [171.49,241640.322,9948], + [171.91,241663.533,9949], + [172.32,241408.088,9938], + [172.88,243776.555,10040], + [173.72,241431.366,9939], + [174.18,241454.577,9940], + [174.6,243799.766,10041], + [175.03,243822.977,10042], + [175.45,241477.788,9941], + [175.89,243846.188,10043], + [176.34,243869.455,10044], + [176.76,241501,9942], + [178.37,243892.666,10045], + [178.8,243915.877,10046], + [179.25,241524.211,9943], + [179.69,243939.088,10047], + [180.12,243962.3,10048], + [181.03,243985.511,10049], + [181.56,244008.722,10050], + [182,244032,10051], + [182.44,244055.211,10052], + [182.93,244078.422,10053], + [183.38,244101.633,10054], + [183.84,241547.422,9944], + [184.41,244124.844,10055], + [184.87,243753.344,10039], + [185.34,241384.877,9937], + [185.82,243730.133,10038], + [186.28,243706.922,10037], + [187.33,241361.666,9936], + [228.22,241338.455,9935], + [228.43,243683.644,10036], + [228.61,243660.433,10035], + [228.84,241315.244,9934], + [229.06,241292.033,9933], + [229.27,243637.222,10034], + [229.5,243614.011,10033], + [229.69,241268.822,9932], + [229.88,244148.055,10056], + [230.08,241686.744,9950], + [230.25,241709.955,9951], + [230.42,241733.166,9952], + [230.59,241756.444,9953], + [230.77,244287.377,10062], + [230.94,244310.588,10063], + [231.12,244333.8,10064], + [231.28,243196.022,10015], + [231.78,243567.588,10031], + [231.98,241199.122,9929], + [232.15,241222.333,9930], + [232.32,243590.8,10032], + [232.5,241245.544,9931], + [232.69,244194.533,10058], + [232.9,244217.744,10059], + [233.09,244240.955,10060], + [233.3,244264.166,10061], + [233.5,244449.922,10069], + [233.69,244473.133,10070], + [233.88,244496.344,10071], + [234.07,244357.077,10065], + [234.27,244380.288,10066], + [234.47,244403.5,10067], + [234.7,244426.711,10068], + [235.54,240874.044,9915], + [309.69,244542.822,10073], + [309.93,244566.033,10074], + [310.03,244589.244,10075], + [310.13,244612.455,10076], + [310.31,244635.666,10077], + [310.41,244658.877,10078], + [310.59,244519.611,10072], + [502.34,244705.366,10080], + [502.54,244728.577,10081], + [502.63,244751.788,10082], + [502.79,244775,10083], + [502.88,244798.211,10084], + [502.97,244821.422,10085], + [503.07,244682.155,10079], + [638.06,244867.911,10087], + [638.23,244891.122,10088], + [638.32,244960.755,10091], + [638.47,244914.333,10089], + [638.58,244937.544,10090], + [638.66,244844.7,10086], + [638.77,244983.966,10092], + [818.46,245030.444,10094], + [818.66,245053.655,10095], + [818.75,245076.866,10096], + [818.87,245100.077,10097], + [819.04,245123.288,10098], + [819.12,245146.5,10099], + [819.22,245007.233,10093], + [957.28,245192.988,10101], + [957.49,245216.2,10102], + [957.57,245239.411,10103], + [957.75,245262.622,10104], + [957.86,245285.833,10105], + [958.05,245309.044,10106], + [958.16,245169.777,10100], + [1107.13,245355.522,10108], + [1107.32,245378.733,10109], + [1107.44,245401.944,10110], + [1107.6,245425.155,10111], + [1107.71,245448.366,10112], + [1107.84,245471.577,10113], + [1108.03,245332.311,10107], + [1261.86,245518.066,10115], + [1262.11,245541.277,10116], + [1262.25,245564.488,10117], + [1262.41,245587.7,10118], + [1262.5,245610.911,10119], + [1262.6,245634.122,10120], + [1262.76,245494.855,10114], + [1424.35,245750.233,10125], + [1424.53,245773.444,10126], + [1424.63,245796.655,10127], + [1424.72,245657.388,10121], + [1424.91,245680.6,10122], + [1424.99,245703.811,10123], + [1425.09,245727.022,10124], + [1588.68,245843.144,10129], + [1588.91,245866.355,10130], + [1589.03,245889.566,10131], + [1589.19,245912.777,10132], + [1589.28,245935.988,10133], + [1589.37,245959.2,10134], + [1589.47,245819.933,10128], + [1753.91,246005.688,10136], + [1754.11,246028.9,10137], + [1754.21,246052.111,10138], + [1754.37,246075.322,10139], + [1754.46,246098.533,10140], + [1754.57,246121.744,10141], + [1754.71,245982.477,10135], + [1921.03,246168.222,10143], + [1921.34,246191.433,10144], + [1921.44,246214.644,10145], + [1921.66,246237.855,10146], + [1921.78,246261.066,10147], + [1921.9,246284.277,10148], + [1922.1,246145.011,10142], + [2087.59,246330.766,10150], + [2087.85,246353.977,10151], + [2087.97,246377.188,10152], + [2088.12,246400.4,10153], + [2088.21,246423.611,10154], + [2088.3,246446.822,10155], + [2088.38,246307.555,10149], + [2255.17,246493.3,10157], + [2255.32,246516.511,10158], + [2255.41,246586.144,10161], + [2255.54,246539.722,10159], + [2255.63,246562.933,10160], + [2255.71,246470.088,10156], + [2255.9,246609.355,10162], + [2424.94,246655.844,10164], + [2425.25,246679.055,10165], + [2425.35,246702.266,10166], + [2425.51,246725.477,10167], + [2425.59,246748.688,10168], + [2425.66,246771.9,10169], + [2425.73,246632.633,10163], + [2575.38,246818.377,10171], + [2575.52,246841.588,10172], + [2575.59,246864.8,10173], + [2575.72,246888.011,10174], + [2575.78,246911.222,10175], + [2575.88,246934.433,10176], + [2575.95,246795.166,10170], + [2749.89,246957.711,10177], + [2750.14,246980.922,10178], + [2750.25,247004.133,10179], + [2750.44,247027.344,10180], + [2750.56,247050.555,10181], + [2750.66,247073.766,10182], + [2750.78,247096.977,10183], + [2900.56,247120.255,10184], + [2900.73,247143.466,10185], + [2900.84,247166.677,10186], + [2900.99,247189.888,10187], + [2901.07,247213.1,10188], + [2901.15,247236.311,10189], + [2901.31,247259.522,10190], + [3071.44,247282.788,10191], + [3071.62,247306,10192], + [3071.7,247329.211,10193], + [3071.87,247352.422,10194], + [3071.95,247375.633,10195], + [3072.03,247398.844,10196], + [3072.17,247422.055,10197], + [3212.59,247468.544,10199], + [3212.79,247445.333,10198], + [3212.9,247514.966,10201], + [3213.05,247491.755,10200], + [3213.15,247538.177,10202], + [3213.24,247561.388,10203], + [3213.38,247584.6,10204], + [3378.91,247631.077,10206], + [3379.09,247654.288,10207], + [3379.18,247607.866,10205], + [3379.3,247677.5,10208], + [3379.4,247723.922,10210], + [3379.48,247747.133,10211], + [3379.6,247700.711,10209], + [3543.59,247770.411,10212], + [3544.84,247886.466,10217], + [3544.94,247909.677,10218], + [3545.05,247793.622,10213], + [3545.15,247816.833,10214], + [3545.24,247840.044,10215], + [3545.31,247863.255,10216], + [3716.75,248072.211,10225], + [3717.96,248049,10224], + [3718.09,247932.944,10219], + [3718.38,248025.788,10223], + [3718.49,248002.577,10222], + [3718.6,247979.366,10221], + [3718.68,247956.155,10220], + [3878.51,248118.7,10227], + [3878.66,248141.911,10228], + [3878.72,248165.122,10229], + [3878.87,248188.333,10230], + [3878.94,248211.544,10231], + [3879,248234.755,10232], + [3879.09,248095.488,10226], + [4024.78,248258.033,10233], + [4024.93,248397.3,10239], + [4025.03,248281.244,10234], + [4025.12,248304.455,10235], + [4025.22,248374.088,10238], + [4025.29,248327.666,10236], + [4025.37,248350.877,10237], + [4186.69,248420.566,10240], + [4186.87,248443.777,10241], + [4186.95,248466.988,10242], + [4187.18,248490.2,10243], + [4187.28,248513.411,10244], + [4187.41,248536.622,10245], + [4187.53,248559.833,10246], + [4347.91,248583.111,10247], + [4348.73,248606.322,10248], + [4348.86,248629.533,10249], + [4348.99,248652.744,10250], + [4349.12,248675.955,10251], + [4349.21,248699.166,10252], + [4349.28,248722.377,10253], + [4532.82,248768.855,10255], + [4533,248792.066,10256], + [4533.07,248815.277,10257], + [4533.21,248838.488,10258], + [4533.27,248861.7,10259], + [4533.4,248884.911,10260], + [4533.52,248745.644,10254], + [4692.91,248908.188,10261], + [4693.07,249047.455,10267], + [4693.18,248931.4,10262], + [4693.27,248954.611,10263], + [4693.34,249024.244,10266], + [4693.5,248977.822,10264], + [4693.58,249001.033,10265], + [4866.71,249070.722,10268], + [4866.88,249093.933,10269], + [4866.96,249117.144,10270], + [4867.09,249140.355,10271], + [4867.16,249163.566,10272], + [4867.24,249186.777,10273], + [4867.31,249209.988,10274], + [5018.38,249326.111,10279], + [5018.5,249349.322,10280], + [5018.59,249372.533,10281], + [5018.65,249233.266,10275], + [5018.75,249302.9,10278], + [5018.82,249256.477,10276], + [5018.9,249279.688,10277], + [5184.16,249395.811,10282], + [5184.35,249419.022,10283], + [5184.44,249442.233,10284], + [5184.53,249465.444,10285], + [5184.59,249488.655,10286], + [5184.72,249511.866,10287], + [5184.79,249535.077,10288], + [5338.05,249558.344,10289], + [5338.18,249581.555,10290], + [5338.29,249604.766,10291], + [5338.36,249627.977,10292], + [5338.5,249651.188,10293], + [5338.57,249674.4,10294], + [5338.65,249697.611,10295], + [5503.97,249720.888,10296], + [5504.18,249744.1,10297], + [5504.28,249767.311,10298], + [5504.46,249790.522,10299], + [5504.56,249813.733,10300], + [5504.65,249836.944,10301], + [5504.84,249860.155,10302], + [5650.21,249883.422,10303], + [5650.35,249906.633,10304], + [5650.45,249929.844,10305], + [5650.53,249953.055,10306], + [5650.65,249976.266,10307], + [5650.72,249999.477,10308], + [5650.8,250022.688,10309], + [5812.98,250045.966,10310], + [5813.17,250069.177,10311], + [5813.27,250092.388,10312], + [5813.4,250115.6,10313], + [5813.47,250138.811,10314], + [5813.55,250162.022,10315], + [5813.69,250185.233,10316], + [5974.33,250208.5,10317], + [5974.48,250347.766,10323], + [5974.59,250324.555,10322], + [5974.67,250301.344,10321], + [5974.8,250254.922,10319], + [5974.93,250231.711,10318], + [5975.03,250278.133,10320], + [6137.69,250371.044,10324], + [6137.88,250394.255,10325], + [6137.97,250417.466,10326], + [6138.11,250440.677,10327], + [6138.19,250510.311,10330], + [6138.28,250487.1,10329], + [6138.43,250463.888,10328], + [6304.4,250533.588,10331], + [6304.56,250556.8,10332], + [6304.64,250580.011,10333], + [6304.77,250603.222,10334], + [6304.9,250626.433,10335], + [6304.99,250649.644,10336], + [6305.13,250672.855,10337], + [6462.38,250696.122,10338], + [6463.72,250719.333,10339], + [6463.87,250742.544,10340], + [6463.96,250765.755,10341], + [6464.09,250788.966,10342], + [6464.17,250812.177,10343], + [6464.27,250835.388,10344], + [6629,250997.933,10351], + [6629.24,250881.877,10346], + [6629.38,250905.088,10347], + [6629.53,250974.722,10350], + [6629.8,250928.3,10348], + [6629.94,250951.511,10349], + [6630.08,250858.666,10345], + [6795.28,251021.2,10352], + [6795.44,251044.411,10353], + [6795.52,251067.622,10354], + [6795.66,251090.833,10355], + [6795.75,251114.044,10356], + [6795.85,251137.255,10357], + [6795.94,251160.466,10358], + [6962.14,251253.377,10362], + [6962.27,251276.588,10363], + [6962.33,251299.8,10364], + [6962.44,251323.011,10365], + [6962.5,251206.955,10360], + [6962.56,251230.166,10361], + [6962.68,251183.744,10359], + [7128.44,251346.277,10366], + [7128.59,251485.544,10372], + [7128.66,251462.333,10371], + [7128.79,251439.122,10370], + [7128.87,251369.488,10367], + [7128.95,251392.7,10368], + [7129.03,251415.911,10369], + [7294.6,251508.822,10373], + [7294.77,251532.033,10374], + [7294.87,251601.666,10377], + [7295,251555.244,10375], + [7295.12,251578.455,10376], + [7295.2,251624.877,10378], + [7295.28,251648.088,10379], + [7437.72,251694.577,10381], + [7437.94,251717.788,10382], + [7438.03,251741,10383], + [7438.17,251764.211,10384], + [7438.24,251787.422,10385], + [7438.32,251810.633,10386], + [7438.39,251671.366,10380], + [7602.92,251833.9,10387], + [7603.62,251973.166,10393], + [7604.08,251857.111,10388], + [7604.31,251880.322,10389], + [7604.6,251949.955,10392], + [7604.86,251903.533,10390], + [7605.18,251926.744,10391], + [7771.25,251996.444,10394], + [7771.68,252019.655,10395], + [7771.87,252042.866,10396], + [7772.03,252066.077,10397], + [7772.31,252089.288,10398], + [7772.43,252112.5,10399], + [7772.57,252135.711,10400], + [7933.8,252158.977,10401], + [7933.97,252182.188,10402], + [7934.03,252205.4,10403], + [7934.16,252228.611,10404], + [7934.22,252251.822,10405], + [7934.28,252275.033,10406], + [7934.36,252298.244,10407], + [8092.4,252344.733,10409], + [8092.54,252367.944,10410], + [8092.61,252391.155,10411], + [8092.75,252414.366,10412], + [8092.82,252437.577,10413], + [8093.09,252460.788,10414], + [8093.21,252321.522,10408], + [8254.12,252600.111,10420], + [8254.28,252623.322,10421], + [8254.37,252484.055,10415], + [8254.54,252530.477,10417], + [8254.62,252553.688,10418], + [8267.91,252576.9,10419], + [8268.05,252507.266,10416], + [8466.57,252785.866,10428], + [8466.72,252646.6,10422], + [8466.8,252669.811,10423], + [8466.92,252739.444,10426], + [8466.99,252762.655,10427], + [8467.06,252693.022,10424], + [8467.19,252716.233,10425], + [8586.91,252832.355,10430], + [8587.04,252855.566,10431], + [8587.1,252878.777,10432], + [8587.17,252901.988,10433], + [8587.31,252925.2,10434], + [8587.37,252948.411,10435], + [8587.46,252809.144,10429], + [8760.22,253064.522,10440], + [8760.37,253110.944,10442], + [8760.44,252994.888,10437], + [8760.55,253018.1,10438], + [8760.63,253087.733,10441], + [8760.69,253041.311,10439], + [8760.77,252971.677,10436], + [8967.66,253203.855,10446], + [8967.84,253227.066,10447], + [8968,253250.277,10448], + [8968.13,253134.222,10443], + [8968.23,253296.7,10450], + [8968.4,253157.433,10444], + [8968.49,253180.644,10445], + [8968.58,253273.488,10449], + [9092.01,253459.244,10457], + [9111.41,253412.822,10455], + [9111.57,253319.977,10451], + [9111.65,253343.188,10452], + [9111.78,253366.4,10453], + [9111.85,253389.611,10454], + [9111.93,253436.033,10456], + [9255.93,253482.522,10458], + [9256.09,253528.944,10460], + [9256.16,253552.155,10461], + [9256.34,253575.366,10462], + [9256.41,253598.577,10463], + [9256.49,253621.788,10464], + [9256.62,253505.733,10459], + [9421.78,253668.266,10466], + [9421.96,253691.477,10467], + [9422.04,253714.688,10468], + [9422.12,253737.9,10469], + [9422.31,253761.111,10470], + [9422.4,253784.322,10471], + [9422.5,253645.055,10465], + [9588.17,253830.811,10473], + [9588.44,253854.022,10474], + [9588.53,253877.233,10475], + [9588.81,253900.444,10476], + [9589.1,253923.655,10477], + [9589.18,253946.866,10478], + [9589.26,253807.6,10472], + [9754.88,254062.977,10483], + [9755.11,254086.188,10484], + [9755.25,254109.4,10485], + [9755.38,253993.344,10480], + [9755.45,254016.555,10481], + [9755.53,253970.133,10479], + [9755.6,254039.766,10482], + [9937.53,254225.522,10490], + [9937.72,254132.677,10486], + [9937.82,254155.888,10487], + [9937.94,254179.1,10488], + [9938.07,254248.733,10491], + [9938.15,254271.944,10492], + [9938.22,254202.311,10489], + [10093.97,254434.488,10499], + [10094.12,254295.222,10493], + [10094.19,254318.433,10494], + [10094.3,254341.644,10495], + [10094.36,254364.855,10496], + [10094.43,254388.066,10497], + [10094.5,254411.277,10498], + [10226.07,254480.966,10501], + [10226.2,254504.177,10502], + [10226.3,254527.388,10503], + [10226.35,254550.6,10504], + [10226.47,254573.811,10505], + [10226.53,254597.022,10506], + [10226.6,254457.755,10500], + [10410.06,254643.511,10508], + [10410.21,254666.722,10509], + [10410.28,254736.355,10512], + [10410.41,254689.933,10510], + [10410.47,254713.144,10511], + [10410.55,254620.3,10507], + [10410.62,254759.566,10513], + [10563.9,254806.044,10515], + [10564.03,254829.255,10516], + [10564.1,254852.466,10517], + [10564.22,254875.677,10518], + [10564.29,254898.888,10519], + [10564.35,254922.1,10520], + [10564.49,254782.833,10514], + [10729.24,254968.588,10522], + [10729.44,254991.8,10523], + [10729.63,255015.011,10524], + [10729.77,255038.222,10525], + [10729.97,255084.644,10527], + [10730.2,254945.377,10521], + [10730.31,255061.433,10526], + [10888.53,255131.122,10529], + [10888.82,255154.333,10530], + [10888.93,255177.544,10531], + [10889,255200.755,10532], + [10889.13,255223.966,10533], + [10889.19,255247.177,10534], + [10889.43,255107.911,10528], + [11084.73,255293.666,10536], + [11084.97,255316.877,10537], + [11085.09,255340.088,10538], + [11085.3,255363.3,10539], + [11085.43,255386.511,10540], + [11085.55,255409.722,10541], + [11085.69,255270.455,10535], + [11220.5,255456.211,10543], + [11220.65,255479.422,10544], + [11220.75,255502.633,10545], + [11220.84,255525.844,10546], + [11220.97,255549.055,10547], + [11221.06,255572.266,10548], + [11221.16,255433,10542], + [11379.03,255618.744,10550], + [11379.19,255641.955,10551], + [11379.27,255665.166,10552], + [11379.39,255688.377,10553], + [11379.47,255711.588,10554], + [11379.55,255734.8,10555], + [11379.63,255595.533,10549], + [11524.88,255781.288,10557], + [11525.08,255804.5,10558], + [11525.2,255827.711,10559], + [11525.31,255850.922,10560], + [11525.52,255874.133,10561], + [11525.61,255897.344,10562], + [11525.7,255758.077,10556], + [11688,255943.822,10564], + [11688.25,255967.033,10565], + [11688.36,255990.244,10566], + [11688.55,256013.455,10567], + [11688.65,256036.666,10568], + [11688.76,256059.877,10569], + [11688.88,255920.611,10563], + [11849.95,256106.366,10571], + [11850.16,256129.577,10572], + [11850.28,256152.788,10573], + [11850.42,256176,10574], + [11850.5,256199.211,10575], + [11850.56,256222.422,10576], + [11850.65,256083.155,10570], + [12012.25,256268.9,10578], + [12012.43,256292.111,10579], + [12012.53,256315.322,10580], + [12012.69,256338.533,10581], + [12012.77,256361.744,10582], + [12012.89,256384.955,10583], + [12012.99,256245.688,10577], + [12204.19,256431.444,10585], + [12204.41,256454.655,10586], + [12204.49,256477.866,10587], + [12204.68,256501.077,10588], + [12204.78,256524.288,10589], + [12204.88,256547.5,10590], + [12204.98,256408.233,10584], + [12340.82,256593.988,10592], + [12341.63,256617.2,10593], + [12341.77,256686.833,10596], + [12341.88,256640.411,10594], + [12342,256663.622,10595], + [12342.08,256570.777,10591], + [12342.17,256710.044,10597], + [12512.62,256849.366,10603], + [12512.78,256872.577,10604], + [12516.18,256733.311,10598], + [12516.31,256756.522,10599], + [12516.43,256779.733,10600], + [12516.5,256802.944,10601], + [12516.63,256826.155,10602], + [12671.85,256965.488,10608], + [12672.04,256988.7,10609], + [12672.41,257011.911,10610], + [12672.53,257035.122,10611], + [12672.63,256895.855,10605], + [12672.72,256919.066,10606], + [12672.96,256942.277,10607], + [12840.08,257081.6,10613], + [12840.27,257104.811,10614], + [12840.38,257128.022,10615], + [12840.52,257151.233,10616], + [12840.59,257174.444,10617], + [12840.66,257197.655,10618], + [12840.75,257058.388,10612], + [13003.86,257360.2,10625], + [13004.03,257313.777,10623], + [13004.16,257336.988,10624], + [13004.24,257290.566,10622], + [13004.41,257267.355,10621], + [13004.5,257244.144,10620], + [13004.57,257220.933,10619], + [13150.1,257522.733,10632], + [13150.26,257383.466,10626], + [13150.37,257406.677,10627], + [13150.46,257429.888,10628], + [13150.56,257453.1,10629], + [13150.64,257476.311,10630], + [13150.72,257499.522,10631], + [13313.7,257685.277,10639], + [13314,257569.222,10634], + [13314.09,257592.433,10635], + [13314.16,257615.644,10636], + [13314.27,257638.855,10637], + [13314.37,257662.066,10638], + [13314.47,257546.011,10633], + [13476.18,257731.766,10641], + [13476.34,257754.977,10642], + [13476.44,257778.188,10643], + [13476.52,257801.4,10644], + [13476.63,257824.611,10645], + [13476.71,257847.822,10646], + [13476.78,257708.555,10640], + [13638.15,257894.3,10648], + [13638.41,257917.511,10649], + [13638.58,257987.144,10652], + [13638.87,257940.722,10650], + [13639.47,257963.933,10651], + [13639.6,257871.088,10647], + [13640.33,258010.355,10653], + [13801.11,258056.844,10655], + [13801.31,258080.055,10656], + [13801.58,258103.266,10657], + [13801.71,258126.477,10658], + [13801.85,258149.688,10659], + [13802.07,258172.9,10660], + [13802.21,258033.633,10654], + [13964.66,258312.222,10666], + [13964.8,258335.433,10667], + [13964.89,258289.011,10665], + [13965.01,258196.166,10661], + [13965.08,258219.377,10662], + [13965.15,258242.588,10663], + [13965.28,258265.8,10664], + [14133.5,258358.711,10668], + [14133.65,258405.133,10670], + [14133.72,258428.344,10671], + [14133.85,258451.555,10672], + [14133.93,258474.766,10673], + [14134,258497.977,10674], + [14134.07,258381.922,10669], + [14318.44,258544.455,10676], + [14318.69,258567.666,10677], + [14318.85,258590.877,10678], + [14318.93,258614.088,10679], + [14319,258637.3,10680], + [14319.19,258660.511,10681], + [14319.31,258521.244,10675], + [14491.59,258707,10683], + [14491.75,258730.211,10684], + [14491.84,258753.422,10685], + [14491.97,258776.633,10686], + [14492.04,258799.844,10687], + [14492.11,258846.266,10689], + [14492.25,258823.055,10688], + [14492.31,258683.788,10682], + [14677.09,258869.544,10690], + [14677.25,258892.755,10691], + [14677.33,258915.966,10692], + [14677.46,258939.177,10693], + [14677.53,258962.388,10694], + [14677.6,258985.6,10695], + [14820.59,259055.288,10698], + [14820.78,259078.5,10699], + [14820.94,259101.711,10700], + [14821.09,259124.922,10701], + [14821.16,259148.133,10702], + [14821.25,259008.866,10696], + [14821.33,259032.077,10697], + [14964.22,259287.466,10708], + [14964.37,259310.677,10709], + [14964.44,259171.411,10703], + [14964.56,259194.622,10704], + [14964.63,259217.833,10705], + [14964.7,259241.044,10706], + [14964.96,259264.255,10707], + [15100.84,259333.944,10710], + [15101.09,259357.155,10711], + [15101.2,259380.366,10712], + [15101.35,259403.577,10713], + [15101.45,259426.788,10714], + [15101.59,259450,10715], + [15101.71,259473.211,10716], + [15269.97,259519.7,10718], + [15270.12,259542.911,10719], + [15270.22,259566.122,10720], + [15270.29,259589.333,10721], + [15270.35,259612.544,10722], + [15270.49,259635.755,10723], + [15270.56,259496.488,10717], + [15425.1,259682.233,10725], + [15425.25,259705.444,10726], + [15425.33,259728.655,10727], + [15425.46,259751.866,10728], + [15425.52,259775.077,10729], + [15425.61,259798.288,10730], + [15425.69,259659.022,10724], + [15590.53,259844.777,10732], + [15590.66,259867.988,10733], + [15590.77,259891.2,10734], + [15590.88,259914.411,10735], + [15590.94,259937.622,10736], + [15591.06,259960.833,10737], + [15591.19,259821.566,10731], + [15755.92,260007.322,10739], + [15756.06,260030.533,10740], + [15756.16,260053.744,10741], + [15756.25,260076.955,10742], + [15756.34,260100.166,10743], + [15756.44,260123.377,10744], + [15756.56,259984.111,10738], + [15920.88,260146.644,10745], + [15921,260239.488,10749], + [15921.11,260262.7,10750], + [15921.22,260285.911,10751], + [15921.38,260216.277,10748], + [15921.54,260169.855,10746], + [15921.63,260193.066,10747], + [16087.75,260378.822,10755], + [16087.94,260425.244,10757], + [16088.03,260448.455,10758], + [16088.15,260309.188,10752], + [16088.22,260332.4,10753], + [16088.29,260355.611,10754], + [16088.37,260402.033,10756], + [16254.79,260494.933,10760], + [16254.97,260518.144,10761], + [16255.08,260541.355,10762], + [16255.18,260564.566,10763], + [16255.28,260587.777,10764], + [16255.41,260610.988,10765], + [16255.52,260471.722,10759], + [16421.4,260657.477,10767], + [16421.72,260680.688,10768], + [16421.94,260703.9,10769], + [16422.03,260727.111,10770], + [16422.11,260750.322,10771], + [16422.19,260773.533,10772], + [16422.37,260634.266,10766], + [16563.5,260820.011,10774], + [16563.69,260843.222,10775], + [16563.78,260866.433,10776], + [16563.89,260889.644,10777], + [16564.07,260912.855,10778], + [16564.16,260936.066,10779], + [16564.25,260796.8,10773], + [16725.87,260982.555,10781], + [16726.04,261005.766,10782], + [16726.13,261028.977,10783], + [16726.21,261052.188,10784], + [16726.35,261075.4,10785], + [16726.43,261098.611,10786], + [16726.53,260959.344,10780], + [16889.66,261145.1,10788], + [16889.81,261168.311,10789], + [16889.88,261191.522,10790], + [16889.96,261214.733,10791], + [16890.03,261237.944,10792], + [16890.1,261261.155,10793], + [16890.19,261121.888,10787], + [17051.54,261307.633,10795], + [17051.7,261330.844,10796], + [17051.78,261354.055,10797], + [17051.93,261377.266,10798], + [17052.01,261400.477,10799], + [17052.08,261423.688,10800], + [17052.25,261284.422,10794], + [17221.22,261493.388,10803], + [17221.41,261516.6,10804], + [17221.48,261539.811,10805], + [17221.62,261563.022,10806], + [17221.69,261586.233,10807], + [17221.76,261446.966,10801], + [17221.92,261470.177,10802], + [17379.66,261655.922,10810], + [17379.85,261679.133,10811], + [17379.97,261725.555,10813], + [17380.05,261748.766,10814], + [17380.19,261609.5,10808], + [17380.25,261632.711,10809], + [17380.33,261702.344,10812], + [17546.72,261795.255,10816], + [17546.99,261818.466,10817], + [17547.14,261841.677,10818], + [17547.27,261864.888,10819], + [17547.5,261888.1,10820], + [17547.6,261911.311,10821], + [17547.74,261772.044,10815], + [17711.03,261934.577,10822], + [17712.21,262027.422,10826], + [17712.38,262050.633,10827], + [17712.55,262073.844,10828], + [17712.66,261957.788,10823], + [17712.81,261981,10824], + [17712.91,262004.211,10825], + [17890.29,262213.177,10834], + [17890.45,262097.122,10829], + [17890.56,262120.333,10830], + [17890.63,262143.544,10831], + [17890.76,262166.755,10832], + [17890.84,262236.388,10835], + [17890.91,262189.966,10833], + [18045.28,262352.511,10840], + [18045.44,262375.722,10841], + [18045.55,262398.933,10842], + [18045.63,262282.877,10837], + [18045.75,262306.088,10838], + [18045.84,262259.666,10836], + [18045.97,262329.3,10839], + [18240.02,262515.044,10847], + [18240.16,262422.2,10843], + [18240.22,262445.411,10844], + [18240.35,262468.622,10845], + [18240.42,262538.255,10848], + [18240.5,262561.466,10849], + [18240.63,262491.833,10846], + [18352.12,262607.955,10851], + [18352.27,262631.166,10852], + [18352.35,262654.377,10853], + [18352.47,262677.588,10854], + [18352.55,262700.8,10855], + [18352.61,262724.011,10856], + [18352.69,262584.744,10850], + [18515.84,262793.7,10859], + [18516,262816.911,10860], + [18516.09,262840.122,10861], + [18516.22,262863.333,10862], + [18516.31,262886.544,10863], + [18516.38,262747.277,10857], + [18516.52,262770.488,10858], + [18676.4,263049.088,10870], + [18676.53,262909.822,10864], + [18676.63,262933.033,10865], + [18676.71,262956.244,10866], + [18676.85,262979.455,10867], + [18676.93,263002.666,10868], + [18677.04,263025.877,10869], + [18848.16,263165.2,10875], + [18848.31,263188.411,10876], + [18848.4,263211.622,10877], + [18848.52,263095.566,10872], + [18848.59,263118.777,10873], + [18848.66,263072.355,10871], + [18848.77,263141.988,10874], + [19026.36,263304.533,10881], + [19026.53,263234.9,10878], + [19026.62,263258.111,10879], + [19026.74,263327.744,10882], + [19026.82,263350.955,10883], + [19026.9,263281.322,10880], + [19027.04,263374.166,10884], + [19178.5,263467.077,10888], + [19178.71,263513.5,10890], + [19178.81,263536.711,10891], + [19179.01,263397.444,10885], + [19179.13,263420.655,10886], + [19179.25,263443.866,10887], + [19179.59,263490.288,10889], + [19355.32,263583.188,10893], + [19355.51,263606.4,10894], + [19355.6,263559.977,10892], + [19355.74,263629.611,10895], + [19355.82,263652.822,10896], + [19355.9,263676.033,10897], + [19355.97,263699.244,10898], + [19504.23,263722.522,10899], + [19505.66,263745.733,10900], + [19505.82,263768.944,10901], + [19505.95,263792.155,10902], + [19506.03,263815.366,10903], + [19506.17,263838.577,10904], + [19506.25,263861.788,10905], + [19672.07,263954.688,10909], + [19672.25,263977.9,10910], + [19672.4,264001.111,10911], + [19672.5,263885.055,10906], + [19672.6,263908.266,10907], + [19672.76,263931.477,10908], + [19672.84,264024.322,10912], + [19815.57,264070.811,10914], + [19815.74,264094.022,10915], + [19815.85,264117.233,10916], + [19815.94,264140.444,10917], + [19816.06,264163.655,10918], + [19816.16,264186.866,10919], + [19816.24,264047.6,10913], + [20050.35,264233.344,10921], + [20050.51,264256.555,10922], + [20050.59,264326.188,10925], + [20050.72,264279.766,10923], + [20050.8,264302.977,10924], + [20050.91,264210.133,10920], + [20051.05,264349.4,10926], + [20151.41,264465.522,10931], + [20151.57,264488.733,10932], + [20151.66,264511.944,10933], + [20151.78,264372.677,10927], + [20151.87,264395.888,10928], + [20151.95,264419.1,10929], + [20152.09,264442.311,10930], + [20307.34,264535.222,10934], + [20307.53,264558.433,10935], + [20307.68,264581.644,10936], + [20307.78,264604.855,10937], + [20307.94,264628.066,10938], + [20308.02,264651.277,10939], + [20308.1,264674.488,10940], + [20466.32,264720.966,10942], + [20466.5,264744.177,10943], + [20466.58,264813.811,10946], + [20466.72,264767.388,10944], + [20466.8,264790.6,10945], + [20466.89,264697.755,10941], + [20467.06,264837.022,10947], + [20641.56,264883.511,10949], + [20641.76,264906.722,10950], + [20641.88,264929.933,10951], + [20642.07,264953.144,10952], + [20642.19,264976.355,10953], + [20642.31,264999.566,10954], + [20642.42,264860.3,10948], + [20796,265138.888,10960], + [20796.17,265162.1,10961], + [20796.27,265115.677,10959], + [20796.44,265022.833,10955], + [20796.53,265046.044,10956], + [20796.63,265069.255,10957], + [20796.74,265092.466,10958], + [20980.12,265231.8,10964], + [20980.27,265208.588,10963], + [20980.37,265255.011,10965], + [20980.44,265278.222,10966], + [20980.6,265301.433,10967], + [20980.68,265324.644,10968], + [20980.73,265185.377,10962], + [21133.86,265347.911,10969], + [21134,265394.333,10971], + [21134.09,265417.544,10972], + [21134.25,265440.755,10973], + [21134.34,265463.966,10974], + [21134.44,265487.177,10975], + [21134.66,265371.122,10970], + [21301.55,265533.666,10977], + [21301.71,265556.877,10978], + [21301.82,265580.088,10979], + [21301.9,265603.3,10980], + [21302.02,265626.511,10981], + [21302.09,265649.722,10982], + [21302.22,265510.455,10976], + [21440.19,265696.211,10984], + [21440.37,265719.422,10985], + [21440.49,265742.633,10986], + [21440.57,265765.844,10987], + [21440.71,265789.055,10988], + [21440.78,265812.266,10989], + [21440.92,265673,10983], + [21602.06,265974.8,10996], + [21602.28,265951.588,10995], + [21602.4,265928.377,10994], + [21602.49,265881.955,10992], + [21602.69,265905.166,10993], + [21602.77,265858.744,10991], + [21602.87,265835.533,10990], + [21764.78,266067.711,11000], + [21764.96,266090.922,11001], + [21765.05,266114.133,11002], + [21765.21,266137.344,11003], + [21765.28,265998.077,10997], + [21765.37,266044.5,10999], + [21765.46,266021.288,10998], + [21927.49,266253.455,11008], + [21927.66,266230.244,11007], + [21927.77,266276.666,11009], + [21927.85,266299.877,11010], + [21927.96,266183.822,11005], + [21928.03,266207.033,11006], + [21928.1,266160.611,11004], + [22090.88,266416,11015], + [22091.03,266439.211,11016], + [22091.11,266462.422,11017], + [22091.24,266346.366,11012], + [22091.3,266369.577,11013], + [22091.38,266323.155,11011], + [22091.53,266392.788,11014], + [22321.65,266508.9,11019], + [22321.82,266532.111,11020], + [22321.94,266578.533,11022], + [22322.11,266601.744,11023], + [22322.19,266624.955,11024], + [22322.25,266485.688,11018], + [22322.4,266555.322,11021], + [22433.75,266741.077,11029], + [22433.99,266764.288,11030], + [22434.12,266787.5,11031], + [22437.23,266671.444,11026], + [22437.45,266694.655,11027], + [22437.55,266648.233,11025], + [22437.66,266717.866,11028], + [22615.91,266903.622,11036], + [22616.09,266810.777,11032], + [22616.18,266833.988,11033], + [22616.31,266857.2,11034], + [22616.39,266926.833,11037], + [22616.47,266950.044,11038], + [22616.62,266880.411,11035], + [22763.72,267019.733,11041], + [22763.91,266973.311,11039], + [22764.05,267042.944,11042], + [22764.13,267066.155,11043], + [22764.27,267089.366,11044], + [22764.34,267112.577,11045], + [22764.43,267135.788,11046], + [22764.53,266996.522,11040], + [22925.46,267159.077,11047], + [22925.62,267182.288,11048], + [22925.69,267205.5,11049], + [22925.87,267228.711,11050], + [22925.94,267251.922,11051], + [22926.02,267275.133,11052], + [22926.1,267298.344,11053], + [23088.63,267344.822,11055], + [23088.78,267368.033,11056], + [23088.87,267391.244,11057], + [23089.03,267414.455,11058], + [23089.11,267437.666,11059], + [23089.19,267460.877,11060], + [23089.25,267321.611,11054], + [23253.25,267623.422,11067], + [23253.4,267553.788,11064], + [23253.51,267577,11065], + [23253.59,267600.211,11066], + [23253.67,267530.577,11063], + [23253.82,267507.366,11062], + [23253.91,267484.155,11061], + [23421.5,267669.9,11069], + [23421.66,267693.111,11070], + [23421.74,267716.322,11071], + [23421.88,267739.533,11072], + [23422.03,267785.955,11074], + [23422.13,267762.744,11073], + [23422.25,267646.688,11068], + [23587.5,267809.233,11075], + [23587.66,267832.444,11076], + [23587.77,267855.655,11077], + [23587.97,267878.866,11078], + [23588.06,267902.077,11079], + [23588.16,267925.288,11080], + [23588.29,267948.5,11081], + [23754.12,267994.977,11083], + [23754.28,268018.188,11084], + [23754.35,268041.4,11085], + [23754.44,268064.611,11086], + [23754.59,268087.822,11087], + [23754.68,268111.033,11088], + [23754.88,267971.766,11082], + [23922.3,268157.522,11090], + [23922.44,268180.733,11091], + [23922.55,268203.944,11092], + [23922.63,268227.155,11093], + [23922.77,268250.366,11094], + [23922.84,268273.577,11095], + [23923,268134.311,11089], + [24063.78,268320.066,11097], + [24063.96,268343.277,11098], + [24064.09,268366.488,11099], + [24064.19,268389.7,11100], + [24064.34,268412.911,11101], + [24064.41,268436.122,11102], + [24064.5,268296.855,11096], + [24225.27,268529.022,11106], + [24225.45,268552.233,11107], + [24225.55,268575.444,11108], + [24225.68,268598.655,11109], + [24225.75,268459.388,11103], + [24225.84,268482.6,11104], + [24225.93,268505.811,11105], + [24390.13,268737.988,11115], + [24390.29,268621.933,11110], + [24390.37,268645.144,11111], + [24390.53,268668.355,11112], + [24390.6,268761.2,11116], + [24390.68,268691.566,11113], + [24390.75,268714.777,11114], + [24566.88,268807.677,11118], + [24567.05,268830.888,11119], + [24567.14,268854.1,11120], + [24567.27,268877.311,11121], + [24567.37,268900.522,11122], + [24567.44,268923.733,11123], + [24567.6,268784.466,11117], + [24738.47,268970.222,11125], + [24738.63,268993.433,11126], + [24738.71,269016.644,11127], + [24738.85,269039.855,11128], + [24738.92,269063.066,11129], + [24739,269086.277,11130], + [24739.08,268947.011,11124], + [24879.15,269202.388,11135], + [24879.31,269225.6,11136], + [24879.41,269109.544,11131], + [24879.55,269132.755,11132], + [24879.63,269155.966,11133], + [24879.7,269179.177,11134], + [24879.79,269248.811,11137], + [25054.02,269272.088,11138], + [25055.04,269295.3,11139], + [25055.16,269318.511,11140], + [25055.26,269341.722,11141], + [25055.33,269364.933,11142], + [25055.6,269388.144,11143], + [25055.67,269411.355,11144], + [25212.62,269573.9,11151], + [25212.79,269434.633,11145], + [25212.9,269457.844,11146], + [25213.06,269481.055,11147], + [25213.14,269504.266,11148], + [25213.23,269527.477,11149], + [25213.39,269550.688,11150], + [25380.99,269597.166,11152], + [25381.31,269620.377,11153], + [25381.4,269643.588,11154], + [25381.55,269666.8,11155], + [25381.62,269690.011,11156], + [25381.71,269713.222,11157], + [25381.8,269736.433,11158], + [25548.94,269875.766,11164], + [25549.1,269898.977,11165], + [25549.19,269852.555,11163], + [25549.37,269759.711,11159], + [25549.44,269782.922,11160], + [25549.62,269806.133,11161], + [25549.71,269829.344,11162], + [25713.68,269922.244,11166], + [25714.61,269991.877,11169], + [25714.74,269945.455,11167], + [25714.86,269968.666,11168], + [25714.94,270015.088,11170], + [25715.03,270038.3,11171], + [25715.19,270061.511,11172], + [25715.26,270084.722,11173], + [25888.14,270177.644,11177], + [25888.28,270200.855,11178], + [25888.37,270224.066,11179], + [25888.51,270108.011,11174], + [25888.61,270131.222,11175], + [25888.7,270154.433,11176], + [25888.79,270247.277,11180], + [26047.55,270409.811,11187], + [26047.72,270363.388,11185], + [26047.82,270386.6,11186], + [26047.99,270316.966,11183], + [26048.07,270293.755,11182], + [26048.17,270340.177,11184], + [26048.3,270270.544,11181], + [26214.11,270479.511,11190], + [26214.21,270525.933,11192], + [26214.34,270433.088,11188], + [26214.43,270456.3,11189], + [26214.55,270502.722,11191], + [26214.65,270549.144,11193], + [26214.76,270572.355,11194], + [26380.43,270618.833,11196], + [26380.67,270642.044,11197], + [26380.78,270665.255,11198], + [26380.91,270688.466,11199], + [26381.18,270711.677,11200], + [26381.27,270734.888,11201], + [26381.35,270595.622,11195], + [26524.57,270781.377,11203], + [26524.77,270804.588,11204], + [26524.87,270827.8,11205], + [26524.97,270851.011,11206], + [26525.16,270874.222,11207], + [26525.23,270897.433,11208], + [26525.32,270758.166,11202], + [26687.77,270943.911,11210], + [26687.95,270967.122,11211], + [26688.07,270990.333,11212], + [26688.15,271013.544,11213], + [26688.28,271036.755,11214], + [26688.35,271059.966,11215], + [26688.47,270920.7,11209], + [26850.96,271106.455,11217], + [26851.13,271129.666,11218], + [26851.22,271152.877,11219], + [26851.34,271176.088,11220], + [26851.42,271199.3,11221], + [26851.49,271222.511,11222], + [26851.66,271083.244,11216], + [27012.06,271269,11224], + [27012.22,271292.211,11225], + [27012.3,271315.422,11226], + [27012.43,271338.633,11227], + [27012.5,271361.844,11228], + [27012.58,271385.055,11229], + [27012.72,271245.788,11223], + [27174.88,271454.744,11232], + [27175.05,271477.955,11233], + [27175.15,271501.166,11234], + [27175.28,271524.377,11235], + [27175.36,271408.322,11230], + [27175.44,271431.533,11231], + [27175.52,271547.588,11236], + [27340.27,271594.077,11238], + [27340.44,271617.288,11239], + [27340.62,271640.5,11240], + [27340.73,271663.711,11241], + [27340.9,271686.922,11242], + [27340.97,271710.133,11243], + [27341.07,271570.866,11237], + [27513.14,271756.611,11245], + [27513.37,271779.822,11246], + [27513.49,271803.033,11247], + [27513.67,271826.244,11248], + [27513.75,271849.455,11249], + [27513.84,271872.666,11250], + [27513.94,271733.4,11244], + [27701.13,271942.366,11253], + [27701.32,271965.577,11254], + [27701.43,271988.788,11255], + [27701.53,272012,11256], + [27701.75,272035.211,11257], + [27701.86,271895.944,11251], + [27701.99,271919.155,11252], + [27849.3,272128.111,11261], + [27849.49,272151.322,11262], + [27849.59,272174.533,11263], + [27849.8,272058.477,11258], + [27849.91,272081.688,11259], + [27849.98,272197.744,11264], + [27850.06,272104.9,11260], + [28004.41,272313.866,11269], + [28004.6,272221.022,11265], + [28004.69,272244.233,11266], + [28004.83,272267.444,11267], + [28004.91,272337.077,11270], + [28004.97,272360.288,11271], + [28005.05,272290.655,11268], + [28150.41,272383.566,11272], + [28150.57,272406.777,11273], + [28150.66,272429.988,11274], + [28150.8,272453.2,11275], + [28150.88,272476.411,11276], + [28150.96,272522.833,11278], + [28151.05,272499.622,11277], + [28339.67,272592.522,11281], + [28339.83,272615.733,11282], + [28339.91,272638.944,11283], + [28340.04,272662.155,11284], + [28340.12,272685.366,11285], + [28340.19,272546.1,11279], + [28340.28,272708.577,11286], + [28340.44,272569.311,11280], + [28502.85,272731.866,11287], + [28504.1,272755.077,11288], + [28504.23,272778.288,11289], + [28504.31,272801.5,11290], + [28504.44,272824.711,11291], + [28504.53,272847.922,11292], + [28504.63,272871.133,11293], + [28671.16,272940.822,11296], + [28671.43,272964.033,11297], + [28671.54,272987.244,11298], + [28671.69,273010.455,11299], + [28671.76,273033.666,11300], + [28671.87,272894.4,11294], + [28671.97,272917.611,11295], + [28840.35,273080.155,11302], + [28840.53,273103.366,11303], + [28840.63,273126.577,11304], + [28840.77,273149.788,11305], + [28840.85,273173,11306], + [28840.94,273196.211,11307], + [28841.01,273056.944,11301], + [29004.41,273312.322,11312], + [29004.55,273335.533,11313], + [29004.62,273358.744,11314], + [29004.74,273242.688,11309], + [29004.82,273265.9,11310], + [29004.91,273219.477,11308], + [29005.05,273289.111,11311], + [29165.59,273474.866,11319], + [29165.75,273382.022,11315], + [29165.85,273405.233,11316], + [29166,273428.444,11317], + [29166.08,273498.077,11320], + [29166.16,273521.288,11321], + [29166.25,273451.655,11318], + [29311.41,273660.611,11327], + [29311.63,273683.822,11328], + [29311.76,273544.555,11322], + [29311.88,273614.188,11325], + [29312.07,273637.4,11326], + [29312.14,273567.766,11323], + [29312.22,273590.977,11324], + [29474.61,273776.733,11332], + [29474.77,273846.366,11335], + [29474.9,273707.1,11329], + [29475.06,273730.311,11330], + [29475.13,273799.944,11333], + [29475.21,273823.155,11334], + [29475.3,273753.522,11331], + [29636.09,273892.855,11337], + [29636.25,273916.066,11338], + [29636.34,273939.277,11339], + [29636.47,273962.488,11340], + [29636.55,273985.7,11341], + [29636.63,274008.911,11342], + [29636.72,273869.644,11336], + [29799.95,274055.388,11344], + [29800.1,274078.6,11345], + [29800.19,274101.811,11346], + [29800.31,274125.022,11347], + [29800.38,274148.233,11348], + [29800.46,274171.444,11349], + [29800.6,274032.177,11343], + [29964.7,274217.933,11351], + [29964.85,274241.144,11352], + [29964.94,274264.355,11353], + [29965.06,274287.566,11354], + [29965.15,274310.777,11355], + [29965.22,274333.988,11356], + [29965.36,274194.722,11350], + [30130.13,274380.466,11358], + [30130.31,274403.677,11359], + [30130.39,274426.888,11360], + [30130.47,274450.1,11361], + [30130.65,274473.311,11362], + [30130.73,274496.522,11363], + [30130.87,274357.255,11357], + [30295.18,274612.644,11368], + [30295.38,274659.066,11370], + [30295.5,274635.855,11369], + [30295.69,274543.011,11365], + [30295.84,274566.222,11366], + [30295.97,274519.8,11364], + [30296.09,274589.433,11367], + [30462.77,274682.333,11371], + [30463.07,274705.544,11372], + [30463.24,274775.177,11375], + [30463.35,274728.755,11373], + [30463.71,274798.388,11376], + [30463.84,274821.6,11377], + [30463.98,274751.966,11374], + [30679.69,274868.088,11379], + [30679.93,274891.3,11380], + [30680.08,274914.511,11381], + [30680.27,274937.722,11382], + [30680.4,274960.933,11383], + [30680.5,274984.144,11384], + [30680.62,274844.877,11378], + [30774.94,275030.633,11386], + [30775.11,275053.844,11387], + [30775.21,275077.055,11388], + [30775.45,275100.266,11389], + [30775.63,275123.477,11390], + [30775.75,275146.688,11391], + [30775.87,275007.422,11385], + [30950.08,275286.011,11397], + [30950.25,275309.222,11398], + [30950.35,275169.955,11392], + [30950.5,275193.166,11393], + [30950.59,275216.377,11394], + [30950.69,275239.588,11395], + [30950.84,275262.8,11396], + [31111.47,275332.5,11399], + [31111.65,275355.711,11400], + [31111.74,275378.922,11401], + [31111.83,275402.133,11402], + [31112,275425.344,11403], + [31112.07,275448.555,11404], + [31112.16,275471.766,11405], + [31261.41,275518.244,11407], + [31261.55,275541.455,11408], + [31261.63,275564.666,11409], + [31261.75,275587.877,11410], + [31261.84,275611.088,11411], + [31261.92,275634.3,11412], + [31262.01,275495.033,11406], + [31424.69,275680.788,11414], + [31424.87,275704,11415], + [31424.95,275727.211,11416], + [31425.07,275750.422,11417], + [31425.15,275773.633,11418], + [31425.24,275796.844,11419], + [31425.31,275657.577,11413], + [31589.23,275843.322,11421], + [31589.38,275866.533,11422], + [31589.46,275936.166,11425], + [31589.6,275889.744,11423], + [31589.69,275912.955,11424], + [31589.77,275820.111,11420], + [31589.93,275959.377,11426], + [31756.16,276005.866,11428], + [31756.33,276029.077,11429], + [31756.42,276052.288,11430], + [31756.59,276075.5,11431], + [31756.66,276098.711,11432], + [31756.75,276121.922,11433], + [31756.85,275982.655,11427], + [31921.65,276168.411,11435], + [31921.8,276191.622,11436], + [31921.88,276214.833,11437], + [31922,276238.044,11438], + [31922.07,276261.255,11439], + [31922.16,276284.466,11440], + [31922.29,276145.2,11434], + [32092.05,276330.944,11442], + [32092.16,276354.155,11443], + [32092.24,276377.366,11444], + [32092.36,276400.577,11445], + [32092.43,276423.788,11446], + [32092.49,276447,11447], + [32092.61,276307.733,11441], + [32265.74,276493.488,11449], + [32265.89,276516.7,11450], + [32265.97,276539.911,11451], + [32266.09,276563.122,11452], + [32266.16,276586.333,11453], + [32266.22,276609.544,11454], + [32266.35,276470.277,11448], + [32399.35,276725.655,11459], + [32399.48,276748.866,11460], + [32399.55,276772.077,11461], + [32399.66,276656.022,11456], + [32399.74,276679.233,11457], + [32399.84,276632.811,11455], + [32400,276702.444,11458], + [32587.97,276795.355,11462], + [32589.35,276888.2,11466], + [32589.47,276911.411,11467], + [32589.57,276818.566,11463], + [32589.69,276841.777,11464], + [32589.77,276864.988,11465], + [32589.91,276934.622,11468], + [32589.99,276957.833,11469], + [32753,277004.322,11471], + [32753.18,277027.533,11472], + [32753.28,277050.744,11473], + [32753.43,277073.955,11474], + [32753.52,277097.166,11475], + [32753.6,277120.377,11476], + [32753.7,276981.111,11470], + [32920.49,277190.077,11479], + [32920.74,277213.288,11480], + [32920.86,277236.5,11481], + [32921.03,277259.711,11482], + [32921.12,277282.922,11483], + [32921.22,277143.655,11477], + [32921.34,277166.866,11478], + [33088.34,277306.188,11484], + [33088.57,277329.4,11485], + [33088.69,277445.455,11490], + [33088.85,277352.611,11486], + [33088.98,277375.822,11487], + [33089.06,277399.033,11488], + [33089.17,277422.244,11489], + [33254.69,277491.944,11492], + [33254.94,277515.155,11493], + [33255.03,277538.366,11494], + [33255.21,277561.577,11495], + [33255.31,277584.788,11496], + [33255.41,277608,11497], + [33255.52,277468.733,11491], + [33397.55,277654.477,11499], + [33397.69,277677.688,11500], + [33397.79,277700.9,11501], + [33397.91,277724.111,11502], + [33397.99,277747.322,11503], + [33398.06,277770.533,11504], + [33398.13,277631.266,11498], + [33559.9,277933.077,11511], + [33560.06,277793.811,11505], + [33560.16,277817.022,11506], + [33560.24,277840.233,11507], + [33560.37,277863.444,11508], + [33560.43,277886.655,11509], + [33560.51,277909.866,11510], + [33723.06,277979.566,11513], + [33723.25,278002.777,11514], + [33723.35,278025.988,11515], + [33723.46,278049.2,11516], + [33723.66,278072.411,11517], + [33723.75,278095.622,11518], + [33723.88,277956.355,11512], + [33885.02,278142.1,11520], + [33885.16,278165.311,11521], + [33885.23,278188.522,11522], + [33885.31,278211.733,11523], + [33885.43,278234.944,11524], + [33885.5,278258.155,11525], + [33885.57,278118.888,11519], + [34050.42,278374.277,11530], + [34050.56,278397.488,11531], + [34050.65,278420.7,11532], + [34050.8,278304.644,11527], + [34050.88,278327.855,11528], + [34050.96,278281.433,11526], + [34051.04,278351.066,11529], + [34213.18,278536.811,11537], + [34213.43,278443.966,11533], + [34213.57,278467.177,11534], + [34213.72,278490.388,11535], + [34213.87,278560.022,11538], + [34214.02,278583.233,11539], + [34214.13,278513.6,11536], + [34350.88,278606.511,11540], + [34351.11,278629.722,11541], + [34351.24,278652.933,11542], + [34351.36,278676.144,11543], + [34351.48,278699.355,11544], + [34351.61,278722.566,11545], + [34520.88,278792.255,11548], + [34521.05,278815.466,11549], + [34521.13,278838.677,11550], + [34521.26,278861.888,11551], + [34521.34,278885.1,11552], + [34521.41,278745.833,11546], + [34521.53,278769.044,11547], + [34676.78,278931.577,11554], + [34676.94,278954.788,11555], + [34677.02,278978,11556], + [34677.13,279001.211,11557], + [34677.21,279024.422,11558], + [34677.27,279047.633,11559], + [34677.34,278908.366,11553], + [34845.22,279094.122,11561], + [34845.37,279117.333,11562], + [34845.45,279140.544,11563], + [34845.57,279163.755,11564], + [34845.65,279186.966,11565], + [34845.72,279210.177,11566], + [34845.8,279070.911,11560] + ] +} diff --git a/test/audio-quality/traces/relay-mic.json b/test/audio-quality/traces/relay-mic.json new file mode 100644 index 00000000000..05a9987c258 --- /dev/null +++ b/test/audio-quality/traces/relay-mic.json @@ -0,0 +1,1764 @@ +{ + "version": 2, + "source": "https://cdn.moq.dev/anon, in headless Chromium capturing its fake audio device, recorded 2026-09-29", + "description": "A browser publisher through the public relay: getUserMedia, the capture worklet, and WebCodecs Opus, with the relay's round trip in between.", + "transport": "webtransport", + "rtt": null, + "config": { + "codec": "opus", + "container": { + "kind": "legacy" + }, + "sampleRate": 48000, + "numberOfChannels": 2, + "bitrate": 64000 + }, + "duration": 35000, + "arrivals": [ + [0,2324.745,0], + [16.53,2344.745,1], + [38.87,2364.745,2], + [58.56,2384.745,3], + [80.79,2404.745,4], + [90.63,2424.745,5], + [123.14,2444.745,6], + [133.15,2464.745,7], + [154.04,2484.745,8], + [175.89,2504.745,9], + [196.55,2524.745,10], + [217.73,2544.745,11], + [239.26,2564.745,12], + [249.41,2584.744,13], + [284.74,2604.745,14], + [293.14,2624.745,15], + [316.97,2644.745,16], + [341.57,2664.745,17], + [369.77,2684.745,18], + [383.67,2704.745,19], + [399.94,2724.745,20], + [412.31,2744.745,21], + [448.56,2764.745,22], + [462.16,2784.745,23], + [481.56,2804.745,24], + [500.93,2824.745,25], + [521.09,2844.744,26], + [549.96,2864.745,27], + [563.49,2884.745,28], + [571.93,2904.745,29], + [604.02,2924.745,30], + [614.12,2944.745,31], + [636.18,2964.745,32], + [655.33,2984.745,33], + [680.99,3004.745,34], + [699.9,3024.745,35], + [743.93,3044.745,36], + [744.29,3064.745,37], + [766.77,3084.745,38], + [773.76,3104.745,39], + [798.74,3124.745,40], + [821.71,3144.745,41], + [841.92,3164.745,42], + [862.21,3184.745,43], + [882.59,3204.745,44], + [900.28,3224.745,45], + [926.38,3244.745,46], + [937.7,3264.745,47], + [953.6,3284.745,48], + [974.8,3304.745,49], + [996.59,3324.745,50], + [1017.6,3344.745,51], + [1038.36,3364.744,52], + [1048.84,3384.745,53], + [1081.81,3404.745,54], + [1092.23,3424.745,55], + [1113.34,3444.745,56], + [1134.38,3464.745,57], + [1155.59,3484.745,58], + [1178.33,3504.745,59], + [1198.64,3524.745,60], + [1208.71,3544.745,61], + [1241.21,3564.745,62], + [1251.7,3584.745,63], + [1273.61,3604.745,64], + [1295.2,3624.745,65], + [1317.27,3644.745,66], + [1337.5,3664.745,67], + [1358.88,3684.745,68], + [1371.88,3704.745,69], + [1405.88,3724.745,70], + [1413.55,3744.745,71], + [1438.38,3764.745,72], + [1459.06,3784.745,73], + [1479,3804.745,74], + [1500.5,3824.745,75], + [1525.47,3844.745,76], + [1537.33,3864.745,77], + [1563.29,3884.745,78], + [1574.71,3904.745,79], + [1593.71,3924.745,80], + [1615.52,3944.745,81], + [1636.41,3964.745,82], + [1658.21,3984.745,83], + [1679.43,4004.745,84], + [1689.69,4024.745,85], + [1724.05,4044.745,86], + [1733.2,4064.745,87], + [1754.76,4084.745,88], + [1775.61,4104.745,89], + [1798.86,4124.745,90], + [1818.39,4144.745,91], + [1845.12,4164.745,92], + [1857.11,4184.745,93], + [1884.52,4204.745,94], + [1892.91,4224.745,95], + [1921.56,4244.745,96], + [1940.71,4264.745,97], + [1957.94,4284.745,98], + [1980.16,4304.745,99], + [2000.9,4324.745,100], + [2012,4344.745,101], + [2047.71,4364.745,102], + [2053.75,4384.744,103], + [2075.34,4404.744,104], + [2097.69,4424.745,105], + [2119.7,4444.745,106], + [2139.53,4464.745,107], + [2159.56,4484.745,108], + [2173.4,4504.745,109], + [2206.64,4524.745,110], + [2213.55,4544.745,111], + [2237.86,4564.745,112], + [2258.59,4584.745,113], + [2282.83,4604.745,114], + [2297.83,4624.745,115], + [2319.55,4644.745,116], + [2329.47,4664.745,117], + [2361.44,4684.745,118], + [2376.64,4704.745,119], + [2394.76,4724.745,120], + [2420.92,4744.745,121], + [2436.81,4764.745,122], + [2460.55,4784.745,123], + [2479.14,4804.745,124], + [2489.11,4824.745,125], + [2521.52,4844.745,126], + [2531.84,4864.745,127], + [2553.61,4884.745,128], + [2574.44,4904.745,129], + [2596.02,4924.745,130], + [2617.52,4944.745,131], + [2638.58,4964.745,132], + [2649.87,4984.745,133], + [2681.42,5004.745,134], + [2693.43,5024.745,135], + [2713.46,5044.745,136], + [2734.76,5064.745,137], + [2756.99,5084.745,138], + [2778.25,5104.745,139], + [2806,5124.745,140], + [2817.2,5144.745,141], + [2845.81,5164.745,142], + [2854.02,5184.745,143], + [2879.26,5204.745,144], + [2897.34,5224.745,145], + [2923.3,5244.745,146], + [2940.44,5264.745,147], + [2969.34,5284.745,148], + [2976.18,5304.745,149], + [3010.24,5324.745,150], + [3013.92,5344.745,151], + [3035.22,5364.745,152], + [3056.81,5384.745,153], + [3076.8,5404.745,154], + [3098.04,5424.745,155], + [3119.11,5444.745,156], + [3129.03,5464.745,157], + [3161.53,5484.745,158], + [3172.28,5504.745,159], + [3193.96,5524.745,160], + [3214.4,5544.745,161], + [3235.97,5564.745,162], + [3256.9,5584.745,163], + [3278.24,5604.745,164], + [3288.81,5624.745,165], + [3320.78,5644.745,166], + [3332.48,5664.745,167], + [3353.17,5684.745,168], + [3374.69,5704.745,169], + [3396.59,5724.745,170], + [3419.17,5744.745,171], + [3438.88,5764.745,172], + [3448.31,5784.745,173], + [3485.27,5804.745,174], + [3491.39,5824.745,175], + [3513.07,5844.745,176], + [3534.63,5864.745,177], + [3555.46,5884.745,178], + [3576.88,5904.745,179], + [3600.12,5924.745,180], + [3609.53,5944.745,181], + [3641.17,5964.745,182], + [3651.9,5984.745,183], + [3673.49,6004.745,184], + [3695.59,6024.745,185], + [3716.17,6044.745,186], + [3737.64,6064.745,187], + [3759.28,6084.745,188], + [3769.84,6104.745,189], + [3803.56,6124.745,190], + [3812.66,6144.745,191], + [3833.34,6164.745,192], + [3855.37,6184.745,193], + [3876.13,6204.745,194], + [3900.31,6224.745,195], + [3920.87,6244.745,196], + [3931.56,6264.745,197], + [3969,6284.745,198], + [3976.17,6304.745,199], + [3995.25,6324.745,200], + [4016.06,6344.745,201], + [4039.69,6364.745,202], + [4061.94,6384.745,203], + [4081.93,6404.745,204], + [4095.02,6424.744,205], + [4129.21,6444.744,206], + [4141.96,6464.744,207], + [4159.78,6484.744,208], + [4181.74,6504.744,209], + [4205.51,6524.745,210], + [4222.18,6544.745,211], + [4242.58,6564.745,212], + [4253.75,6584.745,213], + [4283.58,6604.745,214], + [4296.46,6624.745,215], + [4316.04,6644.745,216], + [4336.97,6664.745,217], + [4357.46,6684.745,218], + [4378.55,6704.745,219], + [4400.38,6724.745,220], + [4410.74,6744.745,221], + [4442.89,6764.745,222], + [4453.32,6784.745,223], + [4474.75,6804.745,224], + [4496.09,6824.745,225], + [4516.89,6844.745,226], + [4538.28,6864.745,227], + [4561.66,6884.745,228], + [4571.86,6904.745,229], + [4606.17,6924.745,230], + [4618.99,6944.745,231], + [4643.04,6964.745,232], + [4660.18,6984.745,233], + [4678.34,7004.745,234], + [4702.41,7024.745,235], + [4724.15,7044.745,236], + [4735.25,7064.745,237], + [4767.56,7084.745,238], + [4775.88,7104.745,239], + [4797.39,7124.745,240], + [4817.46,7144.745,241], + [4838.72,7164.745,242], + [4860.78,7184.745,243], + [4884.25,7204.745,244], + [4893.92,7224.745,245], + [4927.98,7244.745,246], + [4938.37,7264.745,247], + [4959.11,7284.745,248], + [4980.63,7304.745,249], + [5006.03,7324.745,250], + [5021.33,7344.745,251], + [5046.1,7364.745,252], + [5052.44,7384.745,253], + [5086.43,7404.745,254], + [5095.18,7424.745,255], + [5122.14,7444.745,256], + [5137.73,7464.745,257], + [5159.36,7484.745,258], + [5179.19,7504.745,259], + [5202.47,7524.745,260], + [5210.44,7544.745,261], + [5245.55,7564.745,262], + [5254.5,7584.745,263], + [5277.09,7604.745,264], + [5297.06,7624.745,265], + [5318.03,7644.745,266], + [5338.72,7664.745,267], + [5360.22,7684.745,268], + [5370.72,7704.745,269], + [5402.89,7724.745,270], + [5413.28,7744.745,271], + [5434.61,7764.745,272], + [5455.65,7784.745,273], + [5476.71,7804.745,274], + [5498.66,7824.745,275], + [5519.25,7844.745,276], + [5530.03,7864.745,277], + [5562.89,7884.745,278], + [5572.77,7904.745,279], + [5594.59,7924.745,280], + [5615.39,7944.745,281], + [5637.25,7964.745,282], + [5658.91,7984.745,283], + [5680.14,8004.745,284], + [5693.58,8024.745,285], + [5728.24,8044.745,286], + [5738.1,8064.745,287], + [5754.95,8084.745,288], + [5775.96,8104.745,289], + [5796.54,8124.745,290], + [5828.16,8144.745,291], + [5852.67,8164.745,292], + [5858.34,8184.745,293], + [5893.3,8204.745,294], + [5905.38,8224.745,295], + [5923.08,8244.745,296], + [5941.58,8264.745,297], + [5970.14,8284.745,298], + [6004.44,8304.745,299], + [6015.33,8324.745,300], + [6023.34,8344.745,301], + [6051.55,8364.745,302], + [6058.03,8384.745,303], + [6086.87,8404.745,304], + [6113.36,8424.745,305], + [6135.62,8444.745,306], + [6152.66,8464.745,307], + [6170.19,8484.745,308], + [6183.62,8504.745,309], + [6218.41,8524.745,310], + [6238.54,8544.745,311], + [6277.71,8564.745,312], + [6289.04,8584.745,313], + [6289.32,8604.745,314], + [6302.76,8624.745,315], + [6354.31,8644.745,316], + [6376.12,8664.745,317], + [6385.07,8684.745,318], + [6385.53,8704.745,319], + [6396.94,8724.745,320], + [6427.9,8744.745,321], + [6446.93,8764.745,322], + [6459.51,8784.745,323], + [6483.94,8804.745,324], + [6497.64,8824.745,325], + [6534.07,8844.745,326], + [6536.19,8864.745,327], + [6556.34,8884.745,328], + [6582.03,8904.745,329], + [6600.72,8924.745,330], + [6617.41,8944.745,331], + [6646.89,8964.745,332], + [6660.89,8984.745,333], + [6690.03,9004.745,334], + [6696.5,9024.745,335], + [6727.05,9044.745,336], + [6740.59,9064.745,337], + [6770.2,9084.745,338], + [6782.53,9104.745,339], + [6811.18,9124.745,340], + [6812.75,9144.745,341], + [6875.05,9164.745,342], + [6915.56,9184.745,343], + [6955.68,9204.745,344], + [6956.06,9224.745,345], + [6960.73,9244.745,346], + [6961.72,9264.745,347], + [6966.06,9284.745,348], + [6977.21,9304.745,349], + [7014.69,9324.745,350], + [7020.27,9344.745,351], + [7045.12,9364.745,352], + [7080.42,9384.745,353], + [7088.71,9404.745,354], + [7103.48,9424.745,355], + [7157.58,9444.745,356], + [7157.97,9464.745,357], + [7179.94,9484.745,358], + [7195.72,9504.745,359], + [7204.87,9524.745,360], + [7230.26,9544.745,361], + [7249.69,9564.745,362], + [7276.37,9584.745,363], + [7290.15,9604.745,364], + [7321.94,9624.745,365], + [7337.89,9644.745,366], + [7350.89,9664.745,367], + [7376.19,9684.745,368], + [7389.08,9704.745,369], + [7438.18,9724.745,370], + [7438.5,9744.745,371], + [7459.15,9764.745,372], + [7470.24,9784.745,373], + [7501.81,9804.745,374], + [7510.25,9824.745,375], + [7560.51,9844.745,376], + [7573.98,9864.745,377], + [7580.71,9884.745,378], + [7609.84,9904.745,379], + [7613.38,9924.745,380], + [7622.24,9944.745,381], + [7652.93,9964.745,382], + [7663.46,9984.745,383], + [7685.13,10004.745,384], + [7706.59,10024.745,385], + [7734.78,10044.745,386], + [7752.29,10064.745,387], + [7771.87,10084.745,388], + [7780.34,10104.745,389], + [7815.23,10124.745,390], + [7822.97,10144.745,391], + [7844.47,10164.745,392], + [7867.55,10184.745,393], + [7887.53,10204.745,394], + [7909.25,10224.745,395], + [7940.42,10244.745,396], + [7946.36,10264.745,397], + [7972.52,10284.745,398], + [7994.24,10304.745,399], + [8009.22,10324.745,400], + [8034.42,10344.745,401], + [8047.44,10364.745,402], + [8069.04,10384.745,403], + [8097.35,10404.745,404], + [8106.03,10424.745,405], + [8139.49,10444.745,406], + [8145.06,10464.745,407], + [8177.67,10484.745,408], + [8188.15,10504.745,409], + [8209.38,10524.744,410], + [8229.46,10544.744,411], + [8250.23,10564.744,412], + [8260.28,10584.744,413], + [8292.84,10604.744,414], + [8303.52,10624.744,415], + [8325.19,10644.744,416], + [8347.74,10664.744,417], + [8375.09,10684.744,418], + [8390.02,10704.744,419], + [8409.69,10724.745,420], + [8420.53,10744.745,421], + [8452.44,10764.745,422], + [8464.09,10784.745,423], + [8484.45,10804.745,424], + [8505.84,10824.745,425], + [8527.18,10844.745,426], + [8548.19,10864.745,427], + [8569.14,10884.745,428], + [8579.77,10904.745,429], + [8613.44,10924.745,430], + [8623.16,10944.745,431], + [8644.72,10964.745,432], + [8666.02,10984.745,433], + [8687.23,11004.745,434], + [8708.54,11024.745,435], + [8730.54,11044.745,436], + [8740.94,11064.745,437], + [8772.76,11084.745,438], + [8783.44,11104.745,439], + [8805.15,11124.745,440], + [8827.06,11144.745,441], + [8848.43,11164.745,442], + [8875.22,11184.745,443], + [8894.25,11204.745,444], + [8903.74,11224.745,445], + [8932.69,11244.745,446], + [8951.23,11264.745,447], + [8967.19,11284.745,448], + [8989.07,11304.745,449], + [9014.3,11324.745,450], + [9028.79,11344.745,451], + [9049.97,11364.745,452], + [9060.53,11384.745,453], + [9092.59,11404.745,454], + [9104.07,11424.745,455], + [9124.4,11444.745,456], + [9145.61,11464.745,457], + [9167.22,11484.745,458], + [9193.52,11504.745,459], + [9210.05,11524.745,460], + [9220.27,11544.745,461], + [9252.12,11564.745,462], + [9264.11,11584.745,463], + [9284.44,11604.745,464], + [9305.84,11624.745,465], + [9326.97,11644.745,466], + [9347.84,11664.745,467], + [9369.13,11684.745,468], + [9379.46,11704.745,469], + [9411.81,11724.745,470], + [9423.47,11744.745,471], + [9443.53,11764.745,472], + [9464.91,11784.745,473], + [9487.03,11804.745,474], + [9508.34,11824.745,475], + [9529.37,11844.745,476], + [9540.44,11864.745,477], + [9572.81,11884.745,478], + [9583.1,11904.745,479], + [9611.71,11924.745,480], + [9630.78,11944.745,481], + [9651.86,11964.745,482], + [9670.19,11984.745,483], + [9690.71,12004.745,484], + [9701.97,12024.745,485], + [9732.03,12044.745,486], + [9742.03,12064.745,487], + [9769.29,12084.745,488], + [9786.52,12104.745,489], + [9807.66,12124.745,490], + [9831.64,12144.745,491], + [9851.36,12164.745,492], + [9871.03,12184.745,493], + [9897.81,12204.745,494], + [9908.14,12224.745,495], + [9930.15,12244.745,496], + [9949.59,12264.745,497], + [9979,12284.745,498], + [9998.72,12304.745,499], + [10012.69,12324.745,500], + [10027.22,12344.745,501], + [10056.15,12364.745,502], + [10067.18,12384.745,503], + [10087.2,12404.745,504], + [10116.02,12424.745,505], + [10131.27,12444.745,506], + [10153.25,12464.745,507], + [10180.82,12484.745,508], + [10186.06,12504.745,509], + [10218.92,12524.745,510], + [10232.14,12544.745,511], + [10248.27,12564.745,512], + [10272.27,12584.745,513], + [10294.34,12604.745,514], + [10308.91,12624.745,515], + [10335.43,12644.745,516], + [10351.22,12664.745,517], + [10397.18,12684.745,518], + [10397.61,12704.745,519], + [10423.97,12724.745,520], + [10443.56,12744.745,521], + [10454.58,12764.745,522], + [10479.71,12784.745,523], + [10497.21,12804.745,524], + [10506.87,12824.745,525], + [10548.34,12844.745,526], + [10550.89,12864.745,527], + [10572.92,12884.745,528], + [10612.44,12904.745,529], + [10634.27,12924.745,530], + [10641.4,12944.745,531], + [10733.73,12964.745,532], + [10735.82,12984.745,533], + [10755.88,13004.745,534], + [10756.77,13024.745,535], + [10761.41,13044.745,536], + [10777.55,13064.745,537], + [10780.15,13084.745,538], + [10789.19,13104.745,539], + [10815,13124.745,540], + [10829.51,13144.745,541], + [10853.84,13164.745,542], + [10864.53,13184.745,543], + [10884.49,13204.745,544], + [10906.02,13224.745,545], + [10927.5,13244.745,546], + [10948.28,13264.745,547], + [10970.81,13284.745,548], + [10980.9,13304.745,549], + [11015.89,13324.745,550], + [11025.84,13344.745,551], + [11044.63,13364.745,552], + [11066.77,13384.745,553], + [11102.83,13404.745,554], + [11129.58,13424.745,555], + [11140.81,13444.745,556], + [11148.97,13464.745,557], + [11183.4,13484.745,558], + [11204.62,13504.745,559], + [11234.13,13524.745,560], + [11238.59,13544.745,561], + [11261.01,13564.745,562], + [11282.53,13584.745,563], + [11296.55,13604.745,564], + [11305.35,13624.745,565], + [11342.18,13644.745,566], + [11350.14,13664.745,567], + [11369.53,13684.745,568], + [11388.52,13704.745,569], + [11407.11,13724.745,570], + [11428.29,13744.745,571], + [11450.28,13764.745,572], + [11459.62,13784.745,573], + [11491.94,13804.745,574], + [11502.94,13824.745,575], + [11524.53,13844.745,576], + [11545.75,13864.745,577], + [11568.57,13884.745,578], + [11602.66,13904.745,579], + [11621.11,13924.745,580], + [11629.3,13944.745,581], + [11666.22,13964.745,582], + [11677.65,13984.745,583], + [11694.01,14004.745,584], + [11718.39,14024.745,585], + [11734.83,14044.745,586], + [11755.08,14064.745,587], + [11802.39,14084.745,588], + [11808.72,14104.745,589], + [11845.81,14124.745,590], + [11847.44,14144.745,591], + [11847.73,14164.745,592], + [11868.5,14184.745,593], + [11903.06,14204.745,594], + [11918.23,14224.745,595], + [11942.19,14244.745,596], + [11954.33,14264.745,597], + [11982.12,14284.745,598], + [11993.61,14304.745,599], + [12010.03,14324.745,600], + [12032.87,14344.745,601], + [12052.44,14364.745,602], + [12087.72,14384.745,603], + [12098.55,14404.745,604], + [12108.81,14424.745,605], + [12138.69,14444.745,606], + [12154.22,14464.745,607], + [12175.03,14484.745,608], + [12194.24,14504.745,609], + [12282.15,14524.745,610], + [12332.19,14544.745,611], + [12360.18,14564.745,612], + [12451.8,14584.745,613], + [12531.83,14604.745,614], + [12683.12,14624.745,615], + [12683.84,14644.745,616], + [12694.85,14664.745,617], + [12695.43,14684.745,618], + [12716.44,14724.745,620], + [12719.15,14704.745,619], + [12798.54,14744.745,621], + [12802.88,14764.745,622], + [12815.83,14784.745,623], + [12816.4,14804.745,624], + [12816.99,14824.745,625], + [12817.26,14844.745,626], + [12826.02,14864.745,627], + [12826.47,14884.745,628], + [12826.78,14904.745,629], + [12827.66,14924.745,630], + [12827.97,14944.745,631], + [12829.15,14964.745,632], + [12829.38,14984.745,633], + [12829.62,15004.745,634], + [12829.89,15024.745,635], + [12830.14,15044.745,636], + [12830.42,15064.745,637], + [12854.89,15084.745,638], + [12861.39,15104.745,639], + [12885.99,15124.745,640], + [12905.95,15144.745,641], + [12927.51,15164.745,642], + [12952.18,15184.745,643], + [12974.49,15204.745,644], + [12981.92,15224.745,645], + [13012.3,15244.745,646], + [13021.81,15264.745,647], + [13051.53,15284.745,648], + [13080.22,15304.745,649], + [13098.98,15324.745,650], + [13112.16,15344.745,651], + [13150.78,15364.745,652], + [13151.26,15384.745,653], + [13178.52,15404.745,654], + [13182.64,15424.745,655], + [13206.18,15444.745,656], + [13230.7,15464.745,657], + [13245.95,15484.745,658], + [13278.9,15504.745,659], + [13311.47,15524.745,660], + [13323.11,15544.745,661], + [13344.96,15564.745,662], + [13355.49,15584.745,663], + [13372.67,15604.745,664], + [13388.43,15624.745,665], + [13407.59,15644.745,666], + [13432.84,15664.745,667], + [13451.96,15684.745,668], + [13465.4,15704.745,669], + [13496.84,15724.745,670], + [13508.34,15744.745,671], + [13526.87,15764.745,672], + [13543.02,15784.745,673], + [13580.31,15804.745,674], + [13587.09,15824.745,675], + [13611.06,15844.745,676], + [13623.06,15864.745,677], + [13659.03,15884.745,678], + [13665.41,15904.745,679], + [13687.12,15924.745,680], + [13712.53,15944.745,681], + [13727.86,15964.745,682], + [13760.34,15984.745,683], + [13771.49,16004.745,684], + [13787.6,16024.745,685], + [13818.52,16044.745,686], + [13833.63,16064.745,687], + [13847.19,16084.745,688], + [13872.12,16104.745,689], + [13887.81,16124.745,690], + [13910.15,16144.745,691], + [13933.8,16164.745,692], + [13940.68,16184.745,693], + [13970.79,16204.745,694], + [13981.33,16224.745,695], + [14009.81,16244.745,696], + [14025.07,16264.745,697], + [14045.11,16284.745,698], + [14073.75,16304.745,699], + [14087.62,16324.745,700], + [14104.05,16344.745,701], + [14135.34,16364.745,702], + [14140.66,16384.745,703], + [14167.45,16404.745,704], + [14187.02,16424.745,705], + [14211.64,16444.745,706], + [14230.84,16464.745,707], + [14250.61,16484.745,708], + [14261.1,16504.745,709], + [14295.06,16524.745,710], + [14314.29,16544.745,711], + [14327.08,16564.745,712], + [14352.14,16584.745,713], + [14370.44,16604.745,714], + [14398.46,16624.745,715], + [14408.61,16644.745,716], + [14422.25,16664.745,717], + [14453.64,16684.745,718], + [14464.58,16704.745,719], + [14484.72,16724.745,720], + [14503.41,16744.745,721], + [14529.94,16764.745,722], + [14549.86,16784.745,723], + [14574.08,16804.745,724], + [14578.49,16824.745,725], + [14610.43,16844.745,726], + [14623.51,16864.745,727], + [14660.34,16884.745,728], + [14672.86,16904.745,729], + [14683.66,16924.745,730], + [14707.01,16944.745,731], + [14725.82,16964.745,732], + [14744.59,16984.745,733], + [14770.94,17004.745,734], + [14782.34,17024.745,735], + [14805.1,17044.745,736], + [14825.07,17064.745,737], + [14852.14,17084.745,738], + [14875.42,17104.745,739], + [14888.42,17124.745,740], + [14897.26,17144.745,741], + [14930.65,17164.745,742], + [14945.52,17184.745,743], + [14967.65,17204.745,744], + [14989.06,17224.745,745], + [15006.61,17244.745,746], + [15034.49,17264.745,747], + [15054.65,17284.745,748], + [15063.22,17304.745,749], + [15100.63,17324.745,750], + [15104.33,17344.745,751], + [15127.38,17364.745,752], + [15144.07,17384.745,753], + [15169.18,17404.745,754], + [15188.78,17424.745,755], + [15213.14,17444.745,756], + [15223.88,17464.745,757], + [15254.47,17484.745,758], + [15264.47,17504.745,759], + [15293.2,17524.745,760], + [15312.44,17544.745,761], + [15332.38,17564.745,762], + [15351.06,17584.745,763], + [15371.37,17604.745,764], + [15382.61,17624.745,765], + [15415.22,17644.745,766], + [15423.31,17664.745,767], + [15446.81,17684.745,768], + [15465.96,17704.745,769], + [15487.5,17724.745,770], + [15515.33,17744.745,771], + [15531.84,17764.745,772], + [15540.25,17784.745,773], + [15577.82,17804.745,774], + [15582.69,17824.745,775], + [15607.14,17844.745,776], + [15627.72,17864.745,777], + [15652.97,17884.745,778], + [15679.69,17904.745,779], + [15689.29,17924.745,780], + [15700.97,17944.745,781], + [15730.15,17964.745,782], + [15743.86,17984.745,783], + [15765.91,18004.745,784], + [15789.95,18024.745,785], + [15806.11,18044.745,786], + [15835.46,18064.745,787], + [15849.26,18084.745,788], + [15863.13,18104.745,789], + [15897.47,18124.745,790], + [15910.25,18144.745,791], + [15925.78,18164.745,792], + [15948.27,18184.745,793], + [15973.72,18204.745,794], + [15989.05,18224.745,795], + [16021,18244.745,796], + [16027.06,18264.745,797], + [16054.81,18284.745,798], + [16064.04,18304.745,799], + [16082.87,18324.745,800], + [16111.78,18344.745,801], + [16175.69,18364.745,802], + [16181.22,18384.745,803], + [16199.99,18404.745,804], + [16200.38,18424.745,805], + [16226.21,18444.745,806], + [16231.28,18464.745,807], + [16248.06,18484.745,808], + [16266.75,18504.745,809], + [16289.65,18524.745,810], + [16308.06,18544.745,811], + [16331.99,18564.745,812], + [16346.34,18584.745,813], + [16372.28,18604.745,814], + [16384.81,18624.745,815], + [16409.17,18644.745,816], + [16428.72,18664.745,817], + [16451.56,18684.745,818], + [16471.49,18704.745,819], + [16490.24,18724.744,820], + [16503.59,18744.744,821], + [16538.94,18764.744,822], + [16543.86,18784.744,823], + [16562.74,18804.744,824], + [16585.99,18824.744,825], + [16605.67,18844.744,826], + [16631.43,18864.744,827], + [16648.22,18884.744,828], + [16662.86,18904.744,829], + [16693.09,18924.744,830], + [16701.08,18944.744,831], + [16729.03,18964.744,832], + [16746.9,18984.744,833], + [16768.94,19004.744,834], + [16790.95,19024.744,835], + [16810.13,19044.744,836], + [16822.26,19064.744,837], + [16850.33,19084.744,838], + [16863.34,19104.745,839], + [16886.06,19124.745,840], + [16904.81,19144.745,841], + [16930.92,19164.745,842], + [16950.69,19184.745,843], + [16968.87,19204.745,844], + [16979.43,19224.745,845], + [17014.25,19244.745,846], + [17023.69,19264.745,847], + [17044.06,19284.745,848], + [17065.96,19304.745,849], + [17093.69,19324.745,850], + [17107.68,19344.745,851], + [17135.9,19364.745,852], + [17144.09,19384.745,853], + [17170.83,19404.745,854], + [17182.14,19424.745,855], + [17206.22,19444.745,856], + [17223.94,19464.745,857], + [17255.64,19484.745,858], + [17273.94,19504.745,859], + [17290.37,19524.745,860], + [17301.68,19544.745,861], + [17335.52,19564.745,862], + [17340.09,19584.745,863], + [17364.03,19604.745,864], + [17391.93,19624.745,865], + [17407.31,19644.745,866], + [17433.54,19664.745,867], + [17448.95,19684.745,868], + [17463.24,19704.745,869], + [17494.03,19724.745,870], + [17502.15,19744.745,871], + [17522,19764.745,872], + [17546.28,19784.745,873], + [17568.71,19804.745,874], + [17594.83,19824.745,875], + [17611.31,19844.745,876], + [17623.48,19864.745,877], + [17660.58,19884.745,878], + [17669.5,19904.745,879], + [17686.73,19924.745,880], + [17713.12,19944.745,881], + [17728.88,19964.745,882], + [17750.34,19984.745,883], + [17769.96,20004.745,884], + [17780.31,20024.745,885], + [17812.25,20044.745,886], + [17823.72,20064.745,887], + [17848.38,20084.745,888], + [17869.72,20104.745,889], + [17888.93,20124.745,890], + [17918.28,20144.745,891], + [17935.94,20164.745,892], + [17942.34,20184.745,893], + [17974.57,20204.745,894], + [17990.8,20224.745,895], + [18007.68,20244.745,896], + [18028.65,20264.745,897], + [18046.36,20284.745,898], + [18074.06,20304.745,899], + [18089.4,20324.745,900], + [18098.12,20344.745,901], + [18130.6,20364.745,902], + [18144.64,20384.745,903], + [18164.89,20404.745,904], + [18186.75,20424.745,905], + [18204.84,20444.745,906], + [18228.94,20464.745,907], + [18254.34,20484.745,908], + [18261.08,20504.745,909], + [18291.64,20524.745,910], + [18303.61,20544.745,911], + [18321.26,20564.745,912], + [18346.66,20584.745,913], + [18367.45,20604.745,914], + [18387.37,20624.745,915], + [18412.18,20644.745,916], + [18420.23,20664.745,917], + [18457.21,20684.745,918], + [18465.16,20704.745,919], + [18487.44,20724.745,920], + [18505.69,20744.745,921], + [18528.34,20764.745,922], + [18551.11,20784.745,923], + [18569.35,20804.745,924], + [18581.07,20824.745,925], + [18613.65,20844.745,926], + [18623.19,20864.745,927], + [18640.87,20884.745,928], + [18666.65,20904.745,929], + [18684.64,20924.745,930], + [18712,20944.745,931], + [18730.96,20964.745,932], + [18737.5,20984.745,933], + [18780.64,21004.745,934], + [18787.29,21024.745,935], + [18805.08,21044.745,936], + [18826.11,21064.745,937], + [18857.3,21084.745,938], + [18875.96,21104.745,939], + [18893.22,21124.745,940], + [18903.55,21144.745,941], + [18932.99,21164.745,942], + [18940.84,21184.745,943], + [18962.22,21204.745,944], + [18987.97,21224.745,945], + [19004.69,21244.745,946], + [19027.34,21264.745,947], + [19055.22,21284.745,948], + [19066.23,21304.745,949], + [19099.7,21324.745,950], + [19104.93,21344.745,951], + [19127.88,21364.745,952], + [19146.53,21384.745,953], + [19170.06,21404.745,954], + [19187.59,21424.745,955], + [19214.95,21444.745,956], + [19220.57,21464.745,957], + [19254.45,21484.745,958], + [19269.09,21504.745,959], + [19289.28,21524.745,960], + [19306.27,21544.745,961], + [19329.89,21564.745,962], + [19349.15,21584.745,963], + [19374.47,21604.745,964], + [19385.59,21624.745,965], + [19414.19,21644.745,966], + [19424.64,21664.745,967], + [19445.22,21684.745,968], + [19465.8,21704.745,969], + [19486.59,21724.745,970], + [19506.53,21744.745,971], + [19532.21,21764.745,972], + [19538.52,21784.745,973], + [19572.13,21804.745,974], + [19580.68,21824.745,975], + [19606.17,21844.745,976], + [19627.29,21864.745,977], + [19645.94,21884.745,978], + [19667.35,21904.745,979], + [19688.23,21924.745,980], + [19701.83,21944.745,981], + [19740.31,21964.745,982], + [19746.73,21984.745,983], + [19767.14,22004.745,984], + [19788.09,22024.745,985], + [19809.81,22044.745,986], + [19828.34,22064.745,987], + [19852.69,22084.745,988], + [19861.76,22104.745,989], + [19897.88,22124.745,990], + [19905.31,22144.745,991], + [19925.62,22164.745,992], + [19952.78,22184.745,993], + [19973.52,22204.745,994], + [19988.46,22224.745,995], + [20010.23,22244.745,996], + [20018.75,22264.745,997], + [20053.92,22284.745,998], + [20061.94,22304.745,999], + [20092.01,22324.745,1000], + [20108.16,22344.745,1001], + [20126.81,22364.745,1002], + [20150.27,22384.745,1003], + [20172.22,22404.745,1004], + [20186.41,22424.745,1005], + [20231.36,22444.745,1006], + [20233.46,22464.745,1007], + [20245.02,22484.745,1008], + [20276.72,22504.745,1009], + [20291.19,22524.745,1010], + [20307.55,22544.745,1011], + [20329.33,22564.745,1012], + [20338.57,22584.745,1013], + [20377.56,22604.745,1014], + [20398.18,22624.745,1015], + [20416.47,22644.745,1016], + [20424.66,22664.745,1017], + [20445.69,22684.745,1018], + [20473.22,22704.745,1019], + [20489.61,22724.745,1020], + [20499.79,22744.745,1021], + [20561.76,22764.745,1022], + [20566.46,22784.745,1023], + [20569.67,22804.745,1024], + [20595.55,22824.745,1025], + [20608.5,22844.745,1026], + [20629.6,22864.745,1027], + [20649.08,22884.745,1028], + [20663.03,22904.745,1029], + [20701.24,22924.745,1030], + [20708.01,22944.745,1031], + [20726.02,22964.745,1032], + [20747.29,22984.745,1033], + [20765.89,23004.745,1034], + [20787.19,23024.745,1035], + [20812.89,23044.745,1036], + [20821.8,23064.745,1037], + [20852.03,23084.745,1038], + [20862.96,23104.745,1039], + [20889.65,23124.745,1040], + [20905.12,23144.745,1041], + [20927.56,23164.745,1042], + [20946.2,23184.745,1043], + [20970.25,23204.745,1044], + [20978.92,23224.745,1045], + [21019.22,23244.745,1046], + [21027.17,23264.745,1047], + [21046.14,23284.745,1048], + [21064.84,23304.745,1049], + [21085.66,23324.745,1050], + [21109.81,23344.745,1051], + [21132.46,23364.745,1052], + [21153.24,23384.745,1053], + [21185.29,23404.745,1054], + [21193.65,23424.745,1055], + [21204.17,23444.745,1056], + [21230.16,23464.745,1057], + [21255.79,23484.745,1058], + [21271.3,23504.745,1059], + [21289.11,23524.745,1060], + [21297.08,23544.745,1061], + [21331.6,23564.745,1062], + [21339.07,23584.745,1063], + [21362.55,23604.745,1064], + [21387,23624.745,1065], + [21406.25,23644.745,1066], + [21425.71,23664.745,1067], + [21450.5,23684.745,1068], + [21460.08,23704.745,1069], + [21496.71,23724.745,1070], + [21508.12,23744.745,1071], + [21535.12,23764.745,1072], + [21553.94,23784.745,1073], + [21566.16,23804.745,1074], + [21592.14,23824.745,1075], + [21610.49,23844.745,1076], + [21623.01,23864.745,1077], + [21650.72,23884.745,1078], + [21665.94,23904.745,1079], + [21687.99,23924.745,1080], + [21705.19,23944.745,1081], + [21725.18,23964.745,1082], + [21749.41,23984.745,1083], + [21772.69,24004.745,1084], + [21777.97,24024.745,1085], + [21813.3,24044.745,1086], + [21821.05,24064.745,1087], + [21846.72,24084.745,1088], + [21864.73,24104.745,1089], + [21887.8,24124.745,1090], + [21909.22,24144.745,1091], + [21927.68,24164.745,1092], + [21944.41,24184.745,1093], + [21975.99,24204.745,1094], + [21981.84,24224.745,1095], + [22005.43,24244.745,1096], + [22050.17,24264.745,1097], + [22050.56,24284.745,1098], + [22067.53,24304.745,1099], + [22088.01,24324.745,1100], + [22099.11,24344.745,1101], + [22130.08,24364.745,1102], + [22140.17,24384.745,1103], + [22166.05,24404.745,1104], + [22186.64,24424.745,1105], + [22208.59,24444.745,1106], + [22236.03,24464.745,1107], + [22247.54,24484.745,1108], + [22262.98,24504.745,1109], + [22297.67,24524.745,1110], + [22302.87,24544.745,1111], + [22324.64,24564.745,1112], + [22343.21,24584.745,1113], + [22364.76,24604.745,1114], + [22388.21,24624.745,1115], + [22409.63,24644.745,1116], + [22420.59,24664.745,1117], + [22452.99,24684.745,1118], + [22463.58,24704.745,1119], + [22481.88,24724.745,1120], + [22509.56,24744.745,1121], + [22527.5,24764.745,1122], + [22545.19,24784.745,1123], + [22570.21,24804.745,1124], + [22582.88,24824.745,1125], + [22615.07,24844.745,1126], + [22622.19,24864.745,1127], + [22660.94,24884.745,1128], + [22674.41,24904.745,1129], + [22692.94,24924.745,1130], + [22715.66,24944.745,1131], + [22730.52,24964.745,1132], + [22748.88,24984.745,1133], + [22774.78,25004.745,1134], + [22780.86,25024.745,1135], + [22818.93,25044.745,1136], + [22832.89,25064.745,1137], + [22849.34,25084.745,1138], + [22874.8,25104.745,1139], + [22892.87,25124.745,1140], + [22901.09,25144.745,1141], + [22931.56,25164.745,1142], + [22940.98,25184.745,1143], + [22965.55,25204.745,1144], + [22982.46,25224.745,1145], + [23003.8,25244.745,1146], + [23039.55,25264.745,1147], + [23052.64,25284.745,1148], + [23058.4,25304.745,1149], + [23095.96,25324.745,1150], + [23106.78,25344.745,1151], + [23133.42,25364.745,1152], + [23144.36,25384.745,1153], + [23167.58,25404.745,1154], + [23187.89,25424.745,1155], + [23210.56,25444.745,1156], + [23224.16,25464.745,1157], + [23254.64,25484.745,1158], + [23265.24,25504.745,1159], + [23289.8,25524.745,1160], + [23305.65,25544.745,1161], + [23325.92,25564.745,1162], + [23351.41,25584.745,1163], + [23373.47,25604.745,1164], + [23387.58,25624.745,1165], + [23413.25,25644.745,1166], + [23419.11,25664.745,1167], + [23445.47,25684.745,1168], + [23465.53,25704.745,1169], + [23494.52,25724.745,1170], + [23509.51,25744.745,1171], + [23530.47,25764.745,1172], + [23541.52,25784.745,1173], + [23568.65,25804.745,1174], + [23582.87,25824.745,1175], + [23605.06,25844.745,1176], + [23627.37,25864.745,1177], + [23649.82,25884.745,1178], + [23677.29,25904.745,1179], + [23695.42,25924.745,1180], + [23705.53,25944.745,1181], + [23744.03,25964.745,1182], + [23750.62,25984.745,1183], + [23770.56,26004.745,1184], + [23791.64,26024.745,1185], + [23812.78,26044.745,1186], + [23833.56,26064.745,1187], + [23848.75,26084.745,1188], + [23859.92,26104.745,1189], + [23897.21,26124.745,1190], + [23905.83,26144.745,1191], + [23931,26164.745,1192], + [23953.71,26184.745,1193], + [23973.83,26204.745,1194], + [23990.6,26224.745,1195], + [24013.61,26244.745,1196], + [24030.32,26264.745,1197], + [24057.11,26284.745,1198], + [24074.47,26304.745,1199], + [24096.37,26324.745,1200], + [24112.47,26344.745,1201], + [24135.78,26364.745,1202], + [24147.33,26384.745,1203], + [24168.06,26404.745,1204], + [24193.34,26424.745,1205], + [24215.89,26444.745,1206], + [24225.75,26464.745,1207], + [24246.2,26484.745,1208], + [24276.68,26504.745,1209], + [24288.53,26524.745,1210], + [24311.25,26544.745,1211], + [24331.06,26564.745,1212], + [24342.69,26584.745,1213], + [24393.66,26604.745,1214], + [24394.7,26624.745,1215], + [24421.53,26644.745,1216], + [24431.37,26664.745,1217], + [24445.93,26684.745,1218], + [24472.03,26704.745,1219], + [24494.1,26724.745,1220], + [24504.47,26744.745,1221], + [24536.87,26764.745,1222], + [24545.84,26784.745,1223], + [24564.44,26804.745,1224], + [24589.21,26824.745,1225], + [24609.4,26844.745,1226], + [24632.79,26864.745,1227], + [24654.19,26884.745,1228], + [24662.9,26904.745,1229], + [24693.98,26924.745,1230], + [24701.06,26944.745,1231], + [24726.84,26964.745,1232], + [24748.64,26984.745,1233], + [24767.44,27004.745,1234], + [24797.2,27024.745,1235], + [24819.72,27044.745,1236], + [24829.07,27064.745,1237], + [24850.47,27084.745,1238], + [24867.72,27104.745,1239], + [24887.34,27124.745,1240], + [24906.53,27144.745,1241], + [24928.09,27164.745,1242], + [24947.66,27184.745,1243], + [24967.13,27204.745,1244], + [24979.17,27224.745,1245], + [25023.84,27244.745,1246], + [25028.97,27264.745,1247], + [25047.78,27284.745,1248], + [25071.99,27304.745,1249], + [25097.85,27324.745,1250], + [25113.73,27344.745,1251], + [25133.9,27364.745,1252], + [25142.75,27384.745,1253], + [25175.37,27404.745,1254], + [25185.5,27424.745,1255], + [25214.49,27444.745,1256], + [25229.84,27464.745,1257], + [25249.69,27484.745,1258], + [25271.05,27504.745,1259], + [25292.97,27524.745,1260], + [25305.13,27544.745,1261], + [25330.65,27564.745,1262], + [25342.11,27584.745,1263], + [25363.18,27604.745,1264], + [25386.65,27624.745,1265], + [25411.79,27644.745,1266], + [25436.82,27664.745,1267], + [25451.47,27684.745,1268], + [25462.09,27704.745,1269], + [25495.34,27724.745,1270], + [25504.92,27744.745,1271], + [25525.29,27764.745,1272], + [25554.46,27784.745,1273], + [25570.49,27804.745,1274], + [25589.7,27824.745,1275], + [25611.96,27844.745,1276], + [25628.56,27864.745,1277], + [25659.13,27884.745,1278], + [25672.93,27904.745,1279], + [25689.77,27924.745,1280], + [25706.4,27944.745,1281], + [25734.28,27964.745,1282], + [25753.91,27984.745,1283], + [25770.07,28004.745,1284], + [25786.1,28024.745,1285], + [25814.82,28044.745,1286], + [25824.55,28064.745,1287], + [25854.02,28084.745,1288], + [25865.66,28104.745,1289], + [25898.96,28124.745,1290], + [25911.78,28144.745,1291], + [25936.31,28164.745,1292], + [25939.53,28184.745,1293], + [25976.37,28204.745,1294], + [25985.36,28224.745,1295], + [26007.09,28244.745,1296], + [26035.46,28264.745,1297], + [26047.05,28284.745,1298], + [26068.22,28304.745,1299], + [26091.13,28324.745,1300], + [26108.54,28344.745,1301], + [26130.51,28364.745,1302], + [26147.74,28384.745,1303], + [26164.17,28404.745,1304], + [26184.77,28424.745,1305], + [26210.68,28444.745,1306], + [26226.63,28464.745,1307], + [26252.03,28484.745,1308], + [26257.62,28504.745,1309], + [26297.75,28524.745,1310], + [26306.53,28544.745,1311], + [26329.25,28564.745,1312], + [26347.22,28584.745,1313], + [26365.19,28604.745,1314], + [26387.06,28624.745,1315], + [26411.18,28644.745,1316], + [26420.76,28664.745,1317], + [26454.2,28684.745,1318], + [26463.83,28704.745,1319], + [26486.19,28724.745,1320], + [26502.83,28744.745,1321], + [26526.8,28764.745,1322], + [26548.26,28784.745,1323], + [26571.25,28804.745,1324], + [26585.75,28824.745,1325], + [26614.65,28844.745,1326], + [26631.75,28864.745,1327], + [26648.63,28884.745,1328], + [26667.74,28904.745,1329], + [26690.03,28924.745,1330], + [26713.28,28944.745,1331], + [26732.05,28964.745,1332], + [26738.81,28984.745,1333], + [26776.54,29004.745,1334], + [26789.47,29024.745,1335], + [26807.59,29044.745,1336], + [26838.52,29064.745,1337], + [26844.18,29084.745,1338], + [26872.27,29104.745,1339], + [26898.55,29124.745,1340], + [26907.36,29144.745,1341], + [26939.59,29164.745,1342], + [26942.64,29184.745,1343], + [26964.44,29204.745,1344], + [26989.94,29224.745,1345], + [27007.91,29244.745,1346], + [27033.69,29264.745,1347], + [27059.46,29284.745,1348], + [27066.43,29304.745,1349], + [27098,29324.745,1350], + [27110.91,29344.745,1351], + [27125.27,29364.745,1352], + [27145.16,29384.745,1353], + [27170.99,29404.745,1354], + [27213.57,29424.745,1355], + [27215.77,29444.745,1356], + [27218.53,29464.745,1357], + [27256.02,29484.745,1358], + [27262.27,29504.745,1359], + [27288.41,29524.745,1360], + [27311.98,29544.745,1361], + [27335.71,29564.745,1362], + [27353.25,29584.745,1363], + [27367.69,29604.745,1364], + [27378.59,29624.745,1365], + [27412,29644.745,1366], + [27421.5,29664.745,1367], + [27449.03,29684.745,1368], + [27472.77,29704.745,1369], + [27488.81,29724.745,1370], + [27514.43,29744.745,1371], + [27540.24,29764.745,1372], + [27546.29,29784.745,1373], + [27576.06,29804.745,1374], + [27587.97,29824.745,1375], + [27609,29844.745,1376], + [27629.24,29864.745,1377], + [27658.08,29884.745,1378], + [27671.68,29904.745,1379], + [27692.02,29924.745,1380], + [27697.62,29944.745,1381], + [27746.64,29964.745,1382], + [27747.31,29984.745,1383], + [27768.29,30004.745,1384], + [27793.66,30024.745,1385], + [27805.45,30044.745,1386], + [27834.9,30064.745,1387], + [27848.76,30084.745,1388], + [27868.08,30104.745,1389], + [27899.23,30124.745,1390], + [27905.76,30144.745,1391], + [27924.47,30164.745,1392], + [27950.41,30184.745,1393], + [27964.39,30204.745,1394], + [27989.23,30224.745,1395], + [28011.61,30244.745,1396], + [28029.47,30264.745,1397], + [28059.93,30284.745,1398], + [28069.34,30304.745,1399], + [28087.37,30324.745,1400], + [28104.95,30344.745,1401], + [28130.36,30364.745,1402], + [28148.51,30384.745,1403], + [28179.84,30404.745,1404], + [28232.78,30424.745,1405], + [28235.91,30444.745,1406], + [28236.37,30464.745,1407], + [28249.16,30484.745,1408], + [28268.32,30504.745,1409], + [28288.95,30524.745,1410], + [28311.84,30544.745,1411], + [28329.74,30564.745,1412], + [28336.01,30584.745,1413], + [28371.04,30604.745,1414], + [28389.23,30624.745,1415], + [28404.53,30644.745,1416], + [28436.23,30664.745,1417], + [28452.4,30684.745,1418], + [28472.75,30704.745,1419], + [28504.17,30724.745,1420], + [28504.5,30744.745,1421], + [28536.56,30764.745,1422], + [28542.55,30784.745,1423], + [28567.82,30804.745,1424], + [28593.77,30824.745,1425], + [28616.89,30844.745,1426], + [28630.41,30864.745,1427], + [28667.61,30884.745,1428], + [28668.33,30904.745,1429], + [28694.44,30924.745,1430], + [28702.15,30944.745,1431], + [28730.61,30964.745,1432], + [28747.68,30984.745,1433], + [28776.28,31004.745,1434], + [28788.25,31024.745,1435], + [28809.94,31044.745,1436], + [28826.78,31064.745,1437], + [28860.98,31084.745,1438], + [28877.76,31104.745,1439], + [28885.77,31124.745,1440], + [28906.71,31144.745,1441], + [28928.09,31164.745,1442], + [28954.15,31184.745,1443], + [28978.52,31204.745,1444], + [28982.46,31224.745,1445], + [29025.44,31244.745,1446], + [29034.44,31264.745,1447], + [29055.74,31284.745,1448], + [29069.68,31304.745,1449], + [29092.09,31324.745,1450], + [29108.25,31344.745,1451], + [29129.72,31364.745,1452], + [29140.09,31384.745,1453], + [29176.63,31404.745,1454], + [29194.42,31424.745,1455], + [29211.42,31444.745,1456], + [29231.97,31464.745,1457], + [29256.81,31484.745,1458], + [29269.43,31504.745,1459], + [29304.09,31524.745,1460], + [29307.56,31544.745,1461], + [29340.84,31564.745,1462], + [29361.66,31584.745,1463], + [29372.77,31604.745,1464], + [29404.91,31624.745,1465], + [29420.14,31644.745,1466], + [29432.52,31664.745,1467], + [29466.02,31684.745,1468], + [29478.46,31704.745,1469], + [29511.13,31724.745,1470], + [29523.84,31744.745,1471], + [29524.63,31764.745,1472], + [29544.81,31784.745,1473], + [29572.58,31804.745,1474], + [29592.4,31824.745,1475], + [29612.09,31844.745,1476], + [29623.78,31864.745,1477], + [29656.72,31884.745,1478], + [29670.59,31904.745,1479], + [29694.69,31924.745,1480], + [29707.62,31944.745,1481], + [29732.81,31964.745,1482], + [29751.53,31984.745,1483], + [29774.58,32004.745,1484], + [29779.78,32024.745,1485], + [29826.26,32044.745,1486], + [29826.69,32064.745,1487], + [29844.53,32084.745,1488], + [29868.37,32104.745,1489], + [29889.2,32124.745,1490], + [29910.3,32144.745,1491], + [29927.41,32164.745,1492], + [29943.92,32184.745,1493], + [29974.88,32204.745,1494], + [29986.15,32224.745,1495], + [30010.55,32244.745,1496], + [30029.15,32264.745,1497], + [30052.4,32284.745,1498], + [30070.1,32304.745,1499], + [30088.61,32324.745,1500], + [30102.3,32344.745,1501], + [30133.39,32364.745,1502], + [30143.52,32384.745,1503], + [30175.86,32404.745,1504], + [30188.72,32424.745,1505], + [30215.64,32444.745,1506], + [30232.71,32464.745,1507], + [30255.94,32484.745,1508], + [30264.34,32504.745,1509], + [30294.22,32524.745,1510], + [30307.52,32544.745,1511], + [30328.74,32564.745,1512], + [30346.08,32584.745,1513], + [30373.62,32604.745,1514], + [30392.43,32624.745,1515], + [30423.02,32644.745,1516], + [30428.04,32664.745,1517], + [30453.97,32684.745,1518], + [30464.41,32704.745,1519], + [30485.18,32724.745,1520], + [30508.08,32744.745,1521], + [30531.86,32764.745,1522], + [30547.62,32784.745,1523], + [30571.14,32804.745,1524], + [30582.96,32824.745,1525], + [30619.22,32844.745,1526], + [30624.19,32864.745,1527], + [30644.94,32884.745,1528], + [30672.69,32904.745,1529], + [30690,32924.745,1530], + [30718.78,32944.745,1531], + [30728.33,32964.745,1532], + [30741.84,32984.745,1533], + [30774.84,33004.745,1534], + [30780.37,33024.745,1535], + [30813.11,33044.745,1536], + [30829.81,33064.745,1537], + [30847.98,33084.745,1538], + [30878.75,33104.745,1539], + [30901.28,33124.745,1540], + [30902.58,33144.745,1541], + [30934.74,33164.745,1542], + [30964.14,33184.745,1543], + [30972.14,33204.745,1544], + [30987.01,33224.745,1545], + [31005.79,33244.745,1546], + [31030.16,33264.745,1547], + [31051.55,33284.745,1548], + [31057.52,33304.745,1549], + [31096.49,33324.745,1550], + [31107.38,33344.745,1551], + [31133.6,33364.745,1552], + [31167.45,33384.745,1553], + [31171.58,33404.745,1554], + [31193.78,33424.745,1555], + [31234.72,33444.745,1556], + [31247.71,33464.745,1557], + [31261.3,33484.745,1558], + [31273.39,33504.745,1559], + [31300.96,33524.745,1560], + [31315.84,33544.745,1561], + [31341.75,33564.745,1562], + [31358.54,33584.745,1563], + [31372.28,33604.745,1564], + [31383.66,33624.745,1565], + [31423.57,33644.745,1566], + [31429.9,33664.745,1567], + [31446.09,33684.745,1568], + [31467.75,33704.745,1569], + [31490.14,33724.745,1570], + [31514.66,33744.745,1571], + [31531.77,33764.745,1572], + [31546.41,33784.745,1573], + [31583.51,33804.745,1574], + [31590.83,33824.745,1575], + [31606.84,33844.745,1576], + [31629.36,33864.745,1577], + [31648.56,33884.745,1578], + [31683.84,33904.745,1579], + [31695.71,33924.745,1580], + [31708.11,33944.745,1581], + [31739.5,33964.745,1582], + [31749.13,33984.745,1583], + [31769.66,34004.745,1584], + [31786.4,34024.745,1585], + [31810.02,34044.745,1586], + [31829.59,34064.745,1587], + [31862.28,34084.745,1588], + [31869.93,34104.745,1589], + [31892.07,34124.745,1590], + [31910.09,34144.745,1591], + [31922.6,34164.745,1592], + [31955.69,34184.745,1593], + [31977.59,34204.745,1594], + [31990.16,34224.745,1595], + [32013.18,34244.745,1596], + [32022.53,34264.745,1597], + [32055.84,34284.745,1598], + [32061.38,34304.745,1599], + [32087.96,34324.745,1600], + [32111.78,34344.745,1601], + [32138.59,34364.745,1602], + [32145.59,34384.745,1603], + [32174.11,34404.745,1604], + [32182.26,34424.745,1605], + [32212.87,34444.745,1606], + [32225.2,34464.745,1607], + [32251.73,34484.745,1608], + [32270.94,34504.745,1609], + [32289.26,34524.745,1610], + [32309.45,34544.745,1611], + [32330.4,34564.745,1612], + [32342.56,34584.745,1613], + [32373.28,34604.745,1614], + [32382.06,34624.745,1615], + [32409.91,34644.745,1616], + [32428.18,34664.745,1617], + [32446.94,34684.745,1618], + [32468.71,34704.745,1619], + [32488.75,34724.745,1620], + [32502.42,34744.745,1621], + [32532.14,34764.745,1622], + [32542,34784.745,1623], + [32566.9,34804.745,1624], + [32588.19,34824.745,1625], + [32607.52,34844.745,1626], + [32628.47,34864.745,1627], + [32649.21,34884.745,1628], + [32665.42,34904.745,1629], + [32704.83,34924.745,1630], + [32714.33,34944.745,1631], + [32725.23,34964.745,1632], + [32756.48,34984.745,1633], + [32766.21,35004.745,1634], + [32786.23,35024.745,1635], + [32808.31,35044.745,1636], + [32843.97,35064.745,1637], + [32848.81,35084.745,1638], + [32871.66,35104.744,1639], + [32884.82,35124.744,1640], + [32909.82,35144.744,1641], + [32925.68,35164.744,1642], + [32953.94,35184.744,1643], + [32978.05,35204.744,1644], + [32984.49,35224.744,1645], + [33016.7,35244.744,1646], + [33023.36,35264.744,1647], + [33055.98,35284.744,1648], + [33069.43,35304.744,1649], + [33090.6,35324.744,1650], + [33108.59,35344.744,1651], + [33133.73,35364.744,1652], + [33138.11,35384.744,1653], + [33180.16,35404.744,1654], + [33198.44,35424.744,1655], + [33215.02,35444.744,1656], + [33244.39,35464.744,1657], + [33253.67,35484.744,1658], + [33277.44,35504.744,1659], + [33289.75,35524.744,1660], + [33308.04,35544.744,1661], + [33336.46,35564.744,1662], + [33348.34,35584.744,1663], + [33379.66,35604.744,1664], + [33394.03,35624.744,1665], + [33467.68,35644.744,1666], + [33487.94,35664.744,1667], + [33488.19,35684.744,1668], + [33488.42,35704.744,1669], + [33494.14,35724.744,1670], + [33504.19,35744.744,1671], + [33523.07,35764.744,1672], + [33551.59,35784.744,1673], + [33566.06,35804.744,1674], + [33596.17,35824.744,1675], + [33613.97,35844.744,1676], + [33625.5,35864.744,1677], + [33658.05,35884.745,1678], + [33663.83,35904.745,1679], + [33690.91,35924.745,1680], + [33723.63,35944.745,1681], + [33726.52,35964.745,1682], + [33754.03,35984.745,1683], + [33774.93,36004.745,1684], + [33788.19,36024.745,1685], + [33810.69,36044.745,1686], + [33820.76,36064.745,1687], + [33843.05,36084.745,1688], + [33870.92,36104.745,1689], + [33888.27,36124.745,1690], + [33923.34,36144.745,1691], + [33930.84,36164.745,1692], + [33938.01,36184.745,1693], + [33981.11,36204.745,1694], + [33984.38,36224.745,1695], + [34008.41,36244.745,1696], + [34029.34,36264.745,1697], + [34056.3,36284.745,1698], + [34074.65,36304.745,1699], + [34094.26,36324.745,1700], + [34107.72,36344.745,1701], + [34141.99,36364.745,1702], + [34148.11,36384.745,1703], + [34174.3,36404.745,1704], + [34187.58,36424.745,1705], + [34210.09,36444.745,1706], + [34238.46,36464.745,1707], + [34253.78,36484.745,1708], + [34263.96,36504.745,1709], + [34313.14,36524.745,1710], + [34314.44,36544.745,1711], + [34328.59,36564.745,1712], + [34353.72,36584.745,1713], + [34375.72,36604.745,1714], + [34395.25,36624.745,1715], + [34430.81,36644.745,1716], + [34474.11,36664.745,1717], + [34510.88,36684.745,1718], + [34511.25,36704.745,1719], + [34511.44,36724.745,1720], + [34521.42,36744.745,1721], + [34537.44,36764.745,1722], + [34552.49,36784.745,1723], + [34584.16,36804.745,1724], + [34593.16,36824.745,1725], + [34613.93,36844.745,1726], + [34625.03,36864.745,1727], + [34650.06,36884.745,1728], + [34667.72,36904.745,1729], + [34693.82,36924.745,1730], + [34721.68,36944.745,1731], + [34743.8,36964.745,1732], + [34748.39,36984.745,1733], + [34780.86,37004.745,1734], + [34797.81,37024.745,1735], + [34815.28,37044.745,1736], + [34832.37,37064.745,1737], + [34859.22,37084.745,1738], + [34880.69,37104.745,1739], + [34898.72,37124.745,1740], + [34909.97,37144.745,1741], + [34936.79,37164.745,1742], + [34951.06,37184.745,1743], + [34967.18,37204.745,1744] + ] +} diff --git a/test/drain/README.md b/test/drain/README.md new file mode 100644 index 00000000000..a5b821233b5 --- /dev/null +++ b/test/drain/README.md @@ -0,0 +1,54 @@ +# Relay drain harness + +A viewer watching a live track through a relay that drains moves to a sibling +relay behind the same name without missing a group. This is the whole drain +story at once: the relay's GOAWAY, the JS client's migration on it, and the +origin handing the path over while the old session still serves. + +## Running + +```bash +just test drain +``` + +```bash +just test drain --timeout 120 +``` + +`cargo build -p moq-relay` builds the relay from this checkout. `DRAIN_PROFILE` +picks its cargo profile, and `RELAY_BIN` points at a prebuilt relay instead. +Ports come from the shared reservation (see [the harness contract](../README.md)). +A failing run keeps its run directory with both relay logs; a passing run +deletes its own. + +## Shape + +`run.sh` starts relay B, then relay A clustered to B, and hands both to +`drain.ts`, which plays every other part: + +- **The name.** A TCP proxy stands in for DNS. The viewer dials it and lands on + A. Pointing it at B before draining A is the fleet's DNS withdrawal: nothing + new resolves to A, while the sessions already there stay. +- **The publisher** sits on B and writes one group every 100 ms, each carrying + its own sequence number. A pulls the track through the cluster, so both relays + serve the same groups. +- **The viewer** is `@moq/net` over WebSocket, following whichever broadcast the + path routes to the way a player does: subscribe to the new one, drop the old. + +Once the viewer has read groups through A, the driver withdraws A from the name +and sends it SIGTERM, which fires the same trigger an embedder's drain hook +does. The run passes when: + +- the viewer reads fresh groups through B within a few seconds, well inside A's + 20 s drain window, having dialed A exactly once; +- every group from the first read through the last arrived, from one relay or + the other; +- the viewer leaves A at its 2 s handover cap, and A then exits on its own, + logging that every session left rather than that its deadline forced one out. + +The viewer subscribes with a 1 s latency budget, as the interop subscribers do. +After the swap, B has to subscribe upstream afresh once A drops its pull, and the +budget is what reaches back to a group in flight across the swap. With no budget, +a group boundary that lands inside the swap loses that group. +[JS group-boundary handover](/quest/m1/js-group-handover.md) removes the +budget by carrying the subscription across the swap. diff --git a/test/drain/drain.ts b/test/drain/drain.ts new file mode 100644 index 00000000000..59716364ebc --- /dev/null +++ b/test/drain/drain.ts @@ -0,0 +1,235 @@ +/** + * Drives one relay drain end to end: a viewer watching a live track through relay A, behind a + * stand-in for DNS, migrates to relay B when A drains, without missing a group. + * + * The "name" the viewer dials is a TCP proxy owned by this script. Pointing it at B before A + * drains is the fleet's DNS withdrawal: nothing new resolves to A, while sessions already on A + * stay put. A then gets SIGTERM, sends every session a GOAWAY, and the viewer redials the same + * URL, which now lands on B. The publisher sits on B and A pulls it through the cluster, so both + * relays carry the same track with the same group numbers. + * + * bun drain.ts --a-port 4470 --b-port 4471 --proxy-port 4472 --a-pid 1234 --timeout 60 + * + * Exits 0 once the viewer has migrated, read groups on both relays with none missing in + * between, and left A on its own; 1 otherwise. + * + * @module + */ +import * as net from "node:net"; +import { parseArgs } from "node:util"; +import * as Moq from "@moq/net"; + +const { values } = parseArgs({ + options: { + "a-port": { type: "string" }, + "b-port": { type: "string" }, + "proxy-port": { type: "string" }, + "a-pid": { type: "string" }, + timeout: { type: "string", default: "60" }, + }, +}); + +const aPort = Number(values["a-port"]); +const bPort = Number(values["b-port"]); +const proxyPort = Number(values["proxy-port"]); +const aPid = Number(values["a-pid"]); +const timeoutMs = Number(values.timeout) * 1000; +if (![aPort, bPort, proxyPort, aPid, timeoutMs].every((n) => Number.isInteger(n) && n > 0)) { + console.error("usage: drain.ts --a-port P --b-port P --proxy-port P --a-pid PID [--timeout S]"); + process.exit(2); +} + +// Groups the viewer must read on each relay: enough to prove the track is live there, +// not merely that one group slipped through. +const GROUPS_PER_RELAY = 10; +// A new group every 100ms, one frame each carrying its own sequence number. +const GROUP_INTERVAL_MS = 100; +// How long the viewer's old session may keep serving after the GOAWAY. Well inside the relay's +// 20s drain window, so the relay's exit log can prove the viewer left on its own. +const HANDOVER = Moq.Time.Milli(2000); +// The viewer's latency budget, as a player sets one (the interop subscribers use the same). +// Following the route means resubscribing on B, and B has to subscribe upstream afresh once A +// drops its pull; the budget is what lets that resubscribe reach back to a group that was in +// flight across the swap instead of starting at the next one. With none, a group boundary +// landing inside the swap drops that group. +const MAX_AGE = Moq.Time.Milli(1000); + +const path = Moq.Path.from("drain"); +const trackName = "seq"; + +function log(message: string) { + console.error(`[drain] ${message}`); +} + +// Resolve once `pred` holds, re-checking every tick; fails with `what` at the deadline. +async function until(what: string, pred: () => boolean, ms = timeoutMs): Promise { + const deadline = performance.now() + ms; + while (!pred()) { + if (performance.now() > deadline) throw new Error(`timed out waiting for ${what}`); + await new Promise((resolve) => setTimeout(resolve, 20)); + } +} + +// ── the name: a TCP proxy standing in for DNS ───────────────────────────────── +interface Backend { + port: number; + /** Connections ever sent here. */ + dialed: number; + /** Connections still open. */ + open: number; +} +const a: Backend = { port: aPort, dialed: 0, open: 0 }; +const b: Backend = { port: bPort, dialed: 0, open: 0 }; +let resolved = a; + +const proxy = net.createServer((client) => { + const backend = resolved; + backend.dialed++; + backend.open++; + const upstream = net.connect(backend.port, "127.0.0.1"); + client.pipe(upstream).pipe(client); + let closed = false; + const close = () => { + if (closed) return; + closed = true; + backend.open--; + client.destroy(); + upstream.destroy(); + }; + for (const socket of [client, upstream]) { + socket.on("error", close); + socket.on("close", close); + } +}); +await new Promise((resolve, reject) => { + proxy.once("error", reject); + proxy.listen(proxyPort, "127.0.0.1", resolve); +}); + +// ── publisher on B ──────────────────────────────────────────────────────────── +const published = new Moq.Origin.Producer(); +const broadcast = published.createBroadcast(path); +const track = broadcast.createTrack(trackName); +broadcast.announce(); +const publisher = new Moq.Connection({ url: new URL(`http://127.0.0.1:${bPort}/`), publish: published.consume() }); + +let lastPublished = -1; +const ticker = setInterval(() => { + const group = track.appendGroup(); + group.writeString(String(group.sequence)); + group.close(); + lastPublished = group.sequence; +}, GROUP_INTERVAL_MS); + +// ── viewer through the name ─────────────────────────────────────────────────── +/** Which relay delivered each group: the generation of the broadcast it was read from. */ +const seen = new Map>(); +let generation = -1; +/** A group whose payload is not its sequence number: fails the run however the swap goes. */ +let corrupt: string | undefined; + +const watched = new Moq.Origin.Producer(); +const viewer = new Moq.Connection({ + url: new URL(`http://127.0.0.1:${proxyPort}/`), + consume: watched, + goaway: { handover: HANDOVER }, +}); +const request = watched.request(path, { announced: true }); + +async function read(sub: Moq.Track.Subscriber, gen: number): Promise { + try { + for (;;) { + const group = await sub.recvGroup(); + if (!group) return; + const text = await group.readString(); + if (text !== String(group.sequence)) { + corrupt ??= `generation ${gen}: group ${group.sequence} carried ${text}`; + return; + } + let gens = seen.get(group.sequence); + if (!gens) { + gens = new Set(); + seen.set(group.sequence, gens); + } + gens.add(gen); + } + } catch (err) { + // The broadcast it was read from went away; its successor picks up from here. + log(`generation ${gen} ended: ${err instanceof Error ? err.message : String(err)}`); + } +} + +// Follow whichever broadcast the path routes to, as a player does: subscribe to the new one and +// drop the old one each time the route changes. +// Subscribed before the first peek, so no route change can land between reading it and listening. +let current: { broadcast: Moq.Broadcast.Consumer; sub: Moq.Track.Subscriber } | undefined; +const follow = (active: Moq.Broadcast.Consumer | undefined) => { + if (!active || active === current?.broadcast) return; + generation++; + log(`watching generation ${generation}`); + const sub = active.track(trackName).subscribe({ maxAge: MAX_AGE }); + void read(sub, generation); + current?.sub.close(); + current = { broadcast: active, sub }; +}; +const unfollow = request.active.subscribe(follow); +follow(request.active.peek()); + +const readOn = (gen: number) => [...seen.values()].filter((gens) => gens.has(gen)).length; +const newestOn = (gen: number) => Math.max(-1, ...[...seen].filter(([, gens]) => gens.has(gen)).map(([seq]) => seq)); + +let failure: Error | undefined; +try { + await until(`${GROUPS_PER_RELAY} groups through relay A`, () => generation === 0 && readOn(0) >= GROUPS_PER_RELAY); + if (a.dialed !== 1 || b.dialed !== 0) throw new Error(`expected one dial to A, saw A=${a.dialed} B=${b.dialed}`); + + log("withdrawing A from the name, then draining it"); + resolved = b; + process.kill(aPid, "SIGTERM"); + + // Counted past the last group A delivered: B's first answer can include recent groups the + // viewer already had, which prove nothing about B being live. + // Bounded well inside A's 20s drain window: a viewer that only reconnects once A cuts it off + // at the deadline would otherwise pass, just late. + await until( + `${GROUPS_PER_RELAY} new groups through relay B`, + () => generation >= 1 && newestOn(generation) >= newestOn(0) + GROUPS_PER_RELAY, + HANDOVER * 3, + ); + // The old session leaves at the handover cap, before A's own deadline would cut it. + await until("the viewer to leave relay A", () => a.open === 0, HANDOVER * 3); + + if (generation !== 1) throw new Error(`expected exactly one migration, saw ${generation}`); + if (a.dialed !== 1) throw new Error(`the viewer redialed the withdrawn relay (${a.dialed} dials)`); + if (b.dialed < 1) throw new Error("the viewer never dialed relay B"); + + const sequences = [...seen.keys()].sort((x, y) => x - y); + const first = sequences[0] ?? 0; + const last = sequences[sequences.length - 1] ?? 0; + const missing: number[] = []; + for (let seq = first; seq <= last; seq++) { + if (!seen.has(seq)) missing.push(seq); + } + const both = sequences.filter((seq) => seen.get(seq)?.size === 2).length; + log(`read groups ${first}..${last} (published up to ${lastPublished}); ${both} arrived from both relays`); + if (missing.length > 0) throw new Error(`dropped groups across the migration: ${missing.join(", ")}`); + if (corrupt) throw new Error(`corrupt group from ${corrupt}`); + log("migrated without a dropped group"); +} catch (err) { + failure = err instanceof Error ? err : new Error(String(err)); +} finally { + unfollow(); + current?.sub.close(); + clearInterval(ticker); + request.close(); + viewer.close(); + publisher.close(); + broadcast.close(); + proxy.close(); +} + +if (failure) { + console.error(`error: ${failure.message}`); + process.exit(1); +} +process.exit(0); diff --git a/test/drain/package.json b/test/drain/package.json new file mode 100644 index 00000000000..ed14027cc4b --- /dev/null +++ b/test/drain/package.json @@ -0,0 +1,8 @@ +{ + "name": "@moq/drain-test", + "private": true, + "type": "module", + "dependencies": { + "@moq/net": "workspace:*" + } +} diff --git a/test/drain/relay.toml b/test/drain/relay.toml new file mode 100644 index 00000000000..f64e9090294 --- /dev/null +++ b/test/drain/relay.toml @@ -0,0 +1,27 @@ +# Relay config for the drain harness. +# Anonymous access, self-signed localhost cert, QUIC + HTTP on 127.0.0.1. +# +# run.sh starts two relays from this file, overriding the ports on the command +# line (`--listen` / `--web-http-listen`), and clusters the draining one to the +# other with `--cluster-connect`. Only the settings both share live here. + +# Long enough that a client still on the draining relay at the deadline is a +# failure the harness reports, not a close it races. +drain_timeout = "20s" + +[log] +level = "info" + +[listen] +# QUIC on UDP. 127.0.0.1 avoids IPv6 flakiness on CI runners. +bind = "127.0.0.1:4470" +tls.generate = ["localhost", "127.0.0.1"] + +[web.http] +# HTTP and WebSocket on TCP, also serving /certificate.sha256, which is how the +# cluster dial over `http://` pins the peer's self-signed certificate. +listen = "127.0.0.1:4470" + +[auth] +# Allow anonymous access to everything. +public = "**" diff --git a/test/drain/run.sh b/test/drain/run.sh new file mode 100755 index 00000000000..d70344cb4ac --- /dev/null +++ b/test/drain/run.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# A relay drain, end to end: a JS viewer watching a live track through relay A +# migrates to relay B when A drains, without a dropped group. See README.md. +set -euo pipefail + +DRAIN_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$DRAIN_DIR/../.." && pwd) + +# Run directory, reserved ports, and process-group ownership. See test/README.md. +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DRAIN_DIR/../lib/harness.sh" + +# Captured before the parse below consumes it, so the rerun command carries every +# flag and every environment override this run was actually given. +RERUN="$(harness_env DRAIN_TIMEOUT DRAIN_PROFILE RELAY_BIN)just test drain$(harness_argv "$@")" + +TIMEOUT="${DRAIN_TIMEOUT:-60}" +PROFILE="${DRAIN_PROFILE:-debug}" +RELAY="${RELAY_BIN:-}" + +while [[ $# -gt 0 ]]; do + case "$1" in + --timeout) + if [[ $# -lt 2 || -z "${2:-}" || "$2" == -* ]]; then + echo "error: --timeout requires a value" >&2 + exit 2 + fi + TIMEOUT="$2" + shift 2 + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +if [[ ! "$TIMEOUT" =~ ^[1-9][0-9]*$ ]]; then + echo "error: timeout must be a positive whole number of seconds (got '$TIMEOUT')" >&2 + exit 2 +fi + +harness_begin drain "$RERUN" + +for tool in cargo bun curl; do + command -v "$tool" >/dev/null 2>&1 || { + echo "error: $tool not found; run inside 'nix develop'" >&2 + exit 1 + } +done + +# ── build ─────────────────────────────────────────────────────────────────── +echo "building moq-relay ($PROFILE)..." +flag=() +[[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") +(cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay) +TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" +[[ -n "$RELAY" ]] || RELAY="$TARGET_BASE/$PROFILE/moq-relay" + +(cd "$WORKSPACE" && bun install --frozen-lockfile) + +# ── relays ────────────────────────────────────────────────────────────────── +# Start one relay on a reserved port: `start [extra relay args...]`. +# Sets PID and PORT for the caller. +start() { + local name="$1" + shift + harness_port "$name" + PORT="$HARNESS_PORT" + local url="http://127.0.0.1:$PORT" + # The reservation covers other harness runs, not the rest of the machine. + if harness_probe "$url/certificate.sha256"; then + echo "error: something is already listening on 127.0.0.1:$PORT (stale relay?)" >&2 + exit 1 + fi + + echo "starting relay $name on 127.0.0.1:$PORT..." + harness_spawn "relay-$name" "$HARNESS_RUN/relay-$name.log" "$RELAY" "$DRAIN_DIR/relay.toml" \ + --listen "127.0.0.1:$PORT" --web-http-listen "127.0.0.1:$PORT" "$@" + PID="$HARNESS_PID" + if ! harness_ready "$url/certificate.sha256" 30 "$PID"; then + echo "relay $name never became ready" >&2 + sed 's/^/ relay: /' "$HARNESS_RUN/relay-$name.log" >&2 || true + exit 1 + fi + harness_endpoint "relay-$name" "$url" +} + +# B survives and holds the publisher; A drains and serves the track by pulling it +# from B, the way a sibling in a fleet does. +start b +B_PORT="$PORT" +start a --cluster-connect "http://127.0.0.1:$B_PORT/" +A_PORT="$PORT" +A_PID="$PID" + +harness_port proxy +PROXY_PORT="$HARNESS_PORT" +harness_endpoint name "http://127.0.0.1:$PROXY_PORT" + +# ── run ───────────────────────────────────────────────────────────────────── +# Spawned rather than run in the foreground so a SIGTERM lands while the shell is +# in `wait`, where a trap can run. +status=0 +harness_spawn driver - bun "$DRAIN_DIR/drain.ts" \ + --a-port "$A_PORT" --b-port "$B_PORT" --proxy-port "$PROXY_PORT" --a-pid "$A_PID" --timeout "$TIMEOUT" +harness_wait "$HARNESS_PID" || status=$? + +# The driver saw the viewer leave A; A itself has to notice every session is gone +# and exit cleanly, well before its 20s deadline would have forced anyone out. +if [[ $status -eq 0 ]]; then + deadline=$((SECONDS + 10)) + while ! harness_exited "$A_PID" && ((SECONDS < deadline)); do + sleep 0.1 + done + if ! harness_exited "$A_PID"; then + echo "error: relay A was still running 10s after its last session left" >&2 + status=1 + elif ! harness_wait "$A_PID"; then + echo "error: relay A exited with a failure" >&2 + status=1 + elif ! grep -q "drain complete: every session left" "$HARNESS_RUN/relay-a.log"; then + echo "error: relay A did not report that every session left on its own" >&2 + status=1 + else + echo "relay A drained: every session left" + fi +fi + +if [[ $status -ne 0 ]]; then + for name in a b; do + echo "── relay $name log ──" >&2 + sed 's/^/ /' "$HARNESS_RUN/relay-$name.log" >&2 || true + done +fi + +exit $status diff --git a/test/justfile b/test/justfile index 081229e8e62..a67b2ffee07 100644 --- a/test/justfile +++ b/test/justfile @@ -67,6 +67,27 @@ media *args: drill *args: just rs test -p moq-relay --test drills "$@" +# Plays a tone in headless Chromium through moq-shaper and grades what the listener +# heard (underruns, skip-aheads, silence, the resolved delay) against +# audio-quality/budgets.json. The full matrix is 24 rows at 60 s; nightly runs it with +# --enforce. Narrow it with `--profiles mild --codecs opus --rings plain --duration 20`. +# `--replays` grades only the recorded traces, in seconds, as PRs touching the player do. +# See audio-quality/README.md. + +# Grade browser audio playout over an impaired path. +[positional-arguments] +audio-quality *args: + ./audio-quality/run.sh "$@" + +# Records the arrival traces under audio-quality/traces/ that the replay rows grade: the +# public relay's bbb.hang, a local relay's AAC tone, and a browser microphone through the +# public relay. Needs the network. Narrow it with `--lanes relay-bbb --duration 35`. + +# Re-record the audio quality arrival traces. +[positional-arguments] +audio-quality-record *args: + ./audio-quality/record.sh "$@" + # Removes one recovery behavior at a time in a disposable copy of the tree and # requires the drill covering it to fail, for the reason the mutation names. The # checkout is never modified. `--list` shows the mutations. @@ -96,6 +117,14 @@ ts *args: ts-eit *args: ./ts/eit-roundtrip.sh "$@" +# A JS viewer watching a live track through a relay that drains (SIGTERM, after +# a stand-in for DNS stops pointing at it) migrates to a second relay behind the +# same name without a dropped group. See drain/README.md. + +# Relay drain migration, end to end. +drain *args: + ./drain/run.sh {{ args }} + # Runs the @moq/wasm bindings in headless Chromium against a real relay, one per # protocol flavour. The crate is `#![cfg(target_arch = "wasm32")]`, so nothing in # `just check` or `just rs wasm` gets past compiling it. See wasm/README.md. diff --git a/test/ts/README.md b/test/ts/README.md index 4fb5950b9d8..336ceab8589 100644 --- a/test/ts/README.md +++ b/test/ts/README.md @@ -7,7 +7,9 @@ runs [TSDuck](https://tsduck.io) plus a custom analyzer over it. This is a diagnostic gate, not just a pass/fail: the exporter ([`rs/moq-mux/src/container/ts/export.rs`](../../rs/moq-mux/src/container/ts/export.rs)) -is VBR, inserts no null packets, and paces PCR once per media frame, so several +pads with null packets to the multiplex rate the source recorded (or `--mux-rate`), +leaves a source without one unpadded, never delays media to fit the rate, and +puts a PCR on its own packet every 25 ms of media time, so several broadcast-shape checks are expected to flag. The report quantifies exactly where and by how much. @@ -80,7 +82,7 @@ Severities: **hard** checks fail the run by default; **shape** checks report as | `pcr-presence` | hard | a PCR PID is declared and carries PCR | | `pcr-monotonic` | hard | PCR strictly increases (one 33-bit wrap tolerated), except into a PCR that signals `discontinuity_indicator` | | `duration-fidelity` | hard | exported PCR span tracks the source's duration (round-trip only) | -| `pcr-repetition` | shape | consecutive PCRs within the limit (default 40 ms) | +| `pcr-repetition` | shape | consecutive PCRs within the limit (default 100 ms, TR 101 290 V1.4.1) | | `pcr-jitter` | shape | per-interval PCR jitter vs the nominal bitrate (pcrverify model) | | `null-ratio` | shape | null/stuffing fraction (flags only a pathological excess) | | `service-descriptors` | shape | an SDT naming the service is present | @@ -146,7 +148,7 @@ grades only how evenly the bytes are laid over the PCRs. | `sync` | hard | no invalid sync bytes / transport-error packets | | `continuity` | hard | no discontinuities, and a payload-less packet must not advance the counter (ISO 13818-1 2.4.3.3) | | `pcr-single-pid` | hard | every PCR rides one PID | -| `pcr-value-interval` | hard | no interval above `--repetition-ms` (default 40, TR 101 290), within one time base | +| `pcr-value-interval` | hard | no interval above `--repetition-ms` (default 100, TR 101 290 V1.4.1), within one time base | | `pcr-release-timing` | hard | no more than `--release-pct-max` of intervals arrive further than `--release-ms` from the interval their own values assert, and accumulated drift stays within `--drift-ms`, being the standing lag the sender is allowed to hold; a sample below `--live-min-pcr` PCRs or `--live-cover-pct` of the window is a failure, not a pass (`--live` only) | | `pcr-position` | shape | share of PCR packets within `--adjacent-packets` of the previous one | | `pcr-schedule` | shape | share of PCR intervals, on the busiest PCR PID, whose bytes are within `--schedule-tolerance-pct` (default 1) or one packet of what `--mux-rate` implies (estimated from the capture if not given); hard, at that share, when `--schedule-pct-min` is given | diff --git a/test/ts/compliance.py b/test/ts/compliance.py index 66aae98fe7d..2774948c6f4 100755 --- a/test/ts/compliance.py +++ b/test/ts/compliance.py @@ -72,7 +72,7 @@ class Check: class Thresholds: """IRD limits and T-STD model parameters, all overridable from the CLI.""" - pcr_repetition_ms: float = 40.0 + pcr_repetition_ms: float = 100.0 pcr_jitter_us: float = 500.0 null_ratio_max: float = 0.90 bitrate_cov_max: float = 0.10 diff --git a/test/ts/pcr-timing.py b/test/ts/pcr-timing.py index b318fe39043..9c08abe8893 100755 --- a/test/ts/pcr-timing.py +++ b/test/ts/pcr-timing.py @@ -605,7 +605,7 @@ def main(): ap.add_argument("--live", action="store_true", help="read stdin and stamp arrivals, grading release timing") ap.add_argument("--seconds", type=float, default=45.0, help="live capture window (default 45)") ap.add_argument( - "--repetition-ms", type=float, default=40.0, help="max PCR value interval, TR 101 290 (default 40)" + "--repetition-ms", type=float, default=100.0, help="max PCR value interval, TR 101 290 V1.4.1 (default 100)" ) ap.add_argument( "--release-ms", diff --git a/test/ts/run.sh b/test/ts/run.sh index 0858b89dadc..d30320d1e4c 100755 --- a/test/ts/run.sh +++ b/test/ts/run.sh @@ -7,7 +7,8 @@ # TSDuck + custom analyzer in compliance.py against the capture. The point is to # tell whether what the subscriber emits is something an Integrated # Receiver/Decoder would accept, and to quantify where it diverges (the exporter -# is VBR, emits no null packets, and paces PCR per frame). +# pads to the recorded multiplex rate but never delays media to fit it, and puts a +# PCR every 25 ms of media time). # # Modes: # ./run.sh # generate a clip, round-trip it, analyze