From 53b5813a231a680549037216044db4ec93520e4c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 01:28:48 -0800 Subject: [PATCH 1/5] chore(quest): claim quest/m2/rate-quic.md Quest-Claim: 8f1e29551735ff28 From 9c7d1a90e548537e8941ad3901884772e2109b57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:04:56 -0800 Subject: [PATCH 2/5] feat(relay): enforce bitrate caps on QUIC sessions Token claims and grants carry optional publish_rate and subscribe_rate caps (moq_auth::Rate) as the {"path","rate"} object role. The relay holds QUIC and WebTransport sessions on the tokio and io_uring listeners to them, taking the union's highest cap, applying re-checked caps live, and refusing caps on every transport that cannot enforce them. Every server QUIC connection starts with only the SETUP window, opened once admitted. Co-Authored-By: Claude Opus 5.5 --- doc/bin/relay/auth.md | 22 +- doc/lib/js/auth.md | 8 +- doc/lib/rs/moq-auth.md | 4 +- js/auth/README.md | 12 +- js/auth/api/index.api.md | 179 ++++++++---- js/auth/src/claims.test.ts | 42 ++- js/auth/src/claims.ts | 39 ++- js/auth/src/contract.test.ts | 10 +- js/auth/src/contract.ts | 72 +++-- js/auth/src/interop.test.ts | 4 +- js/auth/src/key.test.ts | 24 +- js/auth/src/key.ts | 15 +- js/auth/src/set.ts | 4 +- js/auth/src/wire.ts | 138 +++++++-- quest/m2/README.md | 1 - quest/m2/quic-sim.md | 1 - quest/m2/rate-grant.md | 17 +- quest/m2/rate-quic.md | 96 ------- quest/m2/rate-websocket.md | 18 +- quest/m2/rtc-pacing.md | 4 +- quest/m2/srt-rate-handle.md | 2 +- rs/moq-auth/CHANGELOG.md | 6 +- rs/moq-auth/rate.json | 80 ++++++ rs/moq-auth/src/claims.rs | 25 +- rs/moq-auth/src/grant.rs | 26 +- rs/moq-auth/src/key.rs | 8 + rs/moq-auth/src/lib.rs | 5 +- rs/moq-auth/src/rate.rs | 110 +++++++ rs/moq-auth/src/serve.rs | 16 +- rs/moq-auth/src/set.rs | 2 + rs/moq-auth/src/wire.rs | 190 ++++++++++-- rs/moq-auth/tests/released_token.rs | 53 ++-- rs/moq-net/CHANGELOG.md | 4 + rs/moq-net/src/transport.rs | 14 + rs/moq-relay/CHANGELOG.md | 6 +- rs/moq-relay/src/auth.rs | 138 ++++++++- rs/moq-relay/src/auth/caps.rs | 176 ++++++++++++ rs/moq-relay/src/auth/refresh.rs | 191 ++++++++++-- rs/moq-relay/src/cluster.rs | 19 +- rs/moq-relay/src/connection.rs | 24 +- rs/moq-relay/src/incoming.rs | 26 +- rs/moq-relay/src/uring.rs | 55 +++- .../tests/integration/bitrate_caps.rs | 272 ++++++++++++++++++ rs/moq-relay/tests/integration/main.rs | 1 + rs/moq-relay/tests/integration/peer_limits.rs | 5 +- rs/moq-tokio/CHANGELOG.md | 2 + rs/moq-tokio/src/quic/backend.rs | 13 +- rs/moq-tokio/src/server.rs | 56 +++- 48 files changed, 1829 insertions(+), 406 deletions(-) delete mode 100644 quest/m2/rate-quic.md create mode 100644 rs/moq-auth/rate.json create mode 100644 rs/moq-auth/src/rate.rs create mode 100644 rs/moq-relay/src/auth/caps.rs create mode 100644 rs/moq-relay/tests/integration/bitrate_caps.rs diff --git a/doc/bin/relay/auth.md b/doc/bin/relay/auth.md index 438c3fe2102..0e34af96e15 100644 --- a/doc/bin/relay/auth.md +++ b/doc/bin/relay/auth.md @@ -221,9 +221,25 @@ everything. Put application data somewhere other than the token. Tokens and key scopes from the older `moq-token` format still work: each `put` and `get` prefix `p` reads as the subtree `p/**`. -Roles are pattern arrays. Object roles, including bitrate caps such as -`{"path":["alice/**"],"rate":8000000}`, are refused in full by token and -auth-server grant parsing. No transport currently enforces bitrate caps. +### Bitrate caps + +A role may cap its bitrate, in bits per second, by naming its patterns as +`{"path":["alice/**"],"rate":8000000}` in a token or an auth-server grant. A +`publish` cap bounds what the session sends the relay, and a `subscribe` cap +what the relay sends it, QUIC overhead and retransmissions included. Each +averages over a second, with one second of the rate as a burst. Older relays +refuse a capped role rather than drop the cap. + +Only QUIC and WebTransport on the tokio or io\_uring listener enforce caps. +WebSocket, iroh, and HTTP `/fetch` and `/announced` refuse a capped grant, +whether it arrives at connect, in band, or at revalidation. When in-band tokens +add grants to one session, each direction takes the highest cap among them, and +a grant giving that direction paths without a cap lifts it. A cap changed at +revalidation applies to the live session. + +Every QUIC session starts with only the flow-control credit its SETUP needs, so +a client cannot bank the full `[quic] receive_window` before its cap applies; +the relay opens the window once it admits the session. ### Path matching diff --git a/doc/lib/js/auth.md b/doc/lib/js/auth.md index 21be4ba17c6..ffa503bec0e 100644 --- a/doc/lib/js/auth.md +++ b/doc/lib/js/auth.md @@ -16,9 +16,11 @@ checks a connection path against a token's claims exactly as `moq auth serve` does. A grant can label its session's stats independently of its authorization root. Tokens are interchangeable with the Rust side. Grants and claims are [`Pattern`](https://www.npmjs.com/package/@moq/pattern) unions: `foo` is one -broadcast, `foo/**` is a subtree, `**` is everything. Parsed claim and grant roles are -pattern string arrays; `authorize` returns reduced `Patterns` for each role. -Object roles, including bitrate caps, are refused during parsing. +broadcast, `foo/**` is a subtree, `**` is everything. A claim or grant role may +also carry a bitrate cap, parsed into `publish_rate` or `subscribe_rate` beside +the pattern arrays; key scopes bound paths only, and `authorize` checks paths +only. Answer a capped grant with `encodeGrant`, which writes the cap where the +relay reads it. ```bash bun add @moq/auth diff --git a/doc/lib/rs/moq-auth.md b/doc/lib/rs/moq-auth.md index d3b3e3488ad..c06f997afad 100644 --- a/doc/lib/rs/moq-auth.md +++ b/doc/lib/rs/moq-auth.md @@ -13,13 +13,13 @@ Everything a party needs to ask for or answer an authorization on answers the relay, in a service that mints tokens for clients, or in your own accept loop that decides in process. -- **Request and grant.** `Request` is the JSON a relay POSTs on each session event (`connect`, `revalidate`, `end`), carrying everything it knows about the session. `Grant` is the answer: publish and subscribe [`Pattern`](https://docs.rs/moq-pattern) unions, an expiry, a revalidation cadence, optional stats tier and label, and whether the session is a cluster peer or an [upstream link](/bin/relay/cluster#upstream-links). `Grant::validate` checks the grant contract; the accept loop must enforce its caps or refuse it. Expiry is exact, with no allowance for clock skew. +- **Request and grant.** `Request` is the JSON a relay POSTs on each session event (`connect`, `revalidate`, `end`), carrying everything it knows about the session. `Grant` is the answer: publish and subscribe [`Pattern`](https://docs.rs/moq-pattern) unions with optional [bitrate caps](/bin/relay/auth#bitrate-caps) (`publish_rate`, `subscribe_rate`), an expiry, a revalidation cadence, optional stats tier and label, and whether the session is a cluster peer or an [upstream link](/bin/relay/cluster#upstream-links). `Grant::validate` checks the grant contract; the accept loop must enforce its caps or refuse it. Expiry is exact, with no allowance for clock skew. - **Lease.** `lease::Producer` and `lease::Consumer` are the handle a session holds for its grant: read it, wait for a change, revoke it, and learn why it ended. Whoever runs the accept loop builds the producer, so an embedder can decide in process with no HTTP. - **Refusals.** `Refusal` carries a `refusal::Reason` (`Refused`, `Forbidden`, or `Expired`) and optional `root` and `tier`. An auth server returns it as JSON with HTTP 403; `Client` returns `Error::Refused(refusal)`. Legacy plain-text 401/403 bodies and unknown reason strings read as `Refused`. Set `root` only from a verified credential, never the dialed path. Embedded relay deciders pass the same shape in `auth::Error::Refused`. - **Client.** `Client` drives a lease against an auth server over `https://`, `unix://`, or loopback `http://`. It revalidates on cadence and retries through an outage until the grant expires, revokes on a 401, 403, or invalid grant, and reports `end` with the session's byte totals. - **Server** (feature `serve`). `serve::Server` is the reference auth server behind `moq auth serve`: JWTs from the query or the moq-transport SETUP token, certificate grants, anonymous rules, tiers, and session caps. `Server::router` mounts it in your own axum service. - **Mesh enrollment.** `mesh` holds the [enrollment contract](/bin/relay/auth#mesh-enrollment): a device proves its Ed25519 endpoint key against a fresh challenge and receives relay URLs and a scoped token. `mesh::Client` (feature `client`) enrolls with an enrollment key and renews before expiry; `mesh::Coordinator` (feature `serve`) answers it. -- **Keys and claims.** Generate HMAC, RSA, ECDSA, or EdDSA keys as JWKs, sign `Claims` (`root`, `publish`, `subscribe`, `exp`), and verify them. `Claims::authorize(path)` scopes verified claims to the path a client dialed, exactly as `moq auth serve` does. Roles hold path pattern unions directly. +- **Keys and claims.** Generate HMAC, RSA, ECDSA, or EdDSA keys as JWKs, sign `Claims` (`root`, `publish`, `subscribe`, their optional `Rate` caps, `exp`), and verify them. `Claims::authorize(path)` scopes verified claims to the path a client dialed, exactly as `moq auth serve` does. Roles hold path pattern unions directly. ```bash cargo add moq-auth diff --git a/js/auth/README.md b/js/auth/README.md index 2937ea29eb6..6da88f74923 100644 --- a/js/auth/README.md +++ b/js/auth/README.md @@ -7,7 +7,9 @@ contract, the relay flags, and worked examples, see the [Authentication Documentation](../../doc/bin/relay/auth.md). Grants and claims name paths with patterns from `@moq/pattern`: `foo` is one -broadcast, `foo/**` is a subtree, `**` is everything. +broadcast, `foo/**` is a subtree, `**` is everything. A role may also carry a +bitrate cap, written `{"path": [...], "rate": bps}` and parsed into +`publish_rate` or `subscribe_rate` beside the patterns. ## Installation @@ -105,13 +107,15 @@ moq-auth verify --key root.jwk --in alice.jwt ### Answering a relay -A server implementing the contract validates what the relay sends and answers a grant: +A server implementing the contract validates what the relay sends and answers a grant. +`encodeGrant` writes a cap into its role, where the relay reads it: ```typescript -import { type Grant, GrantSchema, RequestSchema } from "@moq/auth"; +import { encodeGrant, type Grant, RequestSchema } from "@moq/auth"; const request = RequestSchema.parse(await body.json()); -const grant: Grant = GrantSchema.parse({ subscribe: ["**"], expires: 4102444800, revalidate: 60 }); +const grant: Grant = { subscribe: ["**"], subscribe_rate: 8_000_000, expires: 4102444800, revalidate: 60 }; +return Response.json(encodeGrant(grant)); ``` ### Working example diff --git a/js/auth/api/index.api.md b/js/auth/api/index.api.md index d8321776b27..bf2cf84bca3 100644 --- a/js/auth/api/index.api.md +++ b/js/auth/api/index.api.md @@ -61,23 +61,37 @@ export const ClaimsSchema: ZodMiniPipe; jti: ZodMiniOptional; nbf: ZodMiniOptional; - publish: ZodMiniOptional>>; + publish: ZodMiniOptional>, ZodMiniObject<{ + path: ZodMiniArray>; + rate: ZodMiniNumber; + }, $strict>]>>; put: ZodMiniOptional, ZodMiniArray>]>>; root: ZodMiniDefault>; sub: ZodMiniOptional; - subscribe: ZodMiniOptional>>; -}, $strict>, ZodMiniTransform>, ZodMiniObject<{ + path: ZodMiniArray>; + rate: ZodMiniNumber; + }, $strict>]>>; +}, $strict>, ZodMiniTransform & { + subscribe?: string[] | { + path: string[]; + rate: number; + } | undefined; +}, "get" | "put">, "publish" | "subscribe"> & { publish?: string[]; + publish_rate?: number; subscribe?: string[]; + subscribe_rate?: number; }, { exp?: number | undefined; get?: string | string[] | undefined; @@ -85,27 +99,78 @@ export const ClaimsSchema: ZodMiniPipe>; export type Grant = infer; -export const GrantSchema: ZodMiniObject<{ +export const GrantSchema: ZodMiniPipe; label: ZodMiniOptional>; mounts: ZodMiniOptional, ZodMiniString>>; peer: ZodMiniOptional>; - publish: ZodMiniOptional>>; + publish: ZodMiniOptional>, ZodMiniObject<{ + path: ZodMiniArray>; + rate: ZodMiniNumber; + }, $strict>]>>; revalidate: ZodMiniOptional; root: ZodMiniOptional>; - subscribe: ZodMiniOptional>>; + subscribe: ZodMiniOptional>, ZodMiniObject<{ + path: ZodMiniArray>; + rate: ZodMiniNumber; + }, $strict>]>>; tier: ZodMiniOptional>; upstream: ZodMiniOptional>; -}, $strip>; +}, $strip>, ZodMiniTransform | undefined; + peer?: boolean | undefined; + publish?: string[] | { + path: string[]; + rate: number; + } | undefined; + revalidate?: number | undefined; + root?: string | undefined; + subscribe?: string[] | { + path: string[]; + rate: number; + } | undefined; + tier?: string | undefined; + upstream?: boolean | undefined; +}, "publish" | "subscribe"> & { + publish?: string[]; + publish_rate?: number; + subscribe?: string[]; + subscribe_rate?: number; +}, { + expires?: number | undefined; + label?: string | undefined; + mounts?: Record | undefined; + peer?: boolean | undefined; + publish?: string[] | { + path: string[]; + rate: number; + } | undefined; + revalidate?: number | undefined; + root?: string | undefined; + subscribe?: string[] | { + path: string[]; + rate: number; + } | undefined; + tier?: string | undefined; + upstream?: boolean | undefined; +}>>; export { InvalidPattern } from "@moq/pattern"; @@ -120,11 +185,20 @@ export const Key: { iss?: unknown; jti?: unknown; nbf?: number; - publish?: string[]; + publish?: string[] | { + path: string[]; + rate: number; + }; put?: string | string[]; root?: string; sub?: unknown; - subscribe?: string[]; + subscribe?: string[] | { + path: string[]; + rate: number; + }; + } & { + publish_rate?: number; + subscribe_rate?: number; }) => Promise; verify: (key: PublicKey | SymmetricKey, token: string) => Promise; }; @@ -171,10 +245,7 @@ export const KeySchema: ZodMiniDiscriminatedUnion<[ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -225,10 +296,7 @@ export const KeySchema: ZodMiniDiscriminatedUnion<[ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -275,10 +343,7 @@ export const KeySchema: ZodMiniDiscriminatedUnion<[ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -324,10 +389,7 @@ export const KeySchema: ZodMiniDiscriminatedUnion<[ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -348,11 +410,20 @@ export const KeySet: { iss?: unknown; jti?: unknown; nbf?: number; - publish?: string[]; + publish?: string[] | { + path: string[]; + rate: number; + }; put?: string | string[]; root?: string; sub?: unknown; - subscribe?: string[]; + subscribe?: string[] | { + path: string[]; + rate: number; + }; + } & { + publish_rate?: number; + subscribe_rate?: number; }) => Promise; verify: (set: KeySet, token: string) => Promise; }; @@ -396,10 +467,7 @@ export const KeySetSchema: ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -450,10 +518,7 @@ export const KeySetSchema: ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -500,10 +565,7 @@ export const KeySetSchema: ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -549,10 +611,7 @@ export const KeySetSchema: ZodMiniObject<{ put?: string[] | undefined; root: string; subscribe?: string[] | undefined; - }, "get" | "put"> & { - publish?: string[]; - subscribe?: string[]; - }, { + }, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -756,10 +815,7 @@ export const ScopeSchema: ZodMiniPipe & { - publish?: string[]; - subscribe?: string[]; -}, { +}, "get" | "put">, { get?: string[] | undefined; publish?: string[] | undefined; put?: string[] | undefined; @@ -802,6 +858,25 @@ export function authorize(claims: Claims, path: string): Permissions; export { compareSpecificity } from "@moq/pattern"; +export function encodeGrant(grant: Grant): { + expires?: number; + label?: string; + mounts?: Record; + peer?: boolean; + publish?: string[] | { + path: string[]; + rate: number; + }; + revalidate?: number; + root?: string; + subscribe?: string[] | { + path: string[]; + rate: number; + }; + tier?: string; + upstream?: boolean; +}; + export function patterns(texts: readonly string[] | undefined): Patterns; export function scopeAllows(scope: Scope, claims: Claims): boolean; diff --git a/js/auth/src/claims.test.ts b/js/auth/src/claims.test.ts index 494f31cb177..7ae3c55d439 100644 --- a/js/auth/src/claims.test.ts +++ b/js/auth/src/claims.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import { authorize, type Claims, ClaimsSchema, type Permissions, ScopeSchema } from "./claims.ts"; -import { GrantSchema } from "./contract.ts"; +import { encodeGrant, GrantSchema } from "./contract.ts"; import { encodeGrants } from "./wire.ts"; // These cases mirror the Rust moq-auth crate's claims::tests one-for-one, so both @@ -120,13 +120,37 @@ test("authorize - wildcards rebase as a set", () => { expect(permissions.subscribe.toJSON()).toEqual(["", "**/chat"]); }); -test("roles are pattern lists and refuse object restrictions in full", () => { - for (const role of ["publish", "subscribe"]) { - for (const object of [{ path: "alice/**" }, { path: ["alice/**"], rate: 8000 }]) { - const input = { publish: ["**"], subscribe: ["**"], [role]: object }; - expect(ClaimsSchema.safeParse(input).success).toBe(false); - expect(ScopeSchema.safeParse(input).success).toBe(false); - expect(GrantSchema.safeParse(input).success).toBe(false); - } +test("shared rate vectors parse and re-encode like Rust", async () => { + const vectors = await Bun.file(new URL("../../../rs/moq-auth/rate.json", import.meta.url)).json(); + for (const row of vectors.valid) { + expect(encodeGrants(ClaimsSchema.parse(row.input))).toEqual(row.claims); + expect(encodeGrant(GrantSchema.parse(row.input))).toEqual(row.grant); } + for (const input of vectors.invalid) { + expect(ClaimsSchema.safeParse(input).success).toBe(false); + expect(GrantSchema.safeParse(input).success).toBe(false); + } +}); + +test("a capped role decodes beside its patterns and never encodes as a prefix", () => { + for (const role of ["publish", "subscribe"] as const) { + // Subtrees alone would be written as legacy prefixes, which a cap must not be. + const claims = ClaimsSchema.parse({ + publish: ["alice/**"], + subscribe: ["**"], + [role]: { path: ["alice/**"], rate: 8000 }, + }); + expect(claims[role]).toEqual(["alice/**"]); + expect(claims[`${role}_rate`]).toBe(8000); + const encoded = encodeGrants(claims); + expect(encoded).not.toHaveProperty("put"); + expect(encoded).not.toHaveProperty("get"); + expect(encoded[role]).toEqual({ path: ["alice/**"], rate: 8000 }); + } + expect(() => ClaimsSchema.parse({ publish: { path: ["**"], rate: 0 } })).toThrow(/rate must be a positive integer/); +}); + +test("a key scope bounds paths, never bitrate", () => { + expect(ScopeSchema.safeParse({ publish: { path: ["**"], rate: 8000 } }).success).toBe(false); + expect(ScopeSchema.safeParse({ publish: ["**"], subscribe: { path: ["**"], rate: 8000 } }).success).toBe(false); }); diff --git a/js/auth/src/claims.ts b/js/auth/src/claims.ts index c83fed0894b..ec989177ace 100644 --- a/js/auth/src/claims.ts +++ b/js/auth/src/claims.ts @@ -1,5 +1,6 @@ /** - * The payload of a token: a root, plus the publish/subscribe patterns granted beneath it. + * The payload of a token: a root, plus the publish/subscribe patterns granted beneath it + * and an optional bitrate cap on each. * * @module */ @@ -7,24 +8,9 @@ import { Pattern, Patterns } from "@moq/pattern"; import * as z from "@zod/mini"; import * as Path from "./path.ts"; -import { decodeGrants } from "./wire.ts"; - -/** A list of pattern texts, each validated by `Pattern.parse`. */ -export const PatternListSchema = z.array( - z.string().check( - z.refine( - (text) => { - try { - Pattern.parse(text); - return true; - } catch { - return false; - } - }, - { message: "invalid path pattern" }, - ), - ), -); +import { decodeGrants, decodeRoles, PatternListSchema, WireRoleSchema } from "./wire.ts"; + +export { PatternListSchema } from "./wire.ts"; /** Parse a list of pattern texts into a reduced union. */ export function patterns(texts: readonly string[] | undefined): Patterns { @@ -45,7 +31,8 @@ const ScopeFields = { * * `root` is optional on the wire to match the Rust `moq-auth` crate, which omits it * when the scope sits at the top level. A legacy `put`/`get` prefix scope reads as the - * subtree patterns it meant. Any other field is refused. + * subtree patterns it meant. A scope bounds paths, never bitrate, so a capped role is + * refused. Any other field is refused. */ export const ScopeSchema = z .pipe( @@ -67,6 +54,10 @@ export type Scope = z.output; const ClaimsFields = { ...ScopeFields, + /** Patterns the holder may publish, relative to `root`, optionally with a bitrate cap. */ + publish: z.optional(WireRoleSchema), + /** Patterns the holder may subscribe to, relative to `root`, optionally with a bitrate cap. */ + subscribe: z.optional(WireRoleSchema), /** Expiration time, as a whole unix timestamp in seconds. */ exp: z.optional(z.int()), /** Issued-at time, as a whole unix timestamp in seconds. */ @@ -94,8 +85,10 @@ const PrefixListSchema = z.union([z.string(), z.array(z.string())]); * exactly what it says: `alice` is one broadcast, `alice/**` is a subtree, and `**` is * everything under the root. Legacy `moq-token` claims read too, each `put`/`get` * prefix `p` as the subtree `p/**`, and signing writes that form whenever it says the - * same thing. The registered `iss`, `sub`, and `jti` claims are read and dropped; any - * other field fails verification. + * same thing. A role may instead be `{"path": [...], "rate": bps}`, which parses into + * `publish_rate` or `subscribe_rate` beside the patterns; a capped role is never written + * as a prefix, since an older verifier would drop the cap. The registered `iss`, `sub`, + * and `jti` claims are read and dropped; any other field fails verification. */ export const ClaimsSchema = z .pipe( @@ -105,7 +98,7 @@ export const ClaimsSchema = z put: z.optional(PrefixListSchema), get: z.optional(PrefixListSchema), }), - z.transform(({ iss: _iss, sub: _sub, jti: _jti, ...claims }, ctx) => decodeGrants(claims, ctx)), + z.transform(({ iss: _iss, sub: _sub, jti: _jti, ...claims }, ctx) => decodeRoles(decodeGrants(claims, ctx))), ) .check( // Emptiness, not just presence: `publish: []` grants nothing, and the Rust crate diff --git a/js/auth/src/contract.test.ts b/js/auth/src/contract.test.ts index 5f96d97d8c7..dbe867039eb 100644 --- a/js/auth/src/contract.test.ts +++ b/js/auth/src/contract.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { GrantSchema, RefusalSchema, RequestSchema, TokenSchema } from "./contract.ts"; +import { encodeGrant, GrantSchema, RefusalSchema, RequestSchema, TokenSchema } from "./contract.ts"; // The fixtures below are what the Rust `moq_auth::Request` and `Grant` serialize to, // so a server written against these schemas reads what a relay sends. @@ -125,6 +125,14 @@ test("a grant round trips and is validated", () => { expect(() => GrantSchema.parse({ publish: ["**"], label: "project/*" })).toThrow(/invalid path/); }); +test("a capped grant answers with its cap nested in the role", () => { + const grant = GrantSchema.parse({ publish: { path: ["alice/**"], rate: 8_000_000 }, subscribe: ["**"] }); + expect(grant).toMatchObject({ publish: ["alice/**"], publish_rate: 8_000_000, subscribe: ["**"] }); + expect(encodeGrant(grant)).toEqual({ publish: { path: ["alice/**"], rate: 8_000_000 }, subscribe: ["**"] }); + // A cap alone names nothing. + expect(() => encodeGrant({ publish_rate: 8000 })).toThrow(/name something/); +}); + test("a tier a metrics label would escape is refused", () => { for (const tier of ['a"b', "a\\b", "a\nb", "/edge", "a//b"]) { expect(() => GrantSchema.parse({ publish: ["**"], tier })).toThrow(/invalid tier/); diff --git a/js/auth/src/contract.ts b/js/auth/src/contract.ts index 93d4693566d..f472af26764 100644 --- a/js/auth/src/contract.ts +++ b/js/auth/src/contract.ts @@ -9,7 +9,7 @@ import { Pattern } from "@moq/pattern"; import * as z from "@zod/mini"; -import { PatternListSchema } from "./claims.ts"; +import { decodeRoles, encodeRoles, WireRoleSchema } from "./wire.ts"; const PathSchema = z.string().check( z.refine( @@ -166,34 +166,41 @@ export type Request = z.infer; * A 2xx carrying one of these admits; anything else refuses. A grant that names * nothing is a refusal too, and one that asks to be revalidated must say when it * expires, so an outage always has a bound the server chose. + * + * A role may be `{"path": [...], "rate": bps}`, which parses into `publish_rate` or + * `subscribe_rate` beside the patterns. Answer with {@link encodeGrant}, which writes a + * cap back into its role; a bare `publish_rate` field is ignored by the relay. */ export const GrantSchema = z - .object({ - /** Patterns the session may publish, relative to the root. */ - publish: z.optional(PatternListSchema), - /** Patterns the session may subscribe to, relative to the root. */ - subscribe: z.optional(PatternListSchema), - /** The path the patterns are relative to, replacing the dialed one. Absent means the dialed path. */ - root: z.optional(z.string()), - /** Subtrees read from elsewhere: each path, relative to the root, resolves at the absolute path it maps to. Read-only. */ - mounts: z.optional(z.record(z.string(), z.string())), - /** When the session closes, as whole unix seconds. */ - expires: z.optional(z.int()), - /** How long until the relay asks again, in whole seconds. Zero would be a tight loop. */ - revalidate: z.optional(z.int().check(z.positive())), - /** The stats tier this session's traffic is bucketed under: `/`-joined segments of ASCII letters, digits, `.`, `_`, and `-`. */ - tier: z.optional(TierSchema), - /** The literal path this session's stats record under, independent of its root. Absent records under the empty label. */ - label: z.optional(PathSchema), - /** - * The session is a cluster peer (another relay): what it announces entered the cluster elsewhere, - * and it discovers hidden routes without asking, for peers that predate the opt-in. - * A client certificate alone is not. - */ - peer: z.optional(z.boolean()), - /** The peer is upstream: the relay never offers it a route learned from another upstream. Requires `peer`. */ - upstream: z.optional(z.boolean()), - }) + .pipe( + z.object({ + /** Patterns the session may publish, relative to the root, optionally with a bitrate cap. */ + publish: z.optional(WireRoleSchema), + /** Patterns the session may subscribe to, relative to the root, optionally with a bitrate cap. */ + subscribe: z.optional(WireRoleSchema), + /** The path the patterns are relative to, replacing the dialed one. Absent means the dialed path. */ + root: z.optional(z.string()), + /** Subtrees read from elsewhere: each path, relative to the root, resolves at the absolute path it maps to. Read-only. */ + mounts: z.optional(z.record(z.string(), z.string())), + /** When the session closes, as whole unix seconds. */ + expires: z.optional(z.int()), + /** How long until the relay asks again, in whole seconds. Zero would be a tight loop. */ + revalidate: z.optional(z.int().check(z.positive())), + /** The stats tier this session's traffic is bucketed under: `/`-joined segments of ASCII letters, digits, `.`, `_`, and `-`. */ + tier: z.optional(TierSchema), + /** The literal path this session's stats record under, independent of its root. Absent records under the empty label. */ + label: z.optional(PathSchema), + /** + * The session is a cluster peer (another relay): what it announces entered the cluster elsewhere, + * and it discovers hidden routes without asking, for peers that predate the opt-in. + * A client certificate alone is not. + */ + peer: z.optional(z.boolean()), + /** The peer is upstream: the relay never offers it a route learned from another upstream. Requires `peer`. */ + upstream: z.optional(z.boolean()), + }), + z.transform(decodeRoles), + ) .check( z.refine((grant) => (grant.publish?.length ?? 0) > 0 || (grant.subscribe?.length ?? 0) > 0, { message: "a grant must name something", @@ -208,6 +215,17 @@ export const GrantSchema = z /** A value parsed by {@link GrantSchema}. */ export type Grant = z.infer; +/** + * The JSON an auth server answers with for `grant`, writing each capped role as + * `{"path": [...], "rate": bps}`. Throws on a grant {@link GrantSchema} would refuse. + */ +export function encodeGrant(grant: Grant): z.input { + const { publish, publish_rate, subscribe, subscribe_rate, ...rest } = grant; + const wire = { ...encodeRoles({ publish, publish_rate, subscribe, subscribe_rate }), ...rest }; + GrantSchema.parse(wire); + return wire; +} + /** The reason and verified attribution an auth server returns with HTTP 403. */ export const RefusalSchema = z.object({ /** Unknown reason strings read as `refused`. */ diff --git a/js/auth/src/interop.test.ts b/js/auth/src/interop.test.ts index b7927746d82..d9d71299694 100644 --- a/js/auth/src/interop.test.ts +++ b/js/auth/src/interop.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from "bun:test"; import { authorize } from "./claims.ts"; -import { GrantSchema, RequestSchema } from "./contract.ts"; +import { encodeGrant, GrantSchema, RequestSchema } from "./contract.ts"; import { Key } from "./key.ts"; import { KeySet } from "./set.ts"; @@ -302,6 +302,6 @@ describe("Rust-serialized contract", () => { peer: true, upstream: true, }); - expect(JSON.stringify(grant)).toBe(RUST_GRANT); + expect(JSON.stringify(encodeGrant(grant))).toBe(RUST_GRANT); }); }); diff --git a/js/auth/src/key.test.ts b/js/auth/src/key.test.ts index 86149e7d55b..1ae15cd4389 100644 --- a/js/auth/src/key.test.ts +++ b/js/auth/src/key.test.ts @@ -70,7 +70,8 @@ test("verify refuses unsafe signed payloads without dropping their restrictions" { publish: ["**"], exp: 1000 }, { publish: ["**"], nbf: 1001 }, { publish: ["**"], subscribe: { path: "alice/**", rate: 8000 } }, - { publish: { path: ["alice/**"], rate: 8000 }, subscribe: ["**"] }, + { publish: { path: ["alice/**"], rate: 0 }, subscribe: ["**"] }, + { publish: { path: ["alice/**"] }, subscribe: ["**"] }, ]) { await expect(Key.verify(key, await signed(payload))).rejects.toThrow(); } @@ -831,6 +832,27 @@ test("key scope is enforced when signing and verifying", async () => { await expect(Key.verify(scoped, forged)).rejects.toThrow("exceed the key scope"); }); +test("caps survive signing, and a key scope bounds only their paths", async () => { + const key: Key = { ...Key.parse(encodeJwk(testKey)), scope: { root: "room", publish: ["**"], subscribe: ["**"] } }; + const token = await Key.sign(key, { + root: "room", + publish: { path: ["alice/**"], rate: 8000 }, + subscribe: { path: ["bob/**"], rate: 16000 }, + }); + const claims = await Key.verify(key, token); + expect(claims).toMatchObject({ + publish: ["alice/**"], + publish_rate: 8000, + subscribe: ["bob/**"], + subscribe_rate: 16000, + }); + + // The decoded claims sign again with their caps. + const resigned = await Key.verify(key, await Key.sign(key, claims)); + expect(resigned).toMatchObject({ publish_rate: 8000, subscribe_rate: 16000 }); + await expect(Key.sign(key, { ...claims, publish_rate: 0 })).rejects.toThrow(/rate must be a positive integer/); +}); + test("key scope treats absolute and rooted grants alike", async () => { const scoped: Key = { ...Key.parse(encodeJwk(testKey)), scope: { root: "project", publish: ["live/**"] } }; diff --git a/js/auth/src/key.ts b/js/auth/src/key.ts index 502b56aeb0a..7b3d3945416 100644 --- a/js/auth/src/key.ts +++ b/js/auth/src/key.ts @@ -4,7 +4,7 @@ import * as jose from "jose"; import { type Algorithm, AlgorithmSchema } from "./algorithm.ts"; import { type Claims, ClaimsSchema, ScopeSchema, scopeAllows } from "./claims.ts"; import * as Path from "./path.ts"; -import { encodeGrants } from "./wire.ts"; +import { encodeGrants, nestRates } from "./wire.ts"; /** * A validated key identifier (kid). At most 128 alphanumeric, hyphen, or underscore characters. @@ -194,18 +194,25 @@ function parse(jwk: string): Key { * `iat` is written only when the claims carry one, so an unset field stays off the * wire rather than being stamped with the current time. * + * Takes the wire shape or the decoded {@link Claims}, so a cap may be either a capped + * role or a `publish_rate`/`subscribe_rate` beside it. + * * Throws on a root spelled with an empty segment, a `\` escaping anything but `\` or * `/`, or more than 4096 bytes. Verifying stays lenient on the empty segments, so tokens * minted before stay valid. */ -async function sign(key: Key, claims: z.input): Promise { +async function sign( + key: Key, + claims: z.input & { publish_rate?: number; subscribe_rate?: number }, +): Promise { ensureOperationSupported(key, "sign"); // Scope-check and sign what the schema parsed, never the raw input: an untyped - // caller could pass legacy `put`/`get` fields the scope check would not see. + // caller could pass legacy `put`/`get` fields the scope check would not see. The + // caps of decoded claims, as `verify` returns them, nest back into their roles first. let parsed: Claims; try { - parsed = ClaimsSchema.parse(claims); + parsed = ClaimsSchema.parse(nestRates(claims)); } catch (error) { throw new Error(`Invalid claims: ${error instanceof Error ? error.message : "unknown error"}`); } diff --git a/js/auth/src/set.ts b/js/auth/src/set.ts index 7472f135dc5..5f95fa789d0 100644 --- a/js/auth/src/set.ts +++ b/js/auth/src/set.ts @@ -9,7 +9,7 @@ import * as z from "@zod/mini"; import * as jose from "jose"; -import type { Claims, ClaimsSchema } from "./claims.ts"; +import type { Claims } from "./claims.ts"; import { Key, KeySchema, type PublicKey } from "./key.ts"; /** @@ -58,7 +58,7 @@ function find(set: KeySet, kid: string): Key | undefined { } /** Sign the claims with the first key that permits signing and contains signing material. */ -async function sign(set: KeySet, claims: z.input): Promise { +async function sign(set: KeySet, claims: Parameters[1]): Promise { const key = set.keys.find((key) => key.key_ops.includes("sign") && (key.kty === "oct" || key.d !== undefined)); if (!key) throw new Error("Cannot find signing key"); return await Key.sign(key, claims); diff --git a/js/auth/src/wire.ts b/js/auth/src/wire.ts index 798f840565a..673f0aaff92 100644 --- a/js/auth/src/wire.ts +++ b/js/auth/src/wire.ts @@ -1,21 +1,64 @@ /** - * The JSON encoding of the grants in claims and key scopes. + * The JSON encoding of the grants in claims, key scopes, and auth server grants. * * Grants were once prefix lists named `put` and `get`, and every published `moq-token` - * reader still expects them. Must stay in lockstep with `wire.rs` in the Rust - * `moq-auth` crate. + * reader still expects them. A role with a bitrate cap is written as an object, + * `{"path": [...], "rate": bps}`, which every older reader refuses rather than drop the + * cap, so a capped role is never written as prefixes. Must stay in lockstep with + * `wire.rs` in the Rust `moq-auth` crate. * * @module */ import { Pattern, Patterns } from "@moq/pattern"; +import * as z from "@zod/mini"; -/** One grant in whichever encoding it arrived: legacy `put`/`get` prefix lists, or patterns. */ +/** A list of pattern texts, each validated by `Pattern.parse`. */ +export const PatternListSchema = z.array( + z.string().check( + z.refine( + (text) => { + try { + Pattern.parse(text); + return true; + } catch { + return false; + } + }, + { message: "invalid path pattern" }, + ), + ), +); + +// At most 2^53 - 1, so every JSON reader holds the cap exactly. +const RateSchema = z.number().check( + z.refine((bps) => Number.isSafeInteger(bps) && bps > 0, { + message: "rate must be a positive integer of at most 9007199254740991 bits per second", + }), +); + +/** + * One role in a claims or grant document: a list of patterns, or exactly `path` and + * `rate`, so a misspelled cap is refused rather than read as none. A key scope bounds + * paths, never bitrate, so it takes only the list. + */ +export const WireRoleSchema = z.union([ + PatternListSchema, + z.strictObject({ path: PatternListSchema, rate: RateSchema }), +]); + +/** One role as written. */ +type WireRole = z.output; + +/** The decoded roles: pattern lists, each with an optional cap in bits per second. */ +type Roles = { publish?: string[]; publish_rate?: number; subscribe?: string[]; subscribe_rate?: number }; + +/** One grant in whichever encoding it arrived: legacy `put`/`get` prefix lists, or roles. */ type WireGrants = { put?: string | string[]; get?: string | string[]; - publish?: string[]; - subscribe?: string[]; + publish?: WireRole; + subscribe?: WireRole; }; /** @@ -25,10 +68,7 @@ type WireGrants = { * encodings, and a legacy prefix containing `*` is refused: it had no wildcards, so * reading one now would silently widen the grant. */ -export function decodeGrants( - wire: T, - ctx: { issues: unknown[] }, -): Omit & { publish?: string[]; subscribe?: string[] } { +export function decodeGrants(wire: T, ctx: { issues: unknown[] }): Omit { const { put, get, ...rest } = wire; if (put === undefined && get === undefined) return rest; @@ -51,16 +91,68 @@ export function decodeGrants( } } +/** Split each capped role into its patterns and a `_rate` field beside them. */ +export function decodeRoles( + wire: T, +): Omit & { + // Spelled out rather than `Roles`, so the public `Claims` and `Grant` types document each field. + /** Patterns the holder may publish, relative to the root. */ + publish?: string[]; + /** The most the holder may publish, in bits per second, or no cap when absent. */ + publish_rate?: number; + /** Patterns the holder may subscribe to, relative to the root. */ + subscribe?: string[]; + /** The most the holder may receive from its subscriptions, in bits per second, or no cap when absent. */ + subscribe_rate?: number; +} { + const { publish, subscribe, ...rest } = wire; + const split = (role: WireRole | undefined): { path?: string[]; rate?: number } => + role === undefined || Array.isArray(role) ? { path: role } : role; + const pub = split(publish); + const sub = split(subscribe); + return { + ...(pub.path && { publish: pub.path }), + ...(pub.rate !== undefined && { publish_rate: pub.rate }), + ...(sub.path && { subscribe: sub.path }), + ...(sub.rate !== undefined && { subscribe_rate: sub.rate }), + ...rest, + }; +} + +/** + * Nest each `_rate` field back into its role, so the decoded claims `Key.verify` + * returns parse again. A role already in the capped form is left for the schema to refuse. + */ +export function nestRates< + T extends { publish?: unknown; subscribe?: unknown; publish_rate?: number; subscribe_rate?: number }, +>(claims: T) { + const { publish_rate, subscribe_rate, ...rest } = claims; + return { + ...rest, + ...(publish_rate !== undefined && { publish: { path: rest.publish ?? [], rate: publish_rate } }), + ...(subscribe_rate !== undefined && { subscribe: { path: rest.subscribe ?? [], rate: subscribe_rate } }), + }; +} + +/** Write each role as its patterns, or as the capped object when it has a rate; an empty uncapped role is omitted. */ +export function encodeRoles(roles: Roles): { publish?: WireRole; subscribe?: WireRole } { + const role = (path: string[] | undefined, rate: number | undefined): WireRole | undefined => + rate !== undefined ? { path: path ?? [], rate } : path?.length ? path : undefined; + const publish = role(roles.publish, roles.publish_rate); + const subscribe = role(roles.subscribe, roles.subscribe_rate); + return { ...(publish && { publish }), ...(subscribe && { subscribe }) }; +} + /** - * Write grants the legacy way when every pattern is a subtree, so every published - * `moq-token` reader agrees on what they grant. Anything a prefix can't say is written - * as `publish`/`subscribe`, which an older reader refuses rather than misreads. + * Write grants the legacy way when every pattern is a subtree and no role is capped, so + * every published `moq-token` reader agrees on what they grant. Anything a prefix can't + * say is written as `publish`/`subscribe`, which an older reader refuses rather than misreads. */ -export function encodeGrants( - grants: T, -): Omit & WireGrants { - const { publish, subscribe, ...rest } = grants; - const prefixes = (texts: string[] | undefined) => { +export function encodeGrants(grants: T): Omit & WireGrants { + const { publish, publish_rate, subscribe, subscribe_rate, ...rest } = grants; + const prefixes = (texts: string[] | undefined, rate: number | undefined) => { + // A cap has no prefix form, so one moves the whole document to patterns. + if (rate !== undefined) return undefined; const out: string[] = []; for (const pattern of new Patterns((texts ?? []).map((text) => Pattern.parse(text)))) { const prefix = pattern.asPrefix(); @@ -70,14 +162,10 @@ export function encodeGrants, + /// Patterns the holder may subscribe to, relative to the root. pub subscribe: Patterns, + /// The most the holder may receive from its subscriptions, or no cap when absent. + pub subscribe_rate: Option, + /// The expiration time of the token as a unix timestamp (`exp`). pub expires: Option, @@ -155,6 +163,19 @@ impl Claims { self } + /// Cap what the holder may publish, in bits per second; `None` lifts the cap. + pub fn with_publish_rate(mut self, rate: impl Into>) -> Self { + self.publish_rate = rate.into(); + self + } + + /// Cap what the holder may receive from its subscriptions, in bits per second; `None` + /// lifts the cap. + pub fn with_subscribe_rate(mut self, rate: impl Into>) -> Self { + self.subscribe_rate = rate.into(); + self + } + /// Expire the token at this time. Enforced by [`Key::verify`](crate::Key::verify). /// /// Accepts an `Option` so a caller can pass one through without unwrapping it. @@ -248,6 +269,8 @@ mod tests { fn create_test_claims() -> Claims { Claims { + publish_rate: None, + subscribe_rate: None, root: "test-path".to_string(), publish: patterns(&["test-pub/**"]), subscribe: patterns(&["test-sub/**"]), diff --git a/rs/moq-auth/src/grant.rs b/rs/moq-auth/src/grant.rs index 3ad16c15820..7c461bcd754 100644 --- a/rs/moq-auth/src/grant.rs +++ b/rs/moq-auth/src/grant.rs @@ -1,7 +1,7 @@ +use crate::Rate; use kio::clock::Wall; use moq_pattern::Patterns; use serde::{Deserialize, Serialize}; -use serde_with::{DurationSeconds, serde_as}; use std::collections::BTreeMap; use std::time::Duration; @@ -11,20 +11,25 @@ use std::time::Duration; /// nothing is a refusal too, and one that asks to be revalidated must say when it /// expires, so an outage always has a bound the server chose. [`validate`](Self::validate) /// checks these once at the boundary. -#[serde_as] -#[serde_with::skip_serializing_none] +/// +/// A role with a cap is written `{"path": [...], "rate": bps}`, which every older +/// reader refuses rather than drop the cap. #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -#[serde(default)] +#[serde(from = "crate::wire::Grant", into = "crate::wire::Grant")] #[non_exhaustive] pub struct Grant { /// Patterns the session may publish, relative to the root. - #[serde(skip_serializing_if = "Patterns::is_empty")] pub publish: Patterns, + /// The most the session may publish, or no cap when absent. + pub publish_rate: Option, + /// Patterns the session may subscribe to, relative to the root. - #[serde(skip_serializing_if = "Patterns::is_empty")] pub subscribe: Patterns, + /// The most the session may receive from its subscriptions, or no cap when absent. + pub subscribe_rate: Option, + /// The path the patterns are relative to, replacing the dialed one. This is how a /// server aliases a slug to a canonical id. Absent means the dialed path. pub root: Option, @@ -32,15 +37,12 @@ pub struct Grant { /// Subtrees the session reads from elsewhere: each path, relative to the root, /// resolves at the absolute path it maps to. Read-only: nothing is published /// beneath one. The patterns still name the path relative to the root. - #[serde(skip_serializing_if = "BTreeMap::is_empty")] pub mounts: BTreeMap, /// When the session closes, as unix seconds. - #[serde_as(as = "Option")] pub expires: Option, /// How long until the relay asks again, in seconds. - #[serde_as(as = "Option>")] pub revalidate: Option, /// The stats tier this session's traffic is bucketed under: `/`-joined segments of @@ -53,13 +55,11 @@ pub struct Grant { /// The session is a cluster peer (another relay): what it announces entered /// the cluster elsewhere, not here, and it discovers hidden routes without /// asking, for peers that predate the opt-in. A client certificate alone is not. - #[serde(skip_serializing_if = "std::ops::Not::not")] pub peer: bool, /// The cluster peer is upstream: the relay never offers it a route learned /// from another upstream peer, so it never carries traffic between two. /// Requires `peer`. - #[serde(skip_serializing_if = "std::ops::Not::not")] pub upstream: bool, } @@ -125,6 +125,8 @@ mod tests { #[test] fn round_trips_in_seconds() { let grant = Grant { + publish_rate: None, + subscribe_rate: None, publish: patterns(&["alice/**"]), subscribe: patterns(&["**"]), root: Some("pid/room".into()), @@ -150,6 +152,8 @@ mod tests { #[test] fn serializes_to_the_cross_language_vector() { let grant = Grant { + publish_rate: None, + subscribe_rate: None, publish: patterns(&["alice/**"]), subscribe: patterns(&["**"]), root: Some("pid/room".into()), diff --git a/rs/moq-auth/src/key.rs b/rs/moq-auth/src/key.rs index cb2d4eb318e..0fbdb8f583e 100644 --- a/rs/moq-auth/src/key.rs +++ b/rs/moq-auth/src/key.rs @@ -779,6 +779,8 @@ mod tests { fn create_test_claims() -> Claims { Claims { + publish_rate: None, + subscribe_rate: None, root: "test-path".to_string(), publish: patterns(&["test-pub/**"]), subscribe: patterns(&["test-sub/**"]), @@ -1058,6 +1060,8 @@ mod tests { fn test_key_sign_invalid_claims() { let key = create_test_key(); let invalid_claims = Claims { + publish_rate: None, + subscribe_rate: None, root: "test-path".to_string(), publish: patterns(&[]), subscribe: patterns(&[]), @@ -1236,6 +1240,8 @@ mod tests { fn test_key_verify_token_without_exp() { let key = create_test_key(); let claims = Claims { + publish_rate: None, + subscribe_rate: None, root: "test-path".to_string(), publish: patterns(&["**"]), subscribe: patterns(&["**"]), @@ -1256,6 +1262,8 @@ mod tests { fn test_key_round_trip() { let key = create_test_key(); let original_claims = Claims { + publish_rate: None, + subscribe_rate: None, root: "test-path".to_string(), publish: patterns(&["test-pub/**"]), subscribe: patterns(&["test-sub/**"]), diff --git a/rs/moq-auth/src/lib.rs b/rs/moq-auth/src/lib.rs index 38183768fff..bb48a08b43d 100644 --- a/rs/moq-auth/src/lib.rs +++ b/rs/moq-auth/src/lib.rs @@ -16,7 +16,8 @@ //! its presenter key, that another client verifies with [`KeySet::to_peer_set`]. //! //! Grants and claims name paths with [`Pattern`]s from `moq-pattern`, re-exported here: -//! `foo` is one broadcast, `foo/**` is a subtree, `**` is everything. +//! `foo` is one broadcast, `foo/**` is a subtree, `**` is everything. Each role may also +//! carry a bitrate cap, a [`Rate`]. #![warn(missing_docs)] @@ -30,6 +31,7 @@ mod key; mod key_id; mod path; mod peer; +mod rate; mod request; mod set; mod wire; @@ -55,5 +57,6 @@ pub use key::*; pub use key_id::*; pub use moq_pattern::{InvalidPattern, Pattern, Patterns, Segment, Specificity}; pub use peer::*; +pub use rate::*; pub use request::*; pub use set::*; diff --git a/rs/moq-auth/src/rate.rs b/rs/moq-auth/src/rate.rs new file mode 100644 index 00000000000..62cb5b027e8 --- /dev/null +++ b/rs/moq-auth/src/rate.rs @@ -0,0 +1,110 @@ +use std::num::NonZeroU64; + +/// A bitrate cap on one direction of a session, in bits per second. +/// +/// Positive, and at most 2^53 - 1 so every JSON reader holds it exactly. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize)] +#[serde(try_from = "u64", into = "u64")] +pub struct Rate(NonZeroU64); + +/// A bitrate that is zero or past JSON's exact integer range. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("rate must be a positive integer of at most 9007199254740991 bits per second")] +pub struct InvalidRate; + +impl Rate { + /// The largest rate JSON carries exactly: 2^53 - 1 bits per second. + pub const MAX: Self = Self(NonZeroU64::new((1 << 53) - 1).expect("non-zero")); + + /// A cap of `bps` bits per second. + pub fn from_bps(bps: u64) -> Result { + match NonZeroU64::new(bps) { + Some(bps) if bps <= Self::MAX.0 => Ok(Self(bps)), + _ => Err(InvalidRate), + } + } + + /// This cap in bits per second. + pub fn as_bps(self) -> u64 { + self.0.get() + } +} + +impl TryFrom for Rate { + type Error = InvalidRate; + + fn try_from(bps: u64) -> Result { + Self::from_bps(bps) + } +} + +impl From for u64 { + fn from(rate: Rate) -> Self { + rate.as_bps() + } +} + +impl std::fmt::Display for Rate { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}bps", self.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn refuses_zero_and_inexact_rates() { + assert_eq!(Rate::from_bps(0), Err(InvalidRate)); + assert_eq!(Rate::from_bps(1 << 53), Err(InvalidRate)); + assert_eq!(Rate::from_bps((1 << 53) - 1), Ok(Rate::MAX)); + for json in ["0", "-1", "1.5", "9007199254740992", "null", "\"8000\""] { + assert!(serde_json::from_str::(json).is_err(), "{json}"); + } + assert_eq!(serde_json::from_str::("8000").unwrap().as_bps(), 8000); + } + + /// The vectors `js/auth` reads too, so both languages agree on the capped wire shape. + #[test] + fn shared_wire_vectors() { + let vectors: serde_json::Value = serde_json::from_str(include_str!("../rate.json")).unwrap(); + for row in vectors["valid"].as_array().unwrap() { + let claims: crate::Claims = serde_json::from_value(row["input"].clone()).unwrap(); + claims.validate().unwrap(); + assert_eq!(serde_json::to_value(&claims).unwrap(), row["claims"]); + let grant: crate::Grant = serde_json::from_value(row["input"].clone()).unwrap(); + grant.validate().unwrap(); + assert_eq!(serde_json::to_value(&grant).unwrap(), row["grant"]); + } + for input in vectors["invalid"].as_array().unwrap() { + assert!( + serde_json::from_value::(input.clone()).is_err(), + "{input}" + ); + assert!( + serde_json::from_value::(input.clone()).is_err(), + "{input}" + ); + } + } + + /// A cap survives signing and rebasing, and a key scope bounds paths, never bitrate. + #[test] + fn caps_survive_signing_but_scopes_refuse_them() { + let wire = serde_json::json!({ + "root": "room", + "publish": {"path": ["alice/**"], "rate": 8000}, + "subscribe": {"path": ["bob/**"], "rate": 16000}, + }); + assert!(serde_json::from_value::(wire.clone()).is_err()); + let claims: crate::Claims = serde_json::from_value(wire).unwrap(); + let key = crate::Key::generate(crate::Algorithm::HS256, None).unwrap(); + let claims = key.verify(&key.sign(&claims).unwrap()).unwrap(); + assert_eq!(claims.publish_rate, Some(Rate::from_bps(8000).unwrap())); + assert_eq!(claims.subscribe_rate, Some(Rate::from_bps(16000).unwrap())); + let scope: crate::Scope = + serde_json::from_str(r#"{"root":"room","publish":["**"],"subscribe":["**"]}"#).unwrap(); + assert!(scope.allows(&claims)); + } +} diff --git a/rs/moq-auth/src/serve.rs b/rs/moq-auth/src/serve.rs index b59a4832ebf..bde4808d75e 100644 --- a/rs/moq-auth/src/serve.rs +++ b/rs/moq-auth/src/serve.rs @@ -233,7 +233,7 @@ impl Policy { if jwt.is_some() && request.tls.is_some() { return Err(Denial::TokenAndCertificate); } - let (permissions, expires, certificate, root) = if let Some(jwt) = jwt { + let (permissions, rates, expires, certificate, root) = if let Some(jwt) = jwt { let claims = self.verify(&jwt).await?; let permissions = claims.authorize(&request.path).map_err(|err| match err { crate::Error::RootMismatch(path) => Denial::RootMismatch { @@ -246,20 +246,28 @@ impl Policy { }, other => Denial::InvalidToken(other.to_string()), })?; - (permissions, claims.expires, false, Some(claims.root)) + let rates = (claims.publish_rate, claims.subscribe_rate); + (permissions, rates, claims.expires, false, Some(claims.root)) } else if let Some(peer) = &request.tls { if self.mtls.is_empty() { return Err(Denial::NoMtlsGrant); } - (authorize(&self.mtls, &request.path)?, peer.expires, true, None) + ( + authorize(&self.mtls, &request.path)?, + (None, None), + peer.expires, + true, + None, + ) } else { if self.public.is_empty() { return Err(Denial::NoPublicGrant); } - (authorize(&self.public, &request.path)?, None, false, None) + (authorize(&self.public, &request.path)?, (None, None), None, false, None) }; let mut grant = Grant::new(permissions.publish, permissions.subscribe); + (grant.publish_rate, grant.subscribe_rate) = rates; grant.expires = match (expires, self.expires) { (Some(expires), _) => Some(expires), (None, Some(bound)) => Some(kio::clock::wall().checked_add(bound).ok_or(Denial::ExpiresOutOfRange)?), diff --git a/rs/moq-auth/src/set.rs b/rs/moq-auth/src/set.rs index 3c3cd6d99e5..86f83d75943 100644 --- a/rs/moq-auth/src/set.rs +++ b/rs/moq-auth/src/set.rs @@ -149,6 +149,8 @@ mod tests { fn create_test_claims() -> Claims { Claims { + publish_rate: None, + subscribe_rate: None, root: "test-path".to_string(), publish: patterns(&["test-pub/**"]), subscribe: patterns(&["test-sub/**"]), diff --git a/rs/moq-auth/src/wire.rs b/rs/moq-auth/src/wire.rs index 8309bdb5b8d..1a0e72de723 100644 --- a/rs/moq-auth/src/wire.rs +++ b/rs/moq-auth/src/wire.rs @@ -6,10 +6,17 @@ //! reader, old or new, agrees on what they grant. Anything a prefix can't say is written //! as `publish` and `subscribe`, which an old reader sees as granting nothing and //! refuses. Both encodings are read; one document never mixes them. +//! +//! A role with a bitrate cap is written as an object, `{"path": [...], "rate": bps}`, +//! which every older reader refuses rather than drop the cap, so a capped role is never +//! written as prefixes. +use crate::Rate; use moq_pattern::{Pattern, Patterns}; use serde::{Deserialize, Deserializer, Serialize}; -use serde_with::serde_as; +use serde_with::{DurationSeconds, serde_as}; +use std::collections::BTreeMap; +use std::time::Duration; /// The wire form of [`Claims`](crate::Claims). #[serde_with::skip_serializing_none] @@ -24,9 +31,9 @@ pub(crate) struct Claims { #[serde(deserialize_with = "present")] get: Option, #[serde(deserialize_with = "present")] - publish: Option, + publish: Option, #[serde(deserialize_with = "present")] - subscribe: Option, + subscribe: Option, #[serde_as(as = "Option")] exp: Option, #[serde_as(as = "Option")] @@ -46,7 +53,10 @@ pub(crate) struct Claims { impl From for Claims { fn from(claims: crate::Claims) -> Self { - let grants = Grants::encode(claims.publish, claims.subscribe); + let grants = Grants::encode( + Role::new(claims.publish, claims.publish_rate), + Role::new(claims.subscribe, claims.subscribe_rate), + ); Self { root: claims.root, put: grants.put.map(Prefixes::Many), @@ -76,8 +86,10 @@ impl TryFrom for crate::Claims { let (publish, subscribe) = grants.decode()?; Ok(Self { root: wire.root, - publish, - subscribe, + publish: publish.path, + publish_rate: publish.rate, + subscribe: subscribe.path, + subscribe_rate: subscribe.rate, expires: wire.exp, issued: wire.iat, not_before: wire.nbf, @@ -104,13 +116,14 @@ pub(crate) struct Scope { impl From for Scope { fn from(scope: crate::Scope) -> Self { - let grants = Grants::encode(scope.publish, scope.subscribe); + let grants = Grants::encode(Role::new(scope.publish, None), Role::new(scope.subscribe, None)); + let paths = |role: Option| role.map(|role| role.path); Self { root: scope.root, put: grants.put, get: grants.get, - publish: grants.publish, - subscribe: grants.subscribe, + publish: paths(grants.publish), + subscribe: paths(grants.subscribe), } } } @@ -119,17 +132,18 @@ impl TryFrom for crate::Scope { type Error = String; fn try_from(wire: Scope) -> Result { + // A key scope bounds paths, never bitrate, so its roles are only lists. let grants = Grants { put: wire.put, get: wire.get, - publish: wire.publish, - subscribe: wire.subscribe, + publish: wire.publish.map(|path| Role::new(path, None)), + subscribe: wire.subscribe.map(|path| Role::new(path, None)), }; let (publish, subscribe) = grants.decode()?; Ok(Self { root: wire.root, - publish, - subscribe, + publish: publish.path, + subscribe: subscribe.path, }) } } @@ -161,19 +175,23 @@ fn present<'de, D: Deserializer<'de>, T: Deserialize<'de>>(deserializer: D) -> R struct Grants { put: Option>, get: Option>, - publish: Option, - subscribe: Option, + publish: Option, + subscribe: Option, } impl Grants { - fn encode(publish: Patterns, subscribe: Patterns) -> Self { - let prefixes = |patterns: &Patterns| -> Option> { - patterns + fn encode(publish: Role, subscribe: Role) -> Self { + // A cap has no prefix form, so one moves the whole document to patterns. + let prefixes = |role: &Role| -> Option> { + if role.rate.is_some() { + return None; + } + role.path .iter() .map(|pattern| pattern.as_prefix().map(str::to_string)) .collect() }; - let some = |patterns: Patterns| (!patterns.is_empty()).then_some(patterns); + let some = |role: Role| (!role.is_empty()).then_some(role); match (prefixes(&publish), prefixes(&subscribe)) { (Some(put), Some(get)) => Self { @@ -191,7 +209,7 @@ impl Grants { } } - fn decode(self) -> Result<(Patterns, Patterns), String> { + fn decode(self) -> Result<(Role, Role), String> { let legacy = self.put.is_some() || self.get.is_some(); if !legacy { return Ok((self.publish.unwrap_or_default(), self.subscribe.unwrap_or_default())); @@ -200,13 +218,139 @@ impl Grants { return Err("mixes the legacy put/get fields with publish/subscribe".to_string()); } - let subtrees = |prefixes: Option>| -> Result { - prefixes + let subtrees = |prefixes: Option>| -> Result { + let path = prefixes .unwrap_or_default() .iter() .map(|prefix| Pattern::subtree(prefix).map_err(|err| format!("legacy prefix {prefix:?}: {err}"))) - .collect() + .collect::>()?; + Ok(Role::new(path, None)) }; Ok((subtrees(self.put)?, subtrees(self.get)?)) } } + +/// One role on the wire: a list of patterns, or an object adding a bitrate cap. +/// +/// The object takes exactly `path` and `rate`, so a misspelled cap is refused rather +/// than read as none. +#[derive(Default)] +struct Role { + path: Patterns, + rate: Option, +} + +impl Role { + fn new(path: Patterns, rate: Option) -> Self { + Self { path, rate } + } + + fn is_empty(&self) -> bool { + self.path.is_empty() && self.rate.is_none() + } +} + +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Capped

{ + path: P, + rate: Rate, +} + +impl Serialize for Role { + fn serialize(&self, serializer: S) -> Result { + match self.rate { + None => self.path.serialize(serializer), + Some(rate) => Capped { path: &self.path, rate }.serialize(serializer), + } + } +} + +impl<'de> Deserialize<'de> for Role { + fn deserialize>(deserializer: D) -> Result { + use serde::de::value::{MapAccessDeserializer, SeqAccessDeserializer}; + + // Dispatch on the shape, so a bad pattern or cap reports its own error rather + // than an untagged enum's "matched no variant". + struct Visitor; + impl<'de> serde::de::Visitor<'de> for Visitor { + type Value = Role; + + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("a list of path patterns, or an object of a `path` list and a `rate`") + } + + fn visit_seq>(self, seq: A) -> Result { + Ok(Role::new(Patterns::deserialize(SeqAccessDeserializer::new(seq))?, None)) + } + + fn visit_map>(self, map: A) -> Result { + let Capped { path, rate } = Capped::::deserialize(MapAccessDeserializer::new(map))?; + Ok(Role::new(path, Some(rate))) + } + } + deserializer.deserialize_any(Visitor) + } +} + +/// The wire form of [`Grant`](crate::Grant), whose roles may carry caps. A field it does +/// not know is ignored, so an auth server may answer more than this reader takes. +#[serde_as] +#[serde_with::skip_serializing_none] +#[derive(Serialize, Deserialize, Default)] +#[serde(default)] +pub(crate) struct Grant { + #[serde(skip_serializing_if = "Role::is_empty")] + publish: Role, + #[serde(skip_serializing_if = "Role::is_empty")] + subscribe: Role, + root: Option, + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + mounts: BTreeMap, + #[serde_as(as = "Option")] + expires: Option, + #[serde_as(as = "Option>")] + revalidate: Option, + tier: Option, + label: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + peer: bool, + #[serde(skip_serializing_if = "std::ops::Not::not")] + upstream: bool, +} + +impl From for Grant { + fn from(grant: crate::Grant) -> Self { + Self { + publish: Role::new(grant.publish, grant.publish_rate), + subscribe: Role::new(grant.subscribe, grant.subscribe_rate), + root: grant.root, + mounts: grant.mounts, + expires: grant.expires, + revalidate: grant.revalidate, + tier: grant.tier, + label: grant.label, + peer: grant.peer, + upstream: grant.upstream, + } + } +} + +impl From for crate::Grant { + fn from(wire: Grant) -> Self { + Self { + publish: wire.publish.path, + publish_rate: wire.publish.rate, + subscribe: wire.subscribe.path, + subscribe_rate: wire.subscribe.rate, + root: wire.root, + mounts: wire.mounts, + expires: wire.expires, + revalidate: wire.revalidate, + tier: wire.tier, + label: wire.label, + peer: wire.peer, + upstream: wire.upstream, + } + } +} diff --git a/rs/moq-auth/tests/released_token.rs b/rs/moq-auth/tests/released_token.rs index 67edd931a17..ec90fe30c0c 100644 --- a/rs/moq-auth/tests/released_token.rs +++ b/rs/moq-auth/tests/released_token.rs @@ -5,7 +5,6 @@ //! `publish`/`subscribe`, which 0.14 reads as granting nothing and refuses, rather //! than misreading an exact grant as a prefix. -use base64::Engine; use moq_auth::{Algorithm, Claims, Key}; fn patterns(texts: &[&str]) -> moq_auth::Patterns { @@ -71,34 +70,36 @@ fn patterns_keep_the_published_auth_wire_shape() { } } +/// A capped role is an object every published verifier refuses in full, so an old +/// relay never admits the paths while dropping the cap. #[test] -fn capped_objects_are_refused_in_full_by_current_and_published_auth() { +fn capped_roles_are_refused_in_full_by_published_auth() { let key = Key::generate(Algorithm::HS256, None).unwrap(); let published = moq_auth_published::Key::from_str(&key.to_str().unwrap()).unwrap(); let legacy = moq_token::Key::from_str(&key.to_str().unwrap()).unwrap(); - let jwk: serde_json::Value = serde_json::to_value(&key).unwrap(); - let secret = base64::engine::general_purpose::URL_SAFE_NO_PAD - .decode(jwk["k"].as_str().unwrap()) - .unwrap(); - let signing = jsonwebtoken::EncodingKey::from_secret(&secret); - for role in ["publish", "subscribe"] { - for path in [serde_json::json!("alice/**"), serde_json::json!(["alice/**"])] { - let mut payload = serde_json::json!({"publish":["**"],"subscribe":["**"]}); - payload[role] = serde_json::json!({"path":path,"rate":8_000_000}); - let token = jsonwebtoken::encode( - &jsonwebtoken::Header::new(jsonwebtoken::Algorithm::HS256), - &payload, - &signing, - ) - .unwrap(); - assert!( - key.verify(&token).is_err(), - "must not discard the restriction: {payload}" - ); - assert!(published.verify(&token).is_err()); - assert!(legacy.verify(&token).is_err()); - assert!(serde_json::from_value::(payload.clone()).is_err()); - assert!(serde_json::from_value::(payload).is_err()); - } + let rate = moq_auth::Rate::from_bps(8_000_000).unwrap(); + for publish in [true, false] { + // Subtrees alone would be written as legacy prefixes, which a cap must not be. + let mut claims = Claims::default() + .with_publish(patterns(&["alice/**"])) + .with_subscribe(patterns(&["**"])); + claims = match publish { + true => claims.with_publish_rate(rate), + false => claims.with_subscribe_rate(rate), + }; + let token = key.sign(&claims).unwrap(); + let verified = key.verify(&token).unwrap(); + assert_eq!( + (verified.publish_rate, verified.subscribe_rate), + (claims.publish_rate, claims.subscribe_rate) + ); + assert!(published.verify(&token).is_err(), "{claims:?}"); + assert!(legacy.verify(&token).is_err(), "{claims:?}"); + + let mut grant = moq_auth::Grant::new(claims.publish.clone(), claims.subscribe.clone()); + (grant.publish_rate, grant.subscribe_rate) = (claims.publish_rate, claims.subscribe_rate); + let wire = serde_json::to_value(&grant).unwrap(); + assert_eq!(serde_json::from_value::(wire.clone()).unwrap(), grant); + assert!(serde_json::from_value::(wire).is_err()); } } diff --git a/rs/moq-net/CHANGELOG.md b/rs/moq-net/CHANGELOG.md index 05326cc85f5..456e1513be5 100644 --- a/rs/moq-net/CHANGELOG.md +++ b/rs/moq-net/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `transport::SETUP_RECEIVE_WINDOW`, the connection credit a server offers before it admits a session: enough for the SETUP and one in-band AUTH. + ## [0.3.10](https://github.com/moq-dev/moq/compare/moq-net-v0.3.9...moq-net-v0.3.10) - 2026-10-05 ### Fixed diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index 2420752f0c9..eb5aed715e1 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -152,6 +152,20 @@ pub const MIN_RECEIVE_WINDOW: u64 = { if message > segment { message } else { segment } }; +/// The connection receive window a server offers before it admits the session. +/// +/// It fits the SETUP and one in-band AUTH, each with its framing, which is all a client +/// sends before the reply. Credit cannot be withdrawn, so a session that started with +/// its full window could bank it and spend it after a lower window or a bitrate cap +/// arrives; the server opens the full window only once it admits the session. +pub const SETUP_RECEIVE_WINDOW: u64 = { + // A type and a size prefix, each a varint of at most 8 bytes. + const PREFIXES: u64 = 16; + let setup = crate::setup::MAX_SETUP_SIZE as u64 + PREFIXES; + let auth = ::MAX_SIZE as u64 + PREFIXES; + setup + auth +}; + /// The typed transport condition when no application code was received. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[non_exhaustive] diff --git a/rs/moq-relay/CHANGELOG.md b/rs/moq-relay/CHANGELOG.md index 104c8b3d4d3..65be2e610b6 100644 --- a/rs/moq-relay/CHANGELOG.md +++ b/rs/moq-relay/CHANGELOG.md @@ -7,9 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- QUIC and WebTransport sessions on the tokio and io_uring listeners enforce a grant's bitrate caps: `publish_rate` paces what the relay receives and `subscribe_rate` what it sends. Unions take the highest cap and an uncapped grant lifts it; a re-checked cap reaches the live session. WebSocket, iroh, and HTTP refuse a capped grant at connect, in band, and at revalidation. Every QUIC session starts with only the SETUP's flow-control credit, opened once admitted. + ### Changed -- [**breaking**] Auth token roles hold path patterns directly, matching release. Unsupported object roles, including bitrate caps, are refused by auth parsing before admission. +- [**breaking**] Auth token roles hold path patterns directly, matching release; a capped role is enforced or refused, never dropped. - Auth requests distinguish `webtransport` from native `quic`. Upgrade auth servers before relays: older servers reject the new value. Rust and JavaScript auth parsers now map future transport names to `unknown`. - `listen.timeout` bounds auth admission as part of the handshake, replacing the fixed ten-second admission timeout, so an auth server answering `connect` late is admitted within the deadline and `"0"` waits through admission too. HTTP reads bound admission by the web listener's timeout. `Auth::admit` and `Cluster::admit` no longer time out on their own; the caller owns the deadline. In-band token verification keeps its ten-second bound. diff --git a/rs/moq-relay/src/auth.rs b/rs/moq-relay/src/auth.rs index ac55012c12d..94af67ff6e9 100644 --- a/rs/moq-relay/src/auth.rs +++ b/rs/moq-relay/src/auth.rs @@ -3,8 +3,11 @@ //! process answering [`Admissions`]. Nothing else admits anyone; a verified client //! certificate is reported to whoever decides as a fact. +mod caps; mod refresh; +pub(crate) use caps::{Caps, Pacer}; + use std::sync::Arc; use axum::http; @@ -443,6 +446,10 @@ pub struct Token { pub subscribe: moq_auth::Patterns, /// Patterns the session may publish, relative to `root`. pub publish: moq_auth::Patterns, + /// The most the session may publish, or no cap when absent. + pub publish_rate: Option, + /// The most the session may receive from its subscriptions, or no cap when absent. + pub subscribe_rate: Option, /// The tier this session's stats record under. pub tier: Tier, /// The path this session's stats record under, empty when the grant names none. @@ -482,6 +489,8 @@ impl Token { .map_err(invalid)?, subscribe: grant.subscribe.clone(), publish: grant.publish.clone(), + publish_rate: grant.publish_rate, + subscribe_rate: grant.subscribe_rate, tier: crate::configured_tier(grant.tier.clone()).map_err(invalid)?, label: Path::new(grant.label.as_deref().unwrap_or_default()).map_err(invalid)?, peer: grant.peer, @@ -526,6 +535,11 @@ pub struct Lease { expires: Option, /// The session's stats context, moved to a re-checked tier. stats: moq_net::stats::Session, + /// What holds the connection to its grant's caps, or `None` on a transport that + /// cannot, which refuses any capped grant. + pacer: Option, + /// The caps the connection holds. + caps: Caps, } struct Source { @@ -537,15 +551,22 @@ impl Lease { /// Hold `consumer` for a session that dialed `path`, reducing its grant to /// what the origin scopes by. /// - /// Fails if the grant's paths are too long; see [`Token::new`]. + /// Fails if the grant's paths are too long; see [`Token::new`]. Fails too if the + /// grant caps a bitrate, which nothing here enforces. pub fn new(path: &str, consumer: lease::Consumer) -> Result { let grant = consumer.grant(); - Self::granted(path, &grant, consumer) + Self::granted(path, &grant, consumer, None) } /// Hold `consumer` with its token built from `grant`, a snapshot of it. A later /// update still reaches [`Self::ended`] as a change, which re-checks it. - fn granted(path: &str, grant: &Grant, consumer: lease::Consumer) -> Result { + /// + /// A capped grant needs `pacer` to hold the connection to it, and is refused without. + fn granted(path: &str, grant: &Grant, consumer: lease::Consumer, pacer: Option) -> Result { + let caps = Caps::of(grant); + if caps.is_capped() && pacer.is_none() { + return Err(Error::Forbidden(caps::UNENFORCED.into())); + } Ok(Self { refresh: None, source: None, @@ -554,6 +575,8 @@ impl Lease { consumer, session: None, stats: Default::default(), + pacer, + caps, }) } @@ -596,6 +619,15 @@ impl Lease { &self.token } + /// The caps the connection is held to: the union of its live credentials once + /// [prepared](Self::preparing), which the connection applies before answering SETUP. + pub(crate) fn caps(&self) -> Caps { + match &self.refresh { + Some(refresh) => refresh.caps(), + None => self.caps, + } + } + /// Ask the lease's producer to re-check now. A no-op on a fixed lease. pub fn revalidate(&self) { if let Some(refresh) = &self.refresh { @@ -654,6 +686,15 @@ impl Lease { if fresh.upstream != self.token.upstream { return "upstream changed".into(); } + let caps = Caps::of(&grant); + if caps != self.caps { + let Some(pacer) = &self.pacer else { + return caps::UNENFORCED.into(); + }; + tracing::info!(%caps, "bitrate caps changed"); + pacer.pace(caps); + self.caps = caps; + } if fresh.publish != self.token.publish || fresh.subscribe != self.token.subscribe { match &self.session { Some(session) => { @@ -907,11 +948,18 @@ impl Auth { /// deadline: a listener bounds it with the connection's handshake deadline /// ([`moq_tokio::server::Request::within`]), and dropping the future withdraws /// the admission from the decider. + /// + /// A grant that caps a bitrate is refused: nothing here holds the session to it. pub async fn admit(&self, request: Request) -> Result { + self.admit_paced(request, None).await + } + + /// [Admit](Self::admit) a session whose connection `pacer` holds to a capped grant. + pub(crate) async fn admit_paced(&self, request: Request, pacer: Option) -> Result { // Build the token from the grant the admission checked: re-reading it could // pick up an update that skipped the check. let (consumer, grant) = self.decide(request.clone()).await?; - let mut lease = Lease::granted(&request.path, &grant, consumer)?; + let mut lease = Lease::granted(&request.path, &grant, consumer, pacer)?; lease.source = Some(Source { auth: self.clone(), request, @@ -1194,6 +1242,88 @@ mod tests { } } + /// Nothing holds a session admitted without a pacer to a cap, whatever its transport + /// claims, so a capped grant is refused rather than dropped: at admission, and on a + /// lease built without one. + #[tokio::test(start_paused = true)] + async fn capped_grants_are_refused_without_a_pacer() { + use moq_auth::Transport::*; + let rate = moq_auth::Rate::from_bps(8000).unwrap(); + for transport in [Quic, WebTransport, Iroh, WebSocket, Http, Rtmp, Srt] { + for publish in [false, true] { + let mut grant = Grant::new(Pattern::all(), Pattern::all()); + match publish { + true => grant.publish_rate = Some(rate), + false => grant.subscribe_rate = Some(rate), + } + let (auth, mut admissions) = Auth::embedded("test"); + let request = auth.request(transport, "/room"); + let (result, ()) = tokio::join!(auth.admit(request), async { + admissions + .next() + .await + .unwrap() + .grant(lease::Consumer::fixed(grant.clone())); + }); + let Err(Error::Forbidden(message)) = result else { + panic!("{transport} admitted a cap it cannot enforce"); + }; + assert_eq!(message, caps::UNENFORCED, "{transport}"); + + let (auth, mut admissions) = Auth::embedded("test"); + let request = auth.request(transport, "/room"); + let pacer = Pacer::new(|_| {}); + let (result, ()) = tokio::join!(auth.admit_paced(request, Some(pacer)), async { + admissions + .next() + .await + .unwrap() + .grant(lease::Consumer::fixed(grant.clone())); + }); + assert_eq!(result.expect("paced").caps(), Caps::of(&grant), "{transport}"); + + assert!(matches!( + Lease::new("/room", lease::Consumer::fixed(grant)), + Err(Error::Forbidden(_)) + )); + } + } + } + + /// A lease without in-band auth applies a re-checked cap through its pacer, and ends + /// on one without a pacer. + #[tokio::test(start_paused = true)] + async fn a_rechecked_cap_reaches_the_connection() { + use futures::FutureExt; + let rate = moq_auth::Rate::from_bps(8000).unwrap(); + let uncapped = Grant::new(Pattern::all(), Pattern::all()); + let mut capped = uncapped.clone(); + capped.subscribe_rate = Some(rate); + + let applied = Arc::new(std::sync::Mutex::new(Vec::new())); + let record = applied.clone(); + let pacer = Pacer::new(move |caps| record.lock().unwrap().push(caps)); + let (producer, consumer) = lease::Producer::new(uncapped.clone()); + let (auth, mut admissions) = Auth::embedded("test"); + let request = auth.request(moq_auth::Transport::Quic, "/room"); + let (lease, ()) = tokio::join!(auth.admit_paced(request, Some(pacer)), async { + admissions.next().await.unwrap().grant(consumer); + }); + let mut lease = lease.unwrap(); + for grant in [&capped, &uncapped] { + producer.update(grant.clone()); + assert!(lease.ended().now_or_never().is_none(), "a cap change keeps the lease"); + } + producer.revoke(lease::Reason::Refused); + assert_eq!(lease.ended().await, lease::Reason::Refused); + assert_eq!(*applied.lock().unwrap(), [Caps::of(&capped), Caps::default()]); + + let (producer, consumer) = lease::Producer::new(uncapped); + let mut lease = Lease::new("/room", consumer).unwrap(); + producer.update(capped); + assert_eq!(lease.ended().await.to_string(), caps::UNENFORCED); + } + #[test] fn exactly_one_source() { assert!(config(None, &[]).validate(false).is_err()); diff --git a/rs/moq-relay/src/auth/caps.rs b/rs/moq-relay/src/auth/caps.rs new file mode 100644 index 00000000000..0c3fe335f65 --- /dev/null +++ b/rs/moq-relay/src/auth/caps.rs @@ -0,0 +1,176 @@ +//! The bitrate caps a grant puts on a session, and the QUIC connection that holds it to them. + +use std::{num::NonZeroU64, sync::Arc}; + +use moq_auth::{Grant, Rate}; + +/// Why a capped grant is refused: nothing under the session would hold it to the cap. +pub(crate) const UNENFORCED: &str = "this transport cannot enforce bitrate caps"; + +/// What a session may publish and what it may receive for its subscriptions, each +/// uncapped when `None`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct Caps { + pub publish: Option, + pub subscribe: Option, +} + +impl Caps { + /// The caps one grant names. + pub fn of(grant: &Grant) -> Self { + Self { + publish: grant.publish_rate, + subscribe: grant.subscribe_rate, + } + } + + /// The caps of a union of grants, matching the union's permissive meaning: each + /// direction takes the highest cap among the grants that cap it, and a grant that + /// gives the direction paths without a cap lifts it. A grant naming neither has no say. + pub fn union<'a>(grants: impl IntoIterator + Clone) -> Self { + fn highest<'a>(roles: impl Iterator)>) -> Option { + let mut highest = None; + for (paths, rate) in roles { + match rate { + Some(rate) => highest = highest.max(Some(rate)), + None if !paths.is_empty() => return None, + None => {} + } + } + highest + } + Self { + publish: highest( + grants + .clone() + .into_iter() + .map(|grant| (&grant.publish, grant.publish_rate)), + ), + subscribe: highest(grants.into_iter().map(|grant| (&grant.subscribe, grant.subscribe_rate))), + } + } + + /// Whether either direction is capped. + pub fn is_capped(&self) -> bool { + self.publish.is_some() || self.subscribe.is_some() + } + + /// The QUIC limits that hold a connection to these caps: the relay receives what the + /// session publishes and sends what it subscribes to, in bytes rather than bits. + pub fn limits(&self) -> moq_quic::Limits { + let rate = |cap: Option| match cap { + None => moq_quic::Rate::Unlimited, + Some(rate) => { + moq_quic::Rate::BytesPerSecond(NonZeroU64::new(rate.as_bps().div_ceil(8)).expect("a positive rate")) + } + }; + let mut limits = moq_quic::Limits::default(); + limits.receive_rate = Some(rate(self.publish)); + limits.send_rate = Some(rate(self.subscribe)); + limits + } +} + +impl std::fmt::Display for Caps { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let cap = |rate: Option| rate.map_or_else(|| "uncapped".to_string(), |rate| rate.to_string()); + write!(f, "publish {}, subscribe {}", cap(self.publish), cap(self.subscribe)) + } +} + +/// Applies a session's caps to the QUIC connection carrying it, on whichever runtime +/// drives that connection. +/// +/// Only a transport that can hold a session to its caps has one, so a session without +/// one refuses a capped grant at admission, in band, and at revalidation. +#[derive(Clone)] +pub struct Pacer(Arc); + +impl Pacer { + /// Pace a live connection by calling `apply` with each new set of caps. + pub(crate) fn new(apply: impl Fn(Caps) + Send + Sync + 'static) -> Self { + Self(Arc::new(apply)) + } + + /// Hold the connection to `caps` from now on. + pub(crate) fn pace(&self, caps: Caps) { + (self.0)(caps) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn rate(bps: u64) -> Option { + Some(Rate::from_bps(bps).unwrap()) + } + + fn grant(publish: &[&str], publish_rate: Option, subscribe: &[&str], subscribe_rate: Option) -> Grant { + let patterns = |texts: &[&str]| { + texts + .iter() + .map(|text| text.parse().unwrap()) + .collect::() + }; + let mut grant = Grant::new(patterns(publish), patterns(subscribe)); + grant.publish_rate = publish_rate; + grant.subscribe_rate = subscribe_rate; + grant + } + + #[test] + fn a_union_takes_the_highest_cap_and_an_uncapped_grant_lifts_it() { + let low = grant(&["a/**"], rate(1_000), &["**"], rate(8_000)); + let high = grant(&["b/**"], rate(2_000), &["**"], rate(4_000)); + let caps = Caps::union([&low, &high]); + assert_eq!( + caps, + Caps { + publish: rate(2_000), + subscribe: rate(8_000) + } + ); + + let uncapped = grant(&["c/**"], None, &[], None); + let caps = Caps::union([&low, &high, &uncapped]); + assert_eq!( + caps, + Caps { + publish: None, + subscribe: rate(8_000) + }, + "only the direction it grants" + ); + + let silent = grant(&[], None, &["d"], rate(16_000)); + assert_eq!( + Caps::union([&low, &silent]), + Caps { + publish: rate(1_000), + subscribe: rate(16_000) + } + ); + assert_eq!(Caps::union([]), Caps::default()); + } + + #[test] + fn limits_convert_bits_to_bytes_rounding_up() { + let caps = Caps { + publish: rate(8_000_001), + subscribe: None, + }; + let limits = caps.limits(); + assert_eq!( + limits.receive_rate, + Some(moq_quic::Rate::BytesPerSecond(NonZeroU64::new(1_000_001).unwrap())) + ); + assert_eq!(limits.send_rate, Some(moq_quic::Rate::Unlimited)); + let one = Caps { + publish: None, + subscribe: rate(1), + } + .limits(); + assert_eq!(one.send_rate, Some(moq_quic::Rate::BytesPerSecond(NonZeroU64::MIN))); + } +} diff --git a/rs/moq-relay/src/auth/refresh.rs b/rs/moq-relay/src/auth/refresh.rs index 02da98ed8d6..d1a21674e33 100644 --- a/rs/moq-relay/src/auth/refresh.rs +++ b/rs/moq-relay/src/auth/refresh.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, VecDeque}; use futures::{FutureExt, StreamExt, future::BoxFuture, stream::FuturesUnordered}; use moq_net::{SessionError, auth, setup::Role}; -use super::{Grant, Source, Token, lease}; +use super::{Caps, Grant, Pacer, Source, Token, caps, lease}; /// A connection owns its lifecycle while each credential owns only its grant. pub(super) struct Refresh { @@ -31,6 +31,11 @@ pub(super) struct Refresh { /// queue doubles keeps a push amortized O(1). survivors: usize, next: u64, + /// What holds the connection to the union's caps, or `None` on a transport that + /// cannot, where every capped credential is refused. + pacer: Option, + /// The caps of the union of every live credential, which the connection holds. + caps: Caps, } type Verified = (auth::Request, Result); @@ -108,8 +113,13 @@ impl Refresh { Err(lease::Reason::Expired) => return Err(moq_net::Error::Expired), Err(_) => return Err(moq_net::Error::Unauthorized), } + if lease.pacer.is_none() && Caps::of(&initial.grant).is_capped() { + return Err(moq_net::Error::Unauthorized); + } initial.initial = true; - let this = Self { + // The connection applies the first caps itself, with the window it opens. + let caps = Caps::of(&initial.grant); + let mut this = Self { handle, requests, requests_open: true, @@ -124,16 +134,43 @@ impl Refresh { waiting: VecDeque::new(), survivors: 0, next: 1, + pacer: lease.pacer.clone(), + caps, }; this.authorize(); Ok(this) } + /// The caps the connection is held to. + pub(super) fn caps(&self) -> Caps { + self.caps + } + + /// Why `grant` cannot join this session, if it cannot: a cap the transport cannot + /// enforce, or a grant that does not keep the session's root, routing, and role. + fn refusal(&self, grant: &Grant) -> Option { + if self.unenforced(grant) { + return Some(caps::UNENFORCED.to_string()); + } + (!self.valid(grant)).then(|| { + format!( + "token must retain root {}, connection routing, and usable SETUP role permissions", + self.admitted.root + ) + }) + } + + /// Whether `grant` caps a bitrate this connection cannot be held to. + fn unenforced(&self, grant: &Grant) -> bool { + self.pacer.is_none() && Caps::of(grant).is_capped() + } + fn valid(&self, grant: &Grant) -> bool { let Ok(fresh) = self.admitted.recheck(grant) else { return false; }; - super::still_valid(grant).is_ok() + !self.unenforced(grant) + && super::still_valid(grant).is_ok() && fresh.root == self.admitted.root && fresh.mounts == self.admitted.mounts && fresh.peer == self.admitted.peer @@ -148,7 +185,10 @@ impl Refresh { /// The connection credential's grant. moq-net adds every grant issued to an AUTH token /// itself, and keeps a request token's grant to that request alone. - fn authorize(&self) { + /// + /// The caps cover every live credential, request tokens included: each is a grant the + /// decider issued to this connection. + fn authorize(&mut self) { let mut union = auth::Grant::default(); for credential in self.credentials.values().filter(|credential| credential.initial) { let grant = credential.wire(self.role); @@ -158,6 +198,19 @@ impl Refresh { // Credential timers remove one grant at a time; no single deadline may // expire a union that another credential still covers. self.handle.authorize(&union); + + // The last credential leaving ends the session, which needs no caps. + if self.credentials.is_empty() { + return; + } + let caps = Caps::union(self.credentials.values().map(|credential| &credential.grant)); + if caps != self.caps + && let Some(pacer) = &self.pacer + { + tracing::info!(%caps, "bitrate caps changed"); + pacer.pace(caps); + self.caps = caps; + } } fn request(&mut self, request: auth::Request) { @@ -185,7 +238,11 @@ impl Refresh { // exact snapshot that would be issued. let mut credential = Credential::new(self.owner.clone()); if !self.valid(&credential.grant) { - request.reject(SessionError::Unauthorized, "connection credential no longer valid"); + let reason = match self.unenforced(&credential.grant) { + true => caps::UNENFORCED, + false => "connection credential no longer valid", + }; + request.reject(SessionError::Unauthorized, reason); return; } credential.initial = true; @@ -356,19 +413,15 @@ impl Refresh { // Validate the exact snapshot that is issued, not a later read. let credential = result.map(Credential::new); match credential { - Ok(mut credential) if self.valid(&credential.grant) => { - credential.issued = Some(request.accept(credential.wire(self.role))); - self.credentials.insert(self.next, credential); - self.next += 1; - self.authorize(); - } - Ok(_) => request.reject( - SessionError::Unauthorized, - format!( - "token must retain root {}, connection routing, and usable SETUP role permissions", - self.admitted.root - ), - ), + Ok(mut credential) => match self.refusal(&credential.grant) { + None => { + credential.issued = Some(request.accept(credential.wire(self.role))); + self.credentials.insert(self.next, credential); + self.next += 1; + self.authorize(); + } + Some(refusal) => request.reject(SessionError::Unauthorized, refusal), + }, Err(err) => refuse(request, err), } } @@ -393,6 +446,7 @@ impl Refresh { Change::Withdrawn => lease::Reason::Dropped, Change::Grant(grant) => match super::still_valid(&grant) { Err(reason) => reason, + Ok(()) if self.unenforced(&grant) => caps::UNENFORCED.into(), Ok(()) => lease::Reason::Narrowed, }, }; @@ -733,6 +787,107 @@ mod tests { let _ = supervising.await; } + fn capped(prefix: &str, publish: Option, subscribe: Option) -> Grant { + let mut grant = grant(prefix, 3000); + grant.publish_rate = publish.map(|bps| moq_auth::Rate::from_bps(bps).unwrap()); + grant.subscribe_rate = subscribe.map(|bps| moq_auth::Rate::from_bps(bps).unwrap()); + grant + } + + fn rate(bps: u64) -> Option { + Some(moq_auth::Rate::from_bps(bps).unwrap()) + } + + /// A pacer that records every set of caps it applies. + fn recorder() -> (Pacer, std::sync::Arc>>) { + let applied = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let record = applied.clone(); + (Pacer::new(move |caps| record.lock().unwrap().push(caps)), applied) + } + + /// A session admitted on `initial`, held to its caps by `pacer`. + async fn paced(initial: lease::Consumer, pacer: Option) -> (super::super::Lease, super::super::Admissions) { + let (auth, mut admissions) = super::super::Auth::embedded("test"); + let request = auth.request(moq_auth::Transport::Quic, "/room"); + let (lease, ()) = tokio::join!(auth.admit_paced(request, pacer), async { + admissions.next().await.unwrap().grant(initial); + }); + (lease.unwrap(), admissions) + } + + /// In-band tokens join the caps: the highest holds, an uncapped grant lifts it, and a + /// re-check that caps that grant restores the highest. Each change reaches the + /// live connection. + #[tokio::test(start_paused = true)] + async fn the_union_of_credentials_paces_the_connection() { + let (pacer, applied) = recorder(); + let initial = lease::Consumer::fixed(capped("a/**", Some(1_000), None)); + let (lease, mut admissions) = paced(initial, Some(pacer)).await; + assert_eq!( + lease.caps(), + Caps { + publish: rate(1_000), + subscribe: None + } + ); + let (client, supervising) = sessions(lease).await; + let auth = client.auth(); + + let (higher, ()) = tokio::join!(auth.add("higher"), async { + let higher = capped("a/**", Some(4_000), None); + admissions.next().await.unwrap().grant(lease::Consumer::fixed(higher)); + }); + // Held, since dropping a token withdraws its credential. + let _higher = higher.unwrap(); + let (producer, uncapped) = lease::Producer::new(capped("a/**", None, None)); + let (lifted, ()) = tokio::join!(auth.add("uncapped"), async { + admissions.next().await.unwrap().grant(uncapped); + }); + let _lifted = lifted.unwrap(); + // The auth server caps the grant it had left uncapped. + producer.update(capped("a/**", Some(2_000), None)); + while applied.lock().unwrap().len() < 3 { + kio::clock::sleep(Duration::from_millis(1)).await; + } + let highest = Caps { + publish: rate(4_000), + subscribe: None, + }; + assert_eq!(*applied.lock().unwrap(), [highest, Caps::default(), highest]); + client.abort(moq_net::Error::Cancel); + supervising.abort(); + let _ = supervising.await; + } + + /// Without a pacer, a capped in-band token is refused, and the session keeps its grant. + #[tokio::test(start_paused = true)] + async fn an_unpaced_session_refuses_a_capped_token() { + let (lease, mut admissions) = admission(grant("a/**", 3000)).await; + let (client, supervising) = sessions(lease).await; + let auth = client.auth(); + let (result, ()) = tokio::join!(auth.add("capped"), async { + let capped = capped("a/**", None, Some(8_000)); + admissions.next().await.unwrap().grant(lease::Consumer::fixed(capped)); + }); + assert!(result.is_err(), "the cap was discarded"); + assert!(!supervising.is_finished(), "a refused token ended the session"); + client.abort(moq_net::Error::Cancel); + supervising.abort(); + let _ = supervising.await; + } + + /// Without a pacer, a re-check that adds a cap ends the session rather than drop it. + #[tokio::test(start_paused = true)] + async fn an_unpaced_session_ends_when_rechecked_into_a_cap() { + let (producer, consumer) = lease::Producer::new(grant("a/**", 3000)); + let (lease, _admissions) = paced(consumer, None).await; + let (client, supervising) = sessions(lease).await; + producer.update(capped("a/**", Some(8_000), None)); + let reason = supervising.await.unwrap(); + assert_eq!(reason.to_string(), caps::UNENFORCED); + client.abort(moq_net::Error::Cancel); + } + /// How a lease ends decides how its session's stats count the end: an expired grant as /// `expired`, a shutdown as clean, and any other reason as `application`. #[tokio::test(start_paused = true)] diff --git a/rs/moq-relay/src/cluster.rs b/rs/moq-relay/src/cluster.rs index 71b1496b2e0..62a37f804d4 100644 --- a/rs/moq-relay/src/cluster.rs +++ b/rs/moq-relay/src/cluster.rs @@ -1189,10 +1189,18 @@ impl Cluster { } } - /// The QUIC limits an admitted session widens to, if any: only a `peer` grant does. - #[cfg(any(feature = "quic", all(target_os = "linux", feature = "_uring")))] - pub(crate) fn quic_limits(&self, lease: &auth::Lease) -> Option { - lease.token().peer.then(|| self.peer_quic.limits()) + /// The QUIC limits an admitted session runs under once it is answered: a `peer` + /// grant's wider ones, and the lease's bitrate caps. A receive window left unset opens + /// to the configured one. + pub(crate) fn quic_limits(&self, lease: &auth::Lease) -> moq_quic::Limits { + let mut limits = match lease.token().peer { + true => self.peer_quic.limits(), + false => moq_quic::Limits::default(), + }; + let caps = lease.caps().limits(); + limits.receive_rate = caps.receive_rate; + limits.send_rate = caps.send_rate; + limits } /// The origin ID used by this relay on the wire. @@ -1301,9 +1309,10 @@ impl Cluster { &self, auth: &auth::Auth, request: moq_auth::Request, + pacer: Option, ) -> Result { let result = async { - let lease = auth.admit(request.clone()).await?; + let lease = auth.admit_paced(request.clone(), pacer).await?; self.scope_session(lease, &request) } .await; diff --git a/rs/moq-relay/src/connection.rs b/rs/moq-relay/src/connection.rs index 1ba7e40e312..dfb47c73870 100644 --- a/rs/moq-relay/src/connection.rs +++ b/rs/moq-relay/src/connection.rs @@ -92,7 +92,9 @@ impl Connection { // cannot hold the connection open behind a decision that never comes. // Boxed: the admission future is large, and nesting it inline overflows debug stacks. let admitted = request - .within(Box::pin(lifecycle.admit(request.admission(&lifecycle.auth)))) + .within(Box::pin( + lifecycle.admit(request.admission(&lifecycle.auth), request.pacer()), + )) .await .inspect_err(|_| lifecycle.cluster.refusals.record(RefusalKind::Unavailable)) .map_err(|err| err.context("admission timed out"))?; @@ -112,15 +114,6 @@ impl Connection { stats, } = admitted; - // A peer's QUIC limits widen before SETUP OK, so its first requests see them. - // Other transports keep their own limits. - #[cfg(feature = "quic")] - if let Some(limits) = lifecycle.cluster.quic_limits(&lease) - && let Some(quic) = request.quic() - { - quic.set_limits(limits)?; - } - // Accept the connection. // NOTE: subscribe and publish seem backwards because of how relays work. // We publish the tracks the client is allowed to subscribe to. @@ -143,6 +136,9 @@ impl Connection { return Err(err.into()); } }; + // Before SETUP OK, so the first requests see them: a peer's wider limits, and the + // caps the window then opens under. Other transports keep their own limits. + let request = request.with_quic_limits(lifecycle.cluster.quic_limits(&lease))?; let serving = lifecycle.shutdown.serve(); // Bounded too: a peer that stalls the SETUP reply must not outlive the relay. // Abandoning the reply refuses the session with the timeout code, which its stats count. @@ -272,9 +268,12 @@ impl Lifecycle { .ok_or_else(|| anyhow::anyhow!("connection task panicked")) } + /// Admit `request`, holding its connection to a capped grant through `pacer`, or + /// refusing a capped grant without one. pub(crate) async fn admit( &self, request: moq_auth::Request, + pacer: Option, ) -> Result<(cluster::Admitted, Option), Rejection> { self.span.record("session", &request.id); // Peer-controlled, so recorded escaped rather than displayed raw. @@ -282,7 +281,7 @@ impl Lifecycle { self.span .record("implementation", tracing::field::debug(implementation)); } - let admitted = self.shutdown.admit(self.authorize(request)).await; + let admitted = self.shutdown.admit(self.authorize(request, pacer)).await; // Cluster admission records its own refusals; LAN and drain refusals are recorded here. if let Err(err) = &admitted && let Some(refusal) = err.refusal @@ -295,6 +294,7 @@ impl Lifecycle { async fn authorize( &self, request: moq_auth::Request, + pacer: Option, ) -> Result<(cluster::Admitted, Option), Rejection> { if cluster::Cluster::is_lan_path(&request.path) { let Some(presented) = cluster::Cluster::lan_credential(&request.path) else { @@ -331,7 +331,7 @@ impl Lifecycle { } let admitted = self .cluster - .admit_session(&self.auth, request.clone()) + .admit_session(&self.auth, request.clone(), pacer) .await .map_err(|err| { // Cluster admission owns this refusal; only local/drain refusals remain here. diff --git a/rs/moq-relay/src/incoming.rs b/rs/moq-relay/src/incoming.rs index ac1aeddafa4..169adda7e8d 100644 --- a/rs/moq-relay/src/incoming.rs +++ b/rs/moq-relay/src/incoming.rs @@ -57,12 +57,18 @@ pub trait Sealed: Sized { /// Refuse the session, closing it with `code`. async fn reject(self, code: moq_net::SessionError); - /// The QUIC connection under the request, to widen a peer's limits before SETUP OK. - #[cfg(feature = "quic")] - fn quic(&self) -> Option<&moq_tokio::quic::driver::connection::Connection> { + /// What holds the connection to a capped grant once the session runs, or `None` on + /// a transport that cannot, which refuses one. + fn pacer(&self) -> Option { None } + /// Apply QUIC limits before answering SETUP, which then opens the receive window the + /// handshake held back. A transport without moq-quic ignores them. + fn with_quic_limits(self, _limits: moq_quic::Limits) -> anyhow::Result { + Ok(self) + } + fn auth(&self) -> moq_net::auth::Handle; fn requests(&mut self) -> moq_net::auth::Requests; fn role(&self) -> Option; @@ -103,8 +109,18 @@ impl Sealed for moq_tokio::server::Request { } #[cfg(feature = "quic")] - fn quic(&self) -> Option<&moq_tokio::quic::driver::connection::Connection> { - moq_tokio::server::Request::quic(self) + fn pacer(&self) -> Option { + let quic = moq_tokio::server::Request::quic(self)?.clone(); + Some(auth::Pacer::new(move |caps| { + if let Err(err) = quic.set_limits(caps.limits()) { + tracing::warn!(%err, "failed to apply bitrate caps"); + } + })) + } + + #[cfg(feature = "quic")] + fn with_quic_limits(self, limits: moq_quic::Limits) -> anyhow::Result { + Ok(moq_tokio::server::Request::with_quic_limits(self, limits)?) } fn auth(&self) -> moq_net::auth::Handle { diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 217e2432788..541513d7b27 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -82,6 +82,9 @@ struct Serve { /// `listen.reset.secret`, shared so any worker resets any worker's /// connections, a restarted group's included. reset_secret: Option<[u8; 32]>, + /// `[quic] receive_window`, opened once a session is admitted: until then a + /// connection holds [`moq_net::transport::SETUP_RECEIVE_WINDOW`]. + window: Option, } /// A bound group of io_uring QUIC workers sharing one port. @@ -100,6 +103,8 @@ pub struct Workers { members: Vec, addr: SocketAddr, server: moq_uring::quic::server::Config, + /// `[quic] receive_window`, which admission opens; see [`Serve::window`]. + window: Option, /// The same live resolver used by every worker and the fingerprint endpoint. certificates: moq_tokio::tls::Certificates, /// One counter set per worker, in shard order. Created here rather than on @@ -232,6 +237,8 @@ impl Workers { } let mut server = moq_uring::quic::server::Config::new(tls); server.transport = transport(&quic)?; + let window = server.transport.limits.receive_window; + server.transport.limits.receive_window = NonZeroU64::new(moq_net::transport::SETUP_RECEIVE_WINDOW); // GSO is a property of the socket, not the connection, so it rides the // worker's UDP config rather than the connection's. @@ -253,6 +260,7 @@ impl Workers { members, addr, server, + window, certificates, udp, pin: config.pin, @@ -323,6 +331,7 @@ impl Workers { local: self.addr, timeout: self.timeout, reset_secret: self.reset_secret, + window: self.window, }; let cores = match self.pin { @@ -846,10 +855,24 @@ async fn serve_request( auth_request.alpn = alpn.clone(); auth_request.tls = identity.as_ref().and_then(crate::auth::peer); let admission = lifecycle.clone(); + // The connection is pinned to this worker, so a cap that changes on the shared + // runtime reaches it through `caps`, which a task here applies. + let caps = kio::Producer::new(auth::Caps::default()); + let pacer = { + let caps = caps.clone(); + auth::Pacer::new(move |new| { + if let Ok(mut caps) = caps.write() { + *caps = new; + } + }) + }; // The handshake deadline covers admission too, so a peer cannot hold the connection // open behind a decision that never comes. Dropping the shared task withdraws it. let decided = deadline - .bound(lifecycle.shared(&serve.tokio, async move { admission.admit(auth_request).await })) + .bound(lifecycle.shared( + &serve.tokio, + async move { admission.admit(auth_request, Some(pacer)).await }, + )) .await .inspect_err(|err| { if matches!(err.downcast_ref::(), Some(moq_net::Error::Timeout)) { @@ -871,10 +894,6 @@ async fn serve_request( stats, } = admitted; - // A peer's QUIC limits widen before SETUP OK, so its first requests see them. - if let Some(limits) = serve.cluster.quic_limits(&lease) { - conn.set_limits(limits)?; - } let mut request = request.with_stats(stats).with_limits(serve.cluster.limits(&lease)); if let Some(subscriber) = subscriber { request = request.with_publisher(subscriber); @@ -895,6 +914,32 @@ async fn serve_request( return Err(err.into()); } }; + // Before SETUP OK, so the first requests see them: a peer's wider limits, the caps, + // and then the window the handshake held back, which the caps pace. + let mut limits = serve.cluster.quic_limits(&lease); + limits.receive_window = limits.receive_window.or(serve.window); + conn.set_limits(limits)?; + // Dropping the lease, and with it `pacer`, closes `caps`, which ends this task. + let mut applied = lease.caps(); + if let Ok(mut caps) = caps.write() { + *caps = applied; + } + let pacing = async move { + let caps = caps.consume(); + while let Ok(next) = caps + .wait(|caps| match **caps != applied { + true => std::task::Poll::Ready(**caps), + false => std::task::Poll::Pending, + }) + .await + { + if let Err(err) = conn.set_limits(next.limits()) { + tracing::warn!(%err, "failed to apply bitrate caps"); + } + applied = next; + } + }; + handle.spawn(pacing.in_current_span()); let serving = serve.shutdown.serve(); // Bounded by the drain deadline too, like the tokio path, so a peer that stalls // the SETUP reply cannot hold the drain. That is a tokio timer, so it runs on the diff --git a/rs/moq-relay/tests/integration/bitrate_caps.rs b/rs/moq-relay/tests/integration/bitrate_caps.rs new file mode 100644 index 00000000000..dcfd2dc9138 --- /dev/null +++ b/rs/moq-relay/tests/integration/bitrate_caps.rs @@ -0,0 +1,272 @@ +//! A capped grant over QUIC is admitted and holds its session to its caps, and every +//! session holds only the SETUP window until it is admitted, on the tokio listener and +//! the io_uring workers alike. +//! +//! Over real sockets, so the caps are measured on the wall clock as lower bounds on +//! how long a transfer takes: uncapped, the same transfer takes milliseconds. +#![cfg(feature = "quic")] + +use std::time::{Duration, Instant}; + +use moq_relay::{Config, Relay}; +use moq_tokio::moq_net; +use moq_tokio::quic::driver; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// One megabyte a second, so a fresh cap allows a one-megabyte burst. +const RATE: u64 = 8_000_000; +/// Past the burst and the SETUP window, so the rest waits on the rate. +const FRAME: usize = 2 << 20; +/// The least time [`FRAME`] can take under [`RATE`]: what is left after a full burst and +/// the SETUP window, at the rate, with some slack for the burst a cap starts with. +const LEAST: Duration = Duration::from_millis(700); + +fn config(dir: &std::path::Path) -> Config { + let mut config = super::peer_limits::config(dir); + config.quic.max_streams = None; + config +} + +/// A grant of everything, with `publish` and `subscribe` caps. +fn grant(publish: Option, subscribe: Option) -> moq_auth::Grant { + let all: moq_auth::Patterns = moq_auth::Pattern::all().into(); + let mut grant = moq_auth::Grant::new(all.clone(), all); + grant.publish_rate = publish.map(|bps| moq_auth::Rate::from_bps(bps).unwrap()); + grant.subscribe_rate = subscribe.map(|bps| moq_auth::Rate::from_bps(bps).unwrap()); + grant +} + +fn client() -> moq_tokio::Client { + let mut config = moq_tokio::connect::Config::default(); + config.tls.insecure = Some(true); + config.bind = Some("127.0.0.1:0".parse().expect("parse bind")); + config + .init(Default::default()) + .expect("client init") + .with_reconnect(false) +} + +/// Connect `client`, admitted on `grant`. +async fn connect( + client: moq_tokio::Client, + port: u16, + admissions: &mut moq_relay::auth::Admissions, + grant: moq_auth::Grant, +) -> moq_tokio::Connection { + let url: url::Url = format!("https://localhost:{port}/room").parse().unwrap(); + let connecting = client.connect(url).established(); + let admitting = async { + let admission = admissions.next().await.expect("admission"); + admission.grant(moq_auth::lease::Consumer::fixed(grant)); + }; + let (connection, ()) = kio::clock::timeout(TIMEOUT, async { tokio::join!(connecting, admitting) }) + .await + .expect("connect timeout"); + connection.expect("connect failed") +} + +/// Publish one [`FRAME`] on `test/video`, and how long a subscriber takes to read it. +async fn transfer( + port: u16, + admissions: &mut moq_relay::auth::Admissions, + publisher: moq_auth::Grant, + subscriber: moq_auth::Grant, +) -> Duration { + let origin = moq_tokio::origin::spawn(); + let broadcast = origin.create_broadcast("test").expect("create broadcast"); + broadcast.announce(Default::default()).expect("announce"); + let track = broadcast + .create_track( + "video", + moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), + ) + .expect("create track"); + let _publishing = connect(client().with_publisher(&origin), port, admissions, publisher).await; + + let subscribing = moq_tokio::origin::spawn(); + let consumer = subscribing.consume(); + let _subscribed = connect(client().with_subscriber(subscribing), port, admissions, subscriber).await; + let broadcast_consumer = kio::clock::timeout(TIMEOUT, consumer.request("test", None)) + .await + .expect("request timeout") + .expect("announced broadcast resolves"); + let mut subscription = broadcast_consumer + .track("video") + .unwrap() + .subscribe(None) + .await + .expect("subscribe"); + + let start = Instant::now(); + let mut group = track.append_group().expect("append group"); + group + .write_frame(moq_net::Timestamp::ZERO, vec![7u8; FRAME]) + .expect("write frame"); + group.finish().expect("finish group"); + let mut group = kio::clock::timeout(TIMEOUT, subscription.recv_group()) + .await + .expect("recv_group timeout") + .expect("recv_group failed") + .expect("track closed"); + let frame = kio::clock::timeout(TIMEOUT, group.read_frame()) + .await + .expect("read_frame timeout") + .expect("read_frame failed") + .expect("group closed"); + assert_eq!(frame.payload.len(), FRAME); + start.elapsed() +} + +/// A capped publisher feeds the relay at its rate, and a capped subscriber is fed at its. +async fn held_to_its_caps(mut relay: Relay, port: u16) { + let mut admissions = relay.admissions().expect("embedded auth"); + let running = tokio::spawn(relay.run()); + + let elapsed = transfer(port, &mut admissions, grant(Some(RATE), None), grant(None, None)).await; + assert!(elapsed >= LEAST, "a capped publisher sent {FRAME} bytes in {elapsed:?}"); + + let elapsed = transfer(port, &mut admissions, grant(None, None), grant(None, Some(RATE))).await; + assert!( + elapsed >= LEAST, + "a capped subscriber received {FRAME} bytes in {elapsed:?}" + ); + + running.abort(); + let _ = running.await; +} + +/// Before admission the relay credits only the SETUP window, so a client cannot bank +/// the full one; once admitted, the window opens. +async fn window_opens_once_admitted(mut relay: Relay, port: u16) { + let mut admissions = relay.admissions().expect("embedded auth"); + let running = tokio::spawn(relay.run()); + + let mut connect = moq_tokio::connect::Config::default(); + connect.tls.insecure = Some(true); + let mut tls = connect.tls.build().expect("tls config"); + tls.alpn_protocols = vec![b"moq-lite-06".to_vec()]; + let crypto = driver::crypto::rustls::QuicClientConfig::try_from(tls).expect("quic crypto"); + let endpoint = + driver::endpoint::Endpoint::new(Default::default(), std::net::UdpSocket::bind("127.0.0.1:0").unwrap()) + .expect("endpoint"); + let conn = endpoint + .connect( + &driver::client::Config::new(std::sync::Arc::new(crypto)), + ([127, 0, 0, 1], port).into(), + "localhost", + ) + .expect("connect") + .await + .expect("handshake"); + + // SETUP type, then a one-byte message holding zero parameters. + let mut setup = conn.open_uni(Default::default()).await.expect("open"); + setup.write_all(&[0x01, 0x01, 0x00][..]).await.expect("write"); + setup.finish(); + let admission = kio::clock::timeout(TIMEOUT, admissions.next()) + .await + .expect("admission timeout") + .expect("admission"); + + // Unread until the session runs, so it holds its credit, and the write waits for more. + let banked = 1 << 20; + let mut stream = conn.open_uni(Default::default()).await.expect("open"); + let writing = tokio::spawn(async move { + stream.write_all(&vec![0u8; banked][..]).await.expect("write"); + stream + }); + let blocked = async { + while conn.stats().frame_tx.data_blocked == 0 { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + kio::clock::timeout(TIMEOUT, blocked) + .await + .expect("the client was never blocked on the SETUP window"); + let stats = conn.stats(); + assert!( + stats.udp_tx.bytes < moq_net::transport::SETUP_RECEIVE_WINDOW + (64 << 10), + "sent {} bytes before admission", + stats.udp_tx.bytes + ); + + // Nothing reads the blocked stream before admission, so only opening the window + // credits more. The session then stops that stream, so watch the credit, not the data. + let credited = stats.frame_rx.max_data; + admission.grant(moq_auth::lease::Consumer::fixed(grant(None, None))); + let opened = async { + while conn.stats().frame_rx.max_data == credited { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }; + kio::clock::timeout(TIMEOUT, opened) + .await + .expect("the window never opened once admitted"); + writing.abort(); + + conn.close(driver::VarInt::from_u32(0), "done"); + endpoint.close(driver::VarInt::from_u32(0), "done").await; + running.abort(); + let _ = running.await; +} + +async fn tokio_relay() -> (Relay, u16, tempfile::TempDir) { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let dir = tempfile::tempdir().expect("tempdir"); + let relay = Relay::load(config(dir.path())) + .await + .expect("load relay") + .with_signals(false); + let port = relay.quic_addr().expect("quic listener bound").port(); + (relay, port, dir) +} + +#[tokio::test] +async fn tokio_session_is_held_to_its_caps() { + let (relay, port, _dir) = tokio_relay().await; + held_to_its_caps(relay, port).await; +} + +#[tokio::test] +async fn tokio_window_opens_once_admitted() { + let (relay, port, _dir) = tokio_relay().await; + window_opens_once_admitted(relay, port).await; +} + +#[cfg(all(target_os = "linux", feature = "_uring"))] +async fn uring_relay() -> Option<(Relay, u16, tempfile::TempDir)> { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + match moq_uring::Worker::new(Default::default()) { + Ok(_) => {} + Err(moq_uring::Error::Unsupported(reason)) if std::env::var_os("CI_STRICT").is_none() => { + eprintln!("skipping io_uring relay test: {reason}"); + return None; + } + Err(err) => panic!("io_uring worker setup failed: {err}"), + } + let dir = tempfile::tempdir().expect("tempdir"); + let mut config = config(dir.path()); + config.runtime.workers = Some(1); + config.runtime.pin = false; + config.runtime.io_uring = true; + let relay = Relay::load(config).await.expect("load relay").with_signals(false); + let port = relay.addr().expect("workers bound an address").port(); + Some((relay, port, dir)) +} + +#[cfg(all(target_os = "linux", feature = "_uring"))] +#[tokio::test] +async fn uring_session_is_held_to_its_caps() { + if let Some((relay, port, _dir)) = uring_relay().await { + held_to_its_caps(relay, port).await; + } +} + +#[cfg(all(target_os = "linux", feature = "_uring"))] +#[tokio::test] +async fn uring_window_opens_once_admitted() { + if let Some((relay, port, _dir)) = uring_relay().await { + window_opens_once_admitted(relay, port).await; + } +} diff --git a/rs/moq-relay/tests/integration/main.rs b/rs/moq-relay/tests/integration/main.rs index 5c7951cd6ad..2abc4b57f71 100644 --- a/rs/moq-relay/tests/integration/main.rs +++ b/rs/moq-relay/tests/integration/main.rs @@ -8,6 +8,7 @@ //! `shutdown_signal` owns those signals, which reach every listener in this process. mod auth_lifetime; +mod bitrate_caps; mod cluster_close; mod cluster_drain; mod cluster_idle; diff --git a/rs/moq-relay/tests/integration/peer_limits.rs b/rs/moq-relay/tests/integration/peer_limits.rs index c71d7cdd147..6a13235c642 100644 --- a/rs/moq-relay/tests/integration/peer_limits.rs +++ b/rs/moq-relay/tests/integration/peer_limits.rs @@ -1,5 +1,5 @@ //! A session admitted as a cluster peer widens its QUIC limits before SETUP OK, -//! while a viewer keeps the `--quic-*` ones, on the tokio listener and the +//! while a viewer opens to the `--quic-*` ones, on the tokio listener and the //! io_uring workers alike. #![cfg(feature = "quic")] @@ -140,7 +140,8 @@ async fn peer_widens_and_viewer_does_not(mut relay: Relay, port: u16) { "a viewer keeps the stream limit, opened {}", viewer.streams ); - assert_eq!(viewer.max_data, 0, "a viewer's receive window is unchanged"); + // Both start with only the SETUP window, which admission opens for a viewer too. + assert!(viewer.max_data > 0, "a viewer's receive window opens once admitted"); let _leases = deciding.await.expect("decider"); running.abort(); diff --git a/rs/moq-tokio/CHANGELOG.md b/rs/moq-tokio/CHANGELOG.md index c37a54f514d..00a372c38f8 100644 --- a/rs/moq-tokio/CHANGELOG.md +++ b/rs/moq-tokio/CHANGELOG.md @@ -9,6 +9,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `server::Request::with_quic_limits` sets the QUIC limits `ok` applies before answering SETUP. A server QUIC connection holds only `moq_net::transport::SETUP_RECEIVE_WINDOW` until `ok` opens `[quic] receive_window`, so a client cannot bank credit before it is admitted. + - `server::Request::within` finishes a future, such as admitting the session, within what remains of the handshake deadline, closing the session with the timeout code otherwise. ## [0.19.23](https://github.com/moq-dev/moq/compare/moq-tokio-v0.19.22...moq-tokio-v0.19.23) - 2026-10-06 diff --git a/rs/moq-tokio/src/quic/backend.rs b/rs/moq-tokio/src/quic/backend.rs index 7fe873b77a3..f871fb8ba13 100644 --- a/rs/moq-tokio/src/quic/backend.rs +++ b/rs/moq-tokio/src/quic/backend.rs @@ -553,6 +553,9 @@ fn proto_status(err: &web::proto::ConnectError) -> Option { pub(crate) struct Server { pub quic: driver::endpoint::Endpoint, + /// The configured receive window, opened once a session is admitted: until then a + /// connection holds [`moq_net::transport::SETUP_RECEIVE_WINDOW`]. + pub window: Option, tls: crate::tls::Serve, } @@ -572,7 +575,9 @@ impl Server { tls: served, } = options; let quic = quic.resolve(); - let transport = transport(&quic, "server")?; + let mut transport = transport(&quic, "server")?; + let window = transport.limits.receive_window; + transport.limits.receive_window = std::num::NonZeroU64::new(moq_net::transport::SETUP_RECEIVE_WINDOW); let keylog = quic .keylog .as_deref() @@ -667,7 +672,11 @@ impl Server { let quic = driver::endpoint::Endpoint::new(endpoint_config, socket).map_err(Error::CreateEndpoint)?; - Ok(Self { quic, tls: served }) + Ok(Self { + quic, + window, + tls: served, + }) } pub fn accept(&self) -> impl std::future::Future> + '_ { diff --git a/rs/moq-tokio/src/server.rs b/rs/moq-tokio/src/server.rs index d01c16ff9cd..c655d1ac4a5 100644 --- a/rs/moq-tokio/src/server.rs +++ b/rs/moq-tokio/src/server.rs @@ -654,6 +654,9 @@ impl Server { // tokio::select! does not support cfg directives on arms, so we need to create the futures here. // `Some(None)` once a configured endpoint stops, which only losing its // driver does while we accept. + // The window a QUIC session opens to once admitted. + #[cfg(feature = "quic")] + let window = self.quic.as_ref().and_then(|quic| quic.window); #[cfg(feature = "quic")] let quic_accept = async { #[cfg(feature = "quic")] @@ -751,7 +754,7 @@ impl Server { let (request, deadline) = accept_request(&setup.server, session, deadline).await?; // Only WebTransport carries a request URL; raw QUIC puts the path in the SETUP. let transport = match url { Some(_) => Transport::WebTransport, None => Transport::Quic }; - Ok(Request { transport, url, identity, authority, link, handshake: Box::new(request), quic, deadline, active }) + Ok(Request { transport, url, identity, authority, link, handshake: Box::new(request), quic, window, quic_limits: Default::default(), deadline, active }) }.boxed()); } } @@ -778,6 +781,10 @@ impl Server { handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, + #[cfg(feature = "quic")] + window: None, + #[cfg(feature = "quic")] + quic_limits: Default::default(), deadline, active, }) @@ -951,6 +958,10 @@ impl Listener { handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, + #[cfg(feature = "quic")] + window: None, + #[cfg(feature = "quic")] + quic_limits: Default::default(), deadline, active, }) @@ -1096,6 +1107,10 @@ pub(crate) async fn accept_websocket( handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, + #[cfg(feature = "quic")] + window: None, + #[cfg(feature = "quic")] + quic_limits: Default::default(), deadline, active: options.active, }) @@ -1597,6 +1612,10 @@ async fn stream_request( handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, + #[cfg(feature = "quic")] + window: None, + #[cfg(feature = "quic")] + quic_limits: Default::default(), deadline, active, }; @@ -1662,6 +1681,12 @@ pub struct Request { /// The connection under a QUIC or WebTransport request, for [`Request::quic`]. #[cfg(feature = "quic")] quic: Option, + /// The receive window [`Request::ok`] opens unless [`Request::with_quic_limits`] names one. + #[cfg(feature = "quic")] + window: Option, + /// What [`Request::ok`] applies to the QUIC connection before answering SETUP. + #[cfg(feature = "quic")] + quic_limits: moq_quic::Limits, /// What remains of the handshake deadline, which also bounds [`Request::ok`]. deadline: Option, } @@ -1706,13 +1731,26 @@ impl Request { self } - /// The QUIC connection under this request, to change its limits before answering - /// SETUP. `None` on a transport moq-quic does not drive (qmux streams, iroh). + /// The QUIC connection under this request, to change its limits once the session runs. + /// `None` on a transport moq-quic does not drive (qmux streams, iroh). #[cfg(feature = "quic")] pub fn quic(&self) -> Option<&crate::quic::driver::connection::Connection> { self.quic.as_ref() } + /// Set the QUIC limits [`Self::ok`] applies before answering SETUP, replacing any set + /// before. Ignored on a transport moq-quic does not drive. + /// + /// Until then the connection holds only [`moq_net::transport::SETUP_RECEIVE_WINDOW`]; + /// `ok` opens `receive_window`, or `[quic] receive_window` when it is unset, after the + /// other limits, so a `receive_rate` set here paces the opened window. + #[cfg(feature = "quic")] + pub fn with_quic_limits(mut self, limits: moq_quic::Limits) -> Result { + limits.validate()?; + self.quic_limits = limits; + Ok(self) + } + /// Finish `work` before what remains of the handshake deadline, or close the /// session with the timeout code and return [`moq_net::Error::Timeout`]. /// @@ -1732,6 +1770,14 @@ impl Request { /// on a peer that withholds flow-control credit, which keep-alives would /// otherwise hold open. pub async fn ok(self) -> crate::Result { + // Admitted: open the window the handshake held back, before SETUP OK. + #[cfg(feature = "quic")] + if let Some(quic) = &self.quic { + let mut limits = self.quic_limits; + limits.receive_window = limits.receive_window.or(self.window); + quic.set_limits(limits) + .expect("validated by with_quic_limits and the config"); + } let Request { handshake, deadline, @@ -2563,6 +2609,10 @@ mod tests { handshake: Box::new(request), #[cfg(feature = "quic")] quic: None, + #[cfg(feature = "quic")] + window: None, + #[cfg(feature = "quic")] + quic_limits: Default::default(), deadline, } .with_stats( From bf7a71833d521962ce116a94f8739824ba381d18 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:17:53 -0800 Subject: [PATCH 3/5] fix(relay): let the io_uring pacing task end with its lease Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/uring.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rs/moq-relay/src/uring.rs b/rs/moq-relay/src/uring.rs index 541513d7b27..dc1650f9949 100644 --- a/rs/moq-relay/src/uring.rs +++ b/rs/moq-relay/src/uring.rs @@ -919,13 +919,14 @@ async fn serve_request( let mut limits = serve.cluster.quic_limits(&lease); limits.receive_window = limits.receive_window.or(serve.window); conn.set_limits(limits)?; - // Dropping the lease, and with it `pacer`, closes `caps`, which ends this task. + // The task holds only a consumer: dropping the lease, and with it `pacer`, closes + // `caps`, which ends the task and releases its handle on the connection. let mut applied = lease.caps(); if let Ok(mut caps) = caps.write() { *caps = applied; } + let caps = caps.consume(); let pacing = async move { - let caps = caps.consume(); while let Ok(next) = caps .wait(|caps| match **caps != applied { true => std::task::Poll::Ready(**caps), From b7af8ed0719a105be7948d299dae075205963d57 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 02:18:19 -0800 Subject: [PATCH 4/5] test(relay): time the cap tests on the ambient clock Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/tests/integration/bitrate_caps.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rs/moq-relay/tests/integration/bitrate_caps.rs b/rs/moq-relay/tests/integration/bitrate_caps.rs index dcfd2dc9138..6dd07b48c7b 100644 --- a/rs/moq-relay/tests/integration/bitrate_caps.rs +++ b/rs/moq-relay/tests/integration/bitrate_caps.rs @@ -6,7 +6,7 @@ //! how long a transfer takes: uncapped, the same transfer takes milliseconds. #![cfg(feature = "quic")] -use std::time::{Duration, Instant}; +use std::time::Duration; use moq_relay::{Config, Relay}; use moq_tokio::moq_net; @@ -98,7 +98,7 @@ async fn transfer( .await .expect("subscribe"); - let start = Instant::now(); + let start = kio::clock::now(); let mut group = track.append_group().expect("append group"); group .write_frame(moq_net::Timestamp::ZERO, vec![7u8; FRAME]) @@ -115,7 +115,7 @@ async fn transfer( .expect("read_frame failed") .expect("group closed"); assert_eq!(frame.payload.len(), FRAME); - start.elapsed() + kio::clock::now() - start } /// A capped publisher feeds the relay at its rate, and a capped subscriber is fed at its. @@ -178,7 +178,7 @@ async fn window_opens_once_admitted(mut relay: Relay, port: u16) { }); let blocked = async { while conn.stats().frame_tx.data_blocked == 0 { - tokio::time::sleep(Duration::from_millis(10)).await; + kio::clock::sleep(Duration::from_millis(10)).await; } }; kio::clock::timeout(TIMEOUT, blocked) @@ -197,7 +197,7 @@ async fn window_opens_once_admitted(mut relay: Relay, port: u16) { admission.grant(moq_auth::lease::Consumer::fixed(grant(None, None))); let opened = async { while conn.stats().frame_rx.max_data == credited { - tokio::time::sleep(Duration::from_millis(10)).await; + kio::clock::sleep(Duration::from_millis(10)).await; } }; kio::clock::timeout(TIMEOUT, opened) From 7397309693f66c62c442844109af48113120481e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 1 Nov 2026 14:32:04 -0800 Subject: [PATCH 5/5] fix(relay): build each session future outside the accept loop's frame With the io-uring feature the gate's debug build overflowed the 1 MiB stack in stack::debug_relay_load_and_run_fit_windows_stack: main sat within 64 KiB of the limit and this branch's admission changes tipped it over. Spawning each Connection from its own function keeps the session future, built on the stack before tokio::spawn moves it, out of serve_listening's frame, which every accept stacks on. Co-Authored-By: Claude Opus 5.5 --- rs/moq-relay/src/relay.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/rs/moq-relay/src/relay.rs b/rs/moq-relay/src/relay.rs index fbe07646d15..48b47a5771c 100644 --- a/rs/moq-relay/src/relay.rs +++ b/rs/moq-relay/src/relay.rs @@ -865,14 +865,28 @@ async fn serve_listening( ) -> anyhow::Result<()> { loop { let request = listener.accept().await?; - let conn = Connection::new(request, cluster.clone(), auth.clone()) - .with_id(cluster.next_connection_id()) - .with_shutdown(shutdown.clone()) - .with_sessions(sessions.clone()); - tokio::spawn(conn.run()); + spawn_connection(request, &cluster, &auth, &shutdown, &sessions); } } +/// Spawn the [`Connection`] task for one accepted `request`. +/// +/// Its own frame, so the session future built on the stack before `spawn` moves it +/// to the heap stays out of the accept loop's frame, which every accept stacks on. +fn spawn_connection( + request: moq_tokio::server::Request, + cluster: &cluster::Cluster, + auth: &auth::Auth, + shutdown: &shutdown::Observer, + sessions: &crate::session::Registry, +) { + let conn = Connection::new(request, cluster.clone(), auth.clone()) + .with_id(cluster.next_connection_id()) + .with_shutdown(shutdown.clone()) + .with_sessions(sessions.clone()); + tokio::spawn(conn.run()); +} + #[cfg(test)] mod tests { #[cfg(feature = "stun")]