diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index 0b35ecd6056..fac3aa60944 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -1307,6 +1307,7 @@ export class Publisher { version: this.#session.version, sendOrder: place.rank, cancel: race([unsubscribed, abandoned, excluded.promise]), + group: true, }); } finally { opened.resolve(); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 373acefd8bc..7a1fd417d19 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -2229,6 +2229,7 @@ export class Publisher { version: this.version, sendOrder: place.rank, cancel: race([unsubscribed, abandoned]), + group: true, }); } finally { opened.resolve(); diff --git a/js/net/src/open.test.ts b/js/net/src/open.test.ts new file mode 100644 index 00000000000..92cc423271d --- /dev/null +++ b/js/net/src/open.test.ts @@ -0,0 +1,83 @@ +import { expect, test } from "bun:test"; +import Qmux from "@moq/qmux"; +import { Version } from "./lite/version.ts"; +import { createMockTransportPair } from "./mock.ts"; +import { lines } from "./open.ts"; +import { Writer } from "./stream.ts"; + +const version = Version.DRAFT_07; +const never = new Promise(() => {}); + +// A qmux session pair over an in-memory socket. +function qmuxPair(): { client: WebTransport; server: WebTransport } { + const socket = (readable: ReadableStream, writable: WritableStream) => { + const closed = Promise.withResolvers<{ closeCode: number; reason: string }>(); + return { + url: "", + opened: Promise.resolve({ readable, writable, extensions: "", protocol: "qmux-01" }), + closed: closed.promise, + close: () => closed.resolve({ closeCode: 1000, reason: "" }), + }; + }; + // Copied, since a sender may reuse its buffer once a write resolves. + const pipe = () => + new TransformStream({ transform: (chunk, c) => c.enqueue(chunk.slice()) }); + const up = pipe(); + const down = pipe(); + const config = { idleTimeout: null }; + const client = new Qmux(socket(down.readable, up.writable), { config }); + const server = Qmux.accept(socket(up.readable, down.writable), { config }); + return { client, server }; +} + +function openGroup(quic: WebTransport, sendOrder: number): Promise { + return Writer.tryOpen(quic, { version, sendOrder, cancel: never, group: true }) as Promise; +} + +function orders(quic: WebTransport): number[] { + return lines(quic) + .streams.list() + .map((stream) => stream.sendOrder); +} + +test("group streams are listed lowest send order first until they close or reset", async () => { + const { client, server } = createMockTransportPair(); + const first = await openGroup(client, 1); + const second = await openGroup(client, 3); + const control = await Writer.open(client, { version }); + const third = await openGroup(client, 2); + expect(orders(client)).toEqual([1, 2, 3]); + + second.setPriority(0); + expect(orders(client)).toEqual([0, 1, 2]); + + first.close(); + second.reset(new Error("cancel")); + expect(orders(client)).toEqual([2]); + third.close(); + expect(orders(client)).toEqual([]); + + expect(lines(client).streams.acknowledges).toBe(false); + expect(await control.acknowledged()).toBeUndefined(); + client.close(); + server.close(); +}); + +test("a qmux group stream reports what the peer has acknowledged", async () => { + const { client, server } = qmuxPair(); + expect(lines(client).streams.acknowledges).toBe(true); + const stream = await openGroup(client, 0); + const [listed] = lines(client).streams.list(); + expect(listed).toBe(stream); + expect(await listed.acknowledged()).toBe(true); + + const write = stream.write(new Uint8Array(1024)); + expect(await listed.acknowledged()).toBe(false); + await write; + expect(await listed.acknowledged()).toBe(true); + + stream.reset(new Error("cancel")); + expect(await stream.acknowledged()).toBe(false); + client.close(); + server.close(); +}); diff --git a/js/net/src/open.ts b/js/net/src/open.ts index 49116b5ece4..c1d8cb7ab55 100644 --- a/js/net/src/open.ts +++ b/js/net/src/open.ts @@ -1,4 +1,5 @@ import { race } from "@moq/signals"; +import { transportOf } from "./connection/transport.ts"; /** * The one entry point every stream a session opens goes through. @@ -15,6 +16,10 @@ import { race } from "@moq/signals"; * * Chrome ignores a `sendOrder` set after a create, so there a stream handed to another open * keeps the send order it was created at. + * + * The session also lists the group streams it has open, each with its send order and whether + * the peer has acknowledged everything written to it, so an open waiting for credit can choose + * a lower one to make room with. */ const live = Symbol("live"); @@ -156,15 +161,54 @@ function aborted(signal: AbortSignal): { promise: Promise; done: () => voi type SendStream = WritableStream & { sendOrder?: number }; -/** The lines of one session's opens. */ +/** One of a session's open group streams. */ +export interface OpenStream { + /** The send order the stream has now, where HIGHER values are sent first. */ + readonly sendOrder: number; + /** + * Whether the peer has acknowledged every byte written to it so far, or undefined when the + * transport cannot tell. + */ + acknowledged(): Promise; +} + +/** The group streams a session has open, from when each opens until it is closed or reset. */ +export class OpenStreams { + /** + * Whether the transport reports acknowledgements, known when the session starts: + * `@moq/qmux` does, a browser's WebTransport cannot. + */ + readonly acknowledges: boolean; + readonly #open = new Set(); + + constructor(acknowledges: boolean) { + this.acknowledges = acknowledges; + } + + add(stream: OpenStream) { + this.#open.add(stream); + } + + delete(stream: OpenStream) { + this.#open.delete(stream); + } + + /** The open group streams, the lowest send order first, and the earliest opened among equals. */ + list(): OpenStream[] { + return [...this.#open].sort((a, b) => a.sendOrder - b.sendOrder); + } +} + +/** The lines of one session's opens, and the group streams it has open. */ interface Lines { uni: Line; bi: Line; + streams: OpenStreams; } const sessions = new WeakMap(); -/** The lines `quic`'s opens wait in, shared by everything opening on it. */ +/** The lines `quic`'s opens wait in, and its open group streams, shared by everything opening on it. */ export function lines(quic: WebTransport): Lines { let lines = sessions.get(quic); if (lines) return lines; @@ -185,6 +229,7 @@ export function lines(quic: WebTransport): Lines { (stream.writable as SendStream).sendOrder = sendOrder; }, ), + streams: new OpenStreams(transportOf(quic) === "websocket"), }; sessions.set(quic, lines); return lines; diff --git a/js/net/src/stream.ts b/js/net/src/stream.ts index bdeaa39ad6c..26faae029b4 100644 --- a/js/net/src/stream.ts +++ b/js/net/src/stream.ts @@ -2,7 +2,7 @@ import { fromTransport, StreamCode, toStreamCode, toTransport } from "./error.ts import { sharedStreamCode } from "./ietf/error.ts"; import type { IetfVersion } from "./ietf/version.ts"; import { Version as Lite, type Version as LiteVersion } from "./lite/version.ts"; -import { lines, type Rank } from "./open.ts"; +import { lines, type OpenStream, type OpenStreams, type Rank } from "./open.ts"; import { decodeError } from "./util/decode_error.ts"; import { POW32, toBigInt, toNumber, U64 } from "./util/u64.ts"; import { decodeUtf8 } from "./util/utf8.ts"; @@ -151,8 +151,14 @@ export interface TryOpenOptions extends Omit { /** Give up once this resolves. */ cancel: Promise; + + /** List the stream among the session's open group streams until it is closed or reset. */ + group?: boolean; } +/** A send stream that reports what it sent, as W3C WebTransport's `WebTransportSendStream.getStats()` does. */ +type Stats = WritableStream & { getStats(): Promise<{ bytesAcknowledged: number }> }; + export class Stream { reader: Reader; writer: Writer; @@ -690,11 +696,21 @@ const U62 = (c: Cursor) => c.u62(); const VARINT = (c: Cursor) => c.varint(); // Writer wraps a stream and writes chunks of data -export class Writer { +export class Writer implements OpenStream { #writer: WritableStreamDefaultWriter; #stream: WritableStream; #closed?: Promise; + /** The bytes handed to the stream so far. */ + #written = 0; + #sendOrder = 0; + /** Whether the transport reports acknowledgements, as the session's open streams say. */ + #acknowledges = false; + /** Reset by this side, or failed or stopped by the peer, so never acknowledged. */ + #aborted = false; + /** The session's open group streams, while this one is among them. */ + #listed?: OpenStreams; + // Scratch buffer for each primitive write, sized for the longest (a 9-byte leading-ones varint). #scratch: ArrayBuffer; @@ -705,6 +721,19 @@ export class Writer { this.#scratch = new ArrayBuffer(9); this.#writer = this.#stream.getWriter(); this.version = version; + // The stream ends once it is finished, or with a failure: a reset, a stop, or the session closing. + this.#writer.closed.then( + () => this.#unlist(), + () => { + this.#aborted = true; + this.#unlist(); + }, + ); + } + + /** The send order the stream has now, where HIGHER values are sent first. */ + get sendOrder(): number { + return this.#sendOrder; } /** @@ -718,9 +747,22 @@ export class Writer { * ignored property, the same way an ignored `sendOrder` option does at {@link open}. */ setPriority(sendOrder: number) { + this.#sendOrder = sendOrder; (this.#stream as SendStream).sendOrder = sendOrder; } + /** + * Whether the peer has acknowledged every byte written so far, or undefined when the + * transport cannot tell, as a browser's WebTransport cannot. + */ + async acknowledged(): Promise { + if (!this.#acknowledges) return undefined; + if (this.#aborted) return false; + const written = this.#written; + const stats = await (this.#stream as Stats).getStats(); + return !this.#aborted && stats.bytesAcknowledged >= written; + } + async bool(v: boolean) { await this.write(setUint8(this.#scratch, v ? 1 : 0)); } @@ -767,6 +809,7 @@ export class Writer { } async write(v: Uint8Array) { + this.#written += v.byteLength; // Mirrors Reader.#fill: every write funnels through here, so a STOP_SENDING from the // peer surfaces as a typed code rather than the transport's own error shape. await this.#writer.write(v).catch((err: unknown) => { @@ -783,6 +826,7 @@ export class Writer { } close() { + this.#unlist(); this.#writer.close().catch(() => void 0); } @@ -796,9 +840,16 @@ export class Writer { } reset(reason: unknown) { + this.#aborted = true; + this.#unlist(); this.#writer.abort(withCode(reason, this.version)).catch(() => void 0); } + #unlist() { + this.#listed?.delete(this); + this.#listed = undefined; + } + /** Run `f` against a Writer that buffers in memory, returning everything it wrote as one chunk. */ static async collect(version: StreamVersion, f: (w: Writer) => Promise): Promise { const chunks: Uint8Array[] = []; @@ -824,8 +875,9 @@ export class Writer { */ /** Open a send-only stream, waiting its turn for the peer's stream credit. */ static async open(quic: WebTransport, options: OpenOptions): Promise { - const writable = await lines(quic).uni.open(options.sendOrder ?? 0); - return new Writer(writable, options.version); + const session = lines(quic); + const writable = await session.uni.open(options.sendOrder ?? 0); + return Writer.#opened(writable, options.version, options.sendOrder ?? 0, session.streams); } /** @@ -834,8 +886,26 @@ export class Writer { * to the next open in line. A transport failure, or a rejected `cancel`, still throws. */ static async tryOpen(quic: WebTransport, options: TryOpenOptions): Promise { - const writable = await lines(quic).uni.open(options.sendOrder ?? 0, options.cancel); - return writable && new Writer(writable, options.version); + const session = lines(quic); + const rank = options.sendOrder ?? 0; + // The line ranks the stream it hands out by its last read, which can differ from a read now. + let sendOrder = typeof rank === "number" ? rank : 0; + const read = typeof rank === "number" ? rank : () => (sendOrder = rank()); + const writable = await session.uni.open(read, options.cancel); + if (!writable) return undefined; + const writer = Writer.#opened(writable, options.version, sendOrder, session.streams); + if (options.group) { + writer.#listed = session.streams; + session.streams.add(writer); + } + return writer; + } + + static #opened(writable: SendStream, version: StreamVersion, sendOrder: number, streams: OpenStreams): Writer { + const writer = new Writer(writable, version); + writer.#sendOrder = sendOrder; + writer.#acknowledges = streams.acknowledges; + return writer; } } diff --git a/js/qmux/api/index.api.md b/js/qmux/api/index.api.md index 15abe6e5bf7..c03b10a151b 100644 --- a/js/qmux/api/index.api.md +++ b/js/qmux/api/index.api.md @@ -5,6 +5,10 @@ export interface AcceptOptions extends Pick; } +export interface SendStream extends WritableStream { + getStats(): Promise; + sendOrder: number; +} + export type SendStreamOptions = WebTransportSendStreamOptions & { waitUntilAvailable?: boolean; }; +export interface SendStreamStats { + bytesAcknowledged: number; + bytesSent: number; + bytesWritten: number; +} + export class SessionError extends Error implements Pick { constructor(message: string, options?: { cause?: unknown; @@ -109,13 +124,13 @@ export default class implements WebTransport { }): void; readonly closed: Promise; get congestionControl(): WebTransportCongestionControl; - createBidirectionalStream(options?: SendStreamOptions): Promise; - createUnidirectionalStream(options?: SendStreamOptions): Promise>; + createBidirectionalStream(options?: SendStreamOptions): Promise; + createUnidirectionalStream(options?: SendStreamOptions): Promise; get datagrams(): WebTransportDatagramDuplexStream; getStats(): Promise<{ datagrams: {}; }>; - readonly incomingBidirectionalStreams: ReadableStream; + readonly incomingBidirectionalStreams: ReadableStream; readonly incomingUnidirectionalStreams: ReadableStream>; get protocol(): string; readonly ready: Promise; diff --git a/js/qmux/src/index.ts b/js/qmux/src/index.ts index 3e47d49b8b2..55ffaedeefe 100644 --- a/js/qmux/src/index.ts +++ b/js/qmux/src/index.ts @@ -3,7 +3,17 @@ import type { Version } from "./session.ts"; import Session from "./session.ts"; export { SessionError, StreamError } from "./error.ts"; -export type { AcceptOptions, Config, SendStreamOptions, SessionOptions, StreamCredit, Version } from "./session.ts"; +export type { + AcceptOptions, + BidirectionalStream, + Config, + SendStream, + SendStreamOptions, + SendStreamStats, + SessionOptions, + StreamCredit, + Version, +} from "./session.ts"; export { selectSubprotocol } from "./session.ts"; export type { WebSocketLike, diff --git a/js/qmux/src/session.test.ts b/js/qmux/src/session.test.ts index f5c70e35f37..805d4336584 100644 --- a/js/qmux/src/session.test.ts +++ b/js/qmux/src/session.test.ts @@ -363,6 +363,29 @@ describe("Session integration (scripted peer)", () => { }); } + test("a send stream's bytes count as acknowledged once they reach the socket", async () => { + const { session, peer } = connect(); + await session.ready; + peer.send({ type: "transport_parameters", params: peerParams() }); + await waitFor(() => peer.has("transport_parameters")); + const stream = await session.createUnidirectionalStream(); + const writer = stream.getWriter(); + expect(await stream.getStats()).toEqual({ bytesWritten: 0, bytesSent: 0, bytesAcknowledged: 0 }); + + // The ping response sticks in the socket; our write queues behind it. + peer.blockWrites(); + peer.send({ type: "ping_request", sequence: 1n }); + await settle(); + const write = writer.write(new Uint8Array(10)); + await settle(); + expect(await stream.getStats()).toEqual({ bytesWritten: 10, bytesSent: 0, bytesAcknowledged: 0 }); + + peer.unblockWrites(); + await write; + expect(await stream.getStats()).toEqual({ bytesWritten: 10, bytesSent: 10, bytesAcknowledged: 10 }); + session.close(); + }); + test("aborting a write queued behind a wedged socket doesn't wait for it to drain", async () => { const { session, peer } = connect(); await session.ready; diff --git a/js/qmux/src/session.ts b/js/qmux/src/session.ts index 144a05aa371..b5b13ec17d9 100644 --- a/js/qmux/src/session.ts +++ b/js/qmux/src/session.ts @@ -352,6 +352,30 @@ export interface StreamCredit { blocked: boolean; } +/** What a send stream has sent so far, the shape of W3C WebTransport's `WebTransportSendStreamStats`. */ +export interface SendStreamStats { + /** The bytes written to the stream. */ + bytesWritten: number; + /** The bytes handed to the WebSocket. */ + bytesSent: number; + /** The bytes the peer has acknowledged, which are `bytesSent`: QMux acknowledges data once + * it is passed to the underlying transport. */ + bytesAcknowledged: number; +} + +/** A stream's send half, with W3C `WebTransportSendStream`'s `sendOrder` and `getStats()`. */ +export interface SendStream extends WritableStream { + /** Its priority against the session's other streams; higher sends first. */ + sendOrder: number; + /** What the stream has written, sent, and had acknowledged so far. */ + getStats(): Promise; +} + +/** A bidirectional stream whose send half is a {@link SendStream}. */ +export interface BidirectionalStream extends WebTransportBidirectionalStream { + readonly writable: SendStream; +} + /** Options for opening a QMux Session over WebSocket. */ export interface SessionOptions extends WebTransportOptions { /** Application-level subprotocols to advertise via `Sec-WebSocket-Protocol`. @@ -646,8 +670,8 @@ export default class Session implements WebTransport { #closedReject: (err: Error) => void; /** Bidirectional streams the peer opens, in arrival order. */ - readonly incomingBidirectionalStreams: ReadableStream; - #incomingBidirectionalQueue: IncomingStreamQueue; + readonly incomingBidirectionalStreams: ReadableStream; + #incomingBidirectionalQueue: IncomingStreamQueue; /** Unidirectional streams the peer opens, in arrival order. */ readonly incomingUnidirectionalStreams: ReadableStream>; #incomingUnidirectionalQueue: IncomingStreamQueue>; @@ -763,7 +787,7 @@ export default class Session implements WebTransport { // observe the rejection. this.closed.catch(() => {}); - this.#incomingBidirectionalQueue = new IncomingStreamQueue(); + this.#incomingBidirectionalQueue = new IncomingStreamQueue(); this.incomingBidirectionalStreams = this.#incomingBidirectionalQueue.readable; this.#incomingUnidirectionalQueue = new IncomingStreamQueue>(); @@ -1701,7 +1725,13 @@ export default class Session implements WebTransport { await scheduler.enqueueStream(streamId, bytes); } - async #sendStreamDataWithFlowControl(id: Stream.Id, streamId: bigint, data: Uint8Array, signal: AbortSignal) { + async #sendStreamDataWithFlowControl( + id: Stream.Id, + streamId: bigint, + data: Uint8Array, + signal: AbortSignal, + stats: SendStreamStats, + ) { const flow = this.#streamFlow.get(streamId); if (!flow) throw new Error(`missing flow state for stream ${streamId}`); @@ -1738,13 +1768,16 @@ export default class Session implements WebTransport { flow.sendOffset += BigInt(sendable); offset += sendable; + stats.bytesSent += sendable; + stats.bytesAcknowledged = stats.bytesSent; } } - async #sendStreamData(id: Stream.Id, data: Uint8Array, signal: AbortSignal) { + async #sendStreamData(id: Stream.Id, data: Uint8Array, signal: AbortSignal, stats: SendStreamStats) { const streamId = id.value.value; - await this.#sendStreamDataWithFlowControl(id, streamId, data, signal); + stats.bytesWritten += data.byteLength; + await this.#sendStreamDataWithFlowControl(id, streamId, data, signal, stats); } /** Send the FIN. Routed through the stream's own queue so it stays ordered @@ -1836,7 +1869,7 @@ export default class Session implements WebTransport { /** Open a bidirectional stream, waiting for the peer to grant stream credit * unless `waitUntilAvailable` is false. */ - async createBidirectionalStream(options?: SendStreamOptions): Promise { + async createBidirectionalStream(options?: SendStreamOptions): Promise { await this.#claimStream(Stream.Dir.Bi, options, "Failed to create bidirectional stream."); // close() may have torn down the session while the claim resumed. if (this.#closed) throw this.#closed; @@ -1882,7 +1915,7 @@ export default class Session implements WebTransport { /** Open a send-only stream, waiting for the peer to grant stream credit * unless `waitUntilAvailable` is false. */ - async createUnidirectionalStream(options?: SendStreamOptions): Promise> { + async createUnidirectionalStream(options?: SendStreamOptions): Promise { await this.#claimStream(Stream.Dir.Uni, options, "Failed to create send stream."); // close() may have torn down the session while the claim resumed. if (this.#closed) throw this.#closed; @@ -1906,8 +1939,9 @@ export default class Session implements WebTransport { } /** The writable for a stream's send half. */ - #sendWritable(id: Stream.Id, sendOrder: number): WritableStream { + #sendWritable(id: Stream.Id, sendOrder: number): SendStream { const streamId = id.value.value; + const stats: SendStreamStats = { bytesWritten: 0, bytesSent: 0, bytesAcknowledged: 0 }; let signal!: AbortSignal; const reset = (e: unknown) => { this.#scheduler?.dropStream(streamId, e instanceof Error ? e : new Error("stream aborted")); @@ -1941,7 +1975,7 @@ export default class Session implements WebTransport { ); }, write: async (chunk, controller) => { - await this.#sendStreamData(id, chunk, controller.signal); + await this.#sendStreamData(id, chunk, controller.signal, stats); }, abort: reset, close: async () => { @@ -1960,7 +1994,13 @@ export default class Session implements WebTransport { }, }); this.#attachSendOrder(writable, streamId, sendOrder); - return writable; + // W3C `WebTransportSendStream.getStats()`, so a caller can tell what the peer has. + Object.defineProperty(writable, "getStats", { + configurable: true, + value: async (): Promise => ({ ...stats }), + }); + // Both properties are defined above. + return writable as SendStream; } /** Shared teardown for both terminal transitions: tears down streams, diff --git a/quest/m1/stream-credit/README.md b/quest/m1/stream-credit/README.md index 8b4eb210783..25e9db3ed00 100644 --- a/quest/m1/stream-credit/README.md +++ b/quest/m1/stream-credit/README.md @@ -43,6 +43,5 @@ receiving every group, and nothing hanging. ## Required - [An abandoned subscription setup releases its streams](/quest/m1/stream-credit/release-on-leave.md) - release TRACK and pending SUBSCRIBE when demand leaves during setup -- [Open streams](/quest/m1/stream-credit/open-streams.md) - each session lists its open streams and, where the transport can tell, which are fully acknowledged - [Reset fallback](/quest/m1/stream-credit/reset-fallback.md) - a waiting higher-priority open resets the lowest-priority stream, one reset in flight per waiting open - [Takeover](/quest/m1/stream-credit/takeover.md) - on lite-07 a higher-priority group takes over an acknowledged stream, and a group that lost its stream resumes diff --git a/quest/m1/stream-credit/open-streams.md b/quest/m1/stream-credit/open-streams.md deleted file mode 100644 index 02af3563340..00000000000 --- a/quest/m1/stream-credit/open-streams.md +++ /dev/null @@ -1,30 +0,0 @@ -# [M] Each session knows its open streams and which are acknowledged - -## Goal - -Rust and `js/net` sessions keep a per-session list of their open outgoing -group streams with each stream's priority, and the transports that can tell -report when all of a stream's sent data is acknowledged: moq-quic (raw QUIC -and WebTransport), Rust qmux, `@moq/qmux`, and the memory transport. A -browser's WebTransport cannot report it, and the session knows that at setup -rather than guessing. This is what [reset fallback](/quest/m1/stream-credit/reset-fallback.md) -and [takeover](/quest/m1/stream-credit/takeover.md) choose a stream from. - -## Plan - -Decided 2026-11-05 (maintainer, /quest-plan): split from the [M] "A higher-priority group makes room" quest, which -the agent that declined it on 2026-11-05 found was really XL and had gaps: -the transport could not report when a stream's data is all acknowledged, -nothing defined where a reset group resumes, and no session listed its open -streams. This part starts now; it touches no code -[moq-dev/moq#7670](https://github.com/moq-dev/moq/pull/7670) rewrites. - -The signal that room is needed is the open line's waiting state (the head of -the line is pending), per the maintainer's 2026-11-04 decision on -[moq-dev/moq#7894](https://github.com/moq-dev/moq/pull/7894), not a transport -credit or blocked API. Acknowledgement reporting is a different fact (has the -peer received this stream's bytes), so it is new transport surface; keep it -to what takeover needs. - -Public API: transport traits gain an acknowledged signal where supported. -Wire: none. diff --git a/quest/m1/stream-credit/reset-fallback.md b/quest/m1/stream-credit/reset-fallback.md index fd38c62ccad..603f362d92d 100644 --- a/quest/m1/stream-credit/reset-fallback.md +++ b/quest/m1/stream-credit/reset-fallback.md @@ -22,11 +22,13 @@ RTT from WebTransport's `getStats()` when it reports one, else `Probe.rtt` (as `js/net/src/connection/stats.ts` already falls back), and before any sample the maintainer's original 100 ms. +Choose from the session's open group streams: Rust's +`transport::open::Lines::open_streams` (opened through +`boxed::Session::poll_open_group`) and JS's `lines(quic).streams`, each lowest +ranked first with whether the peer has acknowledged it. They report rank and +acknowledgement only; add the handle that reaches the stream's writer here. + Tests: a cap with no acknowledged stream resets exactly one stream per waiting open, and the higher-priority group starts once credit returns. Public API: none. Wire: none. - -## Required - -- [Open streams](/quest/m1/stream-credit/open-streams.md) - the session's open-stream list to choose the lowest-priority stream from diff --git a/quest/m1/stream-credit/takeover.md b/quest/m1/stream-credit/takeover.md index 73a7014f20b..1ca090c65ff 100644 --- a/quest/m1/stream-credit/takeover.md +++ b/quest/m1/stream-credit/takeover.md @@ -20,6 +20,12 @@ otherwise queue behind the old group's retransmissions. Add the takeover wire to `drafts/draft-lcurley-moq-lite.md` before lite-07 finalizes, and define where a reset or taken-over group resumes. +Choose from the session's open group streams: Rust's +`transport::open::Lines::open_streams` (opened through +`boxed::Session::poll_open_group`) and JS's `lines(quic).streams`, each lowest +ranked first with whether the peer has acknowledged it. They report rank and +acknowledgement only; add the handle that reaches the stream's writer here. + Tests: a browser over WebTransport and a Rust relay, more live groups than the cap, the higher-priority group starting without waiting a round trip; a group that lost its stream resumes without loss. @@ -35,5 +41,4 @@ Public API: none. Wire: lite-07 group stream takeover. ## Required - [Drop set](/quest/m1/drop-set.md) - its hold-the-duplicate rule governs a resumed group's second stream -- [Open streams](/quest/m1/stream-credit/open-streams.md) - which open streams are fully acknowledged - [Reset fallback](/quest/m1/stream-credit/reset-fallback.md) - the fallback a takeover is tried before diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 7353f7bae9c..c6e03a6f2ac 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -7093,6 +7093,10 @@ impl crate::transport::poll::SendStream f inner.follow(waiter); inner.stream.poll_closed(waiter) } + + fn acknowledged(&self) -> Option { + self.inner.lock().unwrap().stream.acknowledged() + } } /// Write one group's frames in the negotiated draft's FETCH object layout. @@ -7673,7 +7677,10 @@ impl GroupServe { } // The current rank, so a move while parked applies to the stream as it opens. let _ = self.rank.poll_moved(waiter); - let stream = match ready!(self.session.poll_open_uni(&mut self.open, waiter, self.rank.options())) { + let stream = match ready!( + self.session + .poll_open_group(&mut self.open, waiter, self.rank.options()) + ) { Ok(stream) => stream, Err(err) => { self.state = GroupState::Done; diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index faebe2cb4bb..fd65541d2d1 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -5797,7 +5797,7 @@ impl GroupServe { priority: handle.send_order().into(), }, }; - let stream = match ready!(session.poll_open_uni(&mut self.open, waiter, options)) { + let stream = match ready!(session.poll_open_group(&mut self.open, waiter, options)) { Ok(stream) => stream, Err(err) => { self.settle_open(false); diff --git a/rs/moq-net/src/transport.rs b/rs/moq-net/src/transport.rs index c96efefdc3d..4d2c12aa27f 100644 --- a/rs/moq-net/src/transport.rs +++ b/rs/moq-net/src/transport.rs @@ -816,6 +816,15 @@ pub mod poll { /// resolves once the FIN is sent. fn poll_closed(&self, waiter: &kio::Waiter) -> Poll>; + /// Whether the peer has acknowledged every byte written so far, or `None` when the + /// transport cannot tell, as a browser's WebTransport cannot. + /// + /// A transport answers for all of its streams or for none. A stream that was reset, or + /// that the peer stopped, is never acknowledged. + fn acknowledged(&self) -> Option { + None + } + /// Write some of the buffer, returning how many bytes were accepted. fn write<'a>(&'a mut self, buf: &'a [u8]) -> Write<'a, Self> { Write { diff --git a/rs/moq-net/src/transport/boxed.rs b/rs/moq-net/src/transport/boxed.rs index 5ed9c2b2170..fde24334d43 100644 --- a/rs/moq-net/src/transport/boxed.rs +++ b/rs/moq-net/src/transport/boxed.rs @@ -7,8 +7,8 @@ use std::any::Any; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, OnceLock, Weak}; -use std::task::Poll; +use std::sync::{Arc, Mutex, OnceLock, Weak}; +use std::task::{Poll, ready}; use bytes::{Buf, Bytes}; @@ -55,6 +55,38 @@ impl Session { }) } + /// Poll opening a unidirectional stream for a group, like [`Self::poll_open_uni`], which the + /// session lists among its open group streams until it is finished, reset, or dropped. + pub(crate) fn poll_open_group( + &self, + open: &mut Open, + waiter: &kio::Waiter, + options: SendOptions<'_, Group>, + ) -> Poll> { + let SendStream(Sender::Own(stream)) = ready!(self.poll_open_uni(open, waiter, options))? else { + unreachable!("a stream is listed only here"); + }; + let listed = Arc::new(Listed { + state: Mutex::new(ListedState { + stream, + group: options.group.map(|group| group.priority.clone()), + priority: options.priority, + }), + }); + let key = self.opens.list(&listed); + Poll::Ready(Ok(SendStream(Sender::Listed(Listing { + listed, + lines: self.opens.clone(), + key: Some(key), + })))) + } + + /// The session's open group streams, the lowest ranked first. + #[cfg(test)] + pub(crate) fn open_streams(&self) -> Vec { + self.opens.open_streams() + } + /// Poll opening a bidirectional stream, its sending half scheduled by `options`, once /// `open` is the most important open waiting for the peer's stream credit. pub(crate) fn poll_open_bi( @@ -194,13 +226,114 @@ impl poll::SendGroup for Group { } /// A boxed outgoing stream. -pub(crate) struct SendStream(Box); +pub(crate) struct SendStream(Sender); + +enum Sender { + Own(Box), + /// A group stream its session lists, so the list can read it too. + Listed(Listing), +} impl SendStream { /// Box a stream, keeping its own send-group type behind the boundary. #[cfg(test)] pub(crate) fn new(stream: impl poll::SendStream + Send + Sync) -> Self { - Self(Box::new(stream)) + Self(Sender::Own(Box::new(stream))) + } + + fn own(stream: impl poll::SendStream + Send + Sync + 'static) -> Self { + Self(Sender::Own(Box::new(stream))) + } + + /// Run `f` on the transport's stream, then leave the session's list if `unlist`. + fn with(&mut self, unlist: bool, f: impl FnOnce(&mut dyn ErasedSend) -> T) -> T { + match &mut self.0 { + Sender::Own(stream) => f(stream.as_mut()), + Sender::Listed(listing) => { + let res = f(listing.listed.lock().stream.as_mut()); + if unlist { + listing.unlist(); + } + res + } + } + } + + fn get(&self) -> Read<'_> { + match &self.0 { + Sender::Own(stream) => Read::Own(stream.as_ref()), + Sender::Listed(listing) => Read::Listed(listing.listed.lock()), + } + } +} + +/// A boxed stream borrowed for a read through `&self`. +enum Read<'a> { + Own(&'a (dyn ErasedSend + 'static)), + Listed(std::sync::MutexGuard<'a, ListedState>), +} + +impl std::ops::Deref for Read<'_> { + type Target = dyn ErasedSend; + + fn deref(&self) -> &Self::Target { + match self { + Read::Own(stream) => *stream, + Read::Listed(state) => state.stream.as_ref(), + } + } +} + +/// An open group stream, shared by the task writing it and its session's list. +pub(super) struct Listed { + state: Mutex, +} + +struct ListedState { + stream: Box, + /// The priority of the send group the stream is in, shared with the group's handles. + group: Option>, + priority: i64, +} + +impl Listed { + fn lock(&self) -> std::sync::MutexGuard<'_, ListedState> { + self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// The stream's rank, and whether the peer acknowledged everything written to it. + pub(super) fn read(&self) -> open::OpenStream { + let state = self.lock(); + open::OpenStream { + rank: open::Rank::new( + state.group.as_ref().map(|group| group.load(Ordering::Relaxed)), + state.priority, + ), + acknowledged: state.stream.acknowledged(), + } + } +} + +/// The writing task's hold on a listed stream, which leaves the list once the stream is +/// finished, reset, or dropped. +struct Listing { + listed: Arc, + lines: Arc, + /// The key the stream is listed under, until it leaves. + key: Option, +} + +impl Listing { + fn unlist(&mut self) { + if let Some(key) = self.key.take() { + self.lines.unlist(key); + } + } +} + +impl Drop for Listing { + fn drop(&mut self) { + self.unlist(); } } @@ -367,6 +500,7 @@ trait ErasedSend: Send + Sync { fn finish(&mut self) -> Result<(), Error>; fn reset(&mut self, code: u32); fn poll_closed(&self, waiter: &kio::Waiter) -> Poll>; + fn acknowledged(&self) -> Option; } trait ErasedRecv: Send + Sync { @@ -404,7 +538,7 @@ fn unbox<'a, G: 'static>(options: SendOptions<'a, Group>) -> SendOptions<'a, G> } fn bi((send, recv): poll::BiStreams, budget: Budget) -> (SendStream, RecvStream) { - (SendStream(Box::new(send)), RecvStream::budgeted(recv, budget)) + (SendStream::own(send), RecvStream::budgeted(recv, budget)) } impl ErasedSession for S { @@ -419,7 +553,7 @@ impl ErasedSession for S { fn poll_open_uni(&self, waiter: &kio::Waiter, options: SendOptions<'_, Group>) -> Poll> { poll::Session::poll_open_uni(self, waiter, unbox(options)) - .map(|res| res.map(|send| SendStream(Box::new(send))).map_err(Error::new)) + .map(|res| res.map(SendStream::own).map_err(Error::new)) } fn poll_open_bi( @@ -544,6 +678,10 @@ impl + Send + Sync> ErasedSend for T { fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { poll::SendStream::poll_closed(self, waiter).map_err(Error::new) } + + fn acknowledged(&self) -> Option { + poll::SendStream::acknowledged(self) + } } impl ErasedRecv for Budgeted { @@ -703,31 +841,41 @@ impl poll::SendStream for SendStream { type Group = Group; fn poll_write(&mut self, waiter: &kio::Waiter, buf: &[u8]) -> Poll> { - self.0.poll_write(waiter, buf) + self.with(false, |stream| stream.poll_write(waiter, buf)) } fn poll_write_buf(&mut self, waiter: &kio::Waiter, buf: &mut B) -> Poll> { - self.0.poll_write_buf(waiter, buf) + self.with(false, |stream| stream.poll_write_buf(waiter, buf)) } fn set_priority(&mut self, priority: i64) { - self.0.set_priority(priority) + if let Sender::Listed(listing) = &self.0 { + listing.listed.lock().priority = priority; + } + self.with(false, |stream| stream.set_priority(priority)) } fn set_group(&mut self, group: &Group) { - self.0.set_group(group) + if let Sender::Listed(listing) = &self.0 { + listing.listed.lock().group = Some(group.priority.clone()); + } + self.with(false, |stream| stream.set_group(group)) } fn finish(&mut self) -> Result<(), Error> { - self.0.finish() + self.with(true, |stream| stream.finish()) } fn reset(&mut self, code: u32) { - self.0.reset(code) + self.with(true, |stream| stream.reset(code)) } fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { - self.0.poll_closed(waiter) + self.get().poll_closed(waiter) + } + + fn acknowledged(&self) -> Option { + self.get().acknowledged() } } @@ -840,6 +988,106 @@ mod test { assert_eq!(accept(&session).window(), AHEAD); } + fn open_group(session: &Session, options: SendOptions<'_, Group>) -> SendStream { + let Poll::Ready(Ok(stream)) = session.poll_open_group(&mut Open::default(), &kio::Waiter::noop(), options) + else { + panic!("the transport opens at once"); + }; + stream + } + + fn ranks(session: &Session) -> Vec { + session.open_streams().into_iter().map(|stream| stream.rank).collect() + } + + fn ungrouped(priority: i64) -> SendOptions<'static, Group> { + SendOptions { group: None, priority } + } + + /// Group streams are listed lowest ranked first, following a moved priority, until they + /// finish, reset, or drop; other streams are never listed. + #[test] + fn group_streams_are_listed_until_they_end() { + let (a, _b) = super::super::memory::pair(Default::default()); + let session = Session::new(a); + let mut first = open_group(&session, ungrouped(1)); + let mut second = open_group(&session, ungrouped(3)); + let _control = session + .poll_open_uni(&mut Open::default(), &kio::Waiter::noop(), ungrouped(0)) + .map(Result::unwrap); + let third = open_group(&session, ungrouped(2)); + let rank = |stream| open::Rank::Ungrouped { stream }; + assert_eq!(ranks(&session), [rank(1), rank(2), rank(3)]); + + poll::SendStream::set_priority(&mut second, 0); + assert_eq!(ranks(&session), [rank(0), rank(1), rank(2)]); + + poll::SendStream::finish(&mut first).unwrap(); + poll::SendStream::reset(&mut second, 1); + assert_eq!(ranks(&session), [rank(2)]); + drop(third); + assert!(session.open_streams().is_empty()); + } + + /// A listed stream reports the peer acknowledging what was written to it, and a reset one + /// never is. + #[test] + fn a_listed_stream_reports_its_acknowledgement() { + let (a, b) = super::super::memory::pair(Default::default()); + let session = Session::new(a); + let mut stream = open_group(&session, ungrouped(0)); + let acknowledged = || session.open_streams()[0].acknowledged; + assert_eq!(acknowledged(), Some(true), "nothing written is all acknowledged"); + + let waiter = kio::Waiter::noop(); + assert!(matches!( + poll::SendStream::poll_write(&mut stream, &waiter, b"group"), + Poll::Ready(Ok(5)) + )); + assert_eq!(acknowledged(), Some(false)); + assert_eq!( + poll::SendStream::acknowledged(&stream), + Some(false), + "the writer reads the same" + ); + + let Poll::Ready(Ok(mut recv)) = poll::Session::poll_accept_uni(&b, &waiter) else { + panic!("the peer accepts the stream"); + }; + assert!(matches!( + poll::RecvStream::poll_read_chunk(&mut recv, &waiter, 16), + Poll::Ready(Ok(Some(_))) + )); + assert_eq!(acknowledged(), Some(true)); + + poll::SendStream::reset(&mut stream, 1); + assert_eq!(poll::SendStream::acknowledged(&stream), Some(false)); + } + + /// A grouped stream ranks by its group's current priority, and follows a move to another + /// group; a transport that cannot tell reports no acknowledgement. + #[test] + fn a_listed_stream_ranks_by_its_group() { + let session = Session::new(SinkSession::new(Default::default()).with_groups()); + let mut low = poll::Session::send_group(&session, 1).expect("the sink opens groups"); + let high = poll::Session::send_group(&session, 5).expect("the sink opens groups"); + let mut stream = open_group( + &session, + SendOptions { + group: Some(&low), + priority: 2, + }, + ); + let read = |session: &Session| session.open_streams()[0]; + assert_eq!(read(&session).rank, open::Rank::Grouped { group: 1, stream: 2 }); + assert_eq!(read(&session).acknowledged, None); + + poll::SendGroup::set_priority(&mut low, 3); + assert_eq!(read(&session).rank, open::Rank::Grouped { group: 3, stream: 2 }); + poll::SendStream::set_group(&mut stream, &high); + assert_eq!(read(&session).rank, open::Rank::Grouped { group: 5, stream: 2 }); + } + /// A stream accepted while the connection window was held back, such as one buffered /// before admission, paces at the window in force once ingest reads it. #[test] diff --git a/rs/moq-net/src/transport/memory.rs b/rs/moq-net/src/transport/memory.rs index fe0f1bec1f2..230c23bbfcd 100644 --- a/rs/moq-net/src/transport/memory.rs +++ b/rs/moq-net/src/transport/memory.rs @@ -616,6 +616,16 @@ impl poll::SendStream for SendStream { } closed.map(Err) } + + /// Whether the reader took every message written so far, which stands in for an + /// acknowledgement as it does for [`poll_closed`](poll::SendStream::poll_closed). + fn acknowledged(&self) -> Option { + let channel = self.channel.lock(); + // A reset on a closed session leaves the channel as it was, so only `done` records it. + let finished = matches!(channel.end, Some(End::Fin)); + let open = channel.end.is_none() && !self.done; + Some((open || finished) && channel.messages.is_empty()) + } } impl Drop for SendStream { diff --git a/rs/moq-net/src/transport/open.rs b/rs/moq-net/src/transport/open.rs index eb643df5b06..4cace43bf5d 100644 --- a/rs/moq-net/src/transport/open.rs +++ b/rs/moq-net/src/transport/open.rs @@ -9,38 +9,59 @@ //! An open can also wait on something other than credit, such as a WebTransport stream's //! header, so the line asks the transport's [`StreamCredit`] whether the peer is what it waits //! for. +//! +//! The session also lists the group streams it has open, each with its rank and whether the +//! peer has acknowledged everything written to it, so an open waiting for credit can choose +//! a lower-ranked stream to make room with. use std::cmp::Reverse; -use std::collections::BTreeMap; -use std::sync::{Arc, Mutex}; +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex, Weak}; use std::task::Poll; use super::boxed::{self, Transport}; use super::poll::SendOptions; use super::{Dir, StreamCredit}; -/// Where an open ranks for credit: the order the transport will send its stream in. +/// Where an open or an open stream ranks: the order the transport sends its stream in. /// /// Mirrors the transport's send scheduling: streams outside every group go before all of /// them, then a higher group priority, then a higher priority within the group. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -enum Rank { +pub(crate) enum Rank { Grouped { group: i64, stream: i64 }, Ungrouped { stream: i64 }, } impl Rank { - fn of(options: &SendOptions<'_, super::boxed::Group>) -> Self { - match options.group { - Some(group) => Self::Grouped { - group: group.priority(), - stream: options.priority, - }, - None => Self::Ungrouped { - stream: options.priority, - }, + /// The rank of a stream at `stream` priority, in a group at `group` priority if any. + pub(super) fn new(group: Option, stream: i64) -> Self { + match group { + Some(group) => Self::Grouped { group, stream }, + None => Self::Ungrouped { stream }, } } + + fn of(options: &SendOptions<'_, super::boxed::Group>) -> Self { + Self::new(options.group.map(super::boxed::Group::priority), options.priority) + } +} + +/// One of a session's open group streams, as its list read it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct OpenStream { + /// Where the stream ranks now. + pub rank: Rank, + /// Whether the peer has acknowledged every byte written to it, or `None` when the + /// transport cannot tell. + pub acknowledged: Option, +} + +/// The group streams a session has open, keyed so each can leave. +#[derive(Default)] +struct Listing { + streams: HashMap>, + next: u64, } /// A place in line: the highest rank first, then the earliest arrival. @@ -63,16 +84,31 @@ impl Line { } /// Say why the line starts waiting: the peer's stream credit, unless the transport reports - /// otherwise. - fn start(credit: Option) { + /// otherwise, and what the session's open group streams could make room with. + fn start(credit: Option, lines: &Lines) { + if !tracing::enabled!(tracing::Level::DEBUG) { + return; + } + let open = lines.open_streams(); + let open_streams = open.len(); + let acknowledged = open.iter().filter(|stream| stream.acknowledged == Some(true)).count(); + let lowest = open.first().map(|stream| stream.rank); match credit { Some(credit) if !credit.blocked => { tracing::debug!( available = credit.available, + open_streams, + acknowledged, + ?lowest, "opens wait in priority order for the transport" ) } - _ => tracing::debug!("out of the peer's stream credit: opens wait in priority order"), + _ => tracing::debug!( + open_streams, + acknowledged, + ?lowest, + "out of the peer's stream credit: opens wait in priority order" + ), } } @@ -105,10 +141,12 @@ impl Line { } } -/// The lines of one session's opens, shared by every task opening on it. +/// The lines of one session's opens, and the group streams it has open, shared by every task +/// opening on it. pub(crate) struct Lines { uni: Mutex, bi: Mutex, + listing: Mutex, /// Whose stream credit the opens wait for. pub(super) transport: Transport, } @@ -118,10 +156,40 @@ impl Lines { Self { uni: Default::default(), bi: Default::default(), + listing: Default::default(), transport, } } + /// List an open group stream, returning the key it leaves with. + pub(super) fn list(&self, stream: &Arc) -> u64 { + let mut listing = self.listing(); + let key = listing.next; + listing.next += 1; + listing.streams.insert(key, Arc::downgrade(stream)); + key + } + + /// Remove the group stream listed under `key`. + pub(super) fn unlist(&self, key: u64) { + self.listing().streams.remove(&key); + } + + /// The session's open group streams, the lowest ranked first. + /// + /// A stream is open from when it opens until it is finished, reset, or dropped. + pub(crate) fn open_streams(&self) -> Vec { + // Read each stream outside the listing's lock, which its owner takes to leave. + let streams: Vec<_> = self.listing().streams.values().filter_map(Weak::upgrade).collect(); + let mut open: Vec<_> = streams.iter().map(|stream| stream.read()).collect(); + open.sort_by_key(|stream| stream.rank); + open + } + + fn listing(&self) -> std::sync::MutexGuard<'_, Listing> { + self.listing.lock().unwrap_or_else(|poisoned| poisoned.into_inner()) + } + fn line(&self, dir: Dir) -> std::sync::MutexGuard<'_, Line> { let line = match dir { Dir::Uni => &self.uni, @@ -159,7 +227,7 @@ impl Open { if let Poll::Ready(res) = open(waiter) { return Poll::Ready(res); } - Line::start(boxed::stream_credit(&lines.transport, dir)); + Line::start(boxed::stream_credit(&lines.transport, dir), lines); let key = line.join(rank); line.waiting.get_mut(&key).expect("just joined").register(waiter); self.place = Some((lines.clone(), dir, key)); diff --git a/rs/moq-tokio/src/quic/driver/send.rs b/rs/moq-tokio/src/quic/driver/send.rs index 9729d836e18..285583d87fc 100644 --- a/rs/moq-tokio/src/quic/driver/send.rs +++ b/rs/moq-tokio/src/quic/driver/send.rs @@ -195,6 +195,12 @@ impl Stream { self.half.priority() } + /// Whether the peer acknowledged every byte written so far + #[cfg(feature = "quic")] + pub(crate) fn acknowledged(&self) -> bool { + self.half.acknowledged() + } + /// Move the stream into a send group, or with `None` back to the ungrouped streams, which /// go before every group /// @@ -247,6 +253,12 @@ impl Finished { self.half.priority() } + /// Whether the peer acknowledged every byte written. See [`Stream::acknowledged()`]. + #[cfg(feature = "quic")] + pub(crate) fn acknowledged(&self) -> bool { + self.half.acknowledged() + } + /// Move the stream into a send group. See [`Stream::set_group()`]. /// /// # Panics @@ -357,6 +369,15 @@ impl Half { conn.inner.send_stream(self.stream.id).priority().unwrap_or_default() } + /// Whether the peer acknowledged every byte written so far; never once reset or stopped. + #[cfg(feature = "quic")] + fn acknowledged(&self) -> bool { + let mut conn = self.stream.conn.state.lock(); + let mut stream = conn.inner.send_stream(self.stream.id); + let offset = stream.offset(); + stream.acked(offset).unwrap_or(false) + } + fn set_group(&mut self, group: Option<&Group>) { let group = group.map(|group| group.id_on(&self.stream.conn.0)); let mut conn = self.stream.conn.state.lock(); diff --git a/rs/moq-tokio/src/quic/web/send.rs b/rs/moq-tokio/src/quic/web/send.rs index 9422c39a4a0..ce08cc5bf08 100644 --- a/rs/moq-tokio/src/quic/web/send.rs +++ b/rs/moq-tokio/src/quic/web/send.rs @@ -258,6 +258,13 @@ impl moq_net::transport::poll::SendStream for Stream { Err(err) => Err(err), }) } + fn acknowledged(&self) -> Option { + Some(match &self.state { + State::Open(stream) => stream.acknowledged(), + State::Finished(stream) => stream.acknowledged(), + State::Reset(_) => false, + }) + } } impl moq_net::transport::poll::SendGroup for send::Group { diff --git a/rs/moq-uring/src/quic/driver/send.rs b/rs/moq-uring/src/quic/driver/send.rs index 423439589e8..4bd9045e294 100644 --- a/rs/moq-uring/src/quic/driver/send.rs +++ b/rs/moq-uring/src/quic/driver/send.rs @@ -147,6 +147,11 @@ impl Stream { self.0.priority() } + /// Whether the peer acknowledged every byte written so far. + pub(crate) fn acknowledged(&self) -> bool { + self.0.acknowledged() + } + /// Move the stream into a send group, or with `None` back to the ungrouped streams, which /// go before every group. /// @@ -192,6 +197,11 @@ impl Finished { self.0.priority() } + /// Whether the peer acknowledged every byte written. See [`Stream::acknowledged()`]. + pub(crate) fn acknowledged(&self) -> bool { + self.0.acknowledged() + } + /// Move the stream into a send group. See [`Stream::set_group()`]. /// /// # Panics @@ -262,6 +272,14 @@ impl Half { .unwrap_or_default() } + /// Whether the peer acknowledged every byte written so far; never once reset or stopped. + fn acknowledged(&self) -> bool { + let mut conn = self.shared.conn.borrow_mut(); + let mut stream = conn.send_stream(self.id); + let offset = stream.offset(); + stream.acked(offset).unwrap_or(false) + } + fn set_group(&mut self, group: Option<&Group>) { let group = group.map(Group::id); let _ = self.shared.conn.borrow_mut().send_stream(self.id).set_group(group); diff --git a/rs/moq-uring/src/quic/web/send.rs b/rs/moq-uring/src/quic/web/send.rs index ba0df503eb9..d1e7a51399e 100644 --- a/rs/moq-uring/src/quic/web/send.rs +++ b/rs/moq-uring/src/quic/web/send.rs @@ -244,6 +244,13 @@ impl moq_net::transport::poll::SendStream for Stream { Err(err) => Err(err), }) } + fn acknowledged(&self) -> Option { + Some(match &self.state { + State::Open(stream) => stream.acknowledged(), + State::Finished(stream) => stream.acknowledged(), + State::Reset(_) => false, + }) + } } impl moq_net::transport::poll::SendGroup for send::Group { diff --git a/rs/moq-uring/src/transport/local.rs b/rs/moq-uring/src/transport/local.rs index 7e774530156..8c2086ce81a 100644 --- a/rs/moq-uring/src/transport/local.rs +++ b/rs/moq-uring/src/transport/local.rs @@ -158,6 +158,10 @@ impl poll::SendStream for Local { fn poll_closed(&self, waiter: &kio::Waiter) -> Poll> { self.0.poll_closed(waiter) } + + fn acknowledged(&self) -> Option { + self.0.acknowledged() + } } impl poll::RecvStream for Local { diff --git a/rs/qmux/src/session.rs b/rs/qmux/src/session.rs index 2bb15a2f0f1..543e6f08566 100644 --- a/rs/qmux/src/session.rs +++ b/rs/qmux/src/session.rs @@ -589,6 +589,18 @@ impl poll::SendStream for SendStream { Err(_) => Poll::Ready(Err(Error::StreamClosed)), } } + + /// Whether every byte written so far went to the transport, which acknowledges it on a + /// reliable transport. + fn acknowledged(&self) -> Option { + if self.reset { + return Some(false); + } + let mut state = self.inner.lock(); + let mut stream = state.conn.send_stream(self.id); + let offset = stream.offset(); + Some(stream.acked(offset).unwrap_or(false)) + } } impl Drop for SendStream { diff --git a/rs/qmux/src/session/tests.rs b/rs/qmux/src/session/tests.rs index 14d7859a211..c71c4e603b3 100644 --- a/rs/qmux/src/session/tests.rs +++ b/rs/qmux/src/session/tests.rs @@ -159,6 +159,32 @@ async fn reading_returns_flow_control_credit() { } } +/// A stream's bytes count as acknowledged once they go to the transport, which a stalled link +/// holds back; a reset stream never is. +#[tokio::test(start_paused = true)] +async fn a_stream_is_acknowledged_once_its_bytes_go_out() { + for &version in Version::ALL { + let (client, server, up, _down) = link(); + let (client, server) = establish(client, server, config(version), config(version)).await; + + let mut send = client.open_uni(SendOptions::default()).await.unwrap(); + assert_eq!(send.acknowledged(), Some(true), "{version:?}: nothing written"); + up.send(Flow::Stall).unwrap(); + // More than the pipe holds, within the peer's stream window. + let data = payload(2 * PIPE, 5); + write_all(&mut send, data.clone()).await; + assert_eq!(send.acknowledged(), Some(false), "{version:?}: the link holds the rest"); + + up.send(Flow::Open).unwrap(); + let mut recv = server.accept_uni().await.unwrap(); + assert_eq!(read_exact(&mut recv, data.len()).await.unwrap(), data, "{version:?}"); + assert_eq!(send.acknowledged(), Some(true), "{version:?}: all of it went out"); + + send.reset(1); + assert_eq!(send.acknowledged(), Some(false), "{version:?}"); + } +} + /// Lease `want` bytes, returning how many were leased and handing them back unread, so they /// keep their credit. async fn lease(recv: &mut RecvStream, want: usize) -> Result {