diff --git a/.claude/skills/verify-resume/SKILL.md b/.claude/skills/verify-resume/SKILL.md index 3e56ae51da6..aaed284d979 100644 --- a/.claude/skills/verify-resume/SKILL.md +++ b/.claude/skills/verify-resume/SKILL.md @@ -50,7 +50,7 @@ Sync commit: `103052ed074a87f00269ecb0d639905482ecfb6f`. Run `git log 103052ed07 | `rs/moq-net/src/model/resume.rs` `Reader::bound_at_end`, `let_go`, `Copy::retains` | `BoundAtEnd`, `LetGo`, `Retains` | | `rs/moq-net/src/model/resume.rs` `Reader::poll_end` | `PollEnd` | | `rs/moq-net/src/model/resume.rs` `Producer::splice`, `serve_from`, the front's end | `Serve`, `FrontFinish`, `FrontFail` | -| `rs/moq-net/src/model/track.rs` `TrackState::is_complete`, `may_land`, `latest_sequence`, `track::Subscriber::has_unread_group` | `Seal`, `Held`, `MayLand`, `Latest`, `HasUnread` | +| `rs/moq-net/src/model/track.rs` `TrackState::is_complete`, `may_land`, `max_key`, `track::Subscriber::has_unread_group` | `Seal`, `Held`, `MayLand`, `Latest`, `HasUnread` | ## Not covered diff --git a/bench/browser/page/cmaf.ts b/bench/browser/page/cmaf.ts index b1c14bdbc1c..fda9d10543d 100644 --- a/bench/browser/page/cmaf.ts +++ b/bench/browser/page/cmaf.ts @@ -1,5 +1,6 @@ /** CMAF fragment decode over fixed audio and video input, with no network in the timed phase. */ import * as CmafDecode from "@moq/hang/cmaf"; +import { Time } from "@moq/net"; import { check } from "../fixture.ts"; import { longTasks, nextTask } from "./observe.ts"; @@ -44,6 +45,9 @@ async function prepare(next: Cmaf): Promise { decoded = undefined; } +// A scale-less frame carries a group key and no time, so samples present at their tfdt. +const UNTIMED = Time.Timestamp.unscaled(0n); + async function decode(): Promise { const segment = init; if (!config || !segment || fragments.length !== config.sizes.length) throw new Error("decode before prepare"); @@ -53,7 +57,7 @@ async function decode(): Promise { for (let iteration = 0; iteration < config.iterations; iteration++) { decoded = fragments.map((fragment) => { bytes += fragment.byteLength; - return CmafDecode.decodeDataSegment(fragment, segment, undefined); + return CmafDecode.decodeDataSegment(fragment, segment, UNTIMED); }); } const end = performance.now(); diff --git a/c/moq/test/probe.c b/c/moq/test/probe.c index a151b7ad0f7..1c9e2be549c 100644 --- a/c/moq/test/probe.c +++ b/c/moq/test/probe.c @@ -495,11 +495,10 @@ int main(void) result reading = {0}; task = moq_track_consumer_read_frame(consumer, on_frame, &reading); const uint8_t hello[] = {'h', 'e', 'l', 'l', 'o'}; - moq_frame frame = moq_frame_default(); - frame.payload.data = hello; - frame.payload.len = sizeof(hello); - moq_timestamp timestamp = {1000, 1000000}; - frame.timestamp = ×tamp; + moq_frame frame = { + .payload = {.data = hello, .len = sizeof(hello)}, + .timestamp = {.value = 1000, .scale = {.has_value = true, .value = 1000000}}, + }; ok(moq_track_producer_write_frame(track, &frame), "write_frame"); moq_frame *received = await(&reading, "read_frame"); moq_task_free(task); @@ -507,7 +506,8 @@ int main(void) CHECK(received->payload.len == sizeof(hello)); CHECK(memcmp(received->payload.data, hello, sizeof(hello)) == 0); /* Lite-07 carries both the Implementation parameter and frame timestamps, at any scale. */ - CHECK(received->timestamp && received->timestamp->value * 1000000 == 1000 * received->timestamp->scale); + CHECK(received->timestamp.scale.has_value && + received->timestamp.value * 1000000 == 1000 * received->timestamp.scale.value); moq_frame_free(received); CHECK(!cancelled.done); diff --git a/cpp/moq/bench/main.cpp b/cpp/moq/bench/main.cpp index f129c126b64..e8a6c01aeda 100644 --- a/cpp/moq/bench/main.cpp +++ b/cpp/moq/bench/main.cpp @@ -90,8 +90,7 @@ Reader read(Case c, std::shared_ptr consumer, const std::vec require(group != nullptr); for (int f = 0; f < c.frames(); ++f) { auto frame = ok(co_await group->read_frame()); - require(frame && frame->payload == payload && frame->timestamp && - frame->timestamp->value == n * c.frames() + f); + require(frame && frame->payload == payload && frame->timestamp.value == n * c.frames() + f); } } diff --git a/cpp/moq/test/probe.cpp b/cpp/moq/test/probe.cpp index f9b4d5de3b0..171f8054ea4 100644 --- a/cpp/moq/test/probe.cpp +++ b/cpp/moq/test/probe.cpp @@ -105,11 +105,11 @@ Coroutine read_one(std::shared_ptr consumer, std::optionalaccept(std::nullopt), "accept datagrams"); CHECK(ok(producer->priority(), "aggregate priority") == 12); CHECK(!ok(stream->demand(), "stream demand")->wanted()); - ok(producer->insert({7, std::nullopt, bytes("seven")}), "insert datagram"); + // A scale-less datagram track has no time: its timestamp is the unscaled sequence. + ok(producer->insert({7, moq::Timestamp{7, std::nullopt}, bytes("seven")}), "insert datagram"); auto received = ok(consumer->recv().get(), "receive datagram"); CHECK(received && received->sequence == 7 && received->payload == bytes("seven")); + CHECK(received->timestamp.value == 7 && !received->timestamp.scale); consumer->cancel(); CHECK(!ok(producer->priority_changed(12).get(), "released demand")); ok(producer->finish(), "finish datagrams"); @@ -137,8 +139,9 @@ void datagram_tracks() ok(broadcast->close(), "close datagram broadcast"); } -// Timestamps are exact records: a 90 kHz tick and a present zero read back as written, an -// untimed frame reads back untimed, and a record past the bounds is refused. +// Timestamps are exact records: a 90 kHz tick and a present zero read back as written, a +// scale-less track's unscaled group key reads back unscaled, and a record past the bounds is +// refused. void timestamp_roundtrip() { auto broadcast = ok(moq::BroadcastProducer::init(), "timestamp broadcast"); @@ -149,8 +152,8 @@ void timestamp_roundtrip() for (uint64_t value : {0, 1}) { ok(track->write_frame({bytes("tick"), moq::Timestamp{value, 90000}}), "write tick"); auto frame = ok(consumer->read_frame().get(), "read tick"); - CHECK(frame && frame->timestamp); - CHECK(frame->timestamp->value == value && frame->timestamp->scale == 90000); + CHECK(frame); + CHECK(frame->timestamp.value == value && frame->timestamp.scale == uint64_t(90000)); } for (const moq::Timestamp &invalid : {moq::Timestamp{0, 0}, moq::Timestamp{0, uint64_t(1) << 62}, moq::Timestamp{uint64_t(1) << 61, 90000}}) { @@ -159,11 +162,13 @@ void timestamp_roundtrip() CHECK(std::holds_alternative(refused.error().get_variant())); } - auto untimed = ok(broadcast->publish_track("untimed", std::nullopt), "publish untimed track"); - auto untimed_consumer = ok(untimed->consume(std::nullopt), "consume untimed track"); - ok(untimed->write_frame({bytes("untimed"), std::nullopt}), "write untimed"); - auto frame = ok(untimed_consumer->read_frame().get(), "read untimed"); - CHECK(frame && !frame->timestamp); + auto untimed = ok(broadcast->publish_track("untimed", std::nullopt), "publish scale-less track"); + auto untimed_consumer = ok(untimed->consume(std::nullopt), "consume scale-less track"); + // An unscaled timestamp on a timed track is refused. + CHECK(!track->write_frame({bytes("tick"), moq::Timestamp{2, std::nullopt}})); + ok(untimed->write_frame({bytes("untimed"), moq::Timestamp{3, std::nullopt}}), "write unscaled"); + auto frame = ok(untimed_consumer->read_frame().get(), "read unscaled"); + CHECK(frame && frame->timestamp.value == 3 && !frame->timestamp.scale); ok(broadcast->close(), "close timestamp broadcast"); } @@ -430,7 +435,7 @@ int main() CHECK(frame.has_value()); CHECK(frame->payload == bytes("hello")); // Lite-07 carries both the Implementation parameter and frame timestamps. - CHECK(frame->timestamp && same_time(*frame->timestamp, {1000, 1000000})); + CHECK(same_time(frame->timestamp, {1000, 1000000})); // A continuation receives the next frame on the executor. { diff --git a/cpp/obs/src/moq-source.cpp b/cpp/obs/src/moq-source.cpp index ebacc0468dc..5bdbbc56973 100644 --- a/cpp/obs/src/moq-source.cpp +++ b/cpp/obs/src/moq-source.cpp @@ -79,10 +79,13 @@ namespace { struct moq_source; // A track timestamp in the nanoseconds OBS stamps media with, floored when the scale does not -// divide a second evenly into nanoseconds. +// divide a second evenly into nanoseconds. Decoded media is always scaled; an unscaled group +// key has no time, so it maps to zero rather than dividing by a missing scale. static uint64_t moq_source_nanoseconds(const moq::Timestamp ×tamp) { - return util_mul_div64(timestamp.value, 1'000'000'000, timestamp.scale); + if (!timestamp.scale) + return 0; + return util_mul_div64(timestamp.value, 1'000'000'000, *timestamp.scale); } enum class Decoded { Pending, Ready, Failed }; diff --git a/cpp/obs/test/moq-audio-encoder-test.cpp b/cpp/obs/test/moq-audio-encoder-test.cpp index 1104dc182bc..5aa7961474c 100644 --- a/cpp/obs/test/moq-audio-encoder-test.cpp +++ b/cpp/obs/test/moq-audio-encoder-test.cpp @@ -278,12 +278,15 @@ void CheckSubscriberDecodes(const char *preset, size_t channels) if (!variant) break; const auto *frame = &variant->frame; + CHECK(frame->timestamp.scale.has_value()); + if (!frame->timestamp.scale) + break; if (!first_decoded) first_decoded = frame->timestamp; const auto *samples = reinterpret_cast(frame->data.data()); const size_t frames = frame->data.size() / sizeof(float) / channels; const int64_t offset = - static_cast(frame->timestamp.value * RATE / frame->timestamp.scale) - (start + RATE); + static_cast(frame->timestamp.value * RATE / *frame->timestamp.scale) - (start + RATE); for (size_t i = 0; i < frames; i++) { const double expected = Tone(offset + static_cast(i)); for (size_t c = 0; c < channels; c++) { @@ -298,7 +301,7 @@ void CheckSubscriberDecodes(const char *preset, size_t channels) // OBS never drains an audio encoder, so the decoder trims the 312-sample pre-skip // and nothing pads the tail: every input sample but that lookahead comes back. // Compared as fractions of a second, since the track reads back at its own scale. - CHECK(first_decoded && first_decoded->value * RATE == sample(0) * first_decoded->scale); + CHECK(first_decoded && first_decoded->value * RATE == sample(0) * *first_decoded->scale); CHECK(decoded == count * encoder.frame_size - 312); CHECK(energy > 0 && error / energy < 0.05); printf(" %s, %zu channels: %zu samples, error %.4f\n", preset, channels, decoded, diff --git a/cpp/obs/test/moq-source-test.cpp b/cpp/obs/test/moq-source-test.cpp index aa0141bcdd5..459c285f69f 100644 --- a/cpp/obs/test/moq-source-test.cpp +++ b/cpp/obs/test/moq-source-test.cpp @@ -1042,8 +1042,12 @@ int main() CHECK(publisher.WriteUntil([&] { return g_output_audio >= audio + 3; })); } else { uint64_t timestamp_us = 2'000'000; + // The scale-less track carries the time in its payload, so each frame's timestamp + // is the unscaled key of the group it opens. + uint64_t key = 0; CHECK(WaitFor([&] { - TestOk(pcm->write_frame(moq::Frame{PcmFrame(timestamp_us, channels), std::nullopt}), + TestOk(pcm->write_frame(moq::Frame{PcmFrame(timestamp_us, channels), + moq::Timestamp{key++, std::nullopt}}), "write pcm"); timestamp_us += 20'000; std::this_thread::sleep_for(std::chrono::milliseconds(10)); @@ -1476,10 +1480,12 @@ int main() g_candidate = {candidate, false}; return true; })); + // Each cut opens a group, so each retry starts later than the last. + uint64_t bad_us = 1'000'000'000; CHECK(publisher.WriteUntil([&] { - TestOk(replacement->write_frame(moq::Frame{BAD_OPUS, Micros(1'000'000'000)}), - "replacement frame"); + TestOk(replacement->write_frame(moq::Frame{BAD_OPUS, Micros(bad_us)}), "replacement frame"); TestOk(replacement->cut(), "replacement audio cut"); + bad_us += 20'000; std::lock_guard lock(g_candidate_mutex); return g_candidate.failed && candidate.expired(); })); diff --git a/dart/moq/README.md b/dart/moq/README.md index d33323de046..35e3d8b8d0d 100644 --- a/dart/moq/README.md +++ b/dart/moq/README.md @@ -45,7 +45,9 @@ The package uses `Future` for asynchronous operations and `Stream` for announcements, and `connect` / `listen` take an options struct like Rust does. Types are spelled without the `Moq` prefix (`Session`, `BroadcastProducer`, `Backoff`). Spans cross as a `Duration` (`stats.rtt`), and media times as an -exact `Timestamp` with `toDuration()`. The lower-level generated API +exact `Timestamp` with `toDuration()`; a track without a timescale has no +time, so its frames carry `Timestamp.unscaled(groupKey)` instead (a datagram, +its sequence). The lower-level generated API remains available through the re-exported `moq_ffi` package. Flutter Web is not supported. Browser applications should use the TypeScript diff --git a/dart/moq/bench/ffi.dart b/dart/moq/bench/ffi.dart index e1626e44531..9ceb210d2f1 100644 --- a/dart/moq/bench/ffi.dart +++ b/dart/moq/bench/ffi.dart @@ -64,7 +64,7 @@ Future throughput( final frame = await group.readFrame(); require( frame != null && - frame.timestamp?.value == n * c.frames + f && + frame.timestamp.value == n * c.frames + f && frame.payload.length == payload.length && memcmp(frame.payload.address, payload.address, payload.length) == 0, diff --git a/dart/moq/lib/json.dart b/dart/moq/lib/json.dart index a9c7da503f8..9815f246709 100644 --- a/dart/moq/lib/json.dart +++ b/dart/moq/lib/json.dart @@ -48,7 +48,7 @@ typedef StreamProducer = MoqJsonStreamProducer; typedef StreamConsumer = MoqJsonStreamConsumer; /// A JSON string read from a track, with the timestamp of the frame that -/// carried it, or null when untimed. +/// carried it, or its unscaled group key on a scale-less track. typedef Value = MoqJsonValue; /// Publishes a sliding window of JSON records on a track it takes over, pushed @@ -75,6 +75,6 @@ typedef WindowEventPop = PopMoqJsonWindowEvent; /// The records in `[start, end)` will never be delivered to this reader. typedef WindowEventSkip = SkipMoqJsonWindowEvent; -/// A [WindowEvent] and the timestamp of the frame that carried it, or null -/// when untimed. +/// A [WindowEvent] and the timestamp of the frame that carried it, or its +/// unscaled group key on a scale-less track. typedef WindowTimed = MoqJsonWindowTimed; diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 05a882e6a27..f012ffbb44b 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -3,10 +3,11 @@ import 'package:moq_ffi/moq_ffi.dart'; // Re-export the UniFFI types without their `Moq` prefix, so consumers spell // them the way the Rust API does. These are type aliases, not wrappers: the // values are the exact same objects, so every generated method applies -// unchanged and the prefixed names stay valid. `Delay` and `Subscription` are -// extension types instead, which are the same objects too but give an omitted -// delay bound the zero moq-net defaults to, where the generated record leaves -// it null. +// unchanged and the prefixed names stay valid. `Delay`, `Subscription`, and +// `Timestamp` are extension types instead, which are the same objects too but +// add constructors: an omitted delay bound is the zero moq-net defaults to, +// where the generated record leaves it null, and a timestamp is built either +// scaled or [Timestamp.unscaled], as in Rust. // // Four types are deliberately not aliased, because the unprefixed name is // already taken where this package is used: @@ -144,14 +145,25 @@ typedef GroupProducer = MoqGroupProducer; /// The read side of a single group: yields timestamped raw frames. typedef GroupConsumer = MoqGroupConsumer; -/// A datagram payload with its own sequence and optional timestamp. +/// A datagram payload with its own sequence and timestamp. typedef Datagram = MoqDatagram; /// A payload plus the timestamp it should be presented at. typedef Frame = MoqFrame; /// An exact media time: `value` units at `scale` units per second. -typedef Timestamp = MoqTimestamp; +/// +/// A scale-less track has no time, so there `scale` is null and `value` is the +/// frame's group key (a datagram's sequence), as [Timestamp.unscaled] builds. +/// Reads return the generated [MoqTimestamp], which this type implements. +extension type Timestamp._(MoqTimestamp _timestamp) implements MoqTimestamp { + /// `value` units at `scale` units per second. + Timestamp({required int value, required int scale}) + : this._(MoqTimestamp(value: value, scale: scale)); + + /// The scale-less timestamp at group key `key`, or a datagram's sequence. + Timestamp.unscaled(int key) : this._(MoqTimestamp(value: key, scale: null)); +} /// How far behind the live edge a subscriber or decoder plays: where it starts, and how late it may fall. extension type Delay._(MoqDelay _delay) implements MoqDelay { diff --git a/dart/moq/lib/src/timestamp.dart b/dart/moq/lib/src/timestamp.dart index 67dcbe62a45..e0499b52176 100644 --- a/dart/moq/lib/src/timestamp.dart +++ b/dart/moq/lib/src/timestamp.dart @@ -1,5 +1,7 @@ import 'package:moq_ffi/moq_ffi.dart'; +import 'aliases.dart'; + final _microsPerSecond = BigInt.from(Duration.microsecondsPerSecond); /// Converts a [MoqTimestamp] to Dart's [Duration]. @@ -7,9 +9,14 @@ extension TimestampDuration on MoqTimestamp { /// This timestamp as a [Duration], rounded down to a whole microsecond. /// /// The timestamp itself is exact; keep it, not the duration, to republish a - /// tick such as 90 kHz. Throws a [RangeError] when the result does not fit - /// in a [Duration]. + /// tick such as 90 kHz. Throws a [StateError] when unscaled, since a + /// scale-less track's key is no time, and a [RangeError] when the result + /// does not fit in a [Duration]. Duration toDuration() { + final scale = this.scale; + if (scale == null) { + throw StateError('unscaled timestamp #$value has no time'); + } // value * 10^6 can exceed 64 bits at the allowed bounds. final units = BigInt.from(scale); final micros = BigInt.from(value) * _microsPerSecond ~/ units; @@ -25,11 +32,11 @@ extension DurationTimestamp on Duration { /// This duration as an exact timestamp at microsecond scale. /// /// Throws an [ArgumentError] when negative. - MoqTimestamp toTimestamp() { + Timestamp toTimestamp() { if (isNegative) { throw ArgumentError.value(this, 'duration', 'must not be negative'); } - return MoqTimestamp( + return Timestamp( value: inMicroseconds, scale: Duration.microsecondsPerSecond, ); diff --git a/dart/moq/test/moq_test.dart b/dart/moq/test/moq_test.dart index 87cf87ddac0..f027816482e 100644 --- a/dart/moq/test/moq_test.dart +++ b/dart/moq/test/moq_test.dart @@ -33,11 +33,26 @@ void main() { expect(producer.priority(), 12); expect(stream.demand().wanted(), isNull); producer.insert( - datagram: Datagram(sequence: 7, payload: utf8.encode('seven')), + datagram: Datagram( + sequence: 7, + timestamp: Timestamp.unscaled(7), + payload: utf8.encode('seven'), + ), ); final datagram = await consumer.recv().timeout(timeout); expect(datagram?.sequence, 7); + expect(datagram?.timestamp.scale, isNull); + expect(datagram?.timestamp.value, 7); expect(datagram?.payload, utf8.encode('seven')); + // An unscaled timestamp names the sequence an append writes at. + expect( + producer.append( + timestamp: Timestamp.unscaled(9), + payload: utf8.encode('nine'), + ), + 9, + ); + expect((await consumer.recv().timeout(timeout))?.sequence, 9); consumer.cancel(); expect(await producer.priorityChanged(last: 12).timeout(timeout), isNull); producer.finish(); @@ -81,14 +96,36 @@ void main() { expect(() => catalog.now(), throwsA(isA())); }); - test('raw writes without timestamps stay untimed', () async { + test('raw writes on a scale-less track carry their group key', () async { final broadcast = BroadcastProducer(); final track = broadcast.publishTrack(name: 'untimed', info: null); final consumer = track.consume(subscription: null); - track.writeFrame(frame: Frame(payload: utf8.encode('payload'))); + track.writeFrame( + frame: Frame( + payload: utf8.encode('payload'), + timestamp: Timestamp.unscaled(0), + ), + ); final frame = await consumer.readFrame().timeout(timeout); - expect(frame?.timestamp, isNull); + expect(frame?.timestamp.scale, isNull); + expect(frame?.timestamp.value, 0); expect(consumer.info().timescale, isNull); + // A scaled timestamp disagrees with the scale-less track. + expect( + () => track.writeFrame( + frame: Frame( + payload: utf8.encode('payload'), + timestamp: Duration.zero.toTimestamp(), + ), + ), + throwsA( + isA().having( + (error) => error.details.kind, + 'kind', + MoqProtocolKind.timestampMismatch, + ), + ), + ); track.finish(); broadcast.close(); }); @@ -617,8 +654,8 @@ void main() { ), ); final frame = await consumer.readFrame().timeout(timeout); - expect(frame?.timestamp?.value, value); - expect(frame?.timestamp?.scale, 90000); + expect(frame?.timestamp.value, value); + expect(frame?.timestamp.scale, 90000); } for (final invalid in [ @@ -655,6 +692,10 @@ void main() { () => const Duration(microseconds: -1).toTimestamp(), throwsArgumentError, ); + // A scale-less track's key is no time. + final unscaled = Timestamp.unscaled(3); + expect([unscaled.value, unscaled.scale], [3, null]); + expect(unscaled.toDuration, throwsStateError); }); test('a broadcast is reachable only while announced', () async { @@ -865,12 +906,12 @@ void main() { // next() yields every state in order; latest() skips the buffered third. moq_json.Value? value = await consumer.next().timeout(timeout); expect(jsonDecode(value!.value), {'n': 1}); - expect(value.timestamp?.toDuration(), const Duration(milliseconds: 1)); + expect(value.timestamp.toDuration(), const Duration(milliseconds: 1)); value = await consumer.next().timeout(timeout); expect(jsonDecode(value!.value), {'n': 2}); value = await consumer.latest().timeout(timeout); expect(jsonDecode(value!.value), {'n': 4}); - expect(value.timestamp?.toDuration(), const Duration(milliseconds: 4)); + expect(value.timestamp.toDuration(), const Duration(milliseconds: 4)); consumer.cancel(); producer.finish(); @@ -911,12 +952,12 @@ void main() { final push = timed!.event as moq_json.WindowEventPush; expect(push.index, n); expect(jsonDecode(push.value), {'n': n}); - expect(timed.timestamp?.toDuration(), Duration(milliseconds: n)); + expect(timed.timestamp.toDuration(), Duration(milliseconds: n)); } final timed = await consumer.next().timeout(timeout); final pop = timed!.event as moq_json.WindowEventPop; expect([pop.start, pop.end], [0, 2]); - expect(timed.timestamp?.toDuration(), const Duration(milliseconds: 3)); + expect(timed.timestamp.toDuration(), const Duration(milliseconds: 3)); // The cut's new group restates record 2, which this reader already has. producer.cut(); @@ -927,7 +968,7 @@ void main() { final pushed = await consumer.next().timeout(timeout); final push = pushed!.event as moq_json.WindowEventPush; expect(push.index, 3); - expect(pushed.timestamp?.toDuration(), const Duration(milliseconds: 4)); + expect(pushed.timestamp.toDuration(), const Duration(milliseconds: 4)); producer.finish(); expect(await consumer.next().timeout(timeout), isNull); diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index a02faba4b81..2c5c866cab9 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -1177,8 +1177,8 @@ class FfiConverterMoqJsonSnapshotConfig { class MoqJsonValue { final String value; - final MoqTimestamp? timestamp; - MoqJsonValue({required this.value, this.timestamp}); + final MoqTimestamp timestamp; + MoqJsonValue({required this.value, required this.timestamp}); } class FfiConverterMoqJsonValue { @@ -1193,7 +1193,7 @@ class FfiConverterMoqJsonValue { ); final value = value_lifted.value; new_offset += value_lifted.bytesRead; - final timestamp_lifted = FfiConverterOptionalMoqTimestamp.read( + final timestamp_lifted = FfiConverterMoqTimestamp.read( Uint8List.view(buf.buffer, new_offset), ); final timestamp = timestamp_lifted.value; @@ -1207,7 +1207,7 @@ class FfiConverterMoqJsonValue { static RustBuffer lower(MoqJsonValue value) { final total_length = FfiConverterString.allocationSize(value.value) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -1220,7 +1220,7 @@ class FfiConverterMoqJsonValue { value.value, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalMoqTimestamp.write( + new_offset += FfiConverterMoqTimestamp.write( value.timestamp, Uint8List.view(buf.buffer, new_offset), ); @@ -1229,7 +1229,7 @@ class FfiConverterMoqJsonValue { static int allocationSize(MoqJsonValue value) { return FfiConverterString.allocationSize(value.value) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + 0; } } @@ -1282,8 +1282,8 @@ class FfiConverterMoqJsonWindowConfig { class MoqJsonWindowTimed { final MoqJsonWindowEvent event; - final MoqTimestamp? timestamp; - MoqJsonWindowTimed({required this.event, this.timestamp}); + final MoqTimestamp timestamp; + MoqJsonWindowTimed({required this.event, required this.timestamp}); } class FfiConverterMoqJsonWindowTimed { @@ -1298,7 +1298,7 @@ class FfiConverterMoqJsonWindowTimed { ); final event = event_lifted.value; new_offset += event_lifted.bytesRead; - final timestamp_lifted = FfiConverterOptionalMoqTimestamp.read( + final timestamp_lifted = FfiConverterMoqTimestamp.read( Uint8List.view(buf.buffer, new_offset), ); final timestamp = timestamp_lifted.value; @@ -1312,7 +1312,7 @@ class FfiConverterMoqJsonWindowTimed { static RustBuffer lower(MoqJsonWindowTimed value) { final total_length = FfiConverterMoqJsonWindowEvent.allocationSize(value.event) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -1325,7 +1325,7 @@ class FfiConverterMoqJsonWindowTimed { value.event, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalMoqTimestamp.write( + new_offset += FfiConverterMoqTimestamp.write( value.timestamp, Uint8List.view(buf.buffer, new_offset), ); @@ -1334,7 +1334,7 @@ class FfiConverterMoqJsonWindowTimed { static int allocationSize(MoqJsonWindowTimed value) { return FfiConverterMoqJsonWindowEvent.allocationSize(value.event) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + 0; } } @@ -1827,11 +1827,11 @@ class FfiConverterMoqContainerInit { class MoqDatagram { final int sequence; - final MoqTimestamp? timestamp; + final MoqTimestamp timestamp; final Uint8List payload; MoqDatagram({ this.sequence = 0, - this.timestamp = null, + required this.timestamp, required this.payload, }); } @@ -1848,7 +1848,7 @@ class FfiConverterMoqDatagram { ); final sequence = sequence_lifted.value; new_offset += sequence_lifted.bytesRead; - final timestamp_lifted = FfiConverterOptionalMoqTimestamp.read( + final timestamp_lifted = FfiConverterMoqTimestamp.read( Uint8List.view(buf.buffer, new_offset), ); final timestamp = timestamp_lifted.value; @@ -1867,7 +1867,7 @@ class FfiConverterMoqDatagram { static RustBuffer lower(MoqDatagram value) { final total_length = FfiConverterUInt64.allocationSize(value.sequence) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + FfiConverterUint8List.allocationSize(value.payload) + 0; final buf = Uint8List(total_length); @@ -1881,7 +1881,7 @@ class FfiConverterMoqDatagram { value.sequence, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalMoqTimestamp.write( + new_offset += FfiConverterMoqTimestamp.write( value.timestamp, Uint8List.view(buf.buffer, new_offset), ); @@ -1894,7 +1894,7 @@ class FfiConverterMoqDatagram { static int allocationSize(MoqDatagram value) { return FfiConverterUInt64.allocationSize(value.sequence) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + FfiConverterUint8List.allocationSize(value.payload) + 0; } @@ -1961,8 +1961,8 @@ class FfiConverterMoqDimensions { class MoqFrame { final Uint8List payload; - final MoqTimestamp? timestamp; - MoqFrame({required this.payload, this.timestamp = null}); + final MoqTimestamp timestamp; + MoqFrame({required this.payload, required this.timestamp}); } class FfiConverterMoqFrame { @@ -1977,7 +1977,7 @@ class FfiConverterMoqFrame { ); final payload = payload_lifted.value; new_offset += payload_lifted.bytesRead; - final timestamp_lifted = FfiConverterOptionalMoqTimestamp.read( + final timestamp_lifted = FfiConverterMoqTimestamp.read( Uint8List.view(buf.buffer, new_offset), ); final timestamp = timestamp_lifted.value; @@ -1991,7 +1991,7 @@ class FfiConverterMoqFrame { static RustBuffer lower(MoqFrame value) { final total_length = FfiConverterUint8List.allocationSize(value.payload) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -2004,7 +2004,7 @@ class FfiConverterMoqFrame { value.payload, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterOptionalMoqTimestamp.write( + new_offset += FfiConverterMoqTimestamp.write( value.timestamp, Uint8List.view(buf.buffer, new_offset), ); @@ -2013,7 +2013,7 @@ class FfiConverterMoqFrame { static int allocationSize(MoqFrame value) { return FfiConverterUint8List.allocationSize(value.payload) + - FfiConverterOptionalMoqTimestamp.allocationSize(value.timestamp) + + FfiConverterMoqTimestamp.allocationSize(value.timestamp) + 0; } } @@ -2233,8 +2233,8 @@ class FfiConverterMoqMediaTrackConfig { class MoqTimestamp { final int value; - final int scale; - MoqTimestamp({required this.value, required this.scale}); + final int? scale; + MoqTimestamp({required this.value, this.scale}); } class FfiConverterMoqTimestamp { @@ -2249,7 +2249,7 @@ class FfiConverterMoqTimestamp { ); final value = value_lifted.value; new_offset += value_lifted.bytesRead; - final scale_lifted = FfiConverterUInt64.read( + final scale_lifted = FfiConverterOptionalUInt64.read( Uint8List.view(buf.buffer, new_offset), ); final scale = scale_lifted.value; @@ -2263,7 +2263,7 @@ class FfiConverterMoqTimestamp { static RustBuffer lower(MoqTimestamp value) { final total_length = FfiConverterUInt64.allocationSize(value.value) + - FfiConverterUInt64.allocationSize(value.scale) + + FfiConverterOptionalUInt64.allocationSize(value.scale) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -2276,7 +2276,7 @@ class FfiConverterMoqTimestamp { value.value, Uint8List.view(buf.buffer, new_offset), ); - new_offset += FfiConverterUInt64.write( + new_offset += FfiConverterOptionalUInt64.write( value.scale, Uint8List.view(buf.buffer, new_offset), ); @@ -2285,7 +2285,7 @@ class FfiConverterMoqTimestamp { static int allocationSize(MoqTimestamp value) { return FfiConverterUInt64.allocationSize(value.value) + - FfiConverterUInt64.allocationSize(value.scale) + + FfiConverterOptionalUInt64.allocationSize(value.scale) + 0; } } @@ -11399,7 +11399,7 @@ class FfiConverterMoqDatagramConsumer { abstract class MoqDatagramProducerInterface { void abort({required int errorCode}); - int append({required MoqTimestamp? timestamp, required Uint8List payload}); + int append({required MoqTimestamp timestamp, required Uint8List payload}); MoqDatagramConsumer consume(); void finish(); MoqDatagramInfo info(); @@ -11456,11 +11456,11 @@ class MoqDatagramProducer implements MoqDatagramProducerInterface { }, moqExceptionErrorHandler); } - int append({required MoqTimestamp? timestamp, required Uint8List payload}) { + int append({required MoqTimestamp timestamp, required Uint8List payload}) { return rustCallWithLifter( (status) => uniffi_moq_ffi_fn_method_moqdatagramproducer_append( uniffiClonePointer(), - FfiConverterOptionalMoqTimestamp.lower(timestamp), + FfiConverterMoqTimestamp.lower(timestamp), FfiConverterUint8List.lower(payload), status, ), @@ -19383,53 +19383,6 @@ class FfiConverterOptionalMoqSubscription { } } -class FfiConverterOptionalMoqTimestamp { - static MoqTimestamp? lift(RustBuffer buf) { - return FfiConverterOptionalMoqTimestamp.read(buf.asUint8List()).value; - } - - static LiftRetVal read(Uint8List buf) { - if (ByteData.view(buf.buffer, buf.offsetInBytes).getInt8(0) == 0) { - return LiftRetVal(null, 1); - } - final result = FfiConverterMoqTimestamp.read( - Uint8List.view(buf.buffer, buf.offsetInBytes + 1), - ); - return LiftRetVal(result.value, result.bytesRead + 1); - } - - static int allocationSize([MoqTimestamp? value]) { - if (value == null) { - return 1; - } - return FfiConverterMoqTimestamp.allocationSize(value) + 1; - } - - static RustBuffer lower(MoqTimestamp? value) { - if (value == null) { - return toRustBuffer(Uint8List.fromList([0])); - } - final buf = Uint8List( - FfiConverterOptionalMoqTimestamp.allocationSize(value), - ); - FfiConverterOptionalMoqTimestamp.write(value, buf); - return toRustBuffer(buf); - } - - static int write(MoqTimestamp? value, Uint8List buf) { - if (value == null) { - buf[0] = 0; - return 1; - } - buf[0] = 1; - return FfiConverterMoqTimestamp.write( - value, - Uint8List.view(buf.buffer, buf.offsetInBytes + 1), - ) + - 1; - } -} - class FfiConverterOptionalMoqTokens { static MoqTokens? lift(RustBuffer buf) { return FfiConverterOptionalMoqTokens.read(buf.asUint8List()).value; @@ -25020,7 +24973,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqgroupconsumer_read_frame() != 26363) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqgroupconsumer_sequence() != 46527) { + if (uniffi_moq_ffi_checksum_method_moqgroupconsumer_sequence() != 38121) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqtrackconsumer_cancel() != 65022) { @@ -25063,7 +25016,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqdatagramproducer_abort() != 25849) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqdatagramproducer_append() != 5184) { + if (uniffi_moq_ffi_checksum_method_moqdatagramproducer_append() != 55110) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqdatagramproducer_consume() != 17555) { @@ -25212,7 +25165,8 @@ void _checkApiChecksums() { 40580) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqmediacontainerproducer_seek() != 6771) { + if (uniffi_moq_ffi_checksum_method_moqmediacontainerproducer_seek() != + 19965) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqmediacontainerproducer_write() != @@ -25225,8 +25179,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqmediagroupconsumer_next() != 3173) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqmediagroupconsumer_sequence() != - 57049) { + if (uniffi_moq_ffi_checksum_method_moqmediagroupconsumer_sequence() != 9984) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqmediatrackconsumer_cancel() != 17311) { @@ -25248,7 +25201,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_finish() != 22633) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_seek() != 37244) { + if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_seek() != 29733) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_set_enabled() != @@ -25256,7 +25209,7 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqmediatrackproducer_write_frame() != - 7616) { + 42981) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqmediatrackstreamproducer_demand() != @@ -25432,7 +25385,7 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqgroupproducer_ready() != 10687) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqgroupproducer_sequence() != 21067) { + if (uniffi_moq_ffi_checksum_method_moqgroupproducer_sequence() != 61101) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqgroupproducer_write_frame() != 38774) { @@ -25441,13 +25394,13 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqtrackproducer_abort() != 37537) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqtrackproducer_append_group() != 45225) { + if (uniffi_moq_ffi_checksum_method_moqtrackproducer_append_group() != 30667) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqtrackproducer_consume() != 30970) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } - if (uniffi_moq_ffi_checksum_method_moqtrackproducer_create_group() != 38978) { + if (uniffi_moq_ffi_checksum_method_moqtrackproducer_create_group() != 2465) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqtrackproducer_demand() != 7311) { @@ -25490,7 +25443,7 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqtrackrequests_requested_fill() != - 55377) { + 19170) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqrequest_accept() != 30910) { diff --git a/dart/moq_ffi/lib/src/uniffi_runtime.dart b/dart/moq_ffi/lib/src/uniffi_runtime.dart index 3b7dd9cf392..66b9e185973 100644 --- a/dart/moq_ffi/lib/src/uniffi_runtime.dart +++ b/dart/moq_ffi/lib/src/uniffi_runtime.dart @@ -8,6 +8,7 @@ import "dart:ffi"; import "dart:io" show Platform, File, Directory; import "dart:isolate"; import "dart:typed_data"; + import "package:ffi/ffi.dart"; class UniffiInternalError implements Exception { @@ -252,6 +253,7 @@ mixin FfiConverterPrimitive on FfiConverter { @override T lower(T value) => value; } + Uint8List createUint8ListFromInt(int value) { int length = value.bitLength ~/ 8 + 1; if (length != 4 && length != 8) { diff --git a/demo/boy/src/game.test.ts b/demo/boy/src/game.test.ts index 0b569a1e885..1dae9c4957b 100644 --- a/demo/boy/src/game.test.ts +++ b/demo/boy/src/game.test.ts @@ -68,7 +68,10 @@ test("status skips values that fail the schema and ends when the track errors", // One finished group per status, so between statuses the loop waits on the next group. const publish = (value: unknown) => { const group = track.appendGroup(); - group.writeFrame({ payload: new TextEncoder().encode(JSON.stringify(value)) }); + group.writeFrame({ + payload: new TextEncoder().encode(JSON.stringify(value)), + timestamp: Moq.Time.Timestamp.unscaled(group.sequence), + }); group.finish(); }; @@ -143,7 +146,12 @@ test("input opens a session that carries commands up and latency down until deac expect(command).toEqual({ type: "buttons", buttons: ["a"], timestamps: [] }); const latency = [{ label: "input", ms: 42 }]; - session.producer.writeFrame({ payload: new TextEncoder().encode(JSON.stringify({ latency })) }); + const reply = session.producer.appendGroup(); + reply.writeFrame({ + payload: new TextEncoder().encode(JSON.stringify({ latency })), + timestamp: Moq.Time.Timestamp.unscaled(reply.sequence), + }); + reply.finish(); await settle(30); expect(game.latency.peek()).toEqual(latency); diff --git a/demo/boy/src/game.ts b/demo/boy/src/game.ts index 71dd55a7224..a9262b96452 100644 --- a/demo/boy/src/game.ts +++ b/demo/boy/src/game.ts @@ -365,7 +365,13 @@ export class Game { // One command per group, so a late one never holds up the next. const value = { ...command.cmd, timestamps: command.timestamps }; - open.producer.writeFrame({ payload: new TextEncoder().encode(JSON.stringify(value)) }); + // The socket track is scale-less, so the frame carries its group's key. + const group = open.producer.appendGroup(); + group.writeFrame({ + payload: new TextEncoder().encode(JSON.stringify(value)), + timestamp: Moq.Time.Timestamp.unscaled(group.sequence), + }); + group.finish(); this.#command.set(undefined); } diff --git a/demo/carla/sim/moq_carla_sim/publish.py b/demo/carla/sim/moq_carla_sim/publish.py index 65255c2777c..a471431d696 100644 --- a/demo/carla/sim/moq_carla_sim/publish.py +++ b/demo/carla/sim/moq_carla_sim/publish.py @@ -336,8 +336,11 @@ def reply(self, command: Command, message: dict[str, Any]) -> None: producer = self._replies.get((command.driver_id, command.session)) if producer is None: return + # A session track is scale-less: each reply is a group keyed one past the last. + largest = producer.largest() + key = 0 if largest is None else largest.group + 1 try: - producer.write_frame(json.dumps(message, separators=(",", ":")).encode()) + producer.write_frame(json.dumps(message, separators=(",", ":")).encode(), moq.Timestamp.unscaled(key)) # uniffi's stubs declare the error a plain class; at runtime it is an Exception. except moq.Error as err: # pyright: ignore[reportGeneralTypeIssues] log.info("%s: reply failed: %s", self._truck, err) diff --git a/demo/carla/sim/tests/conftest.py b/demo/carla/sim/tests/conftest.py index 57ce385d4b1..b0845af8c01 100644 --- a/demo/carla/sim/tests/conftest.py +++ b/demo/carla/sim/tests/conftest.py @@ -33,6 +33,7 @@ def update(self, value: object, timestamp: object) -> None: # Tests that run a truck's endpoint script it in place of this. moq.socket = ModuleType("moq.socket") moq.Timestamp = lambda value, scale: SimpleNamespace(value=value, scale=scale) +moq.Timestamp.unscaled = lambda key: SimpleNamespace(value=key, scale=None) moq.TrackInfo = lambda **kwargs: SimpleNamespace(**kwargs) moq_json = ModuleType("moq.json") diff --git a/demo/carla/sim/tests/test_publish.py b/demo/carla/sim/tests/test_publish.py index 1ccb6b51861..5ce4d2ffa6c 100644 --- a/demo/carla/sim/tests/test_publish.py +++ b/demo/carla/sim/tests/test_publish.py @@ -208,9 +208,13 @@ async def read_frame(self): raise frame return frame - def write_frame(self, payload: bytes) -> None: + def largest(self): + return SimpleNamespace(group=len(self.replies) - 1) if self.replies else None + + def write_frame(self, payload: bytes, timestamp) -> None: if self.finished: raise moq.Error("session closed") + assert timestamp == moq.Timestamp.unscaled(len(self.replies)), "a reply keys the next group" self.replies.append(json.loads(payload)) def finish(self) -> None: diff --git a/demo/carla/web/src/preview.ts b/demo/carla/web/src/preview.ts index ce9ebce3202..c4ef756bd5b 100644 --- a/demo/carla/web/src/preview.ts +++ b/demo/carla/web/src/preview.ts @@ -31,6 +31,8 @@ export function populatePreview(add: (path: Moq.Path.Valid) => Truck) { map: "Town10HD_Opt", pose, }, + // The status track is scale-less, so the write names the group it opens. + at: Moq.Time.Timestamp.unscaled(0n), }); cleanup.push(() => { producer.finish(); diff --git a/demo/carla/web/src/truck.test.ts b/demo/carla/web/src/truck.test.ts index 4c0f275c92c..c4efc83daf1 100644 --- a/demo/carla/web/src/truck.test.ts +++ b/demo/carla/web/src/truck.test.ts @@ -7,6 +7,10 @@ import { handle, SOCKET } from "./paths.ts"; import type { TruckStatus } from "./schemas.ts"; import { shiftGear, Truck } from "./truck.ts"; +// The status track is scale-less, so each write names a group key; keys only have to increase. +let keys = 0n; +const key = () => Moq.Time.Timestamp.unscaled(keys++); + describe("Truck video demand", () => { test("only hovered or focused cameras subscribe, and focus suppresses other previews", async () => { const connection = new Moq.Connection({ enabled: false }); @@ -89,7 +93,7 @@ describe("Truck status", () => { (truck.broadcast.out.active as Moq.Signals.Signal).set( broadcast.consume(), ); - producer.update({ value: status(0.5) }); + producer.update({ value: status(0.5), at: key() }); expect(await available).toBe("available"); expect(truck.status.peek()?.accepted.throttle).toBe(0.5); @@ -121,7 +125,7 @@ describe("Truck status", () => { const { broadcast, producer } = statusBroadcast(); const next = async (update: TruckStatus) => { const changed = truck.status.changed(); - producer.update({ value: update }); + producer.update({ value: update, at: key() }); await changed; }; @@ -164,7 +168,7 @@ describe("Truck status", () => { try { active.set(broadcast.consume()); - producer.update({ value: status(0.5) }); + producer.update({ value: status(0.5), at: key() }); while (truck.statusState.peek() !== "available") await truck.statusState.changed(); const unavailable = truck.statusState.changed(); @@ -198,14 +202,14 @@ describe("Truck status", () => { try { active.set(first.broadcast.consume()); - first.producer.update({ value: status(0.25) }); + first.producer.update({ value: status(0.25), at: key() }); while (truck.statusState.peek() !== "available") await truck.statusState.changed(); const transitions: string[] = []; const unsubscribe = truck.statusState.subscribe((state) => transitions.push(state)); try { active.set(second.broadcast.consume()); - second.producer.update({ value: status(0.75) }); + second.producer.update({ value: status(0.75), at: key() }); while (truck.status.peek()?.accepted.throttle !== 0.75) await truck.status.changed(); await Promise.resolve(); await Promise.resolve(); @@ -466,7 +470,12 @@ describe("Truck sessions", () => { const measured = truck.roundTripMs.changed(); const pipeline = [{ label: "received→decoded", ms: 4 }]; const reply = { sequence: command.sequence, media_pipeline: pipeline }; - stream.producer.writeFrame({ payload: encoder.encode(JSON.stringify(reply)) }); + const group = stream.producer.appendGroup(); + group.writeFrame({ + payload: encoder.encode(JSON.stringify(reply)), + timestamp: Moq.Time.Timestamp.unscaled(group.sequence), + }); + group.finish(); expect(await measured).toBeGreaterThanOrEqual(0); expect(truck.pipeline.peek()).toEqual(pipeline); } finally { @@ -538,6 +547,7 @@ describe("Truck sessions", () => { estopped: true, estop_by: await handle(dial.session), }, + at: key(), }); expect(await read(stream, reader)).toBeUndefined(); } finally { diff --git a/demo/carla/web/src/truck.ts b/demo/carla/web/src/truck.ts index aecdbfbc5d1..e485241166b 100644 --- a/demo/carla/web/src/truck.ts +++ b/demo/carla/web/src/truck.ts @@ -54,7 +54,13 @@ function write(stream: Moq.Socket.TrackStream, command: Command) { // A session that already ended has nothing to send to; its reply reader reports the end. if (stream.producer.closed.peek() !== undefined) return; const message = { ...command.cmd, sequence: command.sequence, timestamps: command.timestamps }; - stream.producer.writeFrame({ payload: encoder.encode(JSON.stringify(message)) }); + // The socket track is scale-less, so the frame carries its group's key. + const group = stream.producer.appendGroup(); + group.writeFrame({ + payload: encoder.encode(JSON.stringify(message)), + timestamp: Moq.Time.Timestamp.unscaled(group.sequence), + }); + group.finish(); } /** diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 9ebee20f16e..065d77330f0 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -316,12 +316,12 @@ starting with `.` stays hidden unless `prefix` names it. ```bash moq --connect https://relay.example.com/anon --broadcast my-stream.hang fetch catalog.json | jq -moq ... fetch video/hd --group 42 --json +moq ... fetch video/hd --group 42000 --json ``` -Writes one group of a track (the newest unless `--group` says otherwise) to -stdout over MoQ: the counterpart of the relay's HTTP `/fetch`. It exits -non-zero when the broadcast or group is not found. [Inspect a +Writes one group of a track to stdout over MoQ, the counterpart of the relay's +HTTP `/fetch`: the newest, or the one that starts at `--group` in the track's +timescale. It exits non-zero when the broadcast or group is not found. [Inspect a relay](/bin/inspect) walks through `announced` and `fetch` next to their `curl` equivalents. @@ -394,9 +394,9 @@ Each run of `moq` announces under a fresh random [instance](/concept/moq-lite#publisher-instances), kept across reconnects. The RTMP, SRT, and WHIP ingests mint one per connection instead, `import rtsp` one per RTSP session, and `import ts --program all` one per program. -No importer continues its group sequences across a restart, so `moq` has no +No importer continues its timeline across a restart, so `moq` has no flag to set a stable instance and refuses one. A publisher that does continue -them sets it on its route in a library, such as `RouteOptions::with_instance` in Rust +its timeline sets one on its route in a library, such as `RouteOptions::with_instance` in Rust or `origin.route(path, { instance: { id, epoch } })` in JavaScript; see [identity and recovery](/concept/identity). diff --git a/doc/bin/inspect.md b/doc/bin/inspect.md index 42f7c2e5d24..0e481349b02 100644 --- a/doc/bin/inspect.md +++ b/doc/bin/inspect.md @@ -53,9 +53,10 @@ curl http://localhost:4443/fetch/anon/demo/bbb.hang/catalog.json | jq ``` Both write the frame payloads of one group back to back: the newest group, or -the one `--group 42` (`?group=42`) names. `moq fetch --json` prints one line -per frame instead, `{"group": 42, "frame": 0, "size": 1234, "payload": ""}`, -which shows where the frames split. +the one that starts at `--group 42` (`?group=42`) in the track's timescale. +`moq fetch --json` prints one line per frame instead, +`{"group": 42, "frame": 0, "size": 1234, "payload": ""}`, with the +group's start, which shows where the frames split. `fetch` takes the track name literally. `/fetch` splits its path on the last `/`, so it cannot reach a track whose name contains one. Both give up after 30 diff --git a/doc/bin/relay/http.md b/doc/bin/relay/http.md index b289fc6a4aa..1a79e3804eb 100644 --- a/doc/bin/relay/http.md +++ b/doc/bin/relay/http.md @@ -13,7 +13,7 @@ operational ones that must stay private. | Endpoint | Returns | | --- | --- | | `GET /announced/` | Broadcasts announced under the prefix, named relative to it. | -| `GET /fetch//?group=N` | One group from the cache, the latest by default, or `404` if the track has no such group. Useful for catch-up and debugging. | +| `GET /fetch//?group=N` | One group from the cache: the one that starts at `N` in the track's timescale, the latest by default, or `404` if the track has no such group. Useful for catch-up and debugging. | | `GET /certificate.sha256` | The fingerprint of the first configured TLS certificate, for pinning a self-signed dev certificate. | | `GET /health` | `200 ok`, unauthenticated, for load balancers. | diff --git a/doc/concept/archives.md b/doc/concept/archives.md index 53a095dacbe..752c1cfe453 100644 --- a/doc/concept/archives.md +++ b/doc/concept/archives.md @@ -43,9 +43,11 @@ each relay by listing it in [`listen.version`](/bin/relay/config#listen), plus `connect.version` for its cluster links. A relay left at its default versions cannot negotiate moq-lite 07. -Recording starts with content the subscription can still receive. A missing -group is given up after the publisher's max age capped by the producer's max -delay, so a gap cannot stall later groups. Only stream tracks are recorded: a +Recording starts with content the subscription can still receive. Groups are +stored in start order. A group the track still owes is given up once newer +content runs the publisher's max age, capped by the producer's max delay, ahead +of it, so a gap cannot stall later groups; one the publisher skipped or dropped +is never waited for. Only stream tracks are recorded: a datagram track is live-only, so an archive's playback never includes it. An archive cannot recover content already lost or evicted before it subscribed. The same max delay (`producer::Config::max_delay`, 10 s by default and in `moq export archive`) @@ -57,7 +59,7 @@ path and [instance](/concept/identity). Restarting a recorder on the same instan and prefix resumes after each track's last stored frame, within that max delay: an interruption longer than it leaves a gap. A prefix already marked complete records nothing. Never append a new publisher run that restarts its -group numbers to an old prefix. `moq export archive` makes this choice for you. +group starts to an old prefix. `moq export archive` makes this choice for you. ## What the store holds @@ -65,8 +67,8 @@ Tracks have separate directories, with each track name percent-encoded as one segment. Within one broadcast prefix: - Each track's `.info` stores its timescale, when timed, and priority. -- Data objects are keyed by their first group, frame index, and presentation - timestamp; untimed tracks omit the timestamp. The sorted key listing is the +- Data objects are keyed by their first group's start, frame index, and + presentation timestamp; untimed tracks omit the timestamp. The sorted key listing is the index, with no manifest or timeline object. - Each track's `end` records its final position or stream error. The broadcast's `.end` marks the recording complete, after its track markers are written. @@ -94,8 +96,8 @@ The host chooses the output path and a fresh instance, as [`moq import archive`](/bin/cli#archive) does. A relay routes that publication like any other broadcast; it does not look in object storage on a cache miss. -Each track starts live at its newest stored sequence and serves older groups -from storage when requested by group position. It advertises no max age, so the +Each track starts live at its newest stored group and serves older groups +from storage when requested by their start. It advertises no max age, so the source's live max age cannot make archived history stale. A growing recording is periodically listed for new tracks and objects. Once `.end` appears, refreshing stops and tracks end as recorded; without it, including when a recorder stopped diff --git a/doc/concept/identity.md b/doc/concept/identity.md index 558ceb34d04..5f73e058273 100644 --- a/doc/concept/identity.md +++ b/doc/concept/identity.md @@ -6,7 +6,7 @@ description: Choose stream instances so restarts, failover, and on-demand work n # Identity and recovery A MoQ name always means the same content. Relays cache groups by broadcast -path, track name, and group sequence, and a subscriber that loses its route +path, track name, and group start, and a subscriber that loses its route resumes by asking for the first frame it lacks. Both are only safe while every source of a name serves the same bytes. This page is the contract publishers, players, and relays share to keep that true across restarts, failover, and work @@ -15,17 +15,18 @@ started on demand. Wire details live in the [moq-lite](/draft/moq-lite) draft. ## Identity is a path and an instance A broadcast's identity is its path plus its **instance**. Every route in an -origin carries an instance: any non-empty text naming one continuous run of -group numbers at its path. Within one instance, group sequences never restart -and never name two different groups, so a reader stitches across every route +origin carries an instance: any non-empty text naming one continuous timeline +of groups at its path. A group is addressed by its start timestamp, and within +one instance starts never go back and never name two different groups, so a +reader stitches across every route and incarnation that shares it. Two instances are two independent streams, never stitched together. An instance names a run, not who publishes it: it is no credential, and auth decides who may publish the path. - **A new stream takes a new instance.** A broadcast that names no instance generates a random one when it is created and keeps it while it is withdrawn and announced again or - reconnects, since its numbering continues. A restarted process, or anything - that starts its group numbers over, is a new broadcast with a new instance. + reconnects, since its timeline continues. A restarted process, or anything + that starts its timestamps over, is a new broadcast with a new instance. Replaying a recording is a new stream too: [`moq import archive`](/bin/cli#archive) republishes under a new instance. - **Replicas share it.** One stream carried through several relays or CDNs @@ -33,19 +34,19 @@ no credential, and auth decides who may publish the path. between them. - **A stable instance is a promise.** A publisher may name its own instance (`RouteOptions::with_instance(instance, epoch)` in Rust, - `origin.route(path, { instance: { id, epoch } })` in JavaScript) only if it continues the sequence space across restarts and + `origin.route(path, { instance: { id, epoch } })` in JavaScript) only if its timestamps keep moving forward across restarts and announces a higher **epoch** each time, for example a counter persisted on disk. Breaking the promise splices unrelated groups into every reader's tracks. Two encoders with identical settings fed the same source still - number and cut their groups independently, so never derive an instance from + cut and stamp their groups independently, so never derive an instance from a hash of settings. - **Reusing a name is a bug.** Publishing different content under an instance, - a track name, or a group sequence that already named something else + a track name, or a group start that already named something else corrupts every cache that holds the old content, and MoQ has no ETag-style invalidation. Use a new instance for a whole new stream, a new track name for one rendition whose decoder configuration changes (see - [hang](/concept/hang#catalog)), and continue past the old sequence when a - track is replaced within one stream. + [hang](/concept/hang#catalog)), and keep timestamps moving past the old + track's when a track is replaced within one stream. Think of a cloud instance ID that survives a reboot: the instance stays the same, and the epoch counts the boots. The epoch is a counter within one @@ -59,7 +60,7 @@ one is, so it fences out an older run. Nothing assumes it is a clock. | One broadcast, reconnecting | The one it generated | The same stream; subscriptions resume | | One stream carried by several relays or CDNs | Shared by every route | Failover resumes from the first missing frame | | A restarted or replaced publisher | A fresh one | A restart, then the new stream from its start | -| A publisher that continues its sequences across restarts | Stable, with a higher epoch each run | The same stream; readers continue without a seam | +| A publisher whose timestamps continue across restarts | Stable, with a higher epoch each run | The same stream; readers continue without a seam | | Independent publishers of one source | One each | Two streams; see [redundant ingest](/setup/redundancy) | | A pool of hosts producing paths on demand | One per host, carried by its jobs; a fresh one per restart | See [work on demand](#work-on-demand) | @@ -70,7 +71,7 @@ A route resolving a path is chosen in this order: 1. The most specific prefix wins. 2. Within one instance, only its highest epoch is eligible, even over a cheaper route. Readers continue across an epoch change without a seam, - because the sequence space continues. + because the timeline continues. 3. Among eligible routes: a route from a peer that has not sent GOAWAY (a draining peer's routes rank below every other, whatever their preference or cost, and serve only as a last resort), then the preferred link, then an @@ -179,7 +180,7 @@ demand, whose instance changes each time its job restarts. A worker can advertise a prefix it serves on demand and answer with a broadcast it already holds. This announces no exact route: the answer belongs to the claim's instance at that path. Every answer there must continue -one sequence space with the same bytes. To serve different content at a path +one timeline with the same bytes. To serve different content at a path already served, start a new claim. Its first subscription stays on the claim without a restart. @@ -254,7 +255,7 @@ Within one stream, recovery is about which groups are still worth having. - **Subscriptions and fetches.** A subscription follows the live track and, after a route change within its instance, continues from the first frame it - lacks. A fetch asks for one group by sequence, which a relay fills upstream + lacks. A fetch asks for one group by its start, which a relay fills upstream when its cache misses. Both are bound to the instance they resolved. - **Publisher max age bounds freshness; reader max delay bounds interest.** A track's `max_age` (`maxAge` in JavaScript, `--max-age` on `moq import`) is the @@ -268,9 +269,9 @@ Within one stream, recovery is about which groups are still worth having. - **Arrivals are unordered.** Each group is its own stream, so a lower group can land after a higher one, and is still delivered if the reader's range and budget cover it. Group order chooses what is sent first, not what is skipped. -- **Duplicates are dropped.** A reader receives each group sequence once, +- **Duplicates are dropped.** A reader receives each group start once, however many routes deliver it or however often a producer re-creates it. A - sequence names one group, so resending one cannot correct it. + start names one group, so resending one cannot correct it. - **A drop ends the subscription's copy of a group.** Where the wire reports drops, the publisher names each group a subscription will not get, and every reader of the live copy, even one still arriving, ends with the publisher's diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 8fe07c7a682..f270c250a90 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -102,8 +102,8 @@ announced for late readers is the application's retention policy, such as the ### Publisher instances -Every route carries an instance: any non-empty text naming one continuous run -of group numbers at its path, plus an epoch, a counter within that instance, +Every route carries an instance: any non-empty text naming one continuous +timeline of groups at its path, plus an epoch, a counter within that instance, like a cloud instance ID that survives a reboot and a count of its boots. A path and an instance name one stream. The path never changes, so authorization and hidden names work exactly as before. @@ -133,9 +133,9 @@ hidden names work exactly as before. announced as a restart. Subscriptions already answered stay on the old stream until dropped or none of its routes remain, which ends them with an error, while new requests get the new one, so a player follows the restart - by requesting the path again. That is how a restarted encoder, whose group - numbers start over, takes the name instead of stalling viewers on the old - sequence. A relay drops its copy of the old stream, so a new subscriber never + by requesting the path again. That is how a restarted encoder, whose + timestamps start over, takes the name instead of stalling viewers on the old + timeline. A relay drops its copy of the old stream, so a new subscriber never sees its cached groups. The instance, the epoch, and the restart message travel on moq-lite 07, which @@ -262,7 +262,7 @@ it can still use, which at the default budget is the latest one, so every group must begin at a point a fresh subscriber can decode from (a keyframe, a full JSON snapshot). A moq-lite 05 or moq-transport publisher starts at the latest group whatever the budget, so a subscription there reads older groups only from -a starting group it names. Groups can also be fetched by sequence number, which is how the +a starting group it names. Groups can also be fetched by start, which is how the [HLS gateway](/bin/hls) and the relay's [HTTP fetch](/bin/relay/http) serve history. @@ -283,7 +283,7 @@ Each subscription carries the knobs that decide behavior under congestion: | **Priority** (-128..127, default 0) | Higher-priority tracks get bandwidth first. Audio above video, base layer above enhancement. The publisher's track priority breaks ties between subscriptions at one priority. From a native Rust publisher over QUIC or WebTransport, subscriptions at the same priority and publisher priority split bandwidth evenly by bytes, streams and datagrams alike, however many groups each has queued. A browser publisher and other transports without send groups are not covered. | | **Order** | Which group to send first when several are pending. Newest first for live (the default), oldest first for catch-up. Needs moq-lite 07 or moq-transport; older moq-lite always sends newest first. | | **Max delay** | How long a non-latest group stays worth delivering before it is skipped. Zero means "live edge only", and raising it is also what asks for history. Unset leaves the publisher's max age alone to bound it. | -| **Ranges** | Which groups to ask for: from a group, a number of groups back from the latest, or the group containing a presentation time, optionally up to an end. Each group arrives whole, from its first frame. The default asks for everything max delay allows. Max delay still caps every range. | +| **Ranges** | Which groups to ask for: from the oldest group, a number of groups back from the latest, or the group containing a time, through an end that includes its group. Each group arrives whole, from its first frame. The default asks for everything max delay allows. Max delay still caps every range. | On moq-lite 05 and later, a publisher explicitly reports groups that will not arrive. This lets a finished subscription settle without waiting for missing groups. @@ -308,19 +308,22 @@ no handler it is reported as dropped once the feed stops waiting for it. One evicted after the subscriber saw it cached, but before it read it, is asked the same way, once: evicted again before it is read, it is reported as dropped. A relay merges its subscribers' ranges into one upstream request, which moq-lite-07 carries range by range and older wire versions as the single span -covering them. A relative or time anchor resolves on the hop that publishes the -track, and over moq-lite-07, which carries a relative or time start upstream and -reports the live edge and media time, so a copy resolves the ends itself. A Rust -moq-transport copy resolves a relative anchor from the Largest Object its live join -reports, accepts time anchors its cache can already resolve, and sends group bounds -upstream; it refuses any other time anchor. A JavaScript one refuses every time -anchor. Older -moq-lite versions accept absolute anchors but refuse time anchors and relative anchors other -than the live and default starts. An untimed track refuses a time anchor. At the live -edge, a time end -may let one group through before its first frame shows it starts past the end: -the first group that has yet to start is handed out without waiting for that -frame, and the next one waits on it. +covering them. + +A group is addressed by its start: where it starts in the track's timescale, +or a bare number that grows with each group on a track without one. A time +start is the group containing that time, the newest starting at or before it, +and a time end includes the group containing it, so neither waits on a frame. +A relative anchor counts the groups the hop holds back from the latest one. It +resolves on the hop that publishes the track, and over moq-lite-07, which +carries it upstream and reports the live edge, so a copy resolves the ends +itself. A Rust moq-transport copy resolves a relative anchor from the Largest +Object its live join reports. Older moq-lite versions refuse relative anchors +other than the live and default starts. Older moq-lite versions and +moq-transport carry a group ID equal to the start, so a time anchor goes +upstream as that ID, where a start selects the first group at or after it +rather than the one containing it. moq-lite 01 and 02 carry no start, so every +subscription there starts at the live group. Order only decides what goes first, never what is skipped, so catching up oldest first needs a max delay that covers the backlog. When a relay serves subscribers of @@ -422,8 +425,8 @@ on the bytes that joiner downloads. The publisher drives the track's `Demand::wanted`: when it passes the largest group written, open a new group with a keyframe or complete snapshot. A Rust moq-json snapshot producer answers this by driving its `refresh()` alongside -updates. It restates the current value at its newest source timestamp, with a -new group number. Existing readers continue into that group too. +updates. It restates the current value in a new group at its newest source +timestamp. Existing readers continue into that group too. Each relay judges a join from its own cache and forwards fresh-group demand upstream when needed, all the way to the publisher. Preserving the ratio across diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index 6a9ace3febc..647f975a8d4 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -128,7 +128,7 @@ inside. The header is the reference: each function carries a doc comment. - **Discovery.** Announcement captures are always a list of wildcard matches, empty when the filter has no wildcards. - **Ending cleanly.** When the input ends, finish every track, then call `moq_publish_catalog_finish()` to finish the catalog, which ends the content. The broadcast stays announced and keeps serving the finished tracks, so a subscriber still catching up can drain them, until `moq_publish_close()` or `moq_route_free()` takes it offline. How long to keep it up is yours to decide. A track still open at the close ends with `UNROUTABLE` (an internal error over moq-transport, which has no such code), and its handle is released. Readers follow a publisher's runs by announcement, not by that error: see [identity and recovery](/concept/identity). - **Timeline breaks.** On a seek or pause, call `moq_publish_media_discontinuity`, then keep timestamps moving forward and resume video on a keyframe. -- **Timestamps.** A `moq_timestamp` is exact: `value` units at `scale` units per second, so a 90 kHz tick survives a read and republish. Frames, datagrams, and JSON values point `timestamp` at theirs, NULL on an untimed track and valid as long as the payload. Stream tracks moq-c creates are timed, so their raw writes need one. Datagram tracks are separate and default to untimed; set their timescale when creating them to require timestamps. Stamp media frames and JSON writes from `moq_publish_catalog_now()`, the broadcast clock every track in the broadcast shares; media on a timeline of its own advertises the gap to the JSON tracks as catalog `delay`. JSON writes are converted to their track's milliseconds. A NULL timestamp on any JSON write (snapshot update, stream append, window push or pop) fails with `MOQ_ERROR_INVALID_POINTER` and publishes nothing. A zero scale or an out-of-range value fails with `MOQ_ERROR_TIMESTAMP_OVERFLOW`. +- **Timestamps.** A `moq_timestamp` is exact: `value` units at `scale` units per second, so a 90 kHz tick survives a read and republish. Frames, datagrams, and JSON values point `timestamp` at theirs, valid as long as the payload. An untimed track has no scale, so there `has_scale` is false and `value` is the frame's group key (a datagram's sequence); every write takes one, with `moq_publish_group_sequence` naming a group's key. Stream tracks moq-c creates are timed, so their raw writes need one. Datagram tracks are separate and default to untimed; set their timescale when creating them to require timestamps. Stamp media frames and JSON writes from `moq_publish_catalog_now()`, the broadcast clock every track in the broadcast shares; media on a timeline of its own advertises the gap to the JSON tracks as catalog `delay`. JSON writes are converted to their track's milliseconds. A NULL timestamp on any write fails with `MOQ_ERROR_INVALID_POINTER`. A zero scale with `has_scale` set, or an out-of-range value, fails with `MOQ_ERROR_TIMESTAMP_OVERFLOW`. - **JSON values.** `moq_consume_json_value` fills `moq_json_value` with the document and its frame's timestamp. A snapshot consumer's callback fires for each state in order, but a group superseded before it is read is skipped. - **JSON windows.** `moq_publish_json_window` publishes a sliding run of records: `moq_publish_json_window_push` appends at the back, `moq_publish_json_window_pop` drops from the front, and `moq_publish_json_window_cut` closes the open group at a natural boundary. The catalog has no window mode, so a reader names the track. `moq_consume_json_window_event` fills `moq_json_window_event` with a push (the record at `start`, in `json`), a pop of `[start, end)`, or a skip of `[start, end)` that this reader joined too late to receive. Set `checkpoint_records` to bound what each new group restates for a window you never pop. - **Decoded reads.** `moq_decode_audio` and `moq_decode_video` deliver read ids in order: a frame, a gap where media was lost, or an end where the publisher stopped its encoder. Check `kind` on what `moq_decode_audio_read` or `moq_decode_video_read` writes. Free each id with `moq_decode_audio_read_free` or `moq_decode_video_read_free` promptly: a held video frame holds a decoder buffer, and the decoder stalls. A decoder reads everything the track still holds unless a live player sets `start` to `MOQ_DECODE_START_LATEST`, which joins at the newest group and skips the cached backlog. A decoder setup refusal reports `MOQ_ERROR_UNSUPPORTED_CONFIG` on its terminal callback, so the caller can try another rendition; later peer or decode failures retain their own error codes. diff --git a/doc/lib/cpp/index.md b/doc/lib/cpp/index.md index 2c51cd02732..bc483a497b6 100644 --- a/doc/lib/cpp/index.md +++ b/doc/lib/cpp/index.md @@ -131,7 +131,8 @@ anywhere, so a raw byte stream needs no other importer. `moq::ContainerFormat::kFmp4` splits each fragment into one frame per sample. Media times are exact `moq::Timestamp{value, scale}` records, `value` units at `scale` units per second, and read back at the track's timescale, so a 90 kHz tick republishes unchanged. -A raw `moq::Frame` with `std::nullopt` is untimed. +An untimed track has no time: its frames carry `moq::Timestamp{key, std::nullopt}`, the +frame's group key (a datagram's sequence), and every write takes one. ```cpp // Publish encoded frames, or raw pixels with the codec inside the binding. diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index a5bc7f01950..51a7b2046a9 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -19,13 +19,13 @@ container consumers. `TrackProducer.audio` / `.video` take a broadcast, an init and a `Named` or `Requested` target. `CatalogProducer(broadcast: ...)` owns catalog properties and sections; it holds the broadcast weakly and fails after the broadcast closes. Publish JSON tracks with `MoqTrackInfo(compression: MoqCompression.deflate)`; a `package:moq/json.dart` consumer reads one with no option of its own. -A raw frame without a timestamp stays untimed, which only an untimed track accepts; JSON writes always take one. The library never substitutes zero or now. +Every write takes a timestamp, and the library never substitutes zero or now. `SnapshotProducer.update`, `StreamProducer.append`, and `WindowProducer.push` and `pop` require a timestamp on the broadcast clock (`CatalogProducer.now()`), the epoch every track in the broadcast shares, so their track needs a timescale. `package:moq/json.dart` consumers return `Value` (`value`, `timestamp`) from `next()`; a snapshot consumer yields each state in order, and `latest()` skips to the newest. Media times are exact `Timestamp` records, `value` units at `scale` units per second, so a 90 kHz tick survives a read and republish. `Duration.toTimestamp()` converts exactly; `Timestamp.toDuration()` rounds down to a whole microsecond. -Raw tracks with a `null` timescale are untimed on both publish and receive. +A raw track with a `null` timescale is untimed: each frame carries `Timestamp.unscaled(key)`, its group's key, whose `scale` is `null`. `MoqTrackInfo.compression` set to `MoqCompression.deflate` on a raw track sends each group DEFLATE-compressed to opt-in `moq-lite-07-wip` peers; other peers, and every reader here, get the original frames. Whole-frame media writes require caller time; container imports keep their own timeline handling. Timed raw/data tracks require caller-supplied timestamps. Use `CatalogProducer.now()` to stamp a locally @@ -133,7 +133,7 @@ print(route.instance()); final mine = route.broadcast(); final track = mine.publishTrack(name: 'video', info: null); final group = track.appendGroup(); -group.writeFrame(frame: Frame(payload: bytes, timestamp: null)); +group.writeFrame(frame: Frame(payload: bytes, timestamp: Timestamp.unscaled(group.sequence()))); group.finish(); track.finish(); mine.close(); // ends the route too diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 27cc55a1ea0..d9b12e693d9 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -18,9 +18,9 @@ Construct `NewAudioTrackProducer` / `NewVideoTrackProducer` from a broadcast, an and a `Named` or `Requested` target. `NewCatalogProducer` holds its broadcast weakly; catalog writes fail with `ErrClosed` after the broadcast closes. Media timestamps are exact `moq.Timestamp` records (`Value` at `Scale` units per second), so a 90 kHz tick survives a read and republish; -`TimestampFromDuration` converts exactly; `TimestampToDuration` rounds down to a whole nanosecond. -A raw frame without a timestamp stays untimed, which only an untimed track accepts; JSON writes always take one. The library never substitutes zero or now. -Raw tracks with a `nil` timescale are untimed on both publish and receive. +`TimestampFromScale` and `TimestampFromDuration` build one exactly; `TimestampToDuration` rounds down to a whole nanosecond. +Every write takes a timestamp, and the library never substitutes zero or now. +A raw track with a `nil` timescale is untimed: each frame carries `moq.Unscaled(key)`, its group's key, whose `Scale` is `nil`. `TrackInfo.Compression` set to `CompressionDeflate` sends each group DEFLATE-compressed to opt-in `moq-lite-07-wip` peers; other peers, and every reader here, get the original frames. Set it on JSON tracks. The JSON producers' `Update`, `Append`, `Push`, and `Pop` take a final `moq.Timestamp` on the broadcast clock, the epoch every track in the broadcast shares, so their track needs a timescale. JSON consumers return a `*Timed` holding the value and that timestamp; `SnapshotConsumer.Latest` @@ -98,8 +98,8 @@ import ( route, _ := client.Route("my-stream.hang", moq.RouteOptions{Hidden: true}) broadcast := route.Broadcast() audio, _ := moqmedia.NewAudioTrackProducer(broadcast, moqmedia.Named{}, moqmedia.AudioInit{Format: moqmedia.AudioFormatOpus, Data: opusInit}) -timestamp := moq.Timestamp{Value: 960, Scale: 48_000} // 20 ms on the Opus sample clock -_ = audio.WriteFrame(moq.Frame{Payload: packet, Timestamp: ×tamp}) +timestamp := moq.TimestampFromScale(960, 48_000) // 20 ms on the Opus sample clock +_ = audio.WriteFrame(moq.Frame{Payload: packet, Timestamp: timestamp}) _ = audio.Cut() // audio has no keyframes, so this is what gives it groups track := "camera" @@ -109,7 +109,7 @@ video, _ := moqvideo.NewProducer(catalogProducer, moqvideo.ProducerOptions{ Output: moqvideo.EncoderOutput{Codec: moq.VideoCodecH264, Kind: moq.AutoEncoder()}, Track: &track, }) -_ = video.Write(moqvideo.Frame{Timestamp: moq.Timestamp{Value: pts, Scale: 90_000}, Data: rgba}) +_ = video.Write(moqvideo.Frame{Timestamp: moq.TimestampFromScale(pts, 90_000), Data: rgba}) _ = route.Announce() // each route mints a fresh instance (route.Instance()) _ = audio.Finish() _ = video.Finish() diff --git a/doc/lib/js/json.md b/doc/lib/js/json.md index ad7d0e53ec8..630b524ebe4 100644 --- a/doc/lib/js/json.md +++ b/doc/lib/js/json.md @@ -21,8 +21,11 @@ Compression is the track's: create it with `compression: "deflate"`, which JSON from, and readers see plain frames either way. Producers take a `Timed` value, `{ value, at }`, where `at` is the capture time written as the frame timestamp, and the snapshot and stream consumers return the same -shape. Nothing fills in now: a timed track needs `at` on every write, and -an untimed track (no `timescale`) takes none and reads back without one. +shape. Nothing fills in now: every write takes `at`. On an untimed track (no +`timescale`) it is `Time.Timestamp.unscaled(key)`: a snapshot or window +restatement opens its group at that key, which must pass every group the +track holds, and a delta or op rides the open group, so it reads back with +that group's key. ```ts import { Snapshot } from "@moq/json"; diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index 0a8b6e00973..4f8bad7637c 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -51,7 +51,7 @@ route.announce(); - **List a broadcast's tracks.** `for await (const event of broadcast.tracks())` reports each track the broadcast lists (`added`, with its `Track.Info`), one `synced`, then `added` and `removed` as tracks come and go, until the broadcast ends; leaving the loop stops listing. Over a session it needs `moq-lite-07-wip`, and syncs when the peer does, so a track added after `synced` is new at the source. A track served only on request is never listed. See [discovery](/concept/moq-lite#discovery). - **One track per name.** Concurrent subscriptions share one request and producer. `createTrack` on a name with a request still queued answers it. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a track that replaces it, created or requested, starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups by writing its first group with an explicit sequence. Sequences are `bigint`, so every ID the wire carries fits, past 2^53 included: `track.createGroup(5n)`, `{ absolute: 10n }`. Frame indices and relative anchors stay `number`. - **Serve history on request.** `track.requests()` opens a handle that serves the groups readers miss: ones their ranges name below the newest group written that the cache no longer holds, or never did. Each run of misses arrives as a `Track.Fill` from `await requests.requestedFill()`, shared by every reader missing it. `accept(sequence)` it into the cache and write the group's frames as first published, `dropGroups(start, end)` what the source lacks (sequences are `bigint`), and serve `wanted()` until it is `undefined`: a reader that leaves or narrows stops wanting its groups. `close()` the fill when done; whatever is still wanted is named dropped. A `fetchGroup` of an uncached group asks the handle at once, even for a group not written yet, which a live write then answers; with no handle open it fails with `NotFound`, as in Rust, and a reader's miss is given up. A reader only asks for what its `delay.max` still wants, so a reader at the live edge never asks. This serves lite FETCH and lite-07 ranged subscriptions from the same track, as Rust's `requests()` does. A moq-transport FETCH is refused, so a moq-transport reader never reaches it. -- **Timing is per track, with no default.** A track that declares a `timescale` carries a `timestamp` on every frame; one without is [untimed](/concept/moq-lite#subscriptions) and its frames carry none. Omitting it means untimed, as in Rust. Nothing stamps a frame for you: the publisher passes the timestamp, and the `Timed` type (`{ value, at }`) carries one into the `@moq/json` producers. +- **Timing is per track, with no default.** A track that declares a `timescale` carries a `timestamp` on every frame; one without is [untimed](/concept/moq-lite#subscriptions) and each frame carries `Time.Timestamp.unscaled(group.sequence)`, its group's key, which has no time: `as`, `asMillis`, and `asMicros` throw on it. A scaled timestamp on an untimed track, an unscaled one on a timed track, or another key throws `TimestampMismatch`. Omitting it means untimed, as in Rust. Nothing stamps a frame for you: the publisher passes the timestamp, and the `Timed` type (`{ value, at }`) carries one into the `@moq/json` producers. - **Durations and instants are `Clock` types.** Every duration (`maxAge`, `delay`, timeouts, RTT) is a `Clock.Duration` and every monotonic time a `Clock.Instant`, from [`@moq/clock`](./clock), which `@moq/net` re-exports as `Clock`: `maxAge: Moq.Clock.Duration.secs(30)`. Neither converts to a number implicitly; read one with an explicit unit, such as `asMillis()`. The wire still carries whole milliseconds, rounded up. Frame timestamps stay `Time.Timestamp`, a tick count at the track's timescale. - **A subscription's `delay` is a `Track.Delay { min, max }`, as in Rust.** Staleness is `delay.max`: it counts from a group's successor, by media time and by wall clock, whichever reaches it first. `delay.min` is the target latency, carried on moq-lite 07; a peer that cannot carry it leaves it `null` (not stated). Passing the old `maxAge` or `maxDelay` key throws a `TypeError` naming `delay.max`. The publisher's own staleness bound is the separate `Track.Info.maxAge`, which caps `delay.max` and fetches. - **Authored tracks default to a zero `maxAge`.** `createTrack` and `accept` take a `Track.InfoInit`: an omitted `maxAge` is zero, so readers get only the live edge; `null` sets no publisher limit, and a `Clock.Duration` is that bound, as `Info::default()` and `with_max_age(None)` do in Rust. A track whose readers reach back for superseded groups (history, a fetch, a range from an absolute start) sets one explicitly; media takes `@moq/hang`'s 30-second `Container.MAX_AGE`. `maxAge` is how long a superseded group stays fresh, never the newest group, so a track that writes one snapshot stays readable to a late joiner. A received `Track.Info` uses `null` for none (`maxAge`, `timescale`, `refresh`), so it serves onward unchanged. @@ -114,7 +114,8 @@ const connection = await Moq.Connection.connect({ }); const endpoint = new Moq.Socket.Endpoint(origin, Moq.Path.from("game"), "viewer"); const session = await endpoint.dialTrack("server", undefined, { signal: abort.signal }); -session.producer.writeFrame({ payload: new TextEncoder().encode("hello") }); // One message, one group. +// A socket session is scale-less, so a message's frame carries its group's key, here the first. +session.producer.writeFrame({ payload: new TextEncoder().encode("hello"), timestamp: Moq.Time.Timestamp.unscaled(0n) }); session.producer.finish(); // Ends this direction; the peer may still reply. const reader = session.consumer.ordered(); for (let frame = await reader.readFrame(); frame; frame = await reader.readFrame()) { diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 1e7440cd31e..dd016e06ee0 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -17,13 +17,13 @@ x86\_64) and desktop JVM (Linux x86\_64/aarch64, macOS arm64, Windows x64). / `.video` take a broadcast, a `Named` or `Requested` target, and an init record. `CatalogProducer(broadcast)` holds its broadcast weakly and fails with `Closed` after it closes. Publish JSON tracks with `TrackInfo(compression = Compression.DEFLATE)`; a `dev.moq.json` consumer reads one with no option of its own. -A raw frame without a timestamp stays untimed, which only an untimed track accepts; JSON writes always take one. The library never substitutes zero or now. +Every write takes a timestamp, and the library never substitutes zero or now. `SnapshotProducer.update`, `StreamProducer.append`, and `WindowProducer.push` and `pop` require a timestamp on the broadcast clock (`CatalogProducer.now()`), the epoch every track in the broadcast shares, so their track needs a timescale. `dev.moq.json` consumers' `values()` emits `Value` (`value`, `timestamp`) and `valuesAs()` emits `Timed`; a snapshot consumer yields each state in order, and `latest()` skips to the newest. A `Timestamp(value, scale)` is exact and reads back at the track's timescale, so a 90 kHz tick republishes unchanged. `Duration.toTimestamp()` is exact at nanosecond scale; `Timestamp.toDuration()` rounds down to a whole nanosecond. -Raw tracks with a `null` timescale are untimed on both publish and receive. +A raw track with a `null` timescale is untimed: each frame carries `Timestamp.unscaled(key)`, its group's key, whose `scale` is `null`. `MoqTrackInfo.compression` set to `DEFLATE` on a raw track sends each group DEFLATE-compressed to opt-in `moq-lite-07-wip` peers; other peers, and every reader here, get the original frames. Whole-frame media writes require caller time; container imports keep their own timeline handling. Timed raw/data tracks require caller-supplied timestamps. Use `CatalogProducer.now()` to stamp a locally diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index 0eef5bf1fa9..1474b95bd02 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -24,10 +24,10 @@ so a raw byte stream needs no other importer. `ContainerFormat.FMP4` splits each fragment into one frame per sample. Media timestamps are exact `moq.Timestamp(value, scale)` records, so a 90 kHz tick survives a read and republish. `Timestamp.from_timedelta` is exact; `.to_timedelta()` rounds down to a whole microsecond. -A raw frame without a timestamp stays untimed, which only an untimed track accepts; JSON writes always take one. The library never substitutes zero or now. +Every write takes a timestamp, and the library never substitutes zero or now. JSON consumers yield `moq.json.Timed(value, timestamp)`; a snapshot consumer yields each state in order, and `await consumer.latest()` skips to the newest. A window consumer's value is a `WindowEventPush`, `WindowEventPop`, or `WindowEventSkip`. -Raw tracks with a `None` timescale are untimed on both publish and receive. +A raw track with a `None` timescale is untimed: each frame carries `moq.Timestamp.unscaled(key)`, its group's key, whose `scale` is `None`. `TrackInfo(compression=moq.Compression.DEFLATE)` sends each group DEFLATE-compressed to opt-in `moq-lite-07-wip` peers; other peers, and every reader here, get the original frames. Set it on JSON tracks. The JSON producers' `update`, `append`, `push`, and `pop` require a `timestamp` on the broadcast clock, the epoch every track in the broadcast shares, so their track needs a timescale. Whole-frame media writes require caller time; container imports keep their own timeline handling. @@ -95,7 +95,7 @@ async def main(): # Raw bytes and JSON events = broadcast.publish_track("events") - events.write_frame(b'{"cmd": "ready"}') + events.write_frame(b'{"cmd": "ready"}', moq.Timestamp.unscaled(0)) # untimed: the first group's key status_track = broadcast.publish_track("status", moq.TrackInfo(timescale=1_000, compression=moq.Compression.DEFLATE)) status = moq.json.SnapshotProducer(broadcast, status_track) status.update({"state": "live", "viewers": 42}, catalog_producer.now()) @@ -171,7 +171,7 @@ async def socket_client(): async with moq.Client("https://cdn.moq.dev/anon", versions=versions, publish=origin, consume=origin): async with moq.socket.Endpoint(origin, "room", "client") as endpoint: stream = await endpoint.dial_track("server") - stream.producer.write_frame(b"command") # One message: a group of one frame. + stream.producer.write_frame(b"command", moq.Timestamp.unscaled(0)) # One message, the first group: key 0. stream.producer.finish() # abort(error_code) ends it with an error instead. async for group in stream.consumer: async for frame in group: diff --git a/doc/lib/rs/moq-json.md b/doc/lib/rs/moq-json.md index e1ff5bee366..5f43ac5e897 100644 --- a/doc/lib/rs/moq-json.md +++ b/doc/lib/rs/moq-json.md @@ -21,14 +21,16 @@ The `poll_*` methods work by default. Enable the `async` Cargo feature to await - **window**: a bounded run of records a reader can join at any point. A new group restates what it keeps explicitly, so a reader that was keeping up is not handed a record twice. Its ops roll on the track's `refresh` ratio the same way. Compression is the track's: create it with `Info::default().with_compression(Compression::Deflate)`, -which JSON benefits from, and readers see plain frames either way. A bare value stays untimed; publish -`moq_net::Timed::from(&value).at(captured)` to retain a capture timestamp. -The timestamp must match the track: an untimed track has no timescale, and a -timed track requires a timestamp on every write. Editing guards take the -same optional capture time when opened, including when they publish on drop. +which JSON benefits from, and readers see plain frames either way. Every write is a +`moq_json::Timed::new(&value, at)`, and `at` must match the track. On a timed track it is the +capture time. An untimed track has no timescale, so there `at` is `Timestamp::unscaled(key)`: +a write that opens a group starts it at that key, which must pass every group the track holds, +and a delta inside the open group carries that group's key. Editing guards take the same +timestamp when opened, including when they publish on drop. Consumers return `Timed`: read `.value` for the decoded value and `.at` for -its optional frame timestamp, with the track's exact scale preserved. Snapshot +its frame timestamp, with the track's exact scale preserved (an unscaled group key on an +untimed track). Snapshot `next()` yields states in order, skipping every group a newer one supersedes: `snapshot::Consumer::new` takes the track and subscribes to it itself at a zero `delay.max`, so nothing can raise its budget. Use `latest()` to skip to the diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 7fec6963e28..1aca55c6077 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -17,15 +17,15 @@ What you use it for, beyond what the concept page already describes: - **An origin outlives the session.** `origin.route(path, options)` returns a `Route` that owns the broadcast to fill (`route.broadcast()`) and controls its announcement: `announce()`, `withdraw()`, `set_cost()`, and dropping the last clone retracts it. `origin.claim(prefix, options)` returns a `Claim` that serves requests beneath a prefix instead: `claim.requested()` yields each one, answered with `request.serve(held)`, `request.route(options)` (the path's own exact route) or `reject`. A `Claim` has the same announcement controls and no broadcast. A standalone broadcast cannot announce; publish one only through a claim's `serve`. A reconnect announces existing routes again, and closing the session does not delete them. - **Routes report what the mesh filled in.** An announcement's `route` gives its `instance()` and `cost()`; an application sets only `RouteOptions`. The hop chain a relay uses for loop detection stays internal. An origin's own hop id is `origin::Config::hop`, random unless a relay names one. -- **Requests can pin an instance.** `consumer.request(path, Some(instance))` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances); `None` takes whichever route wins. The resolved consumer names its instance in `info().instance`, pinned or not. `origin.route(path, options)` serves a random instance minted when it is created and kept across `withdraw()` and `announce()`. Unless one is named, a claim moves to a fresh instance each time a job holding its current one ends, and a job takes the claim's. Set one with `RouteOptions::with_instance(instance, epoch)` only for a publisher that continues its group sequences across restarts and raises the epoch each time. `route.instance()` names it. +- **Requests can pin an instance.** `consumer.request(path, Some(instance))` resolves only through a route announcing that [publisher instance](/concept/moq-lite#publisher-instances); `None` takes whichever route wins. The resolved consumer names its instance in `info().instance`, pinned or not. `origin.route(path, options)` serves a random instance minted when it is created and kept across `withdraw()` and `announce()`. Unless one is named, a claim moves to a fresh instance each time a job holding its current one ends, and a job takes the claim's. Set one with `RouteOptions::with_instance(instance, epoch)` only for a publisher that continues its timeline across restarts, so its group starts keep rising, and raises the epoch each time. `route.instance()` names it. - **Rank several CDNs.** Dial each with its own subscribe handle, `origin.clone().with_preference(origin::Preference::new(rank))` (higher wins; 0 is neutral and a negative rank demotes), all on one origin. Requests use the preferred link that has a route and fail over to the next; see [multiple CDNs](/concept/moq-lite#multiple-cdns). - **Request one instance, or follow them all.** `consumer.request(path, None)` resolves the instance serving a path now, and fails `Unroutable` only once discovery has synced with no route for it. `consumer.follow(path, None)` hands out each publisher instance serving a path from `available()`: the first once announced, then each one that takes over, `None` once the path goes offline, and whatever serves it next, like JS `follow(path)`. `consumer.announced()` lists paths as present, then unavailable or available as a held route loses and recovers its path, and finally absent once the origin withdraws one or offline once this side lets go of it, which says nothing of whether the origin still serves it. A broadcast already resolved stays on the old instance until dropped or its route goes. See [identity and recovery](/concept/identity) to choose between following and holding one instance. - **Discovery follows what you watch.** A client session asks the relay for announcements only under the prefixes its subscribe origin's readers watch, with `announced()`, `request`, or `follow`, stops asking about a prefix 2s after nothing at or beneath it is watched, and asks again on reconnect. A broadcast resolved through a discovered route keeps its path watched until dropped. `request` owns discovery of its path, so no separate watch is needed. A server session, or a client fed into an origin handle marked `with_link(origin::Link::Peer)` or `Link::Upstream`, asks for everything it may, so a relay discovers everything. An HTTP gateway (the moq-hls `Server`, the moq-webrtc WHEP router) watches its whole scope while it lives, so the paths its callers name never open a request each. moq-lite-03/04 sessions, whose announcements mark no initial set, still ask for everything. - **Relay by opting into proxying.** On moq-lite-07 (`moq-lite-07-wip`, opt-in) a session sends its peer only the routes to its origin's own claims, so an application never becomes transit for anyone else. A relay built on the library marks the handle it publishes with `origin.clone().with_proxy(true)` to carry what it learned from other peers too; `moq-relay` does so for every session. Older versions advertise everything either way. - **Dial and accept sockets.** `socket::Endpoint::new(&origin, prefix, id)` is a server, a client, or both: its first dial announces a hidden client endpoint at `/.client/`, and its first accept a hidden server endpoint at `/.server/`, both used only to route sessions, so grants decide which roles it may take. `accept_track()` yields an `Incoming` dial to `accept` or `reject` (dropping it rejects). `dial_track(server, info)` and an accepted dial yield a `socket::TrackStream`, this side's `track::Producer` and a `track::Subscriber` of the peer's half, subscribed reliably and in order until the application updates it, one message per group. A dial resolves once the peer accepts, and an accept checks that the claimed dialer really dialed this endpoint, waiting for its endpoint while the dial stays open. Every hop needs moq-lite-07 or moq-transport: while the origin has an older moq-lite client session whose publish half doesn't opt into hidden broadcasts (`with_hidden(true)`), dials and accepts fail with `Unsupported`. A dial to a peer behind such a hop waits, as for a peer not online yet. Session tracks are never announced or listed. See [Sockets](/concept/socket). -- **One track per name.** Concurrent subscriptions share one request and producer. The application owns sequence numbers: the broadcast keeps nothing of an ended track, so a producer that replaces it starts at 0. A name always means the same content, so the replacement continues past its predecessor's groups with `create_group` at an explicit sequence. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. +- **One track per name.** Concurrent subscriptions share one request and producer. A group is keyed by its start, its first frame's timestamp in the track's timescale, and starts only rise. An untimed track has no timescale, so its frames carry `Timestamp::unscaled(key)`, the group's key, and a timestamp's `scale()` is `None`: conversions such as `as_millis()` fail on it, and it compares with no scaled timestamp. Every frame write takes a `Timestamp`, and one that doesn't match its track is refused with `TimestampMismatch`. `append_group(start)` opens the next one, declaring the range below it skipped, and `create_group` places one without declaring anything. The broadcast keeps nothing of an ended track, and a name always means the same content, so a producer that replaces it starts its groups past its predecessor's. Readers of an aborted track resume onto the replacement when they read through an origin request, whose route always names an instance; otherwise their subscription ends with the error. - **Authored tracks default to a zero `max_age`.** `track::Info::default()` sets `max_age` to `Some(Duration::ZERO)`, the live edge only; a publisher that wants readers to reach history sets a longer bound, or `with_max_age(None)` for no publisher limit. Media tracks from `hang::container::track_info` declare `hang::container::MAX_AGE` (30 seconds). `max_age` is how long a superseded group stays fresh, never the newest group; it frees no bytes, which the cache pool's expiry reclaims. Received metadata keeps absence as no limit, and an origin's `cache_duration` can only cap it lower. -- **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track: `finish(self)` consumes the producer and returns a `track::Finished`, which can still take an older group below the end, be read, or be aborted, but not append. Hold it, and the broadcast stays announced and keeps serving the track until you close it or drop its route; dropping the last handle releases the track, so new readers no longer find it. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. A finished group returns a `group::Finished`, which can be read or aborted to release its cache. +- **End content by finishing tracks.** A broadcast has no finished state. When the input ends, finish each track: `finish(self)` consumes the producer and returns a `track::Finished`, which can still take an older group below the end, be read, or be aborted, but not append. Hold it, and the broadcast stays announced and keeps serving the track until you close it or drop its route; dropping the last handle releases the track, so new readers no longer find it. How long to keep it up for late readers is yours to decide: `moq_mux::Linger` holds a hang broadcast for a fixed window. Ending a group returns a `group::Finished`, which can be read or aborted to release its cache: `finish(end)` ends it whole where its content stops, so a reader can tell the group starting there continues it, and `close()` ends it without one. - **Demand says whether anyone is watching.** `demand()` on a track, group, or broadcast says whether a subscriber is attached. First-party capture does not wait for it: video encodes while enabled and audio keeps raw PCM to encode when someone listens, so a late subscription's `delay.max` or a FETCH finds the recent past. Demand gates bandwidth reservations, transcode rungs, and on-demand video where a publisher opts in, which saves the encode but leaves nothing from before the first subscriber. Holding a broadcast or track consumer is not demand: a track consumer requests nothing until it queries, subscribes, or fetches, and those count until dropped. A shared fetch stays up until its last reader leaves. Demand rolls up: a frame being read keeps its group wanted, a group a subscription or fetch handed out keeps its track wanted until read to its end, and any wanted track keeps its broadcast wanted. - **Use the model without a runtime.** By default `moq-net` is its `poll_*` APIs and origin driver with caller-supplied time. The `async` feature adds their async wrappers, `time::run`, and both wire sessions. - **You drive the session, or `moq-tokio` does.** `connect` and `accept` return a session plus a driver that never reads the clock itself. `moq_net::time::run` polls it on tokio. `moq-tokio` hides that. A custom transport implements `moq_net::transport::poll`, and `moq_net::transport::memory::pair` links two sessions in one process with no sockets, for tests or in-process links. `Server` bounds each accept with one deadline (10 s by default, `with_timeout`) across SETUP, your admission decision (`Handshake::expired`), and moq-lite 01/02's SETUP reply, and opens a receive window the transport held back once you admit (`with_receive_window`). diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index e1cf59e4ba7..82a7a1b64b0 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -21,11 +21,11 @@ Media timestamps are exact `Timestamp` records, `value` units at `scale` units p (a write at another scale rounds down to it). On macOS 13+ and iOS, `try Timestamp(duration)` converts a `Duration` exactly, and `try timestamp.duration` rounds down to a whole attosecond. Publish JSON tracks with `TrackInfo(compression: .deflate)`; a `Json` consumer reads one with no option of its own. -A raw frame without a timestamp stays untimed, which only an untimed track accepts; JSON writes always take one. The library never substitutes zero or now. +Every write takes a timestamp, and the library never substitutes zero or now. `Json.SnapshotProducer.update`, `Json.StreamProducer.append`, and `Json.WindowProducer.push` and `pop` require a timestamp on the broadcast clock (`Media.CatalogProducer.now()`), the epoch every track in the broadcast shares, so their track needs a timescale. `Json` consumers yield `Json.Timed` (`value`, `timestamp`); a snapshot consumer yields each state in order, and `latest()` skips to the newest. -Raw tracks with a `nil` timescale are untimed on both publish and receive. +A raw track with a `nil` timescale is untimed: each frame carries `Timestamp.unscaled(key)`, its group's key, whose `scale` is `nil`. `MoqTrackInfo.compression` set to `.deflate` on a raw track sends each group DEFLATE-compressed to opt-in `moq-lite-07-wip` peers; other peers, and every reader here, get the original frames. Whole-frame media writes require caller time; container imports keep their own timeline handling. Timed raw/data tracks require caller-supplied timestamps. Use `Media.CatalogProducer.now()` to stamp a locally diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index 72ffb076a57..44e6ecfaad2 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -179,6 +179,9 @@ Datagram Tracks have a namespace apart from other Tracks, so a datagram Track an A Group is an ordered stream of Frames within a Track. Each group consists of an append-only list of Frames. +A Group's sequence is its start: its frame 0's timestamp on a timed Track, and on an untimed Track a number that grows with each Group, in no unit. +Sequences therefore order Groups by start without a counter, and a gap between two says nothing on its own. +A receiver addresses a Group by the sequence it carries whatever its frames hold, so a publisher that numbers its Groups 0, 1, 2 is still served, with time Anchors compared against those numbers. A publisher never creates a Group with sequence 2^64-1, so every range of Groups ends within 64 bits; a receiver treats a Group Stream naming it as a malformed Track. A Group is normally served by a dedicated QUIC stream which is closed on completion, reset by the publisher, or cancelled by the subscriber. This ensures that all Frames within a Group arrive reliably and in order. @@ -199,7 +202,7 @@ A Group Stream normally starts at frame 0. Each frame of a timed Track carries a presentation timestamp expressed in the Track's `Timescale` (see [TRACK_INFO](#track-info)), used by the moq-lite layer for [expiration](#expiration) decisions. A Track without a `Timescale` is untimed: its frames carry no timestamp. A timestamp MUST be less than 2^61, so every version can carry it; a receiver treats a larger one as a malformed Track. -Unless a negotiated extension specifies otherwise, a publisher MUST NOT start a timed Group's frame 0 earlier than that of any lower-sequence Group it publishes. +Unless a negotiated extension specifies otherwise, a publisher MUST NOT start a timed Group's frame 0 at or before that of any lower-sequence Group it publishes. A receiver MAY check a received Group against this rule, refusing a violating Group's stream (STOP_SENDING) with TIMESTAMP_MISMATCH. A relay serves a Track's timestamps on as it received them, including ones it did not check, unless a negotiated extension specifies otherwise. @@ -1701,7 +1704,7 @@ Which fields follow, and what the Anchor names: - None (0x0): no fields. As a `Start`, the oldest Group (no floor); as an `End`, the live edge (unbounded). - Absolute (0x1): `Group Sequence`. A `Start` delivers from that Group and an `End` stops just before it. - Relative (0x2): `Groups Back`, the Group that many before the latest, 0 being the latest. Included as an `End`. -- Time (0x3): `Timestamp`, the Group containing that presentation time in the Track's `Timescale` (see [TRACK_INFO](#track-info)). Included as an `End`. On an untimed Track it is refused with TIMESTAMP_MISMATCH. +- Time (0x3): `Timestamp`, the Group containing that time: the one whose sequence, its start (see [Group](#group)), is the highest at or below it. Included as an `End`. Any other value is a PROTOCOL_VIOLATION. The publisher resolves a Relative or Time Anchor against the Track as it holds it. @@ -2467,8 +2470,9 @@ An application-specific payload. - Made TRACK_INFO Publisher Max Age optional, encoded as milliseconds plus one with zero meaning no limit. - Added TRACK_INFO `Refresh` and SUBSCRIBE_UPDATE `Wanted Group`, carrying ratio-based refresh demand through relays. - Added `Compression` to TRACK_INFO: none (0x0) or DEFLATE (0x1), whose Groups are DEFLATE streams per draft-lcurley-moq-flate. FRAME carries an `Inflated Size` when it is not none. -- An untimed Track declares a `Timescale` of 0 in TRACK_INFO, and its FRAME messages omit the timestamp. -- A publisher must not start a timed Group's frame 0 earlier than a lower-sequence Group's, and a receiver may refuse a Group that does. A relay serves every Track's timestamps on as received. +- An untimed Track declares a `Timescale` of 0 in TRACK_INFO, and its FRAME messages omit the timestamp. The published moq-lite-05 and moq-lite-06 treat a `Timescale` of 0 as invalid, so an untimed Track served over them is forced to declare milliseconds, and each FRAME carries its Group sequence as the timestamp. +- A publisher must not start a timed Group's frame 0 at or before a lower-sequence Group's, and a receiver may refuse a Group that does. A relay serves every Track's timestamps on as received. +- A Group's sequence is its start: frame 0's timestamp on a timed Track, and a growing number on an untimed one. A Time Anchor names the Group with the highest sequence at or below it, on an untimed Track too. - Bounded every timestamp below 2^61. A larger one is a malformed Track. - Reserved Group sequence 2^64-1: a publisher never creates it, and a Group Stream naming it is a malformed Track. diff --git a/go/wrapper/README.md b/go/wrapper/README.md index 173e0bb1c48..ccc2ae9ee9a 100644 --- a/go/wrapper/README.md +++ b/go/wrapper/README.md @@ -130,7 +130,8 @@ of records a reader can join at any point; the catalog has no window mode, so th takes only the track. Producers accept any `encoding/json` value; consumers return `json.RawMessage` so callers choose their own decoded type. `Update` / `Append` / `Push` / `Pop` take a final `moq.Timestamp` on the broadcast clock, so their track needs a timescale. -Raw tracks with a nil timescale are untimed; timed tracks require matching timestamps. +Raw tracks with a nil timescale are scale-less: each frame's timestamp is `moq.Unscaled` at its group's key. +Timed tracks require matching timestamps; build one with `moq.TimestampFromScale` or `moq.TimestampFromDuration`. `media.CatalogProducer.Now()` samples and fixes the broadcast clock, the epoch every track in the broadcast shares; stamp media and JSON from it. Opaque bytes go on a plain `TrackProducer`. @@ -177,7 +178,7 @@ See the package doc for details. Create a separate live track with `BroadcastProducer.CreateDatagrams(name, info)` and subscribe with `BroadcastConsumer.Datagrams(name)`. `DatagramProducer.Append(timestamp, payload)` -assigns a sequence, `Insert(Datagram)` preserves one, and `DatagramConsumer.Recv(ctx)` or +assigns a sequence (on a scale-less track, the one `moq.Unscaled(sequence)` names), `Insert(Datagram)` preserves one, and `DatagramConsumer.Recv(ctx)` or `All(ctx)` reads in arrival order. Stream tracks of the same name are independent. Watch aggregate subscriber demand with `Priority()` / `PriorityChanged(ctx, last)`, and serve on-demand datagram tracks with `BroadcastRequests.Datagrams()`. diff --git a/go/wrapper/data_timestamps_test.go b/go/wrapper/data_timestamps_test.go index 13ec9bba9c0..fb56d3a1e96 100644 --- a/go/wrapper/data_timestamps_test.go +++ b/go/wrapper/data_timestamps_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "reflect" "testing" "moq.dev/moq" @@ -11,28 +12,29 @@ import ( ) // A 90 kHz tick has no exact microsecond count, so it survives only as a record. -var tick = moq.Timestamp{Value: 1, Scale: 90_000} +var tick = moq.TimestampFromScale(1, 90_000) func ninetyKHz() *moq.TrackInfo { scale := uint64(90_000) return &moq.TrackInfo{Timescale: &scale} } -// Every producer carries an exact timestamp or a present zero, a raw frame none at -// all too, and the raw reader gets back exactly what was written. JSON writes -// always take one, so they have no untimed case. +// Every producer carries an exact timestamp or a present zero, a raw frame on a +// scale-less track its unscaled group key too, and the raw reader gets back exactly +// what was written. JSON writes need a timescale, so they have no scale-less case. func TestDataTimestamps(t *testing.T) { timestamps := []struct { name string - at *moq.Timestamp + at moq.Timestamp }{ - {"untimed", nil}, - {"zero", &moq.Timestamp{Value: 0, Scale: 90_000}}, - {"tick", &tick}, + // The track's first group is key 0. + {"unscaled", moq.Unscaled(0)}, + {"zero", moq.TimestampFromScale(0, 90_000)}, + {"tick", tick}, } for _, mode := range []string{"frame", "json snapshot", "json stream"} { for _, timestamp := range timestamps { - if timestamp.at == nil && mode != "frame" { + if timestamp.at.Scale == nil && mode != "frame" { continue } t.Run(mode+"/"+timestamp.name, func(t *testing.T) { @@ -44,7 +46,7 @@ func TestDataTimestamps(t *testing.T) { } defer broadcast.Close() var info *moq.TrackInfo - if timestamp.at != nil { + if timestamp.at.Scale != nil { info = ninetyKHz() } track, err := broadcast.PublishTrack("data", info) @@ -68,7 +70,7 @@ func TestDataTimestamps(t *testing.T) { t.Fatal("missing frame") } want := timestamp.at - if (frame.Timestamp == nil) != (want == nil) || want != nil && *frame.Timestamp != *want { + if !reflect.DeepEqual(frame.Timestamp, want) { t.Fatalf("timestamp = %+v, want %+v", frame.Timestamp, want) } if mode == "json snapshot" || mode == "json stream" { @@ -88,7 +90,7 @@ func TestDataTimestamps(t *testing.T) { } // publishData writes one value at timestamp through the producer mode names, then finishes. -func publishData(mode string, broadcast *moq.BroadcastProducer, track *moq.TrackProducer, payload []byte, timestamp *moq.Timestamp) error { +func publishData(mode string, broadcast *moq.BroadcastProducer, track *moq.TrackProducer, payload []byte, timestamp moq.Timestamp) error { switch mode { case "frame": return track.WriteFrame(moq.Frame{Payload: payload, Timestamp: timestamp}) @@ -97,7 +99,7 @@ func publishData(mode string, broadcast *moq.BroadcastProducer, track *moq.Track if err != nil { return err } - if err := producer.Update(map[string]int{"n": 1}, *timestamp); err != nil { + if err := producer.Update(map[string]int{"n": 1}, timestamp); err != nil { return err } return producer.Finish() @@ -106,7 +108,7 @@ func publishData(mode string, broadcast *moq.BroadcastProducer, track *moq.Track if err != nil { return err } - if err := producer.Append(map[string]int{"n": 1}, *timestamp); err != nil { + if err := producer.Append(map[string]int{"n": 1}, timestamp); err != nil { return err } return producer.Finish() @@ -121,9 +123,9 @@ func TestTimestampRefused(t *testing.T) { name string at moq.Timestamp }{ - {"zero scale", moq.Timestamp{Value: 1, Scale: 0}}, - {"scale 2^62", moq.Timestamp{Value: 1, Scale: 1 << 62}}, - {"value 2^61", moq.Timestamp{Value: 1 << 61, Scale: 90_000}}, + {"zero scale", moq.TimestampFromScale(1, 0)}, + {"scale 2^62", moq.TimestampFromScale(1, 1<<62)}, + {"value 2^61", moq.TimestampFromScale(1<<61, 90_000)}, } { for _, mode := range []string{"frame", "json stream"} { t.Run(mode+"/"+bad.name, func(t *testing.T) { @@ -136,7 +138,7 @@ func TestTimestampRefused(t *testing.T) { if err != nil { t.Fatal(err) } - if err := publishData(mode, broadcast, track, []byte("x"), &bad.at); !errors.Is(err, moq.ErrTimeOverflow) { + if err := publishData(mode, broadcast, track, []byte("x"), bad.at); !errors.Is(err, moq.ErrTimeOverflow) { t.Fatalf("err = %v, want ErrTimeOverflow", err) } }) diff --git a/go/wrapper/datagram.go b/go/wrapper/datagram.go index d76649dd885..894b4b7cb7e 100644 --- a/go/wrapper/datagram.go +++ b/go/wrapper/datagram.go @@ -36,7 +36,9 @@ func (p *DatagramProducer) Name() (string, error) { return p.inner.Name() } func (p *DatagramProducer) Info() (DatagramInfo, error) { return p.inner.Info() } // Append sends a datagram and returns its sequence; keep payloads below the path MTU. -func (p *DatagramProducer) Append(timestamp *Timestamp, payload []byte) (uint64, error) { +// The timestamp must match the track: on a scale-less track it is Unscaled at the +// sequence to write. +func (p *DatagramProducer) Append(timestamp Timestamp, payload []byte) (uint64, error) { return p.inner.Append(timestamp, payload) } diff --git a/go/wrapper/example_test.go b/go/wrapper/example_test.go index 607e04aad72..4a910e854e3 100644 --- a/go/wrapper/example_test.go +++ b/go/wrapper/example_test.go @@ -100,8 +100,8 @@ func ExampleClient_Route() { _ = route.Announce() // 20 ms in, counted on the Opus 48 kHz sample clock. - pts := moq.Timestamp{Value: 960, Scale: 48_000} - if err := media.WriteFrame(moq.Frame{Payload: []byte("opus frame"), Timestamp: &pts}); err != nil { + pts := moq.TimestampFromScale(960, 48_000) + if err := media.WriteFrame(moq.Frame{Payload: []byte("opus frame"), Timestamp: pts}); err != nil { log.Fatal(err) } diff --git a/go/wrapper/json/json.go b/go/wrapper/json/json.go index 5df55a07ff1..5d070c06b67 100644 --- a/go/wrapper/json/json.go +++ b/go/wrapper/json/json.go @@ -139,8 +139,9 @@ func (p *StreamProducer) Finish() error { type Timed struct { // Value is the JSON document. Value stdjson.RawMessage - // Timestamp is on the broadcast clock, or nil on an untimed track. - Timestamp *moq.Timestamp + // Timestamp is on the broadcast clock, or moq.Unscaled at the frame's group key on a + // scale-less track. + Timestamp moq.Timestamp } func timed(value *ffi.MoqJsonValue) *Timed { @@ -325,10 +326,10 @@ func (WindowEventSkip) windowEvent() {} type WindowTimed struct { // Event is the change to the window. Event WindowEvent - // Timestamp is on the broadcast clock, or nil on an untimed track. A record - // first learned of from a group's restatement carries that frame's time, not - // its original push time. - Timestamp *moq.Timestamp + // Timestamp is on the broadcast clock, or moq.Unscaled at the frame's group key on a + // scale-less track. A record first learned of from a group's restatement carries that + // frame's time, not its original push time. + Timestamp moq.Timestamp } func windowTimed(value *ffi.MoqJsonWindowTimed) (*WindowTimed, error) { diff --git a/go/wrapper/json/json_test.go b/go/wrapper/json/json_test.go index 312d7e7d730..0613367dbf2 100644 --- a/go/wrapper/json/json_test.go +++ b/go/wrapper/json/json_test.go @@ -72,7 +72,7 @@ func TestTracks(t *testing.T) { } defer snapshotConsumer.Cancel() for viewers := 1; viewers <= 4; viewers++ { - at := moq.Timestamp{Value: uint64(viewers) * 1000, Scale: scale} + at := moq.TimestampFromScale(uint64(viewers)*1000, scale) if err := snapshot.Update(map[string]any{"viewers": viewers}, at); err != nil { t.Fatal(err) } @@ -91,7 +91,7 @@ func TestTracks(t *testing.T) { if decoded["viewers"] != float64(viewers) { t.Fatalf("snapshot = %s, want viewers %d", value.Value, viewers) } - if want := (moq.Timestamp{Value: uint64(viewers) * 1000, Scale: scale}); value.Timestamp == nil || *value.Timestamp != want { + if want := moq.TimestampFromScale(uint64(viewers)*1000, scale); !reflect.DeepEqual(value.Timestamp, want) { t.Fatalf("timestamp = %v, want %v", value.Timestamp, want) } } @@ -111,7 +111,7 @@ func TestTracks(t *testing.T) { t.Fatal(err) } defer streamConsumer.Cancel() - at := moq.Timestamp{Value: 5, Scale: 1_000} + at := moq.TimestampFromScale(5, 1_000) if err := stream.Append(map[string]any{"n": 1}, at); err != nil { t.Fatal(err) } @@ -119,7 +119,7 @@ func TestTracks(t *testing.T) { if err != nil { t.Fatal(err) } - if string(record.Value) != `{"n":1}` || record.Timestamp == nil || *record.Timestamp != at { + if string(record.Value) != `{"n":1}` || !reflect.DeepEqual(record.Timestamp, at) { t.Fatalf("record = %s at %v, want %v", record.Value, record.Timestamp, at) } @@ -153,15 +153,15 @@ func TestWindow(t *testing.T) { } defer windowConsumer.Cancel() - ms := func(n int) *moq.Timestamp { - return &moq.Timestamp{Value: uint64(n) * 1000, Scale: scale} + ms := func(n int) moq.Timestamp { + return moq.TimestampFromScale(uint64(n)*1000, scale) } for n := range 3 { - if err := window.Push(map[string]any{"n": n}, *ms(n)); err != nil { + if err := window.Push(map[string]any{"n": n}, ms(n)); err != nil { t.Fatal(err) } } - if err := window.Pop(2, *ms(3)); err != nil { + if err := window.Pop(2, ms(3)); err != nil { t.Fatal(err) } @@ -185,7 +185,7 @@ func TestWindow(t *testing.T) { if err := window.Cut(); err != nil { t.Fatal(err) } - if err := window.Push(map[string]any{"n": 3}, *ms(4)); err != nil { + if err := window.Push(map[string]any{"n": 3}, ms(4)); err != nil { t.Fatal(err) } if err := window.Finish(); err != nil { diff --git a/go/wrapper/media/read_test.go b/go/wrapper/media/read_test.go index b284cd71e03..842c9297553 100644 --- a/go/wrapper/media/read_test.go +++ b/go/wrapper/media/read_test.go @@ -1,6 +1,7 @@ package media import ( + "reflect" "testing" "time" @@ -9,27 +10,27 @@ import ( ) func TestMediaReadPreservesFramesAndBreaks(t *testing.T) { - tick := moq.Timestamp{Value: 1, Scale: 90_000} + tick := moq.TimestampFromScale(1, 90_000) frame, err := mediaRead(ffi.MoqMediaReadFrame{Frame: ffi.MoqMediaFrame{Payload: []byte("encoded"), Timestamp: tick, Keyframe: true}}) if err != nil { t.Fatal(err) } value, ok := frame.(ReadFrame) - if !ok || string(value.Frame.Payload) != "encoded" || value.Frame.Timestamp != tick || !value.Frame.Keyframe { + if !ok || string(value.Frame.Payload) != "encoded" || !reflect.DeepEqual(value.Frame.Timestamp, tick) || !value.Frame.Keyframe { t.Fatalf("frame: %#v", frame) } gap, err := mediaRead(ffi.MoqMediaReadGap{Timestamp: tick, Duration: 78901 * time.Microsecond}) if err != nil { t.Fatal(err) } - if gap != (ReadGap{Timestamp: tick, Duration: 78901 * time.Microsecond}) { + if !reflect.DeepEqual(gap, ReadGap{Timestamp: tick, Duration: 78901 * time.Microsecond}) { t.Fatalf("gap: %#v", gap) } end, err := mediaRead(ffi.MoqMediaReadEnd{End: tick}) if err != nil { t.Fatal(err) } - if end != (ReadEnd{End: tick}) { + if !reflect.DeepEqual(end, ReadEnd{End: tick}) { t.Fatalf("end: %#v", end) } } diff --git a/go/wrapper/moq_test.go b/go/wrapper/moq_test.go index 43afd81e650..0b05b9da426 100644 --- a/go/wrapper/moq_test.go +++ b/go/wrapper/moq_test.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "errors" "fmt" + "reflect" "runtime" "slices" "sync" @@ -129,7 +130,7 @@ func TestDynamicBroadcastRequest(t *testing.T) { if err != nil { t.Fatal(err) } - if frame == nil || string(frame.Payload) != string(payload) || frame.Timestamp == nil || *frame.Timestamp != *ts(0) { + if frame == nil || string(frame.Payload) != string(payload) || !sameTime(frame.Timestamp, ts(0)) { t.Fatalf("frame = %+v, want payload=%q ts=0", frame, payload) } @@ -248,7 +249,7 @@ func TestEncodeAudioWithOpusObject(t *testing.T) { codec.Destroy() } // One 20 ms Opus frame of silence at 48 kHz mono. - if err := producer.Write(moqaudio.Frame{Timestamp: moq.Timestamp{Scale: 48_000}, Data: make([]byte, 960*4)}); err != nil { + if err := producer.Write(moqaudio.Frame{Timestamp: moq.TimestampFromScale(0, 48_000), Data: make([]byte, 960*4)}); err != nil { t.Fatal(err) } if !codecFirst { @@ -328,7 +329,7 @@ func TestAudioConsumerDelay(t *testing.T) { // Two 20 ms Opus frames of silence at 48 kHz mono. for i := range 2 { - if err := audio.Write(moqaudio.Frame{Timestamp: moq.Timestamp{Value: uint64(i) * 960, Scale: 48_000}, Data: make([]byte, 960*4)}); err != nil { + if err := audio.Write(moqaudio.Frame{Timestamp: moq.TimestampFromScale(uint64(i)*960, 48_000), Data: make([]byte, 960*4)}); err != nil { t.Fatal(err) } } @@ -380,7 +381,7 @@ func TestEncodeAudioFrameDurations(t *testing.T) { t.Fatal(err) } // 2.5 ms of silence at 48 kHz mono f32: exactly one encoded frame. - if err := fine.Write(moqaudio.Frame{Timestamp: moq.Timestamp{Scale: 48_000}, Data: make([]byte, 120*4)}); err != nil { + if err := fine.Write(moqaudio.Frame{Timestamp: moq.TimestampFromScale(0, 48_000), Data: make([]byte, 120*4)}); err != nil { t.Fatal(err) } if err := fine.Finish(); err != nil { @@ -463,7 +464,7 @@ func testDecodeVideoFrame(t *testing.T, surface bool) { t.Fatal(err) } for i := range 10 { - if err := video.Write(moqvideo.Frame{Timestamp: moq.Timestamp{Value: uint64(i) * 1001, Scale: 30_000}, Data: rgba}); err != nil { + if err := video.Write(moqvideo.Frame{Timestamp: moq.TimestampFromScale(uint64(i)*1001, 30_000), Data: rgba}); err != nil { t.Fatal(err) } } @@ -494,7 +495,7 @@ func testDecodeVideoFrame(t *testing.T, surface bool) { // Keep the encoder fed so the decoder sees frames after it joined. for i := 10; i < 40; i++ { - if err := video.Write(moqvideo.Frame{Timestamp: moq.Timestamp{Value: uint64(i) * 1001, Scale: 30_000}, Data: rgba}); err != nil { + if err := video.Write(moqvideo.Frame{Timestamp: moq.TimestampFromScale(uint64(i)*1001, 30_000), Data: rgba}); err != nil { t.Fatal(err) } } @@ -565,7 +566,7 @@ func TestInsertDatagramAndLargest(t *testing.T) { if err != nil { t.Fatal(err) } - if err := track.Insert(moq.Datagram{Sequence: 7, Payload: []byte("seven")}); err != nil { + if err := track.Insert(moq.Datagram{Sequence: 7, Timestamp: moq.Unscaled(7), Payload: []byte("seven")}); err != nil { t.Fatal(err) } datagram, err := consumer.Recv(ctx) @@ -575,7 +576,7 @@ func TestInsertDatagramAndLargest(t *testing.T) { if datagram == nil || datagram.Sequence != 7 || string(datagram.Payload) != "seven" { t.Fatalf("got datagram %+v", datagram) } - if next, err := track.Append(nil, []byte("next")); err != nil || next != 8 { + if next, err := track.Append(moq.Unscaled(8), []byte("next")); err != nil || next != 8 { t.Fatalf("appended at %d, err %v", next, err) } if err := consumer.SetPriority(12); err != nil { @@ -835,7 +836,7 @@ func TestLocalPublishConsumeAudio(t *testing.T) { t.Fatalf("expected a media frame, got %T", read) } frame := &value.Frame - if string(frame.Payload) != string(payload) || moq.TimestampToDuration(frame.Timestamp) != time.Second { + if string(frame.Payload) != string(payload) || durationOf(t, frame.Timestamp) != time.Second { t.Fatalf("frame = %+v, want payload=%q ts=1000000", frame, payload) } @@ -861,7 +862,7 @@ func TestLocalPublishConsumeAudio(t *testing.T) { if err != nil { t.Fatal(err) } - if frame == nil || string(frame.Payload) != string(payload) || moq.TimestampToDuration(frame.Timestamp) != time.Second { + if frame == nil || string(frame.Payload) != string(payload) || durationOf(t, frame.Timestamp) != time.Second { t.Fatalf("decoded frame = %+v, want payload=%q ts=1000000", frame, payload) } } @@ -895,7 +896,7 @@ func TestTrackPublishConsume(t *testing.T) { if frame == nil { t.Fatal("expected a frame") } - if string(frame.Payload) != "hello" || frame.Timestamp == nil || *frame.Timestamp != *ts(12_345) { + if string(frame.Payload) != "hello" || !sameTime(frame.Timestamp, ts(12_345)) { t.Fatalf("frame = %+v, want payload=hello ts=12345", frame) } @@ -921,7 +922,7 @@ func TestTrackPublishConsume(t *testing.T) { if frame == nil { t.Fatal("expected a group frame") } - if string(frame.Payload) != "group" || frame.Timestamp == nil || *frame.Timestamp != *ts(23_456) { + if string(frame.Payload) != "group" || !sameTime(frame.Timestamp, ts(23_456)) { t.Fatalf("frame = %+v, want payload=group ts=23456", frame) } } @@ -1014,7 +1015,7 @@ func TestReadFrameSkipsEmptyThenPopulatedGroups(t *testing.T) { if err != nil { t.Fatal(err) } - if frame == nil || string(frame.Payload) != "populated" || frame.Timestamp == nil || *frame.Timestamp != *ts(2_000) { + if frame == nil || string(frame.Payload) != "populated" || !sameTime(frame.Timestamp, ts(2_000)) { t.Fatalf("frame = %+v, want payload=populated ts=2000", frame) } } @@ -1124,7 +1125,7 @@ func TestDynamicTrackRequest(t *testing.T) { if err != nil { t.Fatal(err) } - if frame == nil || string(frame.Payload) != string(payload) || frame.Timestamp == nil || *frame.Timestamp != *ts(0) { + if frame == nil || string(frame.Payload) != string(payload) || !sameTime(frame.Timestamp, ts(0)) { t.Fatalf("frame = %+v, want payload=%q ts=0", frame, payload) } if err := track.Finish(); err != nil { @@ -1217,10 +1218,12 @@ func TestDynamicTrackRequestCanPublishAudio(t *testing.T) { t.Fatalf("expected a media frame, got %T", read) } frame := &value.Frame - if string(frame.Payload) != string(payload) || moq.TimestampToDuration(frame.Timestamp) != 20*time.Millisecond { + if string(frame.Payload) != string(payload) || durationOf(t, frame.Timestamp) != 20*time.Millisecond { t.Fatalf("frame = %+v, want payload=%q ts=20000", frame, payload) } - // The explicit pause marker is an encoder end even while the track stays open. + // The explicit pause marker is an encoder end even while the track stays open. The marker + // is a group of its own, and group starts strictly increase, so it ends one microsecond past + // the frame's group: the producer doesn't parse how long the packet runs. if err := media.Discontinuity(); err != nil { t.Fatal(err) } @@ -1231,8 +1234,8 @@ func TestDynamicTrackRequestCanPublishAudio(t *testing.T) { if err != nil { t.Fatal(err) } - if value, ok := end.(moqmedia.ReadEnd); !ok || moq.TimestampToDuration(value.End) != 20*time.Millisecond { - t.Fatalf("expected encoder end, got %T", end) + if value, ok := end.(moqmedia.ReadEnd); !ok || durationOf(t, value.End) != 20*time.Millisecond+time.Microsecond { + t.Fatalf("expected encoder end at 20001us, got %T %+v", end, end) } if next, err := mediaConsumer.Next(ctx); next != nil || err != nil { t.Fatalf("after finish: %v %v", next, err) @@ -2024,11 +2027,21 @@ func TestCancellingAHiddenAnswerRefusesTheRequest(t *testing.T) { } } -// us is a raw frame's optional timestamp in microseconds. -func us(v uint64) *uint64 { return &v } - // ts is a presentation time in microseconds, the scale timedInfo declares. -func ts(us uint64) *moq.Timestamp { return &moq.Timestamp{Value: us, Scale: 1_000_000} } +func ts(us uint64) moq.Timestamp { return moq.TimestampFromScale(us, 1_000_000) } + +// sameTime reports whether a and b are the same timestamp, comparing Scale by value. +func sameTime(a, b moq.Timestamp) bool { return reflect.DeepEqual(a, b) } + +// durationOf is t as a time.Duration, failing the test when it has no time. +func durationOf(t *testing.T, ts moq.Timestamp) time.Duration { + t.Helper() + d, err := moq.TimestampToDuration(ts) + if err != nil { + t.Fatal(err) + } + return d +} // timedInfo declares the microsecond timeline used by timed raw fixtures. func timedInfo() *moq.TrackInfo { diff --git a/go/wrapper/publish.go b/go/wrapper/publish.go index 57c68fecb7c..96cdaa622bc 100644 --- a/go/wrapper/publish.go +++ b/go/wrapper/publish.go @@ -211,7 +211,8 @@ func (t *TrackProducer) SkipGroups(start, end uint64) error { return t.inner.SkipGroups(start, end) } -// WriteFrame writes frame as a single-frame group. +// WriteFrame writes frame as a single-frame group, which starts at its timestamp: on a +// scale-less track, moq.Unscaled at the new group's key. func (t *TrackProducer) WriteFrame(frame Frame) error { return t.inner.WriteFrame(frame) } @@ -282,7 +283,8 @@ func (g *GroupProducer) Ready(ctx context.Context, size uint64) error { } // WriteFrame appends frame to the group. It never blocks: without Ready, a writer -// that outpaces its readers sheds the front of the group instead. +// that outpaces its readers sheds the front of the group instead. On a scale-less track +// the frame's timestamp is Unscaled at this group's Sequence. func (g *GroupProducer) WriteFrame(frame Frame) error { return g.inner.WriteFrame(frame) } diff --git a/go/wrapper/records.go b/go/wrapper/records.go index 38fbe185ee4..fdc03bdda23 100644 --- a/go/wrapper/records.go +++ b/go/wrapper/records.go @@ -13,40 +13,59 @@ import ( // Timestamp is an exact media time: Value units at Scale units per second, with Value // at most 2^61 - 1 and Scale non-zero and at most 2^62 - 1. A read carries the track's // scale, so a tick such as 90 kHz survives a read and republish exactly. +// +// A scale-less track has no time, so there Scale is nil and Value is the frame's group key +// (a datagram's sequence); build one with Unscaled. The zero Timestamp is Unscaled(0). +// Scale is a pointer, so == compares its address: compare two with reflect.DeepEqual. type Timestamp = ffi.MoqTimestamp +// Unscaled is the scale-less timestamp at group key key, or a datagram's sequence. +func Unscaled(key uint64) Timestamp { + return Timestamp{Value: key} +} + +// TimestampFromScale is value units at scale units per second; a write refuses one past +// the bounds above with ErrTimeOverflow. +func TimestampFromScale(value, scale uint64) Timestamp { + return Timestamp{Value: value, Scale: &scale} +} + // TimestampFromDuration converts d exactly, at nanosecond scale, refusing a negative d. func TimestampFromDuration(d time.Duration) (Timestamp, error) { if d < 0 { return Timestamp{}, fmt.Errorf("negative timestamp: %v", d) } - return Timestamp{Value: uint64(d), Scale: uint64(time.Second)}, nil + return TimestampFromScale(uint64(d), uint64(time.Second)), nil } // TimestampToDuration converts t to a time.Duration, rounded down to a whole -// nanosecond, saturating past time.Duration's ~292 years. It panics on a zero -// Scale, which no read returns. -func TimestampToDuration(t Timestamp) time.Duration { - if t.Scale == 0 { - panic("moq: timestamp with a zero scale") +// nanosecond, saturating past time.Duration's ~292 years. It fails on an unscaled +// timestamp, which has no time, and on a zero Scale, which no read returns. +func TimestampToDuration(t Timestamp) (time.Duration, error) { + if t.Scale == nil { + return 0, fmt.Errorf("unscaled timestamp #%d has no time", t.Value) + } + scale := *t.Scale + if scale == 0 { + return 0, fmt.Errorf("timestamp with a zero scale") } hi, lo := bits.Mul64(t.Value, uint64(time.Second)) - if hi >= t.Scale { - return time.Duration(math.MaxInt64) + if hi >= scale { + return time.Duration(math.MaxInt64), nil } - ns, _ := bits.Div64(hi, lo, t.Scale) + ns, _ := bits.Div64(hi, lo, scale) if ns > math.MaxInt64 { - return time.Duration(math.MaxInt64) + return time.Duration(math.MaxInt64), nil } - return time.Duration(ns) + return time.Duration(ns), nil } -// Frame is a raw track frame: a payload and its presentation timestamp. -// Timestamp is nil on an untimed track, on both publish and receive. +// Frame is a raw track frame: a payload and its presentation timestamp, on both publish +// and receive. On a scale-less track the timestamp is Unscaled at the frame's group key. type Frame = ffi.MoqFrame // Datagram is a best-effort track datagram as received: sequence number, timestamp -// (nil from an untimed track), and payload. +// (Unscaled at the sequence on a scale-less track), and payload. type Datagram = ffi.MoqDatagram // DatagramInfo fixes a datagram track's publisher priority and optional timestamp scale. diff --git a/go/wrapper/records_internal_test.go b/go/wrapper/records_internal_test.go index 1ef5b175089..ef28a470579 100644 --- a/go/wrapper/records_internal_test.go +++ b/go/wrapper/records_internal_test.go @@ -13,11 +13,11 @@ func TestTimestampFromDuration(t *testing.T) { if err != nil { t.Fatal(err) } - if want := (Timestamp{Value: 1_500_000_000, Scale: 1_000_000_000}); got != want { + if want := TimestampFromScale(1_500_000_000, 1_000_000_000); !reflect.DeepEqual(got, want) { t.Fatalf("TimestampFromDuration(1.5s) = %+v, want %+v", got, want) } - if got := TimestampToDuration(got); got != 1500*time.Millisecond { - t.Fatalf("round trip = %v, want 1.5s", got) + if got, err := TimestampToDuration(got); err != nil || got != 1500*time.Millisecond { + t.Fatalf("round trip = %v, %v, want 1.5s", got, err) } if _, err := TimestampFromDuration(-time.Nanosecond); err == nil { t.Fatal("negative duration converted") @@ -31,33 +31,35 @@ func TestTimestampToDuration(t *testing.T) { in Timestamp want time.Duration }{ - {Timestamp{Value: 0, Scale: 90_000}, 0}, + {TimestampFromScale(0, 90_000), 0}, // 11111.1 ns, 666666666.6 ns, and half a nanosecond all round down. - {Timestamp{Value: 1, Scale: 90_000}, 11_111}, - {Timestamp{Value: 2, Scale: 3}, 666_666_666}, - {Timestamp{Value: 1, Scale: 2_000_000_000}, 0}, - {Timestamp{Value: 1001, Scale: 30_000}, 33_366_666}, - {Timestamp{Value: math.MaxInt64, Scale: 1_000_000_000}, math.MaxInt64}, - {Timestamp{Value: math.MaxInt64/1000 + 1, Scale: 1_000_000}, math.MaxInt64}, - {Timestamp{Value: 1<<61 - 1, Scale: 1}, math.MaxInt64}, - {Timestamp{Value: math.MaxUint64, Scale: math.MaxUint64}, time.Second}, + {TimestampFromScale(1, 90_000), 11_111}, + {TimestampFromScale(2, 3), 666_666_666}, + {TimestampFromScale(1, 2_000_000_000), 0}, + {TimestampFromScale(1001, 30_000), 33_366_666}, + {TimestampFromScale(math.MaxInt64, 1_000_000_000), math.MaxInt64}, + {TimestampFromScale(math.MaxInt64/1000+1, 1_000_000), math.MaxInt64}, + {TimestampFromScale(1<<61-1, 1), math.MaxInt64}, + {TimestampFromScale(math.MaxUint64, math.MaxUint64), time.Second}, } for _, c := range cases { - if got := TimestampToDuration(c.in); got != c.want { - t.Errorf("TimestampToDuration(%+v) = %d, want %d", c.in, got, c.want) + if got, err := TimestampToDuration(c.in); err != nil || got != c.want { + t.Errorf("TimestampToDuration(%+v) = %d, %v, want %d", c.in, got, err, c.want) } } } -// A zero scale, such as an unset Timestamp{}, panics rather than reading as the -// longest duration. -func TestTimestampToDurationZeroScale(t *testing.T) { - defer func() { - if recover() == nil { - t.Fatal("TimestampToDuration(Timestamp{}) did not panic") +// An unscaled timestamp has no time, and a zero scale no duration, so both fail rather +// than reading as some duration. +func TestTimestampToDurationRefused(t *testing.T) { + for _, in := range []Timestamp{Unscaled(7), Unscaled(0), TimestampFromScale(1, 0)} { + if got, err := TimestampToDuration(in); err == nil { + t.Errorf("TimestampToDuration(%+v) = %v, want an error", in, got) } - }() - TimestampToDuration(Timestamp{}) + } + if got := Unscaled(7); got.Scale != nil || got.Value != 7 { + t.Fatalf("Unscaled(7) = %+v", got) + } } // The zero Subscription is moq-net's defaults, a stated zero Delay, and the flags reach the diff --git a/go/wrapper/runtime_bench_test.go b/go/wrapper/runtime_bench_test.go index 09f59eae517..708628e67b7 100644 --- a/go/wrapper/runtime_bench_test.go +++ b/go/wrapper/runtime_bench_test.go @@ -200,8 +200,8 @@ func benchThroughput(b *testing.B, c benchCase) { done <- err return } - want := moq.Timestamp{Value: uint64(n*frames + f), Scale: 1_000_000} - if frame == nil || !bytes.Equal(frame.Payload, payload) || frame.Timestamp == nil || *frame.Timestamp != want { + want := moq.TimestampFromScale(uint64(n*frames+f), 1_000_000) + if frame == nil || !bytes.Equal(frame.Payload, payload) || frame.Timestamp.Value != want.Value || frame.Timestamp.Scale == nil || *frame.Timestamp.Scale != *want.Scale { done <- fmt.Errorf("missing, reordered, or corrupt frame %d of round %d", f, n) return } @@ -215,8 +215,8 @@ func benchThroughput(b *testing.B, c benchCase) { return err } for f := range frames { - timestamp := moq.Timestamp{Value: uint64(n*frames + f), Scale: 1_000_000} - if err := group.WriteFrame(moq.Frame{Payload: payload, Timestamp: ×tamp}); err != nil { + timestamp := moq.TimestampFromScale(uint64(n*frames+f), 1_000_000) + if err := group.WriteFrame(moq.Frame{Payload: payload, Timestamp: timestamp}); err != nil { return err } } diff --git a/go/wrapper/types.go b/go/wrapper/types.go index ca456c12c20..4660e3fed5b 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -67,7 +67,7 @@ type ( AnchorAbsolute = ffi.MoqAnchorAbsolute // AnchorRelative is the group Groups before the latest one, clamped to group 0: 0 is the latest. AnchorRelative = ffi.MoqAnchorRelative - // AnchorTime is the group containing a presentation Timestamp. + // AnchorTime is the group containing a time, an Unscaled Timestamp in the track's timescale; a scaled one is refused. AnchorTime = ffi.MoqAnchorTime // Order is which of a subscription's pending groups the publisher sends first. Order = ffi.MoqOrder diff --git a/js/crypto/README.md b/js/crypto/README.md index 6234988d902..8f5f9da6cca 100644 --- a/js/crypto/README.md +++ b/js/crypto/README.md @@ -30,7 +30,7 @@ track's timescale, as the wire carries it). Announce the broadcast with that ins ```ts import { Sign } from "@moq/crypto"; -import { Path } from "@moq/net"; +import { Path, Time } from "@moq/net"; const { Broadcast, SecretKey } = Sign; @@ -42,8 +42,9 @@ const signed = new Broadcast.Producer(route.broadcast, key, { path, instance: ro const messages = signed.createTrack("messages"); route.announce(); +// A scale-less track, so each frame carries its group's key. const group = messages.appendGroup(); -await group.writeFrame(undefined, payload); +await group.writeFrame(Time.Timestamp.unscaled(group.sequence), payload); group.finish(); ``` diff --git a/js/crypto/src/e2ee/datagram.ts b/js/crypto/src/e2ee/datagram.ts index 49e37bad909..9f0db7fcf19 100644 --- a/js/crypto/src/e2ee/datagram.ts +++ b/js/crypto/src/e2ee/datagram.ts @@ -15,8 +15,11 @@ import { DatagramWindow } from "./window.ts"; export interface Datagram { /** The datagram track's own sequence. */ sequence: bigint; - /** Presentation timestamp, or `undefined` when untimed. Not authenticated. */ - timestamp?: Time.Timestamp; + /** + * Presentation timestamp, or on a scale-less track {@link Time.Timestamp.unscaled} at the + * sequence. Not authenticated. + */ + timestamp: Time.Timestamp; /** Decrypted application bytes. */ plaintext: Uint8Array; } @@ -67,18 +70,26 @@ export class Producer { get closed(): NetDatagram.Producer["closed"] { return this.#inner.closed; } - /** Encrypt a payload at the next datagram sequence, returning that sequence. */ - async append(timestamp: Time.Timestamp | undefined, plaintext: Uint8Array): Promise { - const sequence = this.#next; + /** + * Encrypt a payload at the next datagram sequence, returning that sequence. + * + * A scale-less track has no time, so there `timestamp` is {@link Time.Timestamp.unscaled} at the + * sequence to write, as with {@link insert}. + */ + async append(timestamp: Time.Timestamp, plaintext: Uint8Array): Promise { + const sequence = timestamp.scale === undefined ? BigInt(timestamp.value) : this.#next; await this.insert(sequence, timestamp, plaintext); return sequence; } - /** Encrypt at an explicit unused sequence, refusing predictable failures before allocating it. */ - async insert(sequence: bigint, timestamp: Time.Timestamp | undefined, plaintext: Uint8Array): Promise { + /** + * Encrypt at an explicit unused sequence, refusing predictable failures before allocating it. + * On a scale-less track `timestamp` is {@link Time.Timestamp.unscaled} at `sequence`. + */ + async insert(sequence: bigint, timestamp: Time.Timestamp, plaintext: Uint8Array): Promise { if (this.#finished || this.#aborted) throw this.#aborted ?? new Error("datagram track is closed"); checkGroup(sequence); if (sequence < this.#next) throw new Failure("reuse"); - const at = NetDatagram.rescale(this.#info, timestamp); + const at = NetDatagram.rescaleIn(this.#info, timestamp, sequence); const iv = this.#key.admitSeal(sequence, 0, plaintext.byteLength, MAX_DATAGRAM_PAYLOAD); this.#next = sequence + 1n; const owned = plaintext.slice(); diff --git a/js/crypto/src/e2ee/e2ee.test.ts b/js/crypto/src/e2ee/e2ee.test.ts index 2198307956e..ef143c62f15 100644 --- a/js/crypto/src/e2ee/e2ee.test.ts +++ b/js/crypto/src/e2ee/e2ee.test.ts @@ -844,7 +844,9 @@ test("a refused timestamp does not burn the frame", async () => { const p = await pair("video"); await p.net.info(); const group = p.producer.appendGroup(); - await expect(group.writeFrame(undefined, text("untimed"))).rejects.toThrow("timestamp"); + await expect(group.writeFrame(Time.Timestamp.unscaled(group.sequence), text("untimed"))).rejects.toThrow( + "timestamp", + ); await group.writeFrame(ms(1), text("ok")); group.finish(); expect(await readText(await recvGroup(p.consumer))).toBe("ok"); @@ -860,7 +862,9 @@ test("produce waits for an unaccepted track's timescale", async () => { // The timescale is known from the start, so mismatches spend neither a sequence nor a frame. const group = producer.createGroup(1n); - await expect(group.writeFrame(undefined, text("untimed"))).rejects.toThrow("timestamp"); + await expect(group.writeFrame(Time.Timestamp.unscaled(group.sequence), text("untimed"))).rejects.toThrow( + "timestamp", + ); await group.writeFrame(ms(2), text("frame")); group.finish(); expect(await readText(await recvGroup(consumer))).toBe("frame"); diff --git a/js/crypto/src/e2ee/group.ts b/js/crypto/src/e2ee/group.ts index 94528e43cd2..59f0b327415 100644 --- a/js/crypto/src/e2ee/group.ts +++ b/js/crypto/src/e2ee/group.ts @@ -10,8 +10,11 @@ import { MAX_GROUPED_PAYLOAD, TAG_LEN } from "./limits.ts"; /** A decrypted grouped frame. */ export interface Frame { - /** Presentation timestamp, or `undefined` when untimed. Not authenticated. */ - timestamp?: Time.Timestamp; + /** + * Presentation timestamp, or on a scale-less track {@link Time.Timestamp.unscaled} at the + * group's key. Not authenticated. + */ + timestamp: Time.Timestamp; /** Decrypted application bytes. */ plaintext: Uint8Array; } @@ -57,16 +60,16 @@ export class Producer { * since its transport index could no longer match the nonce. Rejects with {@link Failure} * `oversize`, `exhausted`, or `identity`, or a net error. */ - async writeFrame(timestamp: Time.Timestamp | undefined, plaintext: Uint8Array): Promise { + async writeFrame(timestamp: Time.Timestamp, plaintext: Uint8Array): Promise { if (this.#closed || this.#writer.closed) throw this.#aborted ?? this.#writer.aborted ?? new Error("group is closed"); if (plaintext.byteLength + TAG_LEN > MAX_GROUPED_PAYLOAD) throw new Failure("oversize"); - const at = NetDatagram.rescale(this.#writer.track, timestamp); + const at = NetDatagram.rescaleIn(this.#writer.track, timestamp, this.sequence); const owned = plaintext.slice(); return this.#writer.pump.run(() => this.#write(at, owned)); } - async #write(timestamp: Time.Timestamp | undefined, plaintext: Uint8Array): Promise { + async #write(timestamp: Time.Timestamp, plaintext: Uint8Array): Promise { this.#check(); const frame = this.#next; const iv = this.#writer.key.admitSeal(this.sequence, frame, plaintext.byteLength, MAX_GROUPED_PAYLOAD); diff --git a/js/crypto/src/sign/group.ts b/js/crypto/src/sign/group.ts index adac8df45bf..75f6b100b71 100644 --- a/js/crypto/src/sign/group.ts +++ b/js/crypto/src/sign/group.ts @@ -11,8 +11,11 @@ import type { Scope } from "./scope.ts"; /** A grouped frame whose signature verified. */ export interface Frame { - /** Presentation timestamp at the track's timescale, or `undefined` when untimed. Signed. */ - timestamp?: Time.Timestamp; + /** + * Presentation timestamp at the track's timescale, or on a scale-less track + * {@link Time.Timestamp.unscaled} at the group's key. Signed. + */ + timestamp: Time.Timestamp; /** The application bytes, without the signature. */ payload: Uint8Array; } @@ -64,14 +67,15 @@ export class Producer { /** * Sign `payload` at the next frame index and write it with the signature appended. * - * The bytes are copied, so the caller may reuse its buffer. Rejects with the net error if - * the group is closed, the timestamp does not match the track, or the frame is too large. + * On a scale-less track `timestamp` is {@link Time.Timestamp.unscaled} at the group's key. The + * bytes are copied, so the caller may reuse its buffer. Rejects with the net error if the group + * is closed, the timestamp does not match the track, or the frame is too large. */ - async writeFrame(timestamp: Time.Timestamp | undefined, payload: Uint8Array): Promise { + async writeFrame(timestamp: Time.Timestamp, payload: Uint8Array): Promise { if (this.#closed || this.#writer.closed) throw this.#writer.aborted ?? new Error("group is closed"); const owned = payload.slice(); // Signed at the track's timescale, and written that way so a local reader sees the signed value too. - const at = NetDatagram.rescale(this.#writer.track, timestamp); + const at = NetDatagram.rescaleIn(this.#writer.track, timestamp, this.sequence); return this.#writer.pump.run(async () => { this.#check(); const body = await this.#writer.scope.sign(this.#writer.key, this.sequence, this.#next, at, owned); diff --git a/js/crypto/src/sign/index.ts b/js/crypto/src/sign/index.ts index 8b9e5b4704d..6796e32ed19 100644 --- a/js/crypto/src/sign/index.ts +++ b/js/crypto/src/sign/index.ts @@ -18,8 +18,9 @@ * const messages = signed.createTrack("messages"); * route.announce(); * + * // A scale-less track, so each frame carries its group's key. * const group = messages.appendGroup(); - * await group.writeFrame(undefined, payload); + * await group.writeFrame(Time.Timestamp.unscaled(group.sequence), payload); * group.finish(); * ``` * diff --git a/js/crypto/src/sign/scope.ts b/js/crypto/src/sign/scope.ts index cfeb63b4a78..d86a77ef3bf 100644 --- a/js/crypto/src/sign/scope.ts +++ b/js/crypto/src/sign/scope.ts @@ -32,23 +32,21 @@ export class Scope { * The message signed for `payload` at frame `frame` of group `group`, presented at `timestamp`. * * The timestamp is its integer value at the track's timescale, so a frame keeps its signature - * only on a hop that carries both unchanged. An untimed frame binds zero for both, which no - * timescale can be. A fractional value, which no hop carries, never verifies. + * only on a hop that carries both unchanged. A scale-less frame's timestamp is its group key, + * which `group` already binds, so it binds zero for both, which no timescale can be. A + * fractional value, which no hop carries, never verifies. */ - message( - group: bigint, - frame: number, - timestamp: Time.Timestamp | undefined, - payload: Uint8Array, - ): Uint8Array { - if (timestamp && !Number.isSafeInteger(timestamp.value)) throw new Failure("signature"); + message(group: bigint, frame: number, timestamp: Time.Timestamp, payload: Uint8Array): Uint8Array { + // Never reads a scale-less key, which may be past what a number holds. + const [scale, value] = timestamp.scale === undefined ? [0, 0] : [timestamp.scale, timestamp.value]; + if (!Number.isSafeInteger(value)) throw new Failure("signature"); const out = new Uint8Array(this.#prefix.byteLength + 32 + payload.byteLength); const view = new DataView(out.buffer); out.set(this.#prefix); let offset = this.#prefix.byteLength; view.setBigUint64(offset, group); offset += 8; - for (const field of [frame, timestamp?.scale ?? 0, timestamp?.value ?? 0]) { + for (const field of [frame, scale, value]) { view.setBigUint64(offset, BigInt(field)); offset += 8; } @@ -61,7 +59,7 @@ export class Scope { key: SecretKey, group: bigint, frame: number, - timestamp: Time.Timestamp | undefined, + timestamp: Time.Timestamp, payload: Uint8Array, ): Promise> { const signature = await key.sign(this.message(group, frame, timestamp, payload)); @@ -76,7 +74,7 @@ export class Scope { key: PublicKey, group: bigint, frame: number, - timestamp: Time.Timestamp | undefined, + timestamp: Time.Timestamp, body: Uint8Array, ): Promise { const split = body.byteLength - SIGNATURE_LEN; diff --git a/js/crypto/src/sign/sign.test.ts b/js/crypto/src/sign/sign.test.ts index 8eaf8448b1a..29ec65760bc 100644 --- a/js/crypto/src/sign/sign.test.ts +++ b/js/crypto/src/sign/sign.test.ts @@ -94,10 +94,10 @@ test("frames roundtrip signed, timestamps included", async () => { expect(second?.timestamp?.value).toBe(2); expect(second?.timestamp?.scale).toBe(Time.Timescale.MILLI); - // An untimed frame on a timed track is refused before it is signed, so it spends no index. - await expect(group.writeFrame(undefined, encoder.encode("untimed"))).rejects.toBeInstanceOf( - NetError.TimestampMismatch, - ); + // An unscaled frame on a timed track is refused before it is signed, so it spends no index. + await expect( + group.writeFrame(Time.Timestamp.unscaled(group.sequence), encoder.encode("untimed")), + ).rejects.toBeInstanceOf(NetError.TimestampMismatch); await group.writeFrame(Time.Timestamp.fromMillis(3), encoder.encode("late")); group.finish(); expect(await payload(received)).toBe("late"); @@ -119,15 +119,15 @@ test("a forged frame delivers nothing and ends only its group", async () => { // A relay's forgery: unsigned frames the length of a signed one. const forged = raw.appendGroup(); - forged.writeFrame({ payload: new Uint8Array(80) }); - forged.writeFrame({ payload: new Uint8Array(80) }); + forged.writeFrame({ payload: new Uint8Array(80), timestamp: Time.Timestamp.unscaled(forged.sequence) }); + forged.writeFrame({ payload: new Uint8Array(80), timestamp: Time.Timestamp.unscaled(forged.sequence) }); forged.finish(); const group = await track.recvGroup(); await expect(group?.readFrame()).rejects.toEqual(new Failure("signature")); await expect(group?.readFrame()).rejects.toEqual(new Failure("signature")); const next = signed.createGroup(1n); - await next.writeFrame(undefined, encoder.encode("after")); + await next.writeFrame(Time.Timestamp.unscaled(next.sequence), encoder.encode("after")); next.finish(); expect(await payload(await track.recvGroup())).toBe("after"); @@ -147,7 +147,7 @@ test("a frame replayed into another group or track is refused", async () => { const readB = consumer.subscribe("b", subscription); const group = a.signed.appendGroup(); - await group.writeFrame(undefined, encoder.encode("original")); + await group.writeFrame(Time.Timestamp.unscaled(group.sequence), encoder.encode("original")); group.finish(); const body = (await (await tap.recvGroup())?.readFrame())?.payload; if (!body) throw new Error("expected the signed body"); @@ -155,13 +155,13 @@ test("a frame replayed into another group or track is refused", async () => { // Group 1 of the same track. const replay = a.raw.appendGroup(); - replay.writeFrame({ payload: body }); + replay.writeFrame({ payload: body, timestamp: Time.Timestamp.unscaled(replay.sequence) }); replay.finish(); await expect((await readA.recvGroup())?.readFrame()).rejects.toEqual(new Failure("signature")); // The same group and index of another track. const moved = b.raw.createGroup(0n); - moved.writeFrame({ payload: body }); + moved.writeFrame({ payload: body, timestamp: Time.Timestamp.unscaled(moved.sequence) }); moved.finish(); await expect((await readB.recvGroup())?.readFrame()).rejects.toEqual(new Failure("signature")); @@ -176,7 +176,7 @@ test("a group sequence is never signed twice", async () => { const published = await publish(key, Path.from("mesh/node/ssh")); const track = published.signed.createTrack("stream"); const group = track.appendGroup(); - await group.writeFrame(undefined, encoder.encode("first")); + await group.writeFrame(Time.Timestamp.unscaled(group.sequence), encoder.encode("first")); group.abort(new Error("retry")); // Net would serve the aborted sequence again, but frames of both incarnations would verify, @@ -205,7 +205,7 @@ test("a broadcast signed under another key is refused", async () => { const published = await publish(impostor, path); const stream = published.signed.createTrack("stream"); const group = stream.appendGroup(); - await group.writeFrame(undefined, encoder.encode("hello")); + await group.writeFrame(Time.Timestamp.unscaled(group.sequence), encoder.encode("hello")); group.finish(); const track = (await published.consume(victim.public)).subscribe("stream", subscription); diff --git a/js/crypto/src/sign/vectors.test.ts b/js/crypto/src/sign/vectors.test.ts index 018d098ec6d..59670218cc1 100644 --- a/js/crypto/src/sign/vectors.test.ts +++ b/js/crypto/src/sign/vectors.test.ts @@ -28,9 +28,12 @@ function scope(row: Row): Scope { } // A row's timestamp at its timescale, or undefined when the timescale is 0 (untimed). -function timestamp(row: Row): Time.Timestamp | undefined { +// A scale-less row carries its group's key, which the signature does not bind. +function timestamp(row: Row): Time.Timestamp { const scale = Number(row.timescale); - return scale === 0 ? undefined : new Time.Timestamp(Number(row.timestamp), Time.Timescale(scale)); + return scale === 0 + ? Time.Timestamp.unscaled(BigInt(row.group as number | string)) + : new Time.Timestamp(Number(row.timestamp), Time.Timescale(scale)); } test("keys", async () => { diff --git a/js/hang/api/cmaf.api.md b/js/hang/api/cmaf.api.md index 3f8f50fb814..b9e75d2a6dd 100644 --- a/js/hang/api/cmaf.api.md +++ b/js/hang/api/cmaf.api.md @@ -7,7 +7,7 @@ import { Net } from "@moq/hang"; export class Format implements Format_1 { constructor(init: InitSegment); - decode(payload: Uint8Array, timestamp: Net.Time.Timestamp | undefined): Frame[]; + decode(payload: Uint8Array, timestamp: Net.Time.Timestamp): Frame[]; } export interface InitSegment { @@ -27,7 +27,7 @@ export interface Sample { timestamp: number; } -export function decodeDataSegment(segment: Uint8Array, init: InitSegment, timestamp: Net.Time.Timestamp | undefined): Sample[]; +export function decodeDataSegment(segment: Uint8Array, init: InitSegment, timestamp: Net.Time.Timestamp): Sample[]; export function decodeInitSegment(init: Uint8Array): InitSegment; ``` diff --git a/js/hang/api/container.api.md b/js/hang/api/container.api.md index 852ae5c71db..675e857224d 100644 --- a/js/hang/api/container.api.md +++ b/js/hang/api/container.api.md @@ -34,7 +34,7 @@ export interface ConsumerProps { export const DELAY_MAX: Net.Clock.Duration; export interface Format { - decode(payload: Uint8Array, timestamp: Net.Time.Timestamp | undefined): Frame[]; + decode(payload: Uint8Array, timestamp: Net.Time.Timestamp): Frame[]; end?(frame: Frame): Net.Clock.Duration | undefined; readonly kind?: "audio" | "video" | "data"; } @@ -72,6 +72,7 @@ export namespace Legacy { copyTo(buffer: Uint8Array): void; } export function encodeFrame(source: Uint8Array | Legacy.Source, timestamp: Net.Clock.Duration): Uint8Array; + export function packetDuration(config: AudioConfig): ((payload: Uint8Array) => Net.Clock.Duration | undefined) | undefined; } export * as Loc from "@moq/loc"; diff --git a/js/hang/bench/cmaf.ts b/js/hang/bench/cmaf.ts index 840fa6ccd4f..c44337853b4 100644 --- a/js/hang/bench/cmaf.ts +++ b/js/hang/bench/cmaf.ts @@ -4,8 +4,12 @@ * cost what its bytes do, not what its count claims. */ +import { Time } from "@moq/net"; import { decodeDataSegment, type InitSegment } from "../src/cmaf/decode.ts"; +// A scale-less track's timestamp, its group's key, which carries no time. +const UNTIMED = Time.Timestamp.unscaled(0n); + const runs = 200; /** A box of `type` around `parts`. */ @@ -50,7 +54,7 @@ function time(segment: Uint8Array, hostile: boolean): { samples: number; micros: const start = performance.now(); for (let run = 0; run < runs; run++) { try { - samples = decodeDataSegment(segment, init, undefined).length; + samples = decodeDataSegment(segment, init, UNTIMED).length; if (hostile) throw new Error("a hostile count decoded"); } catch (err) { if (!hostile || !String(err).includes("past the")) throw err; diff --git a/js/hang/src/cmaf/decode.test.ts b/js/hang/src/cmaf/decode.test.ts index 0b2d55cbeff..ba4ec29f61f 100644 --- a/js/hang/src/cmaf/decode.test.ts +++ b/js/hang/src/cmaf/decode.test.ts @@ -2,8 +2,12 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; +import { Time } from "@moq/net"; import { decodeDataSegment, decodeInitSegment, type InitSegment } from "./decode.ts"; +// A scale-less track's timestamp, its group's key, which carries no time. +const UNTIMED = Time.Timestamp.unscaled(0n); + // Shared with moq-mux, which decodes the same fragments, so both readers resolve a sample's size, // duration, and keyframe bit the same way. const FIXTURES = join( @@ -24,7 +28,7 @@ describe("CMAF sample defaults", () => { for (const fixture of cases) { test(fixture.name, () => { const init = decodeInitSegment(new Uint8Array(Buffer.from(fixture.init, "base64"))); - const samples = decodeDataSegment(new Uint8Array(Buffer.from(fixture.fragment, "base64")), init, undefined); + const samples = decodeDataSegment(new Uint8Array(Buffer.from(fixture.fragment, "base64")), init, UNTIMED); expect( samples.map(({ timestamp, duration, keyframe, data }) => ({ timestamp, @@ -87,11 +91,11 @@ describe("CMAF untrusted counts and sizes", () => { }); test("a run without a sample table reads defaults until the mdat ends", () => { - const valid = decodeDataSegment(fragment(box("trun", words(0, 3)), 3), INIT, undefined); + const valid = decodeDataSegment(fragment(box("trun", words(0, 3)), 3), INIT, UNTIMED); expect(valid.map((sample) => sample.duration)).toEqual([33_333, 33_333, 33_333]); // The largest count a u32 holds, over three bytes of media: refused before any sample. - expect(() => decodeDataSegment(fragment(box("trun", words(0, 0xffff_ffff)), 3), INIT, undefined)).toThrow( + expect(() => decodeDataSegment(fragment(box("trun", words(0, 0xffff_ffff)), 3), INIT, UNTIMED)).toThrow( "trun declares 4294967295 samples of 1 bytes, past the 3 mdat bytes left", ); }); @@ -99,7 +103,7 @@ describe("CMAF untrusted counts and sizes", () => { test("a sample table shorter than its count is refused before any sample", () => { // sample_size present: four bytes per entry, two entries for a declared million. const trun = box("trun", words(0x200, 1_000_000, 1, 1)); - expect(() => decodeDataSegment(fragment(trun, 2), INIT, undefined)).toThrow( + expect(() => decodeDataSegment(fragment(trun, 2), INIT, UNTIMED)).toThrow( "trun declares 1000000 samples but its table holds 2", ); }); @@ -107,15 +111,15 @@ describe("CMAF untrusted counts and sizes", () => { test("a box size that cannot advance or that overruns its parent is refused", () => { // size 1 with a 64-bit size of zero: no progress. const stalled = new Uint8Array([0, 0, 0, 1, ...new TextEncoder().encode("free"), 0, 0, 0, 0, 0, 0, 0, 0]); - expect(() => decodeDataSegment(stalled, INIT, undefined)).toThrow("Invalid free box size 0"); + expect(() => decodeDataSegment(stalled, INIT, UNTIMED)).toThrow("Invalid free box size 0"); // Smaller than its own header. - expect(() => decodeDataSegment(words(4, 0x6672_6565), INIT, undefined)).toThrow("Invalid free box size 4"); + expect(() => decodeDataSegment(words(4, 0x6672_6565), INIT, UNTIMED)).toThrow("Invalid free box size 4"); // Larger than the bytes left. - expect(() => decodeDataSegment(words(64, 0x6672_6565), INIT, undefined)).toThrow("Invalid free box size 64"); + expect(() => decodeDataSegment(words(64, 0x6672_6565), INIT, UNTIMED)).toThrow("Invalid free box size 64"); // A child overrunning its parent, though the input continues past it. const overrun = box("moof", words(16, 0x7472_6166)); const mdat = box("mdat", new Uint8Array(32)); - expect(() => decodeDataSegment(new Uint8Array([...overrun, ...mdat]), INIT, undefined)).toThrow( + expect(() => decodeDataSegment(new Uint8Array([...overrun, ...mdat]), INIT, UNTIMED)).toThrow( "Invalid traf box size 16", ); expect(() => decodeInitSegment(stalled)).toThrow("Invalid free box size 0"); diff --git a/js/hang/src/cmaf/decode.ts b/js/hang/src/cmaf/decode.ts index 409b0c2ccec..41d3646f492 100644 --- a/js/hang/src/cmaf/decode.ts +++ b/js/hang/src/cmaf/decode.ts @@ -340,14 +340,11 @@ function isSync(flags: number): boolean { * * @param segment - The moof + mdat data * @param init - Parsed init segment (provides timescale and trex defaults) - * @param timestamp - The moq-net frame timestamp carrying this segment, if its track is timed + * @param timestamp - The moq-net frame timestamp carrying this segment; an unscaled one, a + * scale-less track's group key, carries no time, so the samples keep their `tfdt` times * @returns Array of decoded samples */ -export function decodeDataSegment( - segment: Uint8Array, - init: InitSegment, - timestamp: Time.Timestamp | undefined, -): Sample[] { +export function decodeDataSegment(segment: Uint8Array, init: InitSegment, timestamp: Time.Timestamp): Sample[] { // moof and mdat at the top level; the moof's first byte is where CMAF data offsets count from. const top = boxes(segment); const moof = required(top, "moof", "data segment"); @@ -472,12 +469,14 @@ export function decodeDataSegment( } // A loop, not `Math.min(...ptss)`: a long fragment's sample count can exceed the argument limit. + // A peer may publish a scale-less track, so an unscaled timestamp keeps `tfdt` rather than throw. + const timed = timestamp.scale !== undefined; let earliest = 0; - if (timestamp !== undefined) { + if (timed) { earliest = Number.POSITIVE_INFINITY; for (const pts of ptss) earliest = Math.min(earliest, pts); } - const anchor = timestamp?.asMicros() ?? 0; + const anchor = timed ? timestamp.asMicros() : 0; for (const [i, sample] of samples.entries()) { sample.timestamp = Math.round(anchor + ((ptss[i] - earliest) * 1_000_000) / init.timescale); } diff --git a/js/hang/src/cmaf/format.test.ts b/js/hang/src/cmaf/format.test.ts index 8eada5dd01c..d0ebe3bafaf 100644 --- a/js/hang/src/cmaf/format.test.ts +++ b/js/hang/src/cmaf/format.test.ts @@ -105,7 +105,7 @@ test("CmafFormat times an untimed fragment from its tfdt", () => { [3000, 0], ]); - const frames = new Format(INIT).decode(segment, undefined); + const frames = new Format(INIT).decode(segment, Time.Timestamp.unscaled(0n)); expect(frames.map((f) => f.timestamp.asMicros())).toEqual([1_033_333, 1_100_000, 1_066_667]); }); @@ -117,7 +117,7 @@ test("CmafFormat refuses a run that doesn't start at the next sample", () => { const trun = segment.findLastIndex((_, i) => new TextDecoder().decode(segment.subarray(i, i + 4)) === "trun"); view.setInt32(trun + 12, view.getInt32(trun + 12) + 1); - expect(() => new Format(INIT).decode(segment, undefined)).toThrow(/data_offset/); + expect(() => new Format(INIT).decode(segment, Time.Timestamp.unscaled(0n))).toThrow(/data_offset/); }); function at(ticks: number): Time.Timestamp { diff --git a/js/hang/src/cmaf/format.ts b/js/hang/src/cmaf/format.ts index 97641c70540..e315b544e57 100644 --- a/js/hang/src/cmaf/format.ts +++ b/js/hang/src/cmaf/format.ts @@ -13,8 +13,8 @@ export class Format implements ContainerFormat { this.#init = init; } - /** Decode one CMAF fragment, presenting its earliest sample at the moq-net `timestamp`, or at `tfdt` if untimed. */ - decode(payload: Uint8Array, timestamp: Time.Timestamp | undefined): Frame[] { + /** Decode one CMAF fragment, presenting its earliest sample at the moq-net `timestamp`, or at `tfdt` if it is unscaled. */ + decode(payload: Uint8Array, timestamp: Time.Timestamp): Frame[] { return decodeDataSegment(payload, this.#init, timestamp).map((s) => ({ payload: s.data, timestamp: Duration.micros(s.timestamp), diff --git a/js/hang/src/container/consumer.nonsequential.test.ts b/js/hang/src/container/consumer.nonsequential.test.ts index e54b7e5d21a..c7a9f00b9a2 100644 --- a/js/hang/src/container/consumer.nonsequential.test.ts +++ b/js/hang/src/container/consumer.nonsequential.test.ts @@ -2,6 +2,7 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { Duration, type Installed } from "@moq/clock"; import { Manual } from "@moq/clock/test"; import { type Group, Time, Track, Varint } from "@moq/net"; +import { AudioConfigSchema } from "../catalog/audio.ts"; import { Consumer } from "./consumer.ts"; import { Format as LegacyFormat } from "./legacy.ts"; @@ -134,3 +135,42 @@ test("Consumer reads on without a gap past a group it skipped", async () => { track.finish(); track.close(); }); + +// Legacy frames carry no duration, so an audio reader reads each packet's from its codec. One-packet +// groups sequenced by their starts, as a Rust publisher numbers them, then run back to back without +// waiting on the sequences between them, while a missing one is still a gap. +test("Consumer reads one-packet Opus groups at sparse sequences back to back", async () => { + const track = new Track.Producer("test"); + const config = AudioConfigSchema.parse({ + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48_000, + numberOfChannels: 2, + }); + const consumer = new Consumer(track.subscribe(), { + format: new LegacyFormat(config), + delay: { max: Duration.millis(5000) }, + }); + // One 20 ms CELT packet per group, each read before the next lands. The group starting at 40 ms + // never arrives. + const packet = new Uint8Array([31 << 3, 0]); + const publish = (start: number) => { + const group = track.createGroup(BigInt(start)); + group.writeFrame({ payload: encodeLegacyFrame(start, packet), timestamp: Time.Timestamp.now() }); + group.finish(); + }; + + publish(0); + expect(await poll(consumer.next())).toMatchObject({ kind: "frame", group: 0n }); + const next = consumer.next(); + publish(20_000); + expect(await poll(next)).toMatchObject({ kind: "frame", group: 20_000n }); + // Nothing proves the timeline unbroken past 40 ms, so the reader waits on the missing group. + const after = consumer.next(); + publish(60_000); + expect(await poll(after)).toBe("pending"); + + consumer.close(); + track.finish(); + track.close(); +}); diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index cef95763346..5163029d6d3 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -1623,8 +1623,8 @@ test("Consumer with CmafFormat times an untimed track from tfdt", async () => { }); const group = track.createGroup(0n); - group.writeFrame({ payload: fragment("untimedKey") }); - group.writeFrame({ payload: fragment("untimedDelta") }); + group.writeFrame({ payload: fragment("untimedKey"), timestamp: Time.Timestamp.unscaled(0n) }); + group.writeFrame({ payload: fragment("untimedDelta"), timestamp: Time.Timestamp.unscaled(0n) }); group.finish(); track.finish(); track.close(); diff --git a/js/hang/src/container/format.ts b/js/hang/src/container/format.ts index 16ce122a8d9..0732831b959 100644 --- a/js/hang/src/container/format.ts +++ b/js/hang/src/container/format.ts @@ -6,8 +6,8 @@ import type { Frame } from "./types"; export interface Format { /** The track's media role, when the format knows it; a data track decodes an object with no bytes as its discontinuity marker. */ readonly kind?: "audio" | "video" | "data"; - /** Parse one MoQ frame's payload, given its moq-net timestamp (absent on an untimed track), into decoded media frames. On a data track no frames means the discontinuity marker, so a format returns none only for an object with no bytes. */ - decode(payload: Uint8Array, timestamp: Time.Timestamp | undefined): Frame[]; + /** Parse one MoQ frame's payload, given its moq-net timestamp (unscaled, its group's key, on a scale-less track), into decoded media frames. On a data track no frames means the discontinuity marker, so a format returns none only for an object with no bytes. */ + decode(payload: Uint8Array, timestamp: Time.Timestamp): Frame[]; /** Return the endpoint timestamp carried by empty-payload metadata. */ end?(frame: Frame): Duration | undefined; } diff --git a/js/hang/src/container/legacy.ts b/js/hang/src/container/legacy.ts index 0a08ad42bd5..4096159be49 100644 --- a/js/hang/src/container/legacy.ts +++ b/js/hang/src/container/legacy.ts @@ -5,17 +5,55 @@ export type { BufferedRange, BufferedRanges, Frame } from "./types"; import { Duration } from "@moq/clock"; import type { AudioConfig, VideoConfig } from "../catalog"; +import * as Aac from "../util/aac"; +import * as Opus from "../util/opus"; import type { Format as ContainerFormat } from "./format"; import type { Frame } from "./types"; +/** + * How long each packet of an audio track plays, as its codec states. Legacy frames carry no duration, + * and the hang draft leaves it to the codec. Knowing it, a reader tells a group that ends where the next + * one starts from one followed by lost audio, which sparse group sequences can't say. Mirrors Rust's + * `legacy::Audio`. Undefined when a packet doesn't state its duration without decoding it. + */ +export function packetDuration(config: AudioConfig): ((payload: Uint8Array) => Duration | undefined) | undefined { + const rate = config.sampleRate; + if (!(rate > 0)) return undefined; + if (config.codec === "opus") { + return (payload) => { + const samples = Opus.packetSamples(payload); + return samples === undefined ? undefined : Duration.micros((samples * 1_000_000) / 48_000); + }; + } + // AAC-LC codes 1024 samples per packet. SBR and PS packets decode to twice that, at a rate the + // catalog may or may not double, so they state nothing here. + if (Aac.objectType(config.codec) === 2) { + const duration = Duration.micros((1024 * 1_000_000) / rate); + return (payload) => (payload.byteLength > 0 ? duration : undefined); + } + // Interleaved 32-bit samples, so the payload's length says. + if (config.codec === "pcm" && config.numberOfChannels > 0) { + const frameBytes = 4 * config.numberOfChannels; + return (payload) => { + if (payload.byteLength === 0 || payload.byteLength % frameBytes !== 0) return undefined; + return Duration.micros(((payload.byteLength / frameBytes) * 1_000_000) / rate); + }; + } + return undefined; +} + /** The legacy hang container: a microsecond timestamp varint followed by the raw codec payload. */ export class Format implements ContainerFormat { /** Configure the format for the track's media kind. */ readonly kind: "audio" | "video" | "data"; - /** Configure the format from a catalog entry or an explicit media kind. */ + // How long each audio packet plays, when built from a catalog entry whose codec states it. + readonly #duration?: (payload: Uint8Array) => Duration | undefined; + + /** Configure the format from a catalog entry or an explicit media kind. Only an audio entry states its packets' durations. */ constructor(config: AudioConfig | VideoConfig | "audio" | "video" | "data") { this.kind = typeof config === "string" ? config : "sampleRate" in config ? "audio" : "video"; + if (typeof config !== "string" && "sampleRate" in config) this.#duration = packetDuration(config); } /** Write the final video frame's end timestamp before the group closes. */ @@ -36,7 +74,8 @@ export class Format implements ContainerFormat { decode(frame: Uint8Array): Frame[] { if (this.kind === "data" && frame.byteLength === 0) return []; const [timestamp, data] = Moq.Varint.decode(frame); - return [{ payload: data, timestamp: Duration.micros(timestamp), keyframe: false }]; + const duration = data.byteLength > 0 ? this.#duration?.(data) : undefined; + return [{ payload: data, timestamp: Duration.micros(timestamp), keyframe: false, duration }]; } } diff --git a/js/hang/src/container/types.ts b/js/hang/src/container/types.ts index 69eb6e42bf6..00b22288dd1 100644 --- a/js/hang/src/container/types.ts +++ b/js/hang/src/container/types.ts @@ -11,7 +11,8 @@ export interface Frame { /** * How long this frame occupies the presentation timeline. CMAF carries a - * per-sample duration; Legacy and LOC report later endpoints through Consumer.next().end. + * per-sample duration, and Legacy audio reads one from a codec that states it; + * otherwise Legacy and LOC report later endpoints through Consumer.next().end. * The consumer adds it to `timestamp` to learn how far a group has presented, * so it can advance to a newer group as soon as the gap is covered instead of * waiting out the max delay budget. diff --git a/js/hang/src/report.ts b/js/hang/src/report.ts index 3ccb2489e1e..5a953b71146 100644 --- a/js/hang/src/report.ts +++ b/js/hang/src/report.ts @@ -4,7 +4,7 @@ */ import * as Json from "@moq/json"; -import type * as Moq from "@moq/net"; +import * as Moq from "@moq/net"; import type * as z from "zod/mini"; /** How often a reporter writes, in milliseconds: about once a second, as the hang draft asks. */ @@ -29,6 +29,9 @@ export class Writer { #producer: Json.Snapshot.Producer; #schema: z.ZodMiniType; #closed = false; + // The report track is scale-less, so each write names the next group key: past every group a + // snapshot opened, while a delta carries its open group's key whatever this names. + #key = 0n; /** * Create `name` on `broadcast`, parsing each report with `schema` before it is written, so a @@ -48,7 +51,7 @@ export class Writer { update(value: I): void { if (this.#closed || this.track.demand().wanted.peek() === undefined) return; try { - this.#producer.update({ value: this.#schema.parse(value) }); + this.#producer.update({ value: this.#schema.parse(value), at: Moq.Time.Timestamp.unscaled(this.#key++) }); } catch (err) { console.warn("a report track failed; closing it", err); this.#closed = true; diff --git a/js/json/api/index.api.md b/js/json/api/index.api.md index 47fa3074d55..777fb1ae135 100644 --- a/js/json/api/index.api.md +++ b/js/json/api/index.api.md @@ -71,7 +71,7 @@ export namespace Snapshot { constructor(config: Snapshot.Producer.Config); cut(): void; finish(): Snapshot.Finished; - mutate(fn: (value: T) => void, at?: Time.Timestamp): void; + mutate(fn: (value: T) => void, at: Time.Timestamp): void; update({ ... }: Timed): void; } } @@ -179,7 +179,7 @@ export namespace Window { cut(): void; finish(): Window.Finished; get offset(): number; - pop(count: number, at?: Time.Timestamp): void; + pop(count: number, at: Time.Timestamp): void; push({ ... }: Timed): void; get window(): T[]; } diff --git a/js/json/src/key.test-util.ts b/js/json/src/key.test-util.ts new file mode 100644 index 00000000000..1f17d76728a --- /dev/null +++ b/js/json/src/key.test-util.ts @@ -0,0 +1,24 @@ +// Scale-less writes for tests: a scale-less track's writes name their group keys. +import { Time, type Timed } from "@moq/net"; + +let counter = 0n; + +/** Restart {@link key} at 0, so each test's first group is 0: run it before each test. */ +export function resetKeys(): void { + counter = 0n; +} + +/** The next key of a scale-less track: keys only have to increase, so one counter serves every track a test writes. */ +export function key(): Time.Timestamp { + return Time.Timestamp.unscaled(counter++); +} + +/** `value` at the next key; see {@link key}. */ +export function next(value: T): Timed { + return { value, at: key() }; +} + +/** `value` at scale-less `key`. */ +export function keyed(value: T, key: bigint): Timed { + return { value, at: Time.Timestamp.unscaled(key) }; +} diff --git a/js/json/src/snapshot/codec.test.ts b/js/json/src/snapshot/codec.test.ts index 1864cedece6..99ecc5d4199 100644 --- a/js/json/src/snapshot/codec.test.ts +++ b/js/json/src/snapshot/codec.test.ts @@ -1,9 +1,12 @@ -import { expect, test } from "bun:test"; +import { beforeEach, expect, test } from "bun:test"; import * as z from "zod/mini"; import { MissingSnapshot } from "../error.ts"; +import { next, resetKeys } from "../key.test-util.ts"; import { Decoder } from "./decoder.ts"; import { type Config, Encoder } from "./encoder.ts"; +beforeEach(resetKeys); + type Doc = Record; // Encode a sequence of values, committing each frame, and return `[keyframe, payloadLength]` per @@ -326,11 +329,11 @@ test("update after finish reports the closed track", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1 } }); - producer.update({ value: { a: 2 } }); // a delta, so a group is open + producer.update(next({ a: 1 })); + producer.update(next({ a: 2 })); // a delta, so a group is open producer.finish(); - expect(() => producer.update({ value: { a: 3 } })).toThrow("track is closed"); + expect(() => producer.update(next({ a: 3 }))).toThrow("track is closed"); }); // A root that isn't an object has no recursive merge patch, so it forces a snapshot regardless of diff --git a/js/json/src/snapshot/producer.ts b/js/json/src/snapshot/producer.ts index ed5d45be3db..3d5664213fd 100644 --- a/js/json/src/snapshot/producer.ts +++ b/js/json/src/snapshot/producer.ts @@ -1,6 +1,7 @@ import type { Time, Timed } from "@moq/net"; import * as Moq from "@moq/net"; import { MAX_FRAME_BYTES } from "../limits.ts"; +import { frameTime, openGroup, startsPast } from "../time.ts"; import { type Config as CodecConfig, type Encoded, Encoder } from "./encoder.ts"; @@ -21,8 +22,9 @@ export interface Finished {} * * A group of deltas runs until a joiner wants a fresh one: once the deltas outweigh the track's * `refresh` ratio times the snapshot, a join asks for a group past the largest, and the producer - * restates the value there at the newest timestamp written. Set the ratio when creating the - * track, or joins never ask. + * restates the value there at the newest timestamp written. On a scale-less track that timestamp + * names the group, so while it is the newest group's own key the refresh waits for a later update, + * since two groups never share a key. Set the ratio when creating the track, or joins never ask. * * Set `compression` on that track's info too: JSON is small and repetitive, and moq-net compresses each * delta against the snapshot earlier in its group. @@ -42,9 +44,12 @@ export class Producer { #largest?: bigint; // The newest timestamp written, which a refresh restates the value at. An update may carry an // older timestamp than its group's first frame, and a new group may not start before that one. + // Undefined before the first write. #at?: Time.Timestamp; // Stops watching the track's demand. #unwatch?: () => void; + // A scale-less refresh that waits for a key past the newest group's, which the next write answers. + #deferred = false; constructor(config: Producer.Config) { this.#track = config.track; @@ -58,14 +63,22 @@ export class Producer { // Restate the value in a new group once a joiner wants one past the largest written. #refresh(wanted: bigint | undefined): void { + this.#deferred = false; if (wanted === undefined || this.#largest === undefined || wanted <= this.#largest) return; + // Two groups never share a key, so a scale-less refresh at the newest group's own key waits for + // a later write. Checked before encoding, since a snapshot dropped unwritten resets the encoder. + const at = this.#at; + if (at === undefined || !startsPast(this.#track, at)) { + this.#deferred = true; + return; + } const frame = this.#encoder.snapshot(); if (!frame) return; // Nobody awaits a refresh, so a failed write has no caller to report to. Abort the track rather // than leave its newest group without the value, where a latest-value subscriber would wait. try { - this.#write(frame, this.#at); + this.#write(frame, at); frame.commit(); } catch (err) { this.#close(err instanceof Error ? err : new Error(String(err))); @@ -98,8 +111,9 @@ export class Producer { /** * Publish a new value, emitting a snapshot or delta automatically. No-op if unchanged. * - * `at` is when the value was captured, written as its frame timestamp. Omit it only on an - * untimed track: it is never filled in. + * `at` is when the value was captured, written as its frame timestamp. On a scale-less track it + * is {@link Time.Timestamp.unscaled} at a group key instead: a snapshot opens its group there, + * which must pass every group the track holds, and a delta carries the open group's key. */ update({ value, at }: Timed): void { const frame = this.#encoder.update(value); @@ -110,10 +124,12 @@ export class Producer { // stays the last one written, so a refresh restates that rather than the rejection. this.#write(frame, at); frame.commit(); - if (at && !at.isBefore(this.#at ?? at)) this.#at = at; + if (this.#at === undefined || this.#at.isBefore(at)) this.#at = at; + // A deferred refresh waits for a key past the open group's, which this may be. + if (this.#deferred) this.#refresh(this.#track.demand().wanted.peek()); } - #write(encoded: Encoded, at: Time.Timestamp | undefined): void { + #write(encoded: Encoded, at: Time.Timestamp): void { // Check before touching a group. A keyframe closes the previous group and publishes its // replacement before the frame is written, so discovering the limit inside `writeFrame` would // leave an empty newest group behind: a snapshot consumer jumps to the newest, so the last @@ -121,13 +137,14 @@ export class Producer { if (encoded.payload.byteLength > MAX_FRAME_BYTES) throw new Moq.Error.FrameTooLarge(); if (encoded.keyframe) { - // The previous group is complete; no more frames will be appended to it. Drop the handle - // before opening the next one, so a failure below doesn't leave a closed group behind. + // Opened before the previous group finishes, so a key at or before the newest group's is + // refused with the previous group still open. + const group = openGroup(this.#track, at); + this.#largest = group.sequence; + + // The previous group is complete; no more frames will be appended to it. this.#group?.finish(); this.#group = undefined; - - const group = this.#track.appendGroup(); - this.#largest = group.sequence; try { group.writeFrame({ payload: encoded.payload, timestamp: at }); } catch (err) { @@ -150,12 +167,12 @@ export class Producer { } if (!this.#group) throw new Error("delta with no open group"); - this.#group.writeFrame({ payload: encoded.payload, timestamp: at }); + this.#group.writeFrame({ payload: encoded.payload, timestamp: frameTime(this.#group, at) }); } /** - * Mutate the current value in place and publish the result, captured at `at` (omit it only on an - * untimed track). + * Mutate the current value in place and publish the result, captured at `at` (see + * {@link update}). * * The callback receives a deep clone of the current value: everything published through this * producer so far, composed, falling back to {@link Config.initial} if nothing has been (throws if @@ -177,7 +194,7 @@ export class Producer { * instead fail every later owner's publish the same way, or publish a value its owner was told * failed. */ - mutate(fn: (value: T) => void, at?: Time.Timestamp): void { + mutate(fn: (value: T) => void, at: Time.Timestamp): void { // Start from the last-published value, falling back to the configured initial value. We // don't invent an empty object: mutating with nothing to start from is a usage error. const base = this.#encoder.value ?? this.#initial; diff --git a/js/json/src/snapshot/snapshot.test.ts b/js/json/src/snapshot/snapshot.test.ts index 695870b5dd8..95da4a1471d 100644 --- a/js/json/src/snapshot/snapshot.test.ts +++ b/js/json/src/snapshot/snapshot.test.ts @@ -1,12 +1,15 @@ -import { expect, test } from "bun:test"; +import { beforeEach, expect, test } from "bun:test"; import { Duration } from "@moq/clock"; import { Manual } from "@moq/clock/test"; import { Error as NetError, StreamCode, Time, Track } from "@moq/net"; import { hooks } from "../../../net/src/internal.ts"; +import { key, keyed, next, resetKeys } from "../key.test-util.ts"; import { Consumer } from "./consumer.ts"; import { Encoder } from "./encoder.ts"; import { Producer } from "./producer.ts"; +beforeEach(resetKeys); + type Value = Record; // These tests inspect complete finished timelines through raw ordered reads, so request a @@ -50,10 +53,10 @@ async function structure(track: Track.Ordered): Promise { test("a cut makes the next update a snapshot group", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1, b: 1 } }); - producer.update({ value: { a: 1, b: 2 } }); + producer.update(next({ a: 1, b: 1 })); + producer.update(next({ a: 1, b: 2 })); producer.cut(); - producer.update({ value: { a: 1, b: 3 } }); + producer.update(next({ a: 1, b: 3 })); producer.finish(); // The ratio would have kept every update in one group; the cut rolled it anyway. The replacement @@ -65,12 +68,12 @@ test("a cut makes the next update a snapshot group", async () => { test("a cut republishes an unchanged value", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); producer.cut(); // An unchanged value normally writes nothing. After a cut it must still open the replacement // group, or the value would only exist in a group no new consumer reads. - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); producer.finish(); expect(await structure(track.subscribe({ delay: { max: REPLAY_LATENCY } }).ordered())).toEqual([1, 1]); @@ -79,7 +82,7 @@ test("a cut republishes an unchanged value", async () => { test("a cut opens no replacement group", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); producer.cut(); producer.finish(); @@ -94,12 +97,12 @@ test("a cut is idempotent", async () => { // Nothing published yet, so there is no group to cut. producer.cut(); producer.cut(); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); // And a repeated cut rolls once, not once per call. producer.cut(); producer.cut(); - producer.update({ value: { a: 2 } }); + producer.update(next({ a: 2 })); producer.finish(); expect(await structure(track.subscribe({ delay: { max: REPLAY_LATENCY } }).ordered())).toEqual([1, 1]); @@ -108,11 +111,11 @@ test("a cut is idempotent", async () => { test("a cut is inert without deltas", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: false }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); // With deltas off every frame already closes its own group, so there is never one to cut. producer.cut(); - producer.update({ value: { a: 2 } }); + producer.update(next({ a: 2 })); producer.finish(); expect(await structure(track.subscribe({ delay: { max: REPLAY_LATENCY } }).ordered())).toEqual([1, 1]); @@ -122,9 +125,9 @@ test("deltas off: a snapshot group per change", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: false }); const consumer = new Consumer({ track }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); expect((await consumer.next())?.value).toEqual({ a: 1 }); - producer.update({ value: { a: 2 } }); + producer.update(next({ a: 2 })); expect((await consumer.next())?.value).toEqual({ a: 2 }); producer.finish(); @@ -144,13 +147,13 @@ test("latest abandons a held group when a newer snapshot group rolls", async () const producer = new Producer({ track }); const consumer = new Consumer({ track }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); expect((await consumer.latest())?.value).toEqual({ a: 1 }); // A delta lands in the held group, then a cut rolls a new snapshot group. - producer.update({ value: { a: 2 } }); + producer.update(next({ a: 2 })); producer.cut(); - producer.update({ value: { a: 3 } }); + producer.update(next({ a: 3 })); producer.finish(); expect((await consumer.latest())?.value).toEqual({ a: 3 }); @@ -163,7 +166,7 @@ test("latest reads the newest snapshot group", async () => { const producer = new Producer({ track, deltas: false }); const consumer = new Consumer({ track }); - for (let n = 0; n < 4; n++) producer.update({ value: { a: n } }); + for (let n = 0; n < 4; n++) producer.update(next({ a: n })); expect((await consumer.latest())?.value).toEqual({ a: 3 }); producer.finish(); }); @@ -174,12 +177,12 @@ test("next reads a held group out before a newer snapshot group", async () => { const producer = new Producer({ track }); const consumer = new Consumer({ track }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); expect((await consumer.next())?.value).toEqual({ a: 1 }); - producer.update({ value: { a: 2 } }); + producer.update(next({ a: 2 })); producer.cut(); - producer.update({ value: { a: 3 } }); + producer.update(next({ a: 3 })); producer.finish(); expect((await consumer.next())?.value).toEqual({ a: 2 }); @@ -219,7 +222,10 @@ test("reads skip a group the subscription still owes", async () => { const next = new Consumer({ track }); const put = (sequence: bigint) => { const group = track.createGroup(sequence); - group.writeFrame({ payload: new TextEncoder().encode(`{"n":${sequence}}`) }); + group.writeFrame({ + payload: new TextEncoder().encode(`{"n":${sequence}}`), + timestamp: Time.Timestamp.unscaled(sequence), + }); group.finish(); }; @@ -250,11 +256,11 @@ test("close ends a pending read", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); const consumer = new Consumer({ track }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); expect((await consumer.latest())?.value).toEqual({ a: 1 }); const pending = [consumer.latest(), consumer.next()]; - producer.update({ value: { a: 2 } }); + producer.update(next({ a: 2 })); consumer.close(); expect(await Promise.all(pending)).toEqual([undefined, undefined]); // The subscription the consumer owned is gone. @@ -269,15 +275,15 @@ test("overlapping reads take turns", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); const consumer = new Consumer({ track }); - producer.update({ value: { a: 1, keep: true } }); + producer.update(next({ a: 1, keep: true })); expect((await consumer.next())?.value).toEqual({ a: 1, keep: true }); - const next = consumer.next(); - producer.update({ value: { a: 2, keep: true } }); + const reading = consumer.next(); + producer.update(next({ a: 2, keep: true })); producer.cut(); - producer.update({ value: { b: 3 } }); + producer.update(next({ b: 3 })); const latest = consumer.latest(); - expect((await next)?.value).toEqual({ a: 2, keep: true }); + expect((await reading)?.value).toEqual({ a: 2, keep: true }); expect((await latest)?.value).toEqual({ b: 3 }); producer.finish(); }); @@ -288,16 +294,29 @@ test("live consumer sees each update", async () => { const consumer = new Consumer({ track }); for (let n = 1; n <= 3; n++) { - producer.update({ value: { a: n } }); + producer.update(next({ a: n })); expect((await consumer.next())?.value).toEqual({ a: n }); } }); +test("a key past the safe number range opens its group", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track }); + const consumer = new Consumer({ track }); + const wide = 1n << 53n; + + producer.update(keyed({ a: 1 }, wide)); + expect(track.largest()?.group).toBe(wide); + expect((await consumer.next())?.value).toEqual({ a: 1 }); + producer.update(keyed({ a: 2 }, wide + 1n)); + expect((await consumer.next())?.value).toEqual({ a: 2 }); +}); + test("unchanged value writes nothing", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); - producer.update({ value: { a: 1 } }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); + producer.update(next({ a: 1 })); producer.finish(); expect(await structure(track.subscribe().ordered())).toEqual([1]); @@ -306,9 +325,9 @@ test("unchanged value writes nothing", async () => { test("deltas share one group", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1, b: 1 } }); - producer.update({ value: { a: 1, b: 2 } }); - producer.update({ value: { a: 1, b: 3 } }); + producer.update(next({ a: 1, b: 1 })); + producer.update(next({ a: 1, b: 2 })); + producer.update(next({ a: 1, b: 3 })); producer.finish(); // All updates fit in a single group as snapshot + two deltas. @@ -318,9 +337,9 @@ test("deltas share one group", async () => { test("deltas reconstruct to the final value", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1, b: 1 } }); - producer.update({ value: { a: 1, b: 2 } }); - producer.update({ value: { a: 5, b: 2 } }); + producer.update(next({ a: 1, b: 1 })); + producer.update(next({ a: 1, b: 2 })); + producer.update(next({ a: 5, b: 2 })); producer.finish(); expect((await drain(track)).at(-1)).toEqual({ a: 5, b: 2 }); @@ -336,13 +355,13 @@ test("mutate composes independent owners", async () => { producer.mutate((v) => { v.video = "v1"; - }); + }, key()); expect((await consumer.next())?.value).toEqual({ video: "v1" }); // A second owner starts from the latest value and adds its own key without clobbering the first. producer.mutate((v) => { v.scte35 = { id: 1 }; - }); + }, key()); expect((await consumer.next())?.value).toEqual({ video: "v1", scte35: { id: 1 } }); }); @@ -353,7 +372,7 @@ test("mutate starts from the configured initial value", async () => { producer.mutate((v) => { v.a = 1; - }); + }, key()); expect((await consumer.next())?.value).toEqual({ a: 1 }); }); @@ -365,25 +384,25 @@ test("a rejected mutate leaves the value at the last one written", async () => { const producer = new Producer({ track, initial: {} }); producer.mutate((v) => { v.video = "v1"; - }); + }, key()); // Serializes past the group cache limit, so the frame cannot be published. expect(() => producer.mutate((v) => { v.big = "x".repeat(32 * 1024 * 1024 + 1); - }), + }, key()), ).toThrow(NetError.FrameTooLarge); producer.mutate((v) => { v.scte35 = 42; - }); + }, key()); producer.finish(); expect(await drainLatest(track)).toEqual([{ video: "v1", scte35: 42 }]); }); test("mutate without a prior value or initial throws", () => { const producer = new Producer({ track: new Track.Producer("test") }); - expect(() => producer.mutate(() => {})).toThrow(); + expect(() => producer.mutate(() => {}, key())).toThrow(); }); // Removing a section drops it from the reconstructed value, so a consumer detects the removal. @@ -396,12 +415,12 @@ test("mutate removes a section", async () => { producer.mutate((v) => { v.a = 1; v.scte35 = { id: 1 }; - }); + }, key()); expect((await consumer.next())?.value).toEqual({ a: 1, scte35: { id: 1 } }); producer.mutate((v) => { delete v.scte35; - }); + }, key()); expect((await consumer.next())?.value).toEqual({ a: 1 }); }); @@ -412,13 +431,13 @@ async function settle(): Promise { // Publish `{"n":0}` through `{"n":3}`: a 7-byte snapshot and 21 bytes of deltas. function fill(producer: Producer): void { - for (let n = 0; n <= 3; n++) producer.update({ value: { n } }); + for (let n = 0; n <= 3; n++) producer.update(next({ n })); } test("deltas never roll without a joiner", async () => { const track = new Track.Producer("test").accept({ refresh: 1 }); const producer = new Producer({ track }); - for (let n = 0; n <= 10; n++) producer.update({ value: { n } }); + for (let n = 0; n <= 10; n++) producer.update(next({ n })); producer.finish(); // The deltas outweigh the snapshot ten times over, but nobody joined to ask for a fresh group. @@ -492,6 +511,37 @@ test("a refresh after an older update starts at the newest timestamp", async () ]); }); +// A scale-less refresh at the newest key written would share the open group's key, so it waits for a +// later update, then restates the value there. (Mirrors the Rust +// `a_refresh_after_older_updates_waits_for_a_later_one`.) +test("a scale-less refresh waits for an update past the open group's key", async () => { + const track = new Track.Producer("test").accept({ refresh: 1, maxAge: null }); + const producer = new Producer({ track }); + // A snapshot and deltas, all at the open group's key. + for (let n = 0; n <= 3; n++) producer.update(keyed({ n }, 5n)); + + const joiner = track.subscribe(); + await settle(); + expect(track.largest()?.group).toBe(5n); + + // The next update rides as a delta, and the refresh it unblocks restates the value at its key. + producer.update(keyed({ n: 4 }, 6n)); + producer.finish(); + joiner.close(); + + const ordered = track.subscribe({ delay: { max: REPLAY_LATENCY } }).ordered(); + const restated: [bigint, unknown][] = []; + for (let group = await ordered.nextGroup(); group; group = await ordered.nextGroup()) { + const frame = await group.readFrame(); + restated.push([group.sequence, frame && JSON.parse(new TextDecoder().decode(frame.payload))]); + while ((await group.readFrame()) !== undefined) {} + } + expect(restated).toEqual([ + [5n, { n: 0 }], + [6n, { n: 4 }], + ]); +}); + // A refresh restates the last value written, never one whose update was rejected, and the next // update rides as a delta against it. (Mirrors the Rust // `a_refresh_after_a_rejected_update_restates_the_last_written_value`.) @@ -502,11 +552,11 @@ test("a refresh after a rejected update restates the last written value", async // Serializes past the group cache limit, so the frame cannot be published. const oversized = { big: "x".repeat(32 * 1024 * 1024 + 1) }; - expect(() => producer.update({ value: oversized })).toThrow(NetError.FrameTooLarge); + expect(() => producer.update(next(oversized))).toThrow(NetError.FrameTooLarge); const joiner = track.subscribe(); await settle(); - producer.update({ value: { n: 4 } }); + producer.update(next({ n: 4 })); producer.finish(); joiner.close(); @@ -530,13 +580,13 @@ test("no ratio never refreshes", async () => { test("a compressed refresh opens a fresh group", async () => { const track = new Track.Producer("test").accept({ refresh: 0, compression: "deflate", maxAge: null }); const producer = new Producer({ track }); - producer.update({ value: { a: 1, b: 1 } }); - producer.update({ value: { a: 1, b: 2 } }); + producer.update(next({ a: 1, b: 1 })); + producer.update(next({ a: 1, b: 2 })); const joiner = track.subscribe(); await settle(); // The change after the restatement rides as a delta in the new group. - producer.update({ value: { a: 1, b: 3 } }); + producer.update(next({ a: 1, b: 3 })); producer.finish(); joiner.close(); @@ -553,8 +603,8 @@ test("a compressed refresh opens a fresh group", async () => { test("array change is a wholesale delta", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { list: [1, 2] } }); - producer.update({ value: { list: [1, 2, 3] } }); + producer.update(next({ list: [1, 2] })); + producer.update(next({ list: [1, 2, 3] })); producer.finish(); // The array is replaced wholesale in a delta, so it stays in the same group. @@ -567,7 +617,7 @@ test("a late joiner reads a buffered backlog state by state, or collapses it to const subscriber = new Consumer({ track }); const collapsed = new Consumer({ track }); for (let n = 0; n <= 20; n++) { - producer.update({ value: { n } }); + producer.update(next({ n })); } producer.finish(); @@ -582,7 +632,7 @@ test("frame cap rolls snapshot", async () => { const producer = new Producer({ track, deltas: true }); // First update is the snapshot; deltas fill the group until the frame cap forces a roll. for (let i = 0; i <= 256; i++) { - producer.update({ value: { n: i } }); + producer.update(next({ n: i })); } producer.finish(); @@ -595,11 +645,11 @@ test("a rejected update leaves the previous value readable", async () => { // A snapshot consumer jumps to the newest, so that value would vanish on a failed update. const track = new Track.Producer("test"); const producer = new Producer({ track, deltas: false }); - producer.update({ value: { keep: true } }); + producer.update(next({ keep: true })); // Serializes past the group cache limit, so the frame cannot be published. const oversized = { big: "x".repeat(32 * 1024 * 1024 + 1) }; - expect(() => producer.update({ value: oversized })).toThrow(NetError.FrameTooLarge); + expect(() => producer.update(next(oversized))).toThrow(NetError.FrameTooLarge); producer.finish(); // A reader arriving now still finds the last good value, not an empty superseding group. @@ -617,8 +667,8 @@ test("a delta that would overflow the snapshot rolls a new one instead", async ( const track = new Track.Producer("test"); const producer = new Producer({ track, maxGroupBytes: 64 }); - producer.update({ value: { v: "x".repeat(size) } }); - producer.update({ value: { v: "y".repeat(size) } }); + producer.update(next({ v: "x".repeat(size) })); + producer.update(next({ v: "y".repeat(size) })); producer.finish(); // Two groups, a self-contained snapshot each, rather than one whose frame 0 was evicted. @@ -678,18 +728,45 @@ test("snapshot consumer propagates a non-gap frame failure", async () => { await expect(consumer.next()).rejects.toMatchObject({ code: StreamCode.Internal }); }); -test("an update without a timestamp goes out untimed", async () => { +test("a scale-less snapshot opens its group at its key, and a delta carries that key", async () => { const track = new Track.Producer("test").accept({}); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1, b: "x".repeat(64) } }); + producer.update(keyed({ a: 1, b: "x".repeat(64) }, 5n)); + // A delta names a later key, but rides in the open group under its key. producer.mutate((value) => { value.a = 2; - }); + }, Time.Timestamp.unscaled(9n)); producer.finish(); const group = await track.subscribe().ordered().nextGroup(); - expect((await group?.readFrame())?.timestamp).toBeUndefined(); - expect((await group?.readFrame())?.timestamp).toBeUndefined(); + expect(group?.sequence).toBe(5n); + expect((await group?.readFrame())?.timestamp).toEqual(Time.Timestamp.unscaled(5n)); + expect((await group?.readFrame())?.timestamp).toEqual(Time.Timestamp.unscaled(5n)); +}); + +test("a scale-less snapshot at a key not past the newest group is refused", async () => { + const track = new Track.Producer("test").accept({ maxAge: null }); + const producer = new Producer({ track, deltas: false }); + producer.update(keyed({ n: 0 }, 5n)); + // Two groups never share a key, and a new one must pass the newest. + for (const key of [5n, 4n]) expect(() => producer.update(keyed({ n: 1 }, key))).toThrow(NetError.TimestampMismatch); + producer.update(keyed({ n: 1 }, 8n)); + producer.finish(); + + const ordered = track.subscribe({ delay: { max: REPLAY_LATENCY } }).ordered(); + const sequences: bigint[] = []; + for (let group = await ordered.nextGroup(); group; group = await ordered.nextGroup()) + sequences.push(group.sequence); + expect(sequences).toEqual([5n, 8n]); + expect(await drain(track)).toEqual([{ n: 1 }]); +}); + +test("an unscaled update is refused on a timed track", () => { + const track = new Track.Producer("test").accept({ timescale: Time.Timescale.MILLI }); + const producer = new Producer({ track, deltas: true }); + expect(() => producer.update(keyed({ a: 1 }, 0n))).toThrow(NetError.TimestampMismatch); + producer.update({ value: { a: 1 }, at: Time.Timestamp.fromMillis(1) }); + producer.finish(); }); test("a capture timestamp is written on snapshots and deltas alike", async () => { @@ -704,16 +781,17 @@ test("a capture timestamp is written on snapshots and deltas alike", async () => expect((await group?.readFrame())?.timestamp?.as(Time.Timescale.MILLI)).toBe(2_000); }); -test("an untimed state reads back untimed", async () => { +test("a scale-less state reads back with its group's key", async () => { const track = new Track.Producer("test").accept({}); const producer = new Producer({ track, deltas: true }); - producer.update({ value: { a: 1, b: "x".repeat(64) } }); - producer.update({ value: { a: 2, b: "x".repeat(64) } }); + producer.update(keyed({ a: 1, b: "x".repeat(64) }, 7n)); + producer.update(keyed({ a: 2, b: "x".repeat(64) }, 8n)); producer.finish(); const consumer = new Consumer({ track }); - expect(await consumer.next()).toEqual({ value: { a: 1, b: "x".repeat(64) }, at: undefined }); - expect(await consumer.next()).toEqual({ value: { a: 2, b: "x".repeat(64) }, at: undefined }); + const at = Time.Timestamp.unscaled(7n); + expect(await consumer.next()).toEqual({ value: { a: 1, b: "x".repeat(64) }, at }); + expect(await consumer.next()).toEqual({ value: { a: 2, b: "x".repeat(64) }, at }); }); // Each state carries the timestamp of the frame that produced it, snapshot or delta, so a caller can @@ -745,8 +823,8 @@ test("each state reads back with its frame's timestamp", async () => { // compile time and at run time. test("finishing returns a value that publishes nothing", () => { const producer = new Producer({ track: new Track.Producer("test") }); - producer.update({ value: { a: 1 } }); + producer.update(next({ a: 1 })); const finished = producer.finish(); // @ts-expect-error a finished value takes no update - expect(() => finished.update({ value: { a: 2 } })).toThrow(); + expect(() => finished.update(next({ a: 2 }))).toThrow(); }); diff --git a/js/json/src/stream/codec.test.ts b/js/json/src/stream/codec.test.ts index 58d85b715cd..13851b461f0 100644 --- a/js/json/src/stream/codec.test.ts +++ b/js/json/src/stream/codec.test.ts @@ -1,10 +1,13 @@ -import { expect, spyOn, test } from "bun:test"; +import { beforeEach, expect, spyOn, test } from "bun:test"; import { Group, Track } from "@moq/net"; import * as z from "zod/mini"; +import { next, resetKeys } from "../key.test-util.ts"; import { Decoder } from "./decoder.ts"; import { Encoder } from "./encoder.ts"; import { Producer } from "./producer.ts"; +beforeEach(resetKeys); + type Rec = { n: number }; test("roundtrip in order", () => { @@ -44,7 +47,7 @@ test("a rejected record ends the track without opening a group", async () => { const subscriber = track.subscribe().ordered(); const producer = new Producer({ track }); - expect(() => producer.append({ value: undefined })).toThrow("not representable as JSON"); + expect(() => producer.append(next(undefined))).toThrow("not representable as JSON"); // Nothing was appended, so the log has no group for a consumer to enter and wait in. @@ -68,7 +71,7 @@ test("a failed write on the very first record still ends the track", async () => throw failure; }); try { - expect(() => producer.append({ value: "x" })).toThrow(failure); + expect(() => producer.append(next("x"))).toThrow(failure); } finally { write.mockRestore(); } diff --git a/js/json/src/stream/producer.ts b/js/json/src/stream/producer.ts index c74a4413b70..21c688de383 100644 --- a/js/json/src/stream/producer.ts +++ b/js/json/src/stream/producer.ts @@ -1,6 +1,7 @@ import type * as Moq from "@moq/net"; import { Error as NetError, type Timed } from "@moq/net"; +import { frameTime, openGroup } from "../time.ts"; import { type Config as CodecConfig, Encoder, type Pending } from "./encoder.ts"; /** @@ -40,8 +41,9 @@ export class Producer { * second group. The track is aborted rather than closed cleanly, so a consumer sees the failure * instead of a log that merely looks complete. * - * `at` is when the value was captured, written as its frame timestamp. Omit it only on an - * untimed track: it is never filled in. + * `at` is when the value was captured, written as its frame timestamp. On a scale-less track it + * is {@link Moq.Time.Timestamp.unscaled} at a group key instead: the first record opens the log's + * group there, and every record carries that group's key. */ append({ value, at }: Timed): void { // Encode first, so a value that can't be serialized doesn't publish an empty group that @@ -58,10 +60,10 @@ export class Producer { throw err; } - this.#group ??= this.#track.appendGroup(); + this.#group ??= openGroup(this.#track, at); try { - this.#group.writeFrame({ payload: record.payload, timestamp: at }); + this.#group.writeFrame({ payload: record.payload, timestamp: frameTime(this.#group, at) }); } catch (err) { // The group is live, so the record is a hole in the log and a second group would hand // consumers that gap dressed up as a complete log. diff --git a/js/json/src/stream/stream.test.ts b/js/json/src/stream/stream.test.ts index a80cb0e2cec..ff07e49e212 100644 --- a/js/json/src/stream/stream.test.ts +++ b/js/json/src/stream/stream.test.ts @@ -1,9 +1,12 @@ -import { expect, spyOn, test } from "bun:test"; +import { beforeEach, expect, spyOn, test } from "bun:test"; import { Duration } from "@moq/clock"; import { StreamCode, Time, Track } from "@moq/net"; import * as z from "zod/mini"; +import { next, resetKeys } from "../key.test-util.ts"; import { Consumer, Producer, Rolled } from "./index.ts"; +beforeEach(resetKeys); + type Rec = { n: number }; // Drain every record currently available from a fresh consumer over the (finished) track. @@ -21,7 +24,7 @@ async function drain(track: Track.Subscriber): Promise { test("plaintext roundtrip in order", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); - for (let n = 0; n < 5; n++) producer.append({ value: { n } }); + for (let n = 0; n < 5; n++) producer.append(next({ n })); producer.finish(); expect(await drain(track.subscribe())).toEqual([0, 1, 2, 3, 4]); @@ -30,7 +33,7 @@ test("plaintext roundtrip in order", async () => { test("compressed roundtrip in order", async () => { const track = new Track.Producer("test").accept({ compression: "deflate" }); const producer = new Producer({ track }); - for (let n = 0; n < 20; n++) producer.append({ value: { n } }); + for (let n = 0; n < 20; n++) producer.append(next({ n })); producer.finish(); expect(await drain(track.subscribe())).toEqual(Array.from({ length: 20 }, (_, n) => n)); @@ -40,10 +43,10 @@ test("a record whose worst-case slice outgrows the frame cap is refused, leaving const track = new Track.Producer("test").accept({ compression: "deflate" }); const producer = new Producer({ track }); // 32 MiB of JSON fits the plain cap, but its DEFLATE bound does not. - expect(() => producer.append({ value: "x".repeat(32 * 1024 * 1024 - 2) })).toThrow( + expect(() => producer.append(next("x".repeat(32 * 1024 * 1024 - 2)))).toThrow( expect.objectContaining({ code: StreamCode.FrameTooLarge }), ); - producer.append({ value: "small" }); + producer.append(next("small")); producer.finish(); const consumer = new Consumer({ track: track.subscribe() }); @@ -53,7 +56,7 @@ test("a record whose worst-case slice outgrows the frame cap is refused, leaving test("the whole log rides one group, never rolled", async () => { const track = new Track.Producer("test").accept({ compression: "deflate" }); const producer = new Producer({ track }); - for (let n = 0; n < 50; n++) producer.append({ value: { n } }); + for (let n = 0; n < 50; n++) producer.append(next({ n })); producer.finish(); // A single group holds everything, and the consumer reads it all in order. @@ -70,7 +73,7 @@ test("records with embedded newlines round-trip (JSON escapes the newline)", asy const track = new Track.Producer("test").accept({ compression: "deflate" }); const producer = new Producer<{ s: string }>({ track }); const value = { s: "line1\nline2\ttab" }; - for (let i = 0; i < 4; i++) producer.append({ value: value }); + for (let i = 0; i < 4; i++) producer.append(next(value)); producer.finish(); const consumer = new Consumer<{ s: string }>({ track: track.subscribe() }); @@ -117,7 +120,7 @@ for (const compress of [false, true]) { test(`schema validation stays inside the read (compress=${compress})`, async () => { const track = new Track.Producer("test").accept({ compression: compress ? "deflate" : "none" }); const producer = new Producer({ track }); - producer.append({ value: { n: 1 } }); + producer.append(next({ n: 1 })); producer.finish(); const schema = z.pipe( @@ -128,7 +131,7 @@ for (const compress of [false, true]) { }), ); const consumer = new Consumer({ track: track.subscribe(), schema }); - expect(await consumer.next()).toEqual({ value: 2, at: undefined }); + expect((await consumer.next())?.value).toBe(2); expect(await consumer.next()).toBeUndefined(); }); } @@ -147,7 +150,7 @@ test("a second concurrent read is refused rather than served the first one's gro // group for a second one and fail a perfectly good log. Rust gets this from `&mut self`. const track = new Track.Producer("test"); const producer = new Producer({ track }); - producer.append({ value: { n: 0 } }); + producer.append(next({ n: 0 })); producer.finish(); const consumer = new Consumer({ track: track.subscribe() }); @@ -184,9 +187,9 @@ test("blocked reads leave nothing behind on the pending group read", async () => const reactions = await pendingReactions(async () => { for (let n = 0; n < 1000; n++) { - const next = consumer.next(); - producer.append({ value: { n } }); - expect((await next)?.value.n).toBe(n); + const reading = consumer.next(); + producer.append(next({ n })); + expect((await reading)?.value.n).toBe(n); } }); expect(reactions).toBeLessThan(10); @@ -195,14 +198,17 @@ test("blocked reads leave nothing behind on the pending group read", async () => producer.finish(); }); -test("an untimed record reads back untimed", async () => { +test("a scale-less log opens its group at the first key, and every record carries it", async () => { const track = new Track.Producer("test").accept({}); const producer = new Producer({ track }); - producer.append({ value: 1 }); + producer.append({ value: 1, at: Time.Timestamp.unscaled(4n) }); + producer.append({ value: 2, at: Time.Timestamp.unscaled(9n) }); producer.finish(); const consumer = new Consumer({ track: track.subscribe() }); - expect(await consumer.next()).toEqual({ value: 1, at: undefined }); + const at = Time.Timestamp.unscaled(4n); + expect(await consumer.next()).toEqual({ value: 1, at }); + expect(await consumer.next()).toEqual({ value: 2, at }); expect(await consumer.next()).toBeUndefined(); }); @@ -226,7 +232,7 @@ for (const compress of [false, true]) { test(`a stream has no record ceiling (compress=${compress})`, async () => { const track = new Track.Producer("test").accept({ compression: compress ? "deflate" : "none" }); const producer = new Producer({ track }); - for (let n = 0; n < 9000; n++) producer.append({ value: { n } }); + for (let n = 0; n < 9000; n++) producer.append(next({ n })); producer.finish(); const records = await drain(track.subscribe()); expect(records.length).toBe(9000); @@ -240,7 +246,7 @@ for (const compress of [false, true]) { const slow = new Consumer({ track: track.subscribe() }); const pad = "x".repeat(1024 * 1024); for (let n = 0; n < 100; n++) { - producer.append({ value: { n, pad } }); + producer.append(next({ n, pad })); expect((await fast.next())?.value.n).toBe(n); } await expect(slow.next()).rejects.toMatchObject({ code: StreamCode.Evicted }); @@ -256,9 +262,9 @@ for (const compress of [false, true]) { test("finishing returns a log that appends nothing", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); - producer.append({ value: { n: 0 } }); + producer.append(next({ n: 0 })); const finished = producer.finish(); // @ts-expect-error a finished log appends no record - expect(() => finished.append({ value: { n: 1 } })).toThrow(); + expect(() => finished.append(next({ n: 1 }))).toThrow(); expect(await drain(track.subscribe())).toEqual([0]); }); diff --git a/js/json/src/time.ts b/js/json/src/time.ts new file mode 100644 index 00000000000..728a5f012a8 --- /dev/null +++ b/js/json/src/time.ts @@ -0,0 +1,38 @@ +// How a producer places a write's timestamp on its track: a timed track's groups take the next +// sequence, while a scale-less track's frames carry a group's key, which names its group. +import * as Moq from "@moq/net"; + +/** + * Whether a group whose first frame is at `at` would start past every group on `track`, which a + * new scale-less group must: two groups never share a key. A timed group takes the next sequence, + * so it always does. Mirrors the Rust `starts_past`. + */ +export function startsPast(track: Moq.Track.Producer, at: Moq.Time.Timestamp): boolean { + const key = Moq.Time.keyOf(at); + if (key === undefined) return true; + const largest = track.largest(); + return largest === undefined || key > largest.group; +} + +/** + * Open a group for a write at `at`: the next sequence on a timed track, or on a scale-less one the + * group `at` names, which must pass every group the track holds ({@link Moq.Error.TimestampMismatch} + * otherwise). The sequences it passes are skipped, as `appendGroup` skips them. + */ +export function openGroup(track: Moq.Track.Producer, at: Moq.Time.Timestamp): Moq.Group.Producer { + const key = Moq.Time.keyOf(at); + if (key === undefined) return track.appendGroup(); + if (!startsPast(track, at)) throw new Moq.Error.TimestampMismatch(); + const largest = track.largest(); + if (largest !== undefined && largest.group + 1n < key) track.skipGroups(largest.group + 1n, key); + return track.createGroup(key); +} + +/** + * `at` for a frame of `group`: a scale-less track's frames carry their group's key, so a frame + * appended to an open group carries that group's key, whatever key the caller named. Mirrors the + * Rust `frame_time`. + */ +export function frameTime(group: Moq.Group.Producer, at: Moq.Time.Timestamp): Moq.Time.Timestamp { + return at.scale === undefined ? Moq.Time.Timestamp.unscaled(group.sequence) : at; +} diff --git a/js/json/src/window/consumer.ts b/js/json/src/window/consumer.ts index 8c6fa07b9cd..06118a2e9c1 100644 --- a/js/json/src/window/consumer.ts +++ b/js/json/src/window/consumer.ts @@ -25,9 +25,10 @@ const GAPS: Moq.StreamCode[] = [ * window yields nothing for records already delivered, so this reads as one continuous stream of * {@link Event}s regardless of how the publisher framed them. * - * Each event carries the timestamp of the frame that produced it, absent on an untimed track. The - * wire has no per-record time, so a record a reader first learns of from a group header (one it - * joined after, or missed while behind) gets the header frame's time, not its original push time. + * Each event carries the timestamp of the frame that produced it, which on a scale-less track is + * its group's key. The wire has no per-record time, so a record a reader first learns of from a + * group header (one it joined after, or missed while behind) gets the header frame's time, not + * its original push time. * * @public */ @@ -65,7 +66,8 @@ export class Consumer { // Drain what the last frame produced before reading more, so every queued event is that // frame's and shares its timestamp. const event = this.#decoder.next(); - if (event) return { value: event, at: this.#at }; + // An event follows the frame that produced it, so that frame's timestamp is set. + if (event && this.#at) return { value: event, at: this.#at }; if (!this.#group) { this.#group = await this.#track.nextGroup(); diff --git a/js/json/src/window/producer.ts b/js/json/src/window/producer.ts index 1b7be001787..ecae8f020dd 100644 --- a/js/json/src/window/producer.ts +++ b/js/json/src/window/producer.ts @@ -1,6 +1,7 @@ import type * as Moq from "@moq/net"; import type { Time, Timed } from "@moq/net"; +import { frameTime, openGroup, startsPast } from "../time.ts"; import { type Config, type Encoded, Encoder } from "./encoder.ts"; /** Window producer options, including the destination track. */ @@ -31,8 +32,9 @@ export interface Finished { * * A group of ops runs until a joiner wants a fresh one: once the ops outweigh the track's `refresh` * ratio times the header, a join asks for a group past the largest, and the producer restates the - * window there at the newest timestamp written. Set the ratio when creating the track, or joins - * never ask. + * window there at the newest timestamp written. On a scale-less track that timestamp names the + * group, so while it is the newest group's own key the refresh waits for a later edit, since two + * groups never share a key. Set the ratio when creating the track, or joins never ask. * * @public */ @@ -48,9 +50,12 @@ export class Producer { #largest?: bigint; // The newest timestamp written, which a refresh restates the window at. An edit may carry an // older timestamp than its group's first frame, and a new group may not start before that one. + // Undefined before the first write. #at?: Time.Timestamp; // Stops watching the track's demand. #unwatch?: () => void; + // A scale-less refresh that waits for a key past the newest group's, which the next write answers. + #deferred = false; /** Wrap a track to publish a window into it. */ constructor(config: ProducerConfig) { @@ -65,12 +70,20 @@ export class Producer { // Restate the window in a new group once a joiner wants one past the largest written. #refresh(wanted: bigint | undefined): void { + this.#deferred = false; if (this.#finished || wanted === undefined || this.#largest === undefined || wanted <= this.#largest) return; + // Two groups never share a key, so a scale-less refresh at the newest group's own key waits for + // a later write. Checked before encoding, since a header dropped unwritten resets the encoder. + const at = this.#at; + if (at === undefined || !startsPast(this.#track, at)) { + this.#deferred = true; + return; + } // Nobody awaits a refresh, so a failed write has no caller to report to. Abort the track rather // than leave its newest group without the window, where a joiner would wait. try { - this.#write(this.#encoder.snapshot(), this.#at); + this.#write(this.#encoder.snapshot(), at); } catch (err) { this.#close(err instanceof Error ? err : new Error(String(err))); } @@ -89,8 +102,9 @@ export class Producer { /** * Append one record to the back of the window. * - * `at` is when the record was captured, written as its frame timestamp. Omit it only on an - * untimed track: it is never filled in. + * `at` is when the record was captured, written as its frame timestamp. On a scale-less track it + * is {@link Time.Timestamp.unscaled} at a group key instead: a header opens its group there, which + * must pass every group the track holds, and an op carries the open group's key. */ push({ value, at }: Timed): void { this.#assertOpen(); @@ -104,10 +118,9 @@ export class Producer { * A no-op when the window is already empty, and clamped to what it holds, so a caller can trim * unconditionally. * - * `at` is when the records were retired, written as the frame timestamp. Omit it only on an - * untimed track: it is never filled in. + * `at` is when the records were retired, written as the frame timestamp (see {@link push}). */ - pop(count: number, at?: Time.Timestamp): void { + pop(count: number, at: Time.Timestamp): void { this.#assertOpen(); const frame = this.#encoder.pop(count); if (!frame) return; @@ -115,26 +128,29 @@ export class Producer { this.#advance(at); } - // Remember `at` if it is the newest timestamp written. - #advance(at: Time.Timestamp | undefined): void { - if (at && !at.isBefore(this.#at ?? at)) this.#at = at; + // Remember `at` if it is the newest timestamp written, then answer a deferred refresh it may + // unblock: a scale-less refresh waits for a key past the open group's. + #advance(at: Time.Timestamp): void { + if (this.#at === undefined || this.#at.isBefore(at)) this.#at = at; + if (this.#deferred) this.#refresh(this.#track.demand().wanted.peek()); } #assertOpen(): void { if (this.#finished) throw new Error("track is closed"); } - #write(encoded: Encoded & { commit(): void }, at: Time.Timestamp | undefined): void { + #write(encoded: Encoded & { commit(): void }, at: Time.Timestamp): void { // A throw leaves the edit uncommitted and the retained window unchanged. The next edit opens a // new group, since a consumer may hold the group the frame was meant for. if (encoded.keyframe) { - // The previous group is complete; no more frames will be appended to it. Drop the handle - // before opening the next one, so a failure below doesn't leave a closed group behind. + // Opened before the previous group finishes, so a key at or before the newest group's is + // refused with the previous group still open. + const group = openGroup(this.#track, at); + this.#largest = group.sequence; + + // The previous group is complete; no more frames will be appended to it. this.#group?.finish(); this.#group = undefined; - - const group = this.#track.appendGroup(); - this.#largest = group.sequence; try { group.writeFrame({ payload: encoded.payload, timestamp: at }); } catch (err) { @@ -150,7 +166,7 @@ export class Producer { } if (!this.#group) throw new Error("op with no open group"); - this.#group.writeFrame({ payload: encoded.payload, timestamp: at }); + this.#group.writeFrame({ payload: encoded.payload, timestamp: frameTime(this.#group, at) }); encoded.commit(); } diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index c91204ca030..48f1ab5f4b1 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -1,9 +1,12 @@ -import { expect, test } from "bun:test"; +import { beforeEach, expect, test } from "bun:test"; import { Duration, yieldNow } from "@moq/clock"; import { Error as NetError, StreamCode, Time, type Timed, Track } from "@moq/net"; +import { key, next, resetKeys } from "../key.test-util.ts"; import { Consumer } from "./consumer.ts"; import { Decoder, Encoder, type Event, Producer, type Span } from "./index.ts"; +beforeEach(resetKeys); + type Rec = { n: number }; /** @@ -27,12 +30,12 @@ class Live { } async push(n: number): Promise { - this.producer.push({ value: { n } }); + this.producer.push(next({ n })); await this.read(); } async pop(count: number): Promise { - this.producer.pop(count); + this.producer.pop(count, key()); await this.read(); } @@ -110,11 +113,11 @@ test("concurrent consumer reads are rejected", async () => { test("writes after finish are rejected before encoding", () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); - producer.push({ value: 1 }); + producer.push(next(1)); producer.finish(); - expect(() => producer.push({ value: 2 })).toThrow("track is closed"); - expect(() => producer.pop(1)).toThrow("track is closed"); + expect(() => producer.push(next(2))).toThrow("track is closed"); + expect(() => producer.pop(1, key())).toThrow("track is closed"); expect(producer.window).toEqual([1]); }); @@ -210,7 +213,7 @@ test("compressed round-trip across rolls", async () => { expect(pushed(await live.finish())).toEqual(Array.from({ length: 40 }, (_, n) => n)); }); -// The frame count and first-frame timestamp of each group in a finished track. +// The frame count and first-frame time of each group in a finished track, undefined when scale-less. async function groups(track: Track.Producer): Promise<[number, number | undefined][]> { const ordered = track.subscribe({ delay: { max: Duration.millis(30_000) } }).ordered(); const out: [number, number | undefined][] = []; @@ -218,7 +221,7 @@ async function groups(track: Track.Producer): Promise<[number, number | undefine const first = await group.readFrame(); let frames = first ? 1 : 0; while ((await group.readFrame()) !== undefined) frames++; - out.push([frames, first?.timestamp?.asMillis()]); + out.push([frames, first?.timestamp.scale === undefined ? undefined : first.timestamp.asMillis()]); } return out; } @@ -236,8 +239,8 @@ test("a joiner over the ratio gets a fresh header", async () => { ] as const) { const track = new Track.Producer("test").accept({ refresh: 1, maxAge: null }); const producer = new Producer({ track }); - producer.push({ value: { n: 1000 } }); - for (let n = 0; n < pushes; n++) producer.push({ value: { n } }); + producer.push(next({ n: 1000 })); + for (let n = 0; n < pushes; n++) producer.push(next({ n })); const joiner = track.subscribe(); await settle(); @@ -293,7 +296,7 @@ test("a window refresh after an older edit starts at the newest timestamp", asyn test("a window refresh needs a ratio", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); - for (let n = 0; n < 4; n++) producer.push({ value: { n } }); + for (let n = 0; n < 4; n++) producer.push(next({ n })); const joiner = track.subscribe(); await settle(); @@ -322,8 +325,8 @@ test("the window slides", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); for (let n = 0; n < 5; n++) { - producer.push({ value: { n } }); - if (n >= 2) producer.pop(1); + producer.push(next({ n })); + if (n >= 2) producer.pop(1, key()); } expect(producer.offset).toBe(3); @@ -346,7 +349,7 @@ test("a pop is clamped to the window", async () => { test("an empty pop writes nothing", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track }); - producer.pop(5); + producer.pop(5, key()); producer.finish(); // Nothing was ever pushed, so there is nothing to drop and no group to publish. @@ -367,8 +370,8 @@ test("a lagging consumer is told what it missed", async () => { // The consumer stops reading while the window slides past everything it holds. for (let n = 2; n < 8; n++) { - producer.push({ value: { n } }); - producer.pop(1); + producer.push(next({ n })); + producer.pop(1, key()); } subscriber.withRange({ start: { included: (await subscriber.largest())?.group ?? 0n } }); const events = await live.finish(); @@ -420,8 +423,8 @@ test("consumer resumes at a checkpoint after losing a group", async () => { test("a fresh consumer adopts the current offset", async () => { const track = new Track.Producer("test"); const producer = new Producer({ track, ops: false }); - for (let n = 0; n < 5; n++) producer.push({ value: { n } }); - producer.pop(3); + for (let n = 0; n < 5; n++) producer.push(next({ n })); + producer.pop(3, key()); const subscriber = track.subscribe(); subscriber.withRange({ start: { included: (await subscriber.largest())?.group ?? 0n } }); @@ -551,18 +554,17 @@ test("an uncommitted edit leaves the window unchanged", () => { expect(encoder.window).toEqual([2, 3]); }); -test("edits keep their timestamps, and an edit without one goes out untimed", async () => { - for (const at of [undefined, Time.Timestamp.fromMillis(1_000)]) { - const track = new Track.Producer("test").accept(at ? { timescale: Time.Timescale.MILLI } : {}); +test("edits keep their timestamps, and a scale-less edit its group's key", async () => { + for (const at of [Time.Timestamp.unscaled(3n), Time.Timestamp.fromMillis(1_000)]) { + const track = new Track.Producer("test").accept(at.scale ? { timescale: Time.Timescale.MILLI } : {}); const producer = new Producer({ track }); producer.push({ value: { n: 0 }, at }); - producer.pop(1, at); + producer.pop(1, at.scale ? at : Time.Timestamp.unscaled(4n)); producer.finish(); + // A scale-less header opens its group at its key, and an op carries that key. const group = await track.subscribe().ordered().nextGroup(); - for (let i = 0; i < 2; i++) { - expect((await group?.readFrame())?.timestamp?.as(Time.Timescale.MILLI)).toBe(at?.as(Time.Timescale.MILLI)); - } + for (let i = 0; i < 2; i++) expect((await group?.readFrame())?.timestamp).toEqual(at); } }); @@ -601,28 +603,28 @@ test("events carry their frame's timestamp, and a header's restated records get expect(events.map(millis)).toEqual([400, 400]); }); -test("events on an untimed track have no timestamp", async () => { +test("events on a scale-less track carry their keys", async () => { const track = new Track.Producer("test"); const consumer = new Consumer({ track: track.subscribe() }); const producer = new Producer({ track }); - producer.push({ value: { n: 0 } }); - producer.pop(1); + producer.push(next({ n: 0 })); + producer.pop(1, key()); producer.finish(); const events: Timed>[] = []; for await (const event of consumer) events.push(event); expect(events.length).toBe(2); - expect(events.every((event) => event.at === undefined)).toBeTrue(); + expect(events.every((event) => event.at.scale === undefined)).toBeTrue(); }); // Mirrors the Rust `window::Finished` doctest: the window `finish` returns still reads, but takes // no edit, at compile time and at run time. test("finishing returns a window that reads but takes no edit", () => { const producer = new Producer({ track: new Track.Producer("test") }); - producer.push({ value: { n: 1 } }); + producer.push(next({ n: 1 })); const finished = producer.finish(); expect(finished.window).toEqual([{ n: 1 }]); expect(finished.offset).toBe(0); // @ts-expect-error a finished window takes no record - expect(() => finished.push({ value: { n: 2 } })).toThrow("track is closed"); + expect(() => finished.push(next({ n: 2 }))).toThrow("track is closed"); }); diff --git a/js/net/api/index.api.md b/js/net/api/index.api.md index 81a90222648..5c776f3d18a 100644 --- a/js/net/api/index.api.md +++ b/js/net/api/index.api.md @@ -273,7 +273,7 @@ export namespace Datagram { export interface Datagram { payload: Uint8Array; sequence: bigint; - timestamp?: Time.Timestamp; + timestamp: Time.Timestamp; } export interface Info { priority: number; @@ -283,7 +283,7 @@ export namespace Datagram { constructor(name: string); abort(error: Error): void; accept(properties?: Partial): this; - append(timestamp: Time.Timestamp | undefined, payload: Uint8Array): bigint; + append(timestamp: Time.Timestamp, payload: Uint8Array): bigint; get closed(): Signals.GetPromise; consume(): Datagram.Consumer; finish(): void; @@ -294,7 +294,10 @@ export namespace Datagram { } export function rescale(info: { timescale?: Time.Timescale | null; - }, timestamp: Time.Timestamp | undefined): Time.Timestamp | undefined; + }, timestamp: Time.Timestamp): Time.Timestamp; + export function rescaleIn(info: { + timescale?: Time.Timescale | null; + }, timestamp: Time.Timestamp, key: bigint): Time.Timestamp; } export namespace Error { @@ -366,18 +369,18 @@ export namespace Frame { readChunk(): Promise; readonly sequence: number; readonly size: number; - readonly timestamp: Time.Timestamp | undefined; + readonly timestamp: Time.Timestamp; } export interface Info { size: number; - timestamp?: Time.Timestamp; + timestamp: Time.Timestamp; } export class Producer { abort(error: Error): void; finish(): void; get remaining(): number; readonly size: number; - readonly timestamp: Time.Timestamp | undefined; + readonly timestamp: Time.Timestamp; write(chunk: Uint8Array): void; } } @@ -412,7 +415,7 @@ export namespace Group { export interface Frame { arrived?: Clock.Instant; payload: Uint8Array; - timestamp?: Time.Timestamp; + timestamp: Time.Timestamp; } export interface Info { sequence: bigint; @@ -670,13 +673,15 @@ export namespace Time { static fromMillis(ms: number): Time.Timestamp; isBefore(other: Time.Timestamp): boolean; static now(): Time.Timestamp; - readonly scale: Time.Timescale; + readonly scale: Time.Timescale | undefined; + static unscaled(key: bigint | number): Time.Timestamp; readonly value: number; } + export let keyOf: (timestamp: Time.Timestamp) => bigint | undefined; } export interface Timed { - at?: Time.Timestamp; + at: Time.Timestamp; value: T; } diff --git a/js/net/bench/compression-fanout.ts b/js/net/bench/compression-fanout.ts index f25628e17de..20f2aae8e72 100644 --- a/js/net/bench/compression-fanout.ts +++ b/js/net/bench/compression-fanout.ts @@ -1,5 +1,6 @@ import type { Producer as Group } from "../src/group.ts"; import { hooks } from "../src/internal.ts"; +import { Timestamp } from "../src/time.ts"; import { Producer as Track } from "../src/track.ts"; /** Sweep compressed publishers and public group readers, sharing one inflated copy per group. */ @@ -22,7 +23,8 @@ export function fanout() { const consumers = groups.map((group) => Array.from({ length: readers }, () => group.consume())); let started = performance.now(); for (const group of groups) { - for (let index = 0; index < frames; index++) group.writeFrame({ payload }); + for (let index = 0; index < frames; index++) + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } publish += performance.now() - started; @@ -81,7 +83,7 @@ export function tails() { const started = performance.now(); for (let index = 0; index < frames; index++) { for (const [publisher, group] of groups.entries()) { - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); for (const consumer of consumers[publisher]) { const frame = consumer.tryReadFrameSequence(); if ( diff --git a/js/net/bench/datagrams.ts b/js/net/bench/datagrams.ts index 8ca94efc37b..d8fabe003ed 100644 --- a/js/net/bench/datagrams.ts +++ b/js/net/bench/datagrams.ts @@ -1,5 +1,6 @@ /** Sweep datagram tracks x readers, with a stalled reader sharing each bounded buffer. */ import { Producer } from "../src/datagram.ts"; +import { Timestamp } from "../src/time.ts"; const payload = new Uint8Array(1024); async function measure(tracks: number, readers: number): Promise<{ join: number; delivery: number }> { @@ -11,7 +12,8 @@ async function measure(tracks: number, readers: number): Promise<{ join: number; const start = performance.now(); for (let i = 0; i < 256; i++) { for (const [index, producer] of producers.entries()) { - producer.append(undefined, payload); + // A scale-less datagram's timestamp is its sequence. + producer.append(Timestamp.unscaled(i), payload); for (const consumer of consumers[index]) { if ((await consumer.recv())?.sequence !== BigInt(i)) throw new Error("missing datagram"); } diff --git a/js/net/bench/drift.ts b/js/net/bench/drift.ts index b8cd1ea003b..a8b7ef74c54 100644 --- a/js/net/bench/drift.ts +++ b/js/net/bench/drift.ts @@ -34,9 +34,10 @@ export async function run(): Promise { for (const publishers of [1, 4]) for (const readers of [1, 8]) for (const mode of [0, 1, 2]) { - const stamp = (ms: number, index: number) => + // A scale-less frame carries its group's key instead of a time. + const stamp = (ms: number, index: number, key: bigint) => mode === 0 - ? undefined + ? Time.Timestamp.unscaled(key) : mode === 2 && index % 2 !== 0 ? Time.Timestamp.fromMicros(ms * 1000) : Time.Timestamp.fromMillis(ms); @@ -56,7 +57,7 @@ export async function run(): Promise { const group = track.appendGroup(); group.writeFrame({ payload, - timestamp: stamp(sequence * 100, sequence), + timestamp: stamp(sequence * 100, sequence, BigInt(sequence)), }); group.finish(); } @@ -98,6 +99,7 @@ export async function run(): Promise { Math.floor(iteration / offsets.length) * 100 + offsets[iteration % offsets.length], iteration, + BigInt(retained), ); for (const source of sources) source.writeFrame({ payload, timestamp }); for (const reader of waiting) @@ -105,7 +107,9 @@ export async function run(): Promise { for (const frame of await Promise.all(pending)) if ( frame?.payload[0] !== 7 || - frame.timestamp?.asMillis() !== timestamp?.asMillis() + (timestamp.scale === undefined + ? frame.timestamp.value !== timestamp.value + : frame.timestamp.asMillis() !== timestamp.asMillis()) ) throw new Error("frame missing or reordered"); } diff --git a/js/net/bench/fill-probe.ts b/js/net/bench/fill-probe.ts index d78ffeaa292..5a5fd6f3b39 100644 --- a/js/net/bench/fill-probe.ts +++ b/js/net/bench/fill-probe.ts @@ -5,6 +5,7 @@ import { Duration } from "@moq/clock"; import { Producer as BroadcastProducer } from "../src/broadcast.ts"; import { hooks, staleHole } from "../src/internal.ts"; +import { Timestamp } from "../src/time.ts"; console.debug = () => {}; console.log("runs,readers,step_us"); @@ -15,7 +16,7 @@ for (const runs of [250, 1000, 4000, 16000]) { // 2 and 3 are read up front, so the only gaps any fill is asked for are the drops below. for (const sequence of [2n, 3n]) { const live = track.createGroup(sequence); - live.writeFrame({ payload: new Uint8Array([0]) }); + live.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(live.sequence) }); live.finish(); } const requests = track.requests(); @@ -43,7 +44,7 @@ for (const runs of [250, 1000, 4000, 16000]) { hooks.dropTrackGroups(track, sequence, sequence + 1n, staleHole(60_000)); for (const sub of subs) sub.tryRecvGroup(); const group = track.createGroup(sequence + 1n); - group.writeFrame({ payload: new Uint8Array([0]) }); + group.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); for (const sub of subs) { if (sub.tryRecvGroup()?.sequence !== sequence + 1n) diff --git a/js/net/bench/fills.ts b/js/net/bench/fills.ts index 8991a448c78..b2a97d1c9b2 100644 --- a/js/net/bench/fills.ts +++ b/js/net/bench/fills.ts @@ -12,6 +12,7 @@ import { createMockTransportPair } from "../src/mock"; import { Producer } from "../src/origin"; import * as Path from "../src/path"; import { Reader, Stream } from "../src/stream"; +import { Timestamp } from "../src/time"; console.debug = () => {}; const version = Version.DRAFT_22; @@ -23,7 +24,7 @@ for (const groups of [2, 16, 128]) { const track = publication.broadcast.createTrack("video", { maxAge: Duration.millis(60_000) }); for (let i = 0; i < groups; i++) { const group = track.appendGroup(); - group.writeFrame({ payload: new Uint8Array([i % 256]) }); + group.writeFrame({ payload: new Uint8Array([i % 256]), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } const peers = await Promise.all( diff --git a/js/net/bench/refresh.ts b/js/net/bench/refresh.ts index 219b56f2394..9d40485d114 100644 --- a/js/net/bench/refresh.ts +++ b/js/net/bench/refresh.ts @@ -7,6 +7,7 @@ import { ALPN_07_WIP } from "../src/lite/version.ts"; import { createMockTransportPair } from "../src/mock.ts"; import { Producer } from "../src/origin.ts"; import * as Path from "../src/path.ts"; +import { Timestamp } from "../src/time.ts"; console.debug = () => {}; console.log("retained,tracks,joined,refresh_us"); @@ -39,7 +40,7 @@ for (const retained of [1, 64, 1024]) const baseline = consumers.map((track) => track.subscribe()); for (let i = 0; i < tracks; i++) { const group = producers[i].appendGroup(); - group.writeFrame({ payload: new Uint8Array(3) }); + group.writeFrame({ payload: new Uint8Array(3), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); const received = await baseline[i].recvGroup(); if (!received) throw new Error("baseline group missing"); @@ -50,7 +51,7 @@ for (const retained of [1, 64, 1024]) // track grows, so scanning that cache appears independently of fanout. for (let old = 1; old < retained; old++) { const group = producers[0].appendGroup(); - group.writeFrame({ payload: new Uint8Array(3) }); + group.writeFrame({ payload: new Uint8Array(3), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); const received = await baseline[0].recvGroup(); if (!received) throw new Error("historical group missing"); @@ -61,8 +62,8 @@ for (const retained of [1, 64, 1024]) const samples: number[] = []; for (let round = 0; round < 9; round++) { const group = producers[0].appendGroup(); - group.writeFrame({ payload: new Uint8Array(3) }); - group.writeFrame({ payload: new Uint8Array(4) }); + group.writeFrame({ payload: new Uint8Array(3), timestamp: Timestamp.unscaled(group.sequence) }); + group.writeFrame({ payload: new Uint8Array(4), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); const received = await baseline[0].recvGroup(); if (!received) throw new Error("measured group missing"); diff --git a/js/net/bench/replay.ts b/js/net/bench/replay.ts index 7f78650ad02..7fcb5e0c37b 100644 --- a/js/net/bench/replay.ts +++ b/js/net/bench/replay.ts @@ -1,7 +1,7 @@ /** Sweep cached-group replay to joining subscribers, and arrivals into unread backlogs. */ import { Duration } from "@moq/clock"; import { Manual } from "@moq/clock/test"; -import { Track } from "@moq/net"; +import { Time, Track } from "@moq/net"; type Order = "monotonic" | "reverse" | "mixed"; const ORDERS: Order[] = ["monotonic", "reverse", "mixed"]; @@ -33,7 +33,7 @@ function publish(track: Track.Producer, sequence: number): void { const group = track.createGroup(BigInt(sequence)); const payload = new Uint8Array(PAYLOAD); new DataView(payload.buffer).setUint32(0, sequence); - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Time.Timestamp.unscaled(group.sequence) }); group.finish(); } diff --git a/js/net/bench/resume-history.ts b/js/net/bench/resume-history.ts index db707957199..b52b0f3ea5e 100644 --- a/js/net/bench/resume-history.ts +++ b/js/net/bench/resume-history.ts @@ -26,6 +26,7 @@ import { createMockTransportPair } from "../src/mock.ts"; import { Producer as OriginProducer } from "../src/origin.ts"; import * as Path from "../src/path.ts"; import { Front } from "../src/resume.ts"; +import { Timestamp } from "../src/time.ts"; import { wireOf } from "../src/wire.ts"; const PAYLOAD = 4096; @@ -226,7 +227,7 @@ async function measureOn(clock: Manual, { fronts, names: perRoute, routes, histo for (const name of all) { const track = broadcast.createTrack(name); const group = track.appendGroup(); - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } } diff --git a/js/net/bench/resume.ts b/js/net/bench/resume.ts index a1b59ebedb2..2aa31ae6c49 100644 --- a/js/net/bench/resume.ts +++ b/js/net/bench/resume.ts @@ -11,6 +11,7 @@ import { Producer as BroadcastProducer } from "../src/broadcast.ts"; import { Producer as GroupProducer } from "../src/group.ts"; import { hooks } from "../src/internal.ts"; import { Front } from "../src/resume.ts"; +import { Timestamp } from "../src/time.ts"; import type { Subscriber } from "../src/track.ts"; const readerCounts = [1, 4, 16]; @@ -49,7 +50,7 @@ async function measure(readers: number, swaps: number | undefined, fork = false) // Every route's copy holds the same groups under the same sequences. const write = () => { const group = track.createGroup(sequence); - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); }; sources.push({ broadcast, write }); @@ -145,7 +146,7 @@ async function measureFlaps(readers: number, flaps: number): Promise { const tracks = sources.map((source) => source.createTrack("seq")); const groups = tracks.map((track) => { const group = track.appendGroup(); - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); return group; }); front.serve(handles[0]); @@ -193,10 +194,10 @@ async function measureReplay(readers: number, replayed: number): Promise const producer = new GroupProducer(0n); const replacement = new GroupProducer(0n); for (let index = 0; index < replayed; index++) { - producer.writeFrame({ payload }); - replacement.writeFrame({ payload }); + producer.writeFrame({ payload, timestamp: Timestamp.unscaled(producer.sequence) }); + replacement.writeFrame({ payload, timestamp: Timestamp.unscaled(replacement.sequence) }); } - replacement.writeFrame({ payload }); + replacement.writeFrame({ payload, timestamp: Timestamp.unscaled(replacement.sequence) }); const consumers = Array.from({ length: readers }, () => producer.mirror()); for (const consumer of consumers) { for (let index = 0; index < replayed; index++) { diff --git a/js/net/bench/socket.ts b/js/net/bench/socket.ts index fba778c5310..bcb871268d7 100644 --- a/js/net/bench/socket.ts +++ b/js/net/bench/socket.ts @@ -2,7 +2,7 @@ import { yieldNow } from "@moq/clock"; import { accept } from "../src/connection/accept.ts"; import { connect } from "../src/connection/connect.ts"; -import { Origin, Path, Socket } from "../src/index.ts"; +import { Origin, Path, Socket, Time } from "../src/index.ts"; import * as Lite from "../src/lite/index.ts"; import { createMockTransportPair } from "../src/mock.ts"; @@ -50,7 +50,10 @@ for (const mode of ["local", "lite07"] as const) const opened = performance.now(); await Promise.all( pairs.map(async ([local, remote]) => { - local.producer.writeFrame({ payload: new Uint8Array(1024) }); + local.producer.writeFrame({ + payload: new Uint8Array(1024), + timestamp: Time.Timestamp.unscaled(0n), + }); const frame = await remote.consumer.ordered().readFrame(); if (frame?.payload.length !== 1024) throw new Error("missing socket payload"); }), diff --git a/js/net/src/auth.test.ts b/js/net/src/auth.test.ts index 97fa80c8bb6..f09eb89dccd 100644 --- a/js/net/src/auth.test.ts +++ b/js/net/src/auth.test.ts @@ -13,6 +13,7 @@ import { } from "./auth.ts"; import { accept as acceptSession, connect as connectSession, type Session } from "./connection/index.ts"; import { SessionCode, SessionError, StreamCode, StreamError, toStreamCode } from "./error.ts"; +import { appendFrame } from "./frame.test-util.ts"; import * as Ietf from "./ietf/index.ts"; import { AuthMessage as LiteAuthMessage, LiteAuthWire } from "./lite/auth.ts"; import * as Lite from "./lite/index.ts"; @@ -491,7 +492,7 @@ describe.each([Lite.ALPN_07_WIP])("%s", (protocol) => { const broadcastRoute = origin.route(Path.from("room/x")); const broadcast = broadcastRoute.broadcast; const published = broadcast.createTrack("video"); - published.appendGroup().writeFrame({ payload: new Uint8Array([1]) }); + appendFrame(published, new Uint8Array([1])); const { client, server } = await connect({ serverPublish: origin, protocol }); const requests = server.auth.requests(); await wireOf(client).auth.ready(Path.from("room/x"), "subscribe"); @@ -516,10 +517,7 @@ describe.each([Lite.ALPN_07_WIP])("%s", (protocol) => { const renewing = answered(client.auth.add("renewal")); const renewal = await requests.next(); if (!renewal) throw new Error("missing renewal"); - broadcast - .createTrack("audio") - .appendGroup() - .writeFrame({ payload: new Uint8Array([3]) }); + appendFrame(broadcast.createTrack("audio"), new Uint8Array([3])); const covered = wireOf(client).consume(Path.from("room/x")).track("audio").subscribe(); expect(await covered.recvGroup()).toBeDefined(); covered.close(); @@ -528,7 +526,7 @@ describe.each([Lite.ALPN_07_WIP])("%s", (protocol) => { first.close(); await yieldNow(); expect(track.closed.peek()).toBeUndefined(); - published.appendGroup().writeFrame({ payload: new Uint8Array([2]) }); + appendFrame(published, new Uint8Array([2])); expect(await track.recvGroup()).toBeDefined(); track.close(); renewed.close(); @@ -623,7 +621,7 @@ describe.each([Lite.ALPN_07_WIP])("%s", (protocol) => { const broadcastRoute = origin.route(Path.from("room/x")); const broadcast = broadcastRoute.broadcast; const published = broadcast.createTrack("video"); - published.appendGroup().writeFrame({ payload: new Uint8Array([7]) }); + appendFrame(published, new Uint8Array([7])); const { client, server } = await connect({ serverPublish: origin, protocol }); const requests = server.auth.requests(); const setup = await requests.next(); @@ -658,10 +656,7 @@ describe.each([Lite.ALPN_07_WIP])("%s", (protocol) => { const origin = new OriginProducer(); const broadcastRoute = origin.route(Path.from("room/x")); const broadcast = broadcastRoute.broadcast; - broadcast - .createTrack("video") - .appendGroup() - .writeFrame({ payload: new Uint8Array([1]) }); + appendFrame(broadcast.createTrack("video"), new Uint8Array([1])); const { client, server } = await connect({ serverPublish: origin, protocol }); await waitFor(client.auth.grant, (g) => g !== undefined && g.subscribe.size > 0); @@ -1090,10 +1085,7 @@ test("a fetch abandoned while its token waits never reaches the wire", async () const origin = new OriginProducer(); const broadcastRoute = origin.route(Path.from("room/x")); const broadcast = broadcastRoute.broadcast; - broadcast - .createTrack("video") - .appendGroup() - .writeFrame({ payload: new Uint8Array([1]) }); + appendFrame(broadcast.createTrack("video"), new Uint8Array([1])); const { client, server, transport } = await connect({ serverPublish: origin, protocol: Lite.ALPN_07_WIP }); const requests = server.auth.requests(); const setup = await requests.next(); @@ -1170,10 +1162,7 @@ test.each(Array.from({ length: 17 }, (_, gap) => gap))( const origin = new OriginProducer(); const broadcastRoute = origin.route(Path.from("room/x")); const broadcast = broadcastRoute.broadcast; - broadcast - .createTrack("video") - .appendGroup() - .writeFrame({ payload: new Uint8Array([7]) }); + appendFrame(broadcast.createTrack("video"), new Uint8Array([7])); const { client, server } = await connect({ serverPublish: origin, protocol: Lite.ALPN_07_WIP }); const requests = server.auth.requests(); const setup = await requests.next(); @@ -1312,9 +1301,8 @@ test("a revoked grant resets its subscriptions with UNAUTHORIZED", async () => { })(); await waitFor(client.auth.grant, (g) => g !== undefined && g.subscribe.size > 0); - const frame = { payload: new Uint8Array([1]) }; - downTrack.appendGroup().writeFrame(frame); - upTrack.appendGroup().writeFrame(frame); + appendFrame(downTrack, new Uint8Array([1])); + appendFrame(upTrack, new Uint8Array([1])); // The client subscribes to the server, and the server to the client. const downSub = wireOf(client).consume(Path.from("room/x")).track("video").subscribe().ordered(); diff --git a/js/net/src/cache.test.ts b/js/net/src/cache.test.ts index 0e5900e591d..fcf6f40bb1d 100644 --- a/js/net/src/cache.test.ts +++ b/js/net/src/cache.test.ts @@ -6,6 +6,7 @@ import { StreamCode, StreamError, TooFarBehind } from "./error.ts"; import { hooks, staleHole } from "./internal.ts"; import { RUN_OVERHEAD as ARRIVAL_OVERHEAD } from "./ledger.ts"; import { Tail } from "./tail.ts"; +import { Timestamp } from "./time.ts"; import { Producer as TrackProducer } from "./track.ts"; // A reader continued on a held re-offer keeps the frames the failed copy wrote that it has yet to @@ -18,7 +19,7 @@ test("a re-offer bills the failed copy's frames its readers still hold", async ( const track = producer.accept({ maxAge: null }); hooks.receiveTrack(track); const first = hooks.receiveGroup(track, { sequence: 0n, start: 0 }); - first.writeFrame({ payload: new Uint8Array(10_000) }); + first.writeFrame({ payload: new Uint8Array(10_000), timestamp: Timestamp.unscaled(first.sequence) }); const reader = first.consume(); const arriving = hooks.arrivingGroup(track, 0n); first.abort(new Error("reset")); @@ -52,7 +53,7 @@ function pooled(pool: cache.Pool, name = "test"): TrackProducer { function finished(producer: TrackProducer, size: number): bigint { const group = producer.appendGroup(); - group.writeFrame({ payload: new Uint8Array(size) }); + group.writeFrame({ payload: new Uint8Array(size), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); return group.sequence; } @@ -131,7 +132,8 @@ test("carried debt keeps the live group", async () => { const group = producer.appendGroup(); const reader = group.consume(); - for (let i = 0; i < 3; i++) group.writeFrame({ payload: new Uint8Array(unit) }); + for (let i = 0; i < 3; i++) + group.writeFrame({ payload: new Uint8Array(unit), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); for (let i = 0; i < 3; i++) expect((await reader.readFrame())?.payload.length).toBe(unit); expect(await reader.readFrame()).toBeUndefined(); @@ -200,7 +202,7 @@ test("a shared capacity spares a fresher track and evicts a staler one's oldest" const pool = new cache.Pool({ capacity: 6 * billed(10_000) }); const open = (producer: TrackProducer) => { const group = producer.appendGroup(); - group.writeFrame({ payload: new Uint8Array(10_000) }); + group.writeFrame({ payload: new Uint8Array(10_000), timestamp: Timestamp.unscaled(group.sequence) }); return group; }; const staler = pooled(pool, "staler"); @@ -235,7 +237,7 @@ test("the sweep expires idle groups across tracks without a write", async () => const tracks = [pooled(pool, "a"), pooled(pool, "b")]; const stalled = tracks.map((track) => { const group = track.appendGroup(); // left open - group.writeFrame({ payload: new Uint8Array(1) }); + group.writeFrame({ payload: new Uint8Array(1), timestamp: Timestamp.unscaled(group.sequence) }); finished(track, 1); // the live edge return group; }); @@ -275,7 +277,8 @@ test("a group's tail limit releases what it sheds from the pool", async () => { const producer = pooled(pool); const group = producer.appendGroup(); const frame = new Uint8Array(1024 * 1024); - for (let i = 0; i < 40; i++) group.writeFrame({ payload: frame }); + for (let i = 0; i < 40; i++) + group.writeFrame({ payload: frame, timestamp: Timestamp.unscaled(group.sequence) }); expect(hooks.groupCacheBytes(group)).toBeLessThanOrEqual(32 * 1024 * 1024); expect(pool.used).toBe(hooks.groupCacheBytes(group) + cache.ENTRY_OVERHEAD); }); @@ -292,7 +295,8 @@ test("the debt sheds the front of an open live edge", async () => { const group = producer.appendGroup(); const behind = reader.tryRecvGroup(); const frame = new Uint8Array(64 * 1024); - for (let i = 0; i < 64; i++) group.writeFrame({ payload: frame }); + for (let i = 0; i < 64; i++) + group.writeFrame({ payload: frame, timestamp: Timestamp.unscaled(group.sequence) }); expect(group.closed.peek()).toBeUndefined(); // What a frame write may add before it settles, on top of the capacity. @@ -385,7 +389,7 @@ for (const stale of [false, true]) { let error: unknown; for (let sequence = 2n * runs; sequence < 2n * runs + 64n && error === undefined; sequence++) { const group = producer.createGroup(sequence); - group.writeFrame({ payload: new Uint8Array(16 * 1024) }); + group.writeFrame({ payload: new Uint8Array(16 * 1024), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); try { expect(reader.tryRecvGroup()?.sequence).toBe(sequence); @@ -541,14 +545,14 @@ test("folding an arrival gap wakes the reader parked on it", async () => { // gap below it is the first thing a payment takes: no other signal moves. hooks.dropTrackGroups(producer, 0n, 1n, old()); const live = producer.createGroup(2n); - live.writeFrame({ payload: new Uint8Array(8) }); + live.writeFrame({ payload: new Uint8Array(8), timestamp: Timestamp.unscaled(live.sequence) }); for (let group = reader.tryRecvGroup(); group; group = reader.tryRecvGroup()); for (let run = hooks.tryDrop(reader); run; run = hooks.tryDrop(reader)); let woken = 0; const dispose = hooks.groupChanged(reader, () => woken++); expect(hooks.arrivalRuns(producer)).toBe(2); // Past the write threshold that settles debt. - live.writeFrame({ payload: new Uint8Array(512 * 1024) }); + live.writeFrame({ payload: new Uint8Array(512 * 1024), timestamp: Timestamp.unscaled(live.sequence) }); expect(hooks.arrivalRuns(producer)).toBe(1); await yieldNow(); expect(woken).toBeGreaterThan(0); @@ -712,7 +716,7 @@ test("a held failed fill answer stays billed", async () => { hooks.receiveTrack(producer); const reader = producer.subscribe({ delay: { max: Duration.millis(60_000) } }); const answer = hooks.receiveGroup(producer, { sequence: 1n, start: 0, fill: true }); - answer.writeFrame({ payload: new Uint8Array(10_000) }); + answer.writeFrame({ payload: new Uint8Array(10_000), timestamp: Timestamp.unscaled(answer.sequence) }); const group = await reader.recvGroup(); expect(group?.sequence).toBe(1n); answer.abort(new Error("archive failed")); diff --git a/js/net/src/datagram.test.ts b/js/net/src/datagram.test.ts index 0a1af9eba18..0d47d4f8073 100644 --- a/js/net/src/datagram.test.ts +++ b/js/net/src/datagram.test.ts @@ -1,5 +1,9 @@ import { expect, test } from "bun:test"; import * as Broadcast from "./broadcast.ts"; +import { Timescale, Timestamp } from "./time.ts"; + +/** A scale-less datagram's timestamp: its sequence. */ +const seq = (sequence: bigint) => Timestamp.unscaled(sequence); test("a broadcast keeps same-name datagram and stream tracks independent", async () => { const broadcast = new Broadcast.Producer(); @@ -7,7 +11,7 @@ test("a broadcast keeps same-name datagram and stream tracks independent", async const datagrams = broadcast.createDatagrams("audio"); const reader = broadcast.consume().datagrams("audio"); expect(streams.appendGroup().sequence).toBe(0n); - expect(datagrams.append(undefined, new Uint8Array([1]))).toBe(0n); + expect(datagrams.append(seq(0n), new Uint8Array([1]))).toBe(0n); expect(streams.appendGroup().sequence).toBe(1n); expect((await reader.recv())?.sequence).toBe(0n); reader.close(); @@ -19,14 +23,13 @@ test("a broadcast keeps same-name datagram and stream tracks independent", async import * as Datagram from "./datagram.ts"; import { StreamCode, StreamError, TimestampMismatch } from "./error.ts"; import { hooks } from "./internal.ts"; -import { Timescale, Timestamp } from "./time.ts"; import { wireOf } from "./wire.ts"; test("datagram readers start live, clone their cursor, and drain only an existing finish", async () => { const track = new Datagram.Producer("test").accept(); - track.append(undefined, new Uint8Array([0])); + track.append(seq(0n), new Uint8Array([0])); const reader = track.consume(); - track.append(undefined, new Uint8Array([1])); + track.append(seq(1n), new Uint8Array([1])); track.finish(); const cloned = reader.clone(); expect((await reader.recv())?.sequence).toBe(1n); @@ -44,7 +47,7 @@ test("a shared datagram buffer bounds both count and bytes for stalled readers", const slow = track.consume(); const fast = track.consume(); for (let i = 0; i < 70; i++) { - track.append(undefined, new Uint8Array(size)); + track.append(seq(BigInt(i)), new Uint8Array(size)); expect((await fast.recv())?.sequence).toBe(BigInt(i)); } track.finish(); @@ -56,13 +59,15 @@ test("a shared datagram buffer bounds both count and bytes for stalled readers", }); test("datagram inserts preserve repeats and reordering, and append past the high water mark", async () => { - const track = new Datagram.Producer("test").accept(); + // Timed, so `append` picks the sequence: a scale-less timestamp names its own. + const track = new Datagram.Producer("test").accept({ timescale: Timescale.MILLI }); const reader = track.consume(); - for (const sequence of [10n, 3n, 10n]) track.insert({ sequence, payload: new Uint8Array() }); - expect(track.append(undefined, new Uint8Array())).toBe(11n); + for (const sequence of [10n, 3n, 10n]) + track.insert({ sequence, timestamp: Timestamp.fromMillis(Number(sequence)), payload: new Uint8Array() }); + expect(track.append(Timestamp.fromMillis(11), new Uint8Array())).toBe(11n); for (const sequence of [10n, 3n, 10n, 11n]) expect((await reader.recv())?.sequence).toBe(sequence); - track.insert({ sequence: (1n << 64n) - 1n, payload: new Uint8Array() }); - expect(() => track.append(undefined, new Uint8Array())).toThrow(RangeError); + track.insert({ sequence: (1n << 64n) - 1n, timestamp: Timestamp.fromMillis(0), payload: new Uint8Array() }); + expect(() => track.append(Timestamp.fromMillis(1), new Uint8Array())).toThrow(RangeError); track.finish(); reader.close(); }); @@ -70,7 +75,7 @@ test("datagram inserts preserve repeats and reordering, and append past the high test("refused datagram writes spend neither sequences nor buffer space", async () => { const track = new Datagram.Producer("test").accept({ timescale: Timescale.MILLI }); const reader = track.consume(); - expect(() => track.append(undefined, new Uint8Array())).toThrow(TimestampMismatch); + expect(() => track.append(seq(0n), new Uint8Array())).toThrow(TimestampMismatch); expect(() => track.append(Timestamp.fromMillis(1), new Uint8Array(65536))).toThrow(); expect(track.append(Timestamp.fromMicros(1234), new Uint8Array([7]))).toBe(0n); expect((await reader.recv())?.timestamp).toEqual(Timestamp.fromMillis(1)); @@ -78,10 +83,26 @@ test("refused datagram writes spend neither sequences nor buffer space", async ( reader.close(); }); +test("a scale-less datagram's timestamp names its sequence", async () => { + const track = new Datagram.Producer("test").accept(); + const reader = track.consume(); + expect(() => track.append(Timestamp.fromMillis(1), new Uint8Array())).toThrow(TimestampMismatch); + // Another key than the sequence is a mismatch. + expect(() => track.insert({ sequence: 4n, timestamp: seq(5n), payload: new Uint8Array() })).toThrow( + TimestampMismatch, + ); + expect(track.append(seq(4n), new Uint8Array())).toBe(4n); + const received = await reader.recv(); + expect(received?.sequence).toBe(4n); + expect(received?.timestamp).toEqual(seq(4n)); + track.finish(); + reader.close(); +}); + test("an abort or local close wakes datagram reads with its exact result", async () => { const track = new Datagram.Producer("test").accept(); const reader = track.consume(); - track.append(undefined, new Uint8Array([1])); + track.append(seq(0n), new Uint8Array([1])); const cause = new StreamError(StreamCode.Unauthorized); track.abort(cause); await expect(reader.recv()).rejects.toBe(cause); @@ -100,8 +121,8 @@ test("each datagram reader owns its copy, and edits never reach the shared byte const second = track.consume(); // Two of these fit the 64 KiB bound; a third evicts exactly one. const size = 30_000; - track.append(undefined, new Uint8Array(size).fill(5)); - track.append(undefined, new Uint8Array(size).fill(6)); + track.append(seq(0n), new Uint8Array(size).fill(5)); + track.append(seq(1n), new Uint8Array(size).fill(6)); const edited = await first.recv(); if (!edited) throw new Error("missing datagram"); @@ -114,7 +135,7 @@ test("each datagram reader owns its copy, and edits never reach the shared byte // Replacing a returned payload must not change what eviction subtracts. edited.payload = new Uint8Array(0); - track.append(undefined, new Uint8Array(size).fill(7)); + track.append(seq(2n), new Uint8Array(size).fill(7)); expect((await second.recv())?.payload[0]).toBe(6); expect((await second.recv())?.payload[0]).toBe(7); first.close(); @@ -148,7 +169,7 @@ test("abandoned datagram requests leave neither a queued request nor a name clai expect(request?.name).toBe("test"); if (!request) throw new Error("missing request"); const track = request.accept(); - track.append(undefined, new Uint8Array([1])); + track.append(seq(0n), new Uint8Array([1])); expect((await next.recv())?.payload).toEqual(new Uint8Array([1])); track.finish(); next.close(); @@ -164,7 +185,7 @@ test("creating a datagram track in the turn its queued request closes serves the left.close(); const track = broadcast.createDatagrams("test"); const next = consumer.datagrams("test"); - track.append(undefined, new Uint8Array([2])); + track.append(seq(0n), new Uint8Array([2])); expect((await next.recv())?.payload).toEqual(new Uint8Array([2])); track.finish(); next.close(); @@ -200,7 +221,7 @@ test("passive forwarding drains a finished track without adding demand", async ( expect(producer.priority.peek()).toBe(-7); reader.close(); expect(producer.priority.peek()).toBeUndefined(); - producer.append(undefined, Uint8Array.of(1)); + producer.append(seq(0n), Uint8Array.of(1)); producer.finish(); for (const consumer of [observer, copy]) { expect((await consumer.recv())?.payload).toEqual(Uint8Array.of(1)); diff --git a/js/net/src/datagram.ts b/js/net/src/datagram.ts index 24b93eed6da..0a559368f05 100644 --- a/js/net/src/datagram.ts +++ b/js/net/src/datagram.ts @@ -6,25 +6,31 @@ import { type GetPromise, type Getter, Once, Signal } from "@moq/signals"; import { StreamCode, StreamError } from "./error.ts"; import { hooks } from "./internal.ts"; -import { rescale } from "./rescale.ts"; -import { Timescale, type Timestamp } from "./time.ts"; +import { rescaleIn } from "./rescale.ts"; +import { keyOf, Timescale, type Timestamp } from "./time.ts"; import { checkSequence, max } from "./util/sequence.ts"; -export { rescale } from "./rescale.ts"; +export { rescale, rescaleIn } from "./rescale.ts"; /** A single unreliable payload, addressed within its datagram track. */ export interface Datagram { /** The datagram track's own sequence number. */ sequence: bigint; - /** Presentation timestamp, present exactly when the track has a timescale. */ - timestamp?: Timestamp; + /** + * Presentation timestamp in the track's timescale, or on a scale-less track + * {@link Timestamp.unscaled} at its sequence. + */ + timestamp: Timestamp; /** The datagram payload. */ payload: Uint8Array; } /** Publisher properties fixed for the lifetime of a datagram track. */ export interface Info { - /** Timestamp units per second, or `null` for an untimed track. */ + /** + * Timestamp units per second, or `null` for a scale-less (untimed) track, whose datagrams' + * timestamps are {@link Timestamp.unscaled} at their sequence. + */ timescale: Timescale | null; /** Publisher priority, higher first, breaking ties between subscriber priorities. */ priority: number; @@ -145,9 +151,15 @@ export class Producer { return consume(this.name, this.#state); } - /** Append a payload with the next sequence, returning that sequence. */ - append(timestamp: Timestamp | undefined, payload: Uint8Array): bigint { - const sequence = this.#state.next; + /** + * Append a payload with the next sequence, returning that sequence. + * + * A scale-less track has no time, so its datagram's {@link Timestamp.unscaled} names the + * sequence instead, as its key names a group. A timestamp that doesn't match the track + * throws a `TimestampMismatch`. + */ + append(timestamp: Timestamp, payload: Uint8Array): bigint { + const sequence = keyOf(timestamp) ?? this.#state.next; this.insert({ sequence, timestamp, payload }); return sequence; } @@ -161,7 +173,8 @@ export class Producer { if (!properties) throw new Error("datagram track has not been accepted"); const sequence = checkSequence(datagram.sequence); if (datagram.payload.byteLength > MAX_PAYLOAD) throw new Error("datagram payload too large"); - const timestamp = rescale(properties, datagram.timestamp); + // A scale-less datagram's timestamp is its sequence. + const timestamp = rescaleIn(properties, datagram.timestamp, sequence); // A small view must not pin a peer's larger receive buffer. const payload = new Uint8Array(datagram.payload); state.next = max(state.next, sequence + 1n); diff --git a/js/net/src/fill.test.ts b/js/net/src/fill.test.ts index 9796ef1fffc..61763a19ca7 100644 --- a/js/net/src/fill.test.ts +++ b/js/net/src/fill.test.ts @@ -46,7 +46,8 @@ function serve(fill: Fill, archive: Archive): void { continue; } const group = fill.accept(sequence); - for (const frame of frames) group.writeFrame({ payload: enc.encode(frame) }); + for (const frame of frames) + group.writeFrame({ payload: enc.encode(frame), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } fill.close(); @@ -73,7 +74,7 @@ function resumed(first: bigint): { broadcast: BroadcastProducer; track: Producer const broadcast = new BroadcastProducer(); const track = broadcast.createTrack("video", { maxAge: null }); const group = track.createGroup(first); - group.writeFrame({ payload: enc.encode(`live ${first}`) }); + group.writeFrame({ payload: enc.encode(`live ${first}`), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); return { broadcast, track }; } @@ -135,7 +136,7 @@ test("fill: a fetch of a group about to be written asks the handler at once, and // The producer gets there before the handler answers: the live write is the answer. const group = track.createGroup(7n); - group.writeFrame({ payload: enc.encode("written") }); + group.writeFrame({ payload: enc.encode("written"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); expect(fill.wanted()).toBeUndefined(); expect(await frames(await fetching)).toEqual(["written"]); @@ -215,7 +216,7 @@ test("fill: an answer aborted below the end does not hold the end open", async ( const fill = await requests.requestedFill(); if (!fill) throw new Error("no fill"); const group = fill.accept(0n); - group.writeFrame({ payload: enc.encode("backfill") }); + group.writeFrame({ payload: enc.encode("backfill"), timestamp: Timestamp.unscaled(group.sequence) }); if (parked) { track.finishAt({ group: 2n, frame: 0 }); await yieldNow(); @@ -351,7 +352,7 @@ test("fill: a held answer that fails keeps its frames and its failure", async () const fill = await requests.requestedFill(); if (!fill) throw new Error("no fill"); const answer = fill.accept(1n); - answer.writeFrame({ payload: enc.encode("fill 1") }); + answer.writeFrame({ payload: enc.encode("fill 1"), timestamp: Timestamp.unscaled(answer.sequence) }); const group = await fetching; const failure = new Error("archive failed"); answer.abort(failure); @@ -381,7 +382,7 @@ test("fill: an answer withheld after a probe judged it is asked again", async () if (!fill) throw new Error("no fill"); fill.accept(0n).finish(); const answer = fill.accept(2n); - answer.writeFrame({ payload: enc.encode("fill 2") }); + answer.writeFrame({ payload: enc.encode("fill 2"), timestamp: Timestamp.unscaled(answer.sequence) }); expect((await first)?.sequence).toBe(0n); // Waiting on group 1, the bounded window probes everything it names, group 2 held among it. const second = ordered.nextGroup(); @@ -412,7 +413,7 @@ test("fill: a gap below a group that fails after a probe is asked", async () => const ordered = reader.ordered(); const failed = track.createGroup(2n); const group = track.createGroup(4n); - group.writeFrame({ payload: enc.encode("live 4") }); + group.writeFrame({ payload: enc.encode("live 4"), timestamp: Timestamp.unscaled(group.sequence) }); failed.abort(new StreamError(StreamCode.Cancel)); await yieldNow(); // Judged stale against group 4, so not asked. @@ -438,7 +439,7 @@ test("fill: a gap the newest group rises over after a probe is asked", async () expect(reader.tryRecvGroup()?.sequence).toBe(2n); expect(reader.tryRecvGroup()).toBeUndefined(); const group = track.createGroup(5n); - group.writeFrame({ payload: enc.encode("live 5") }); + group.writeFrame({ payload: enc.encode("live 5"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); expect(reader.tryRecvGroup()?.sequence).toBe(5n); reader.tryRecvGroup(); @@ -590,7 +591,7 @@ test("fill: a group arriving after a refusal reaches the reader the refusal name fill.refuseGroups(2n, 3n, new StreamError(StreamCode.Cancel, { message: "storage busy" })); // Before the waiting fetch looks again. const group = track.createGroup(2n); - group.writeFrame({ payload: enc.encode("late") }); + group.writeFrame({ payload: enc.encode("late"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); fill.close(); expect(await frames(await waiting)).toEqual(["late"]); @@ -665,7 +666,7 @@ test("fill: a stale verdict leaves an outstanding fill owed", async () => { hooks.dropTrackGroups(track, 2n, 3n, staleHole(500)); expect(fill.wanted()).toBe(2n); const group = fill.accept(2n); - group.writeFrame({ payload: enc.encode("filled") }); + group.writeFrame({ payload: enc.encode("filled"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); expect(await frames(await fetching)).toEqual(["filled"]); @@ -686,7 +687,7 @@ test("fill: a group arriving live answers the fill", async () => { // The producer writes it itself before the handler gets to it. const live = track.createGroup(1n); - live.writeFrame({ payload: enc.encode("live") }); + live.writeFrame({ payload: enc.encode("live"), timestamp: Timestamp.unscaled(live.sequence) }); live.finish(); expect(fill.wanted()).toBeUndefined(); expect(() => fill.accept(1n)).toThrow(); @@ -799,7 +800,10 @@ test("fill: a bounded reader keeps waiting on a fresh fill once an older one goe const track = broadcast.createTrack("video", { maxAge: null }); const write = (sequence: bigint) => { const group = track.createGroup(sequence); - group.writeFrame({ payload: enc.encode(`live ${sequence}`) }); + group.writeFrame({ + payload: enc.encode(`live ${sequence}`), + timestamp: Timestamp.unscaled(group.sequence), + }); group.finish(); }; // Holes 0-1 wait on group 2, which arrives first; holes 3-4 on group 5, half a second later. @@ -827,7 +831,7 @@ test("fill: a bounded reader keeps waiting on a fresh fill once an older one goe // Holes 0-1 are past the budget now, but 3-4 are not: the read waits on them. await clock.advance(Duration.millis(600)); const late = fills[1].accept(3n); - late.writeFrame({ payload: enc.encode("old 3") }); + late.writeFrame({ payload: enc.encode("old 3"), timestamp: Timestamp.unscaled(late.sequence) }); late.finish(); expect((await next)?.sequence).toBe(3n); @@ -874,7 +878,7 @@ test("fill: a reader joining a wider fill is owed only its own ranges", async () for (const sequence of [2n, 7n]) { const group = fill.accept(sequence); - group.writeFrame({ payload: enc.encode(`old ${sequence}`) }); + group.writeFrame({ payload: enc.encode(`old ${sequence}`), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } // The wide reader still waits on the rest of the fill; the narrow one has all it reads. @@ -988,7 +992,7 @@ test("fill: a filled group stays wanted only while a reader will still read it", if (!fill) throw new Error("no fill"); expect(fill.wanted()).toBe(0n); const source = fill.accept(0n); - source.writeFrame({ payload: enc.encode("g0") }); + source.writeFrame({ payload: enc.encode("g0"), timestamp: Timestamp.unscaled(source.sequence) }); const taken = await next; expect(taken?.sequence).toBe(0n); @@ -1414,7 +1418,7 @@ test("fill: an OLD answer under the max age is held as stale until forgotten", a const broadcast = new BroadcastProducer(); const track = broadcast.createTrack("video", { maxAge: Duration.millis(10_000) }); const live = track.createGroup(3n); - live.writeFrame({ payload: enc.encode("live 3") }); + live.writeFrame({ payload: enc.encode("live 3"), timestamp: Timestamp.unscaled(live.sequence) }); live.finish(); const requests = track.requests(); const terms: Subscription = { @@ -1471,10 +1475,11 @@ function headless(): { broadcast: BroadcastProducer; track: Producer } { const broadcast = new BroadcastProducer(); const track = broadcast.createTrack("video", { maxAge: null }); const tail = hooks.receiveGroup(track, { sequence: 2n, start: 3 }); - for (const frame of ["c3", "c4"]) tail.writeFrame({ payload: enc.encode(frame) }); + for (const frame of ["c3", "c4"]) + tail.writeFrame({ payload: enc.encode(frame), timestamp: Timestamp.unscaled(2n) }); tail.finish(); const next = track.createGroup(3n); - next.writeFrame({ payload: enc.encode("d0") }); + next.writeFrame({ payload: enc.encode("d0"), timestamp: Timestamp.unscaled(3n) }); next.finish(); return { broadcast, track }; } diff --git a/js/net/src/flate/group.test.ts b/js/net/src/flate/group.test.ts index 58dee17d2f5..b3d58dd9d36 100644 --- a/js/net/src/flate/group.test.ts +++ b/js/net/src/flate/group.test.ts @@ -2,6 +2,7 @@ import { expect, spyOn, test } from "bun:test"; import { StreamCode } from "../error.ts"; import { Producer as Group } from "../group.ts"; import { hooks } from "../internal.ts"; +import { Timestamp } from "../time.ts"; import { Producer as Track } from "../track.ts"; import { Decoder } from "./decoder.ts"; import { Encoder } from "./encoder.ts"; @@ -15,8 +16,8 @@ test("synchronous writers store slices; plain readers share one charged inflated const first = group.mirror(); const second = group.mirror(); const raw = group.mirror(); - group.writeFrame({ payload }); - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); const stored = hooks.groupCacheBytes(group); expect(stored).toBeLessThan(payload.length); @@ -45,7 +46,7 @@ test("received slices are forwarded byte-for-byte and exact declarations fail al const group = new Group(0n); hooks.bindGroupCompression(group, true); const slice = new Encoder().frame(payload); - hooks.writeGroupFrame(group, { payload: slice }, payload.length + 1); + hooks.writeGroupFrame(group, { payload: slice, timestamp: Timestamp.unscaled(0n) }, payload.length + 1); group.finish(); const raw = await hooks.readGroupFrame(group.mirror(), true); expect(raw?.frame.payload).toBe(slice); @@ -60,10 +61,12 @@ test("a slice that fails to decode fails its open group, which takes no more fra hooks.bindGroupCompression(group, true); const raw = group.consume(); const slice = new Encoder().frame(payload); - hooks.writeGroupFrame(group, { payload: slice }, payload.length + 1); + hooks.writeGroupFrame(group, { payload: slice, timestamp: Timestamp.unscaled(0n) }, payload.length + 1); await expect(group.consume().readFrame()).rejects.toThrow(/declared size/); expect(group.closed.peek()).toBeInstanceOf(Error); - expect(() => hooks.writeGroupFrame(group, { payload: slice }, payload.length)).toThrow(); + expect(() => + hooks.writeGroupFrame(group, { payload: slice, timestamp: Timestamp.unscaled(0n) }, payload.length), + ).toThrow(); await expect(raw.readFrame()).rejects.toThrow(/declared size/); }); @@ -82,7 +85,7 @@ for (const readPlain of [false, true]) { const budget = 32 * 1024 * 1024; for (let index = 0; index < budget / bytes.length + 4; index++) { const slice = encoder.frame(bytes); - hooks.writeGroupFrame(group, { payload: slice }, bytes.length); + hooks.writeGroupFrame(group, { payload: slice, timestamp: Timestamp.unscaled(0n) }, bytes.length); const frame = await hooks.readGroupFrame(raw, true); expect(frame?.frame.payload).toBe(slice); expect(decoder.frame(slice, bytes.length)).toEqual(bytes); @@ -97,10 +100,10 @@ for (const readPlain of [false, true]) { test("empty frames preserve the window", async () => { const track = new Track("compressed").accept({ compression: "deflate" }); const group = track.createGroup(0n); - group.writeFrame({ payload }); - group.writeFrame({ payload: new Uint8Array() }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); + group.writeFrame({ payload: new Uint8Array(), timestamp: Timestamp.unscaled(group.sequence) }); const reader = group.mirror(); - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); for (const expected of [payload, new Uint8Array(), payload]) expect((await reader.readFrame())?.payload).toEqual(expected); @@ -112,7 +115,7 @@ test("a cut compressed group preserves its committed frames before returning its const track = new Track("compressed").accept({ compression: "deflate" }); const group = track.appendGroup(); const reader = group.mirror(); - group.writeFrame({ payload }); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); const cut = new Error("cut"); group.abort(cut); expect((await reader.readFrame())?.payload).toEqual(payload); @@ -128,10 +131,12 @@ test("a shared plain read cannot change a later synchronous write's admission", // The largest local frame whose worst-case slice fits the frame cap. let size = 32 * 1024 * 1024; while (!encodable(size)) size--; - group.writeFrame({ payload: new Uint8Array(size) }); + group.writeFrame({ payload: new Uint8Array(size), timestamp: Timestamp.unscaled(group.sequence) }); expect((await reader.readFrame())?.payload.length).toBe(size); // The plain-byte budget is full, but another empty frame still fits, regardless of readers. - expect(() => group.writeFrame({ payload: new Uint8Array() })).not.toThrow(); + expect(() => + group.writeFrame({ payload: new Uint8Array(), timestamp: Timestamp.unscaled(group.sequence) }), + ).not.toThrow(); group.finish(); expect((await reader.readFrame())?.payload.length).toBe(0); track.finish(); @@ -142,7 +147,8 @@ test("refresh measures stored slices without inflating a compressed group", () = const track = new Track("compressed").accept({ compression: "deflate" }); const group = track.appendGroup(); const payload = Uint8Array.from({ length: 1024 }, (_, index) => index % 251); - for (let index = 0; index < 3; index++) group.writeFrame({ payload }); + for (let index = 0; index < 3; index++) + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); const inflating = spyOn(Decoder.prototype, "frame"); try { expect(hooks.overRefresh(group, 1)).toBe(false); diff --git a/js/net/src/flate/group.ts b/js/net/src/flate/group.ts index 0fe9994950d..06db7bdcf94 100644 --- a/js/net/src/flate/group.ts +++ b/js/net/src/flate/group.ts @@ -17,10 +17,10 @@ export interface Stored { // Class identity keeps reactive equality from reading the lazy payload getter. class SliceFrame implements Frame { - readonly timestamp: Timestamp | undefined; + readonly timestamp: Timestamp; readonly arrived: Instant | undefined; #read: () => Uint8Array; - constructor(timestamp: Timestamp | undefined, arrived: Instant | undefined, read: () => Uint8Array) { + constructor(timestamp: Timestamp, arrived: Instant | undefined, read: () => Uint8Array) { this.timestamp = timestamp; this.arrived = arrived; this.#read = read; diff --git a/js/net/src/frame.test-util.ts b/js/net/src/frame.test-util.ts new file mode 100644 index 00000000000..86628c8ad34 --- /dev/null +++ b/js/net/src/frame.test-util.ts @@ -0,0 +1,12 @@ +// Scale-less frames for tests: a scale-less track has no time, so each frame's timestamp is its +// group's key. +import type * as Group from "./group.ts"; +import { Timestamp } from "./time.ts"; +import type * as Track from "./track.ts"; + +/** Append a group to the scale-less `track` and write `payload` as its first frame, leaving it open. */ +export function appendFrame(track: Track.Producer, payload: Uint8Array): Group.Producer { + const group = track.appendGroup(); + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); + return group; +} diff --git a/js/net/src/frame.ts b/js/net/src/frame.ts index f663579dcf5..402df6a288e 100644 --- a/js/net/src/frame.ts +++ b/js/net/src/frame.ts @@ -21,8 +21,11 @@ import { SEGMENT_BYTES } from "./util/limits.ts"; export interface Info { /** The payload size in bytes, which the writes must add up to exactly. */ size: number; - /** Presentation timestamp, present exactly when the track has a timescale (see `Group.Frame.timestamp`). */ - timestamp?: Timestamp; + /** + * Presentation timestamp, or on a scale-less track {@link Timestamp.unscaled} at its group's + * key (see `Group.Frame.timestamp`). + */ + timestamp: Timestamp; } // A frame written in pieces, shared by its group and every mirror. Written whole, it stays the @@ -32,7 +35,7 @@ export interface Info { /** @internal */ export class Chunked implements Frame { readonly size: number; - readonly timestamp: Timestamp | undefined; + readonly timestamp: Timestamp; arrived?: Instant; // Bytes written so far, which a streaming reader waits on. readonly written = new Signal(0); @@ -166,8 +169,8 @@ export let makeProducer: (info: Info, ops: Ops) => Producer; export class Producer { /** The declared payload size in bytes. */ readonly size: number; - /** The frame's presentation timestamp, if its track is timed. */ - readonly timestamp: Timestamp | undefined; + /** The frame's presentation timestamp, or on a scale-less track its group's key. */ + readonly timestamp: Timestamp; #ops: Ops; #done = false; @@ -248,8 +251,8 @@ export class Consumer { readonly sequence: number; /** The payload size in bytes. */ readonly size: number; - /** The frame's presentation timestamp, if its track is timed. */ - readonly timestamp: Timestamp | undefined; + /** The frame's presentation timestamp, or on a scale-less track its group's key. */ + readonly timestamp: Timestamp; #frame: Frame; #offset = 0; diff --git a/js/net/src/group-id.test.ts b/js/net/src/group-id.test.ts index 54754b10c3c..10c04473641 100644 --- a/js/net/src/group-id.test.ts +++ b/js/net/src/group-id.test.ts @@ -8,6 +8,7 @@ import * as Lite from "./lite/index.ts"; import { createMockTransportPair } from "./mock.ts"; import { Producer as OriginProducer } from "./origin.ts"; import * as Path from "./path.ts"; +import { Timestamp } from "./time.ts"; import { wireOf } from "./wire.ts"; // The first group ID a `number` cannot hold, and the largest a QUIC varint carries. @@ -109,8 +110,14 @@ for (const wire of WIRES) { for (const sequence of [PAST_SAFE, MAX_QUIC]) { const group = track.createGroup(sequence); - group.writeFrame({ payload: enc.encode(`first ${sequence}`) }); - group.writeFrame({ payload: enc.encode(`second ${sequence}`) }); + group.writeFrame({ + payload: enc.encode(`first ${sequence}`), + timestamp: Timestamp.unscaled(group.sequence), + }); + group.writeFrame({ + payload: enc.encode(`second ${sequence}`), + timestamp: Timestamp.unscaled(group.sequence), + }); group.finish(); const received = await reader.recvGroup(); @@ -124,7 +131,11 @@ for (const wire of WIRES) { if (datagrams) { const unreliable = remote.datagrams("video"); await unreliable.info(); - datagrams.insert({ sequence: PAST_SAFE + 1n, payload: enc.encode("datagram") }); + datagrams.insert({ + sequence: PAST_SAFE + 1n, + timestamp: Timestamp.unscaled(PAST_SAFE + 1n), + payload: enc.encode("datagram"), + }); const received = await unreliable.recv(); expect(received?.sequence).toBe(PAST_SAFE + 1n); expect(dec.decode(received?.payload)).toBe("datagram"); diff --git a/js/net/src/group.test.ts b/js/net/src/group.test.ts index c603e8ad3b4..0c21112a89c 100644 --- a/js/net/src/group.test.ts +++ b/js/net/src/group.test.ts @@ -94,7 +94,11 @@ test("a frame read late keeps the instant it landed, plain, compressed, or resum // A frame written in with a stamp of its own still gets this node's. const forwarded = new Producer(0n); - forwarded.writeFrame({ payload: new Uint8Array([1]), arrived: first }); + forwarded.writeFrame({ + payload: new Uint8Array([1]), + arrived: first, + timestamp: Timestamp.unscaled(forwarded.sequence), + }); expect(forwarded.consume().tryReadFrame()?.arrived).toEqual(clock.now()); // A resumed read takes the stamp the replacement copy holds, not when it was adopted. @@ -173,15 +177,16 @@ test("a group read late keeps the instant it joined, alone, on a track, or resum } }); -test("zero group PTS keeps its scale and untimed groups stay unstamped", () => { +test("zero group PTS keeps its scale and scale-less groups stay unstamped", () => { const installed = new Manual().install(); try { - for (const timestamp of [undefined, Timestamp.fromMicros(0)]) { + for (const timestamp of [Timestamp.unscaled(0n), Timestamp.fromMicros(0)]) { const producer = new Producer(0n); const consumer = producer.consume(); expect(hooks.groupLatest(consumer)).toBeUndefined(); producer.writeFrame({ payload: new Uint8Array(), timestamp }); - expect(hooks.groupLatest(consumer)).toBe(timestamp); + // A group key is no presentation time. + expect(hooks.groupLatest(consumer)).toBe(timestamp.scale === undefined ? undefined : timestamp); consumer.close(); producer.finish(); } @@ -276,7 +281,7 @@ test("a live group retains its tail, while fast readers keep up and lagging read const slow = producer.consume(); const payload = new Uint8Array(1024 * 1024); for (let sequence = 0; sequence < 128; sequence++) { - producer.writeFrame({ payload }); + producer.writeFrame({ payload, timestamp: Timestamp.unscaled(producer.sequence) }); const frame = await fast.readFrameSequence(); expect(frame?.sequence).toBe(sequence); expect(frame?.payload).toBe(payload); @@ -294,7 +299,8 @@ test("empty frames consume tail storage without a frame count ceiling", async () const producer = new Producer(0n); const slow = producer.consume(); const payload = new Uint8Array(); - for (let index = 0; index < 600_000; index++) producer.writeFrame({ payload }); + for (let index = 0; index < 600_000; index++) + producer.writeFrame({ payload, timestamp: Timestamp.unscaled(producer.sequence) }); expect(hooks.groupFrameCount(producer)).toBe(600_000); expect(hooks.groupCacheBytes(producer)).toBeLessThanOrEqual(32 * 1024 * 1024); await expect(slow.readFrame()).rejects.toMatchObject({ code: StreamCode.Evicted }); @@ -524,14 +530,24 @@ test("buffered frames are still readable after the group closes", async () => { test("a plain frame has no size cap, while a compressed one keeps its whole-frame guard", () => { const producer = new Producer(0n); - producer.writeFrame({ payload: new Uint8Array(32 * 1024 * 1024 + 1) }); + producer.writeFrame({ + payload: new Uint8Array(32 * 1024 * 1024 + 1), + timestamp: Timestamp.unscaled(producer.sequence), + }); expect(hooks.groupFrameCount(producer)).toBe(1); producer.finish(); const compressed = new Producer(1n); hooks.bindGroupCompression(compressed, true); - expect(() => compressed.writeFrame({ payload: new Uint8Array(32 * 1024 * 1024 + 1) })).toThrow(FrameTooLarge); - expect(() => compressed.createFrame({ size: 32 * 1024 * 1024 + 1 })).toThrow(FrameTooLarge); + expect(() => + compressed.writeFrame({ + payload: new Uint8Array(32 * 1024 * 1024 + 1), + timestamp: Timestamp.unscaled(compressed.sequence), + }), + ).toThrow(FrameTooLarge); + expect(() => + compressed.createFrame({ size: 32 * 1024 * 1024 + 1, timestamp: Timestamp.unscaled(compressed.sequence) }), + ).toThrow(FrameTooLarge); expect(hooks.groupFrameCount(compressed)).toBe(0); compressed.finish(); }); @@ -546,16 +562,23 @@ test("a compressed frame is refused unless its worst-case slice fits a peer's fr hooks.bindGroupCompression(compressed, true); const reader = compressed.consume(); // Refused by size alone, even though zeros would compress far below the cap. - expect(() => compressed.writeFrame({ payload: new Uint8Array(largest + 1) })).toThrow(FrameTooLarge); - expect(() => compressed.createFrame({ size: largest + 1 })).toThrow(FrameTooLarge); - compressed.writeFrame({ payload: new Uint8Array(largest) }); + expect(() => + compressed.writeFrame({ + payload: new Uint8Array(largest + 1), + timestamp: Timestamp.unscaled(compressed.sequence), + }), + ).toThrow(FrameTooLarge); + expect(() => + compressed.createFrame({ size: largest + 1, timestamp: Timestamp.unscaled(compressed.sequence) }), + ).toThrow(FrameTooLarge); + compressed.writeFrame({ payload: new Uint8Array(largest), timestamp: Timestamp.unscaled(compressed.sequence) }); const read = await hooks.readGroupFrame(reader, true); expect(read?.frame.payload.byteLength).toBeLessThanOrEqual(MAX); compressed.finish(); // A frame written before the track chose compression is held to the same cap. const early = new Producer(1n); - early.writeFrame({ payload: new Uint8Array(largest + 1) }); + early.writeFrame({ payload: new Uint8Array(largest + 1), timestamp: Timestamp.unscaled(early.sequence) }); expect(() => hooks.bindGroupCompression(early, true)).toThrow(FrameTooLarge); }); @@ -563,7 +586,7 @@ test("closing a reader ends a whole read of a stalled frame", async () => { const producer = new Producer(0n); const aborted = producer.consume(); const closed = producer.consume(); - const open = producer.createFrame({ size: 10 }); + const open = producer.createFrame({ size: 10, timestamp: Timestamp.unscaled(producer.sequence) }); open.write(new Uint8Array(4)); const failing = aborted.readFrame(); const ending = closed.readFrame(); @@ -592,7 +615,7 @@ test("a frame many times the tail streams through a reader that keeps up", async const size = 96 * MiB + 123; const producer = new Producer(0n); const reader = producer.consume(); - const open = producer.createFrame({ size }); + const open = producer.createFrame({ size, timestamp: Timestamp.unscaled(producer.sequence) }); const frame = await reader.nextFrame(); expect(frame?.sequence).toBe(0); expect(frame?.size).toBe(size); @@ -625,7 +648,7 @@ test("a reader that falls a tail behind inside a frame is evicted there", async const producer = new Producer(0n); const slow = producer.consume(); const waiting = producer.consume(); - const open = producer.createFrame({ size: 64 * MiB }); + const open = producer.createFrame({ size: 64 * MiB, timestamp: Timestamp.unscaled(producer.sequence) }); const frame = await slow.nextFrame(); for (let written = 0; written < 40 * MiB; written += MiB) open.write(pattern(written, MiB)); // A reader that had not taken the frame can no longer start it. @@ -639,7 +662,7 @@ test("a whole read of a frame being written collects it as it streams", async () const size = 40 * MiB; const producer = new Producer(0n); const reader = producer.consume(); - const open = producer.createFrame({ size }); + const open = producer.createFrame({ size, timestamp: Timestamp.unscaled(producer.sequence) }); expect(reader.tryReadFrame()).toBeUndefined(); const whole = reader.readFrame(); for (let written = 0; written < size; written += MiB) { @@ -657,7 +680,7 @@ test("a small frame written in pieces reads whole once it completes", async () = const producer = new Producer(0n); const reader = producer.consume(); const readable = reader.readable(); - const open = producer.createFrame({ size: 6 }); + const open = producer.createFrame({ size: 6, timestamp: Timestamp.unscaled(producer.sequence) }); expect(() => producer.writeFrame(textFrame("x"))).toThrow("a frame is still being written"); open.write(new TextEncoder().encode("abc")); expect(reader.tryReadFrame()).toBeUndefined(); @@ -675,7 +698,7 @@ test("a small frame written in pieces reads whole once it completes", async () = test("a whole read grows with the bytes that arrive, never with the declared size", async () => { const producer = new Producer(0n); const reader = producer.consume(); - const open = producer.createFrame({ size: 2 ** 50 }); + const open = producer.createFrame({ size: 2 ** 50, timestamp: Timestamp.unscaled(producer.sequence) }); open.write(new Uint8Array(1)); const whole = reader.readFrame(); await yieldNow(); @@ -688,7 +711,7 @@ test("a whole read of a frame stalled partway fails with Old when its budget giv const reader = producer.consume(); const expired = new Signal(false); hooks.expireGroup(reader, { expired: () => expired.peek(), changed: [expired] }); - const open = producer.createFrame({ size: 10 }); + const open = producer.createFrame({ size: 10, timestamp: Timestamp.unscaled(producer.sequence) }); open.write(new Uint8Array(4)); const whole = reader.readFrame(); await yieldNow(); @@ -701,7 +724,7 @@ test("a frame written whole in one piece stays the writer's buffer", async () => const producer = new Producer(0n); const reader = producer.consume(); const payload = new Uint8Array(3 * 1024 * 1024); - const open = producer.createFrame({ size: payload.byteLength }); + const open = producer.createFrame({ size: payload.byteLength, timestamp: Timestamp.unscaled(producer.sequence) }); open.write(payload); open.finish(); const frame = await reader.nextFrame(); @@ -712,7 +735,7 @@ test("a frame written whole in one piece stays the writer's buffer", async () => test("ending a group mid-frame fails the frame and its readers", async () => { const producer = new Producer(0n); const reader = producer.consume(); - const open = producer.createFrame({ size: 10 }); + const open = producer.createFrame({ size: 10, timestamp: Timestamp.unscaled(producer.sequence) }); open.write(new Uint8Array(4)); const frame = await reader.nextFrame(); expect((await frame?.readChunk())?.byteLength).toBe(4); @@ -747,7 +770,7 @@ test("a group that ends on its own abandons the replacement it asked for", async return new Promise(() => {}); }, }); - producer.writeFrame({ payload: new Uint8Array([0]) }); + producer.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(producer.sequence) }); await consumer.readFrame(); route.set(1); @@ -782,7 +805,7 @@ test.each([false, true])("a group ending beside its replacement discards it (con requested.resolve(); }), }); - producer.writeFrame({ payload: new Uint8Array([0]) }); + producer.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(producer.sequence) }); await consumer.readFrame(); route.set(1); @@ -828,16 +851,26 @@ test("a group recovered twice reads on and settles", async () => { }), }); void consumer.closed; - producer.writeFrame({ payload: new Uint8Array([0]) }); + producer.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(producer.sequence) }); await consumer.readFrame(); // Two failures in a row: each moves the group to the next copy. producer.abort(new Error("route failed")); - replacements[0].writeFrame({ payload: new Uint8Array([0]) }); - replacements[0].writeFrame({ payload: new Uint8Array([1]) }); + replacements[0].writeFrame({ + payload: new Uint8Array([0]), + timestamp: Timestamp.unscaled(replacements[0].sequence), + }); + replacements[0].writeFrame({ + payload: new Uint8Array([1]), + timestamp: Timestamp.unscaled(replacements[0].sequence), + }); expect((await consumer.readFrame())?.payload[0]).toBe(1); replacements[0].abort(new Error("route failed")); - for (const payload of [0, 1, 2]) replacements[1].writeFrame({ payload: new Uint8Array([payload]) }); + for (const payload of [0, 1, 2]) + replacements[1].writeFrame({ + payload: new Uint8Array([payload]), + timestamp: Timestamp.unscaled(replacements[1].sequence), + }); replacements[1].finish(); expect((await consumer.readFrame())?.payload[0]).toBe(2); expect(await consumer.readFrame()).toBeUndefined(); @@ -855,7 +888,7 @@ test("a stalled group whose copy then fails asks again with the failure", async // A stall waits for the route; a failure gets the route's verdict. replace: (_index, failed) => (failed ? Promise.reject(verdict) : new Promise(() => {})), }); - producer.writeFrame({ payload: new Uint8Array([0]) }); + producer.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(producer.sequence) }); await consumer.readFrame(); route.set(1); @@ -891,9 +924,9 @@ test("a drained unfinished group fails with Old when its budget gives up", async // frame is still being written, since that partial frame is not readable yet. test("a debt shed keeps the newest committed frame behind one being written", async () => { const producer = new Producer(0n); - producer.writeFrame({ payload: new Uint8Array(1024) }); + producer.writeFrame({ payload: new Uint8Array(1024), timestamp: Timestamp.unscaled(producer.sequence) }); const reader = producer.consume(); - producer.createFrame({ size: 10 }).write(new Uint8Array(5)); + producer.createFrame({ size: 10, timestamp: Timestamp.unscaled(producer.sequence) }).write(new Uint8Array(5)); expect(hooks.shedGroup(producer, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, [])).toBe(0); expect((await reader.readFrame())?.payload.byteLength).toBe(1024); @@ -903,8 +936,8 @@ test("a debt shed keeps the newest committed frame behind one being written", as // holds that had not taken the shed frames, rather than crediting bytes they still hold. test("a debt shed of a finished group ends the copies that lost a frame", async () => { const producer = new Producer(0n); - producer.writeFrame({ payload: new Uint8Array(1024) }); - producer.writeFrame({ payload: new Uint8Array(1024) }); + producer.writeFrame({ payload: new Uint8Array(1024), timestamp: Timestamp.unscaled(producer.sequence) }); + producer.writeFrame({ payload: new Uint8Array(1024), timestamp: Timestamp.unscaled(producer.sequence) }); producer.finish(); const behind = producer.consume(); const drained = producer.consume(); diff --git a/js/net/src/group.ts b/js/net/src/group.ts index 71bb789eeba..7a199fe7c91 100644 --- a/js/net/src/group.ts +++ b/js/net/src/group.ts @@ -25,7 +25,7 @@ import { } from "./frame.ts"; import { type Expiry, type GroupRecover, hooks, type ReadGroupFrame, type Replacement } from "./internal.ts"; import { Queue } from "./queue.ts"; -import { rescale } from "./rescale.ts"; +import { rescaleIn } from "./rescale.ts"; import type { Timestamp } from "./time.ts"; import type { Info as TrackInfo } from "./track.ts"; import { MAX_FRAME_BYTES } from "./util/limits.ts"; @@ -42,7 +42,7 @@ function frameBytes(frame: Frame): number { } /** - * A frame buffered in a group: its presentation {@link Timestamp}, if any, and payload bytes. + * A frame buffered in a group: its presentation {@link Timestamp} and payload bytes. * * The timestamp carries its own scale, so a track can pick its units; the wire layer * converts it into the track's negotiated timescale. @@ -51,11 +51,12 @@ export interface Frame { /** The plain payload, shared across readers even when the track stores compressed bytes. */ payload: Uint8Array; /** - * Presentation timestamp, present exactly when the track has a timescale. A frame on a - * track without one is untimed and omits it; a write that disagrees with its track is - * refused with {@link TimestampMismatch}. + * Presentation timestamp at the track's timescale. A scale-less track has no time, so its + * frame carries {@link Timestamp.unscaled} at its group's key instead. A write that + * disagrees with its track, or names another group's key, is refused with + * {@link TimestampMismatch}. */ - timestamp?: Timestamp; + timestamp: Timestamp; /** * When the frame landed in this node's group, on the installed clock. * @@ -107,11 +108,11 @@ class GroupState { charge?: Charge; compressed?: boolean; flate?: Compression; - // The owning track's info, which every frame's timestamp must `rescale` to: a timed track's frames - // carry a timestamp its scale holds, an untimed track's carry none. Undefined for a group - // no track has claimed yet, which takes either. + // The owning track's info, which every frame's timestamp must `rescaleIn` to: a timed track's + // frames carry a timestamp its scale holds, a scale-less track's the unscaled group key. + // Undefined for a group no track has claimed yet, which takes either. track?: TrackInfo; - // The first frame's timestamp, retained after reads. Media time only: an untimed group + // The first frame's timestamp, retained after reads. Media time only: a scale-less group // never has one, so it says nothing about whether a frame was written. timestamp?: Timestamp; // The highest frame PTS, including when the decode-order tail presents earlier. @@ -153,22 +154,24 @@ class Landed implements Frame { get payload(): Uint8Array { return this.#frame.payload; } - get timestamp(): Timestamp | undefined { + get timestamp(): Timestamp { return this.#frame.timestamp; } } -// Refuse a frame whose timestamp the track's timeline refuses. -function checkTimed(track: TrackInfo | undefined, frame: Frame) { - if (track) rescale(track, frame.timestamp); +// Refuse a frame whose timestamp the track's timeline refuses, or that names another group's key. +function checkTimed(track: TrackInfo | undefined, sequence: bigint, frame: Frame) { + if (track) rescaleIn(track, frame.timestamp, sequence); } function appendFrame(state: GroupState, frame: Frame) { if (state.closed.peek() !== undefined) throw new Error("group is closed"); - state.timestamp ??= frame.timestamp; - if (frame.timestamp === undefined) state.latest = undefined; - else if (state.latest === undefined || state.latest.isBefore(frame.timestamp)) state.latest = frame.timestamp; + // Media time only: a scale-less frame's key says nothing about when it presents. + const timed = frame.timestamp.scale === undefined ? undefined : frame.timestamp; + state.timestamp ??= timed; + if (timed === undefined) state.latest = undefined; + else if (state.latest === undefined || state.latest.isBefore(timed)) state.latest = timed; state.frames.mutate((frames) => { frames.push(frame); }); @@ -387,7 +390,7 @@ export class Producer { if (state.track !== undefined && (state.track.timescale != null) === timed) return; // Another track already holds it to the opposite timedness, empty or not. if (state.track !== undefined) throw new TimestampMismatch(); - for (const frame of state.frames.peek()) checkTimed(track, frame); + for (const frame of state.frames.peek()) checkTimed(track, state.sequence, frame); state.track = track; }; hooks.admitGroupStart = (group, admit) => { @@ -619,13 +622,13 @@ export class Producer { if (this.#open) throw new Error("a frame is still being written"); if (this.#state.total.peek() === Number.MAX_SAFE_INTEGER) throw new ProtocolViolation("group frame index exceeds the safe integer range"); - checkTimed(this.#state.track, frame); + checkTimed(this.#state.track, this.#state.sequence, frame); if (this.#limit !== undefined && this.#state.total.peek() >= this.#limit) { throw new Error(`frame ${this.#state.total.peek()} is past the track's end`); } // Frame 0 must fit between its neighbors in every track it joined, checked before // any of them records it, so a refusal leaves the group and its tracks as they were. - if (this.#admits && frame.timestamp !== undefined && this.#state.total.peek() === 0) { + if (this.#admits && frame.timestamp.scale !== undefined && this.#state.total.peek() === 0) { const start = frame.timestamp; return this.#admits.map((admit) => admit(start)); } diff --git a/js/net/src/ietf/datagram.test.ts b/js/net/src/ietf/datagram.test.ts index 3f123247e9d..db2124e8298 100644 --- a/js/net/src/ietf/datagram.test.ts +++ b/js/net/src/ietf/datagram.test.ts @@ -310,10 +310,10 @@ for (const alpn of [ALPN.DRAFT_18, ALPN.DRAFT_20, ALPN.DRAFT_22]) { reader.setPriority(5); try { while (datagrams.priority.peek() !== 5) await datagrams.priority.changed(); - datagrams.append(undefined, Uint8Array.of(1)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(1)); if (finish) datagrams.finish(); await reader.info(); - if (!finish) datagrams.append(undefined, Uint8Array.of(2)); + if (!finish) datagrams.append(Timestamp.unscaled(1n), Uint8Array.of(2)); expect((await reader.recv())?.payload).toEqual(Uint8Array.of(1)); // The end waits out the grace a late datagram gets. if (finish) expect(await clock.drive(reader.recv())).toBeUndefined(); @@ -352,7 +352,7 @@ for (const alpn of [ALPN.DRAFT_18, ALPN.DRAFT_20, ALPN.DRAFT_22]) { try { await reader.info(); while (datagrams.priority.peek() !== 5) await datagrams.priority.changed(); - for (let i = 0; i < 64; i++) datagrams.append(undefined, Uint8Array.of(i)); + for (let i = 0; i < 64; i++) datagrams.append(Timestamp.unscaled(i), Uint8Array.of(i)); datagrams.finish(); for (let i = 0; i < 64; i++) expect((await reader.recv())?.payload).toEqual(Uint8Array.of(i)); // The end waits out the grace a late datagram gets. diff --git a/js/net/src/ietf/join.test.ts b/js/net/src/ietf/join.test.ts index 7d6bf381bb0..837194e1e60 100644 --- a/js/net/src/ietf/join.test.ts +++ b/js/net/src/ietf/join.test.ts @@ -10,7 +10,7 @@ import { Producer as OriginProducer } from "../origin.ts"; import * as Path from "../path.ts"; import { Reader, Stream, Writer } from "../stream.ts"; import { Tail } from "../tail.ts"; -import { Timescale } from "../time.ts"; +import { Timescale, Timestamp } from "../time.ts"; import * as track from "../track.ts"; import { wireOf } from "../wire.ts"; import { Fetch, FetchOk } from "./fetch.ts"; @@ -563,7 +563,7 @@ test("a clean end finishes a head nothing claimed", () => { const j = new Join(0n, t); const producer = new track.Producer("catalog").createGroup(0n); expect(j.open(() => producer)).toBe(producer); - producer.writeFrame({ payload: text("c") }); + producer.writeFrame({ payload: text("c"), timestamp: Timestamp.unscaled(producer.sequence) }); j.wrote(); j.finish(producer); expect(j.outstanding).toBe(true); @@ -683,7 +683,7 @@ test("a second fetch stream after the track ended leaves the first head to its o // The second stream, discarding, ends first: the head is still the first stream's. j.finish(undefined); expect(first.closed.peek()).toBeUndefined(); - first.writeFrame({ payload: text("c") }); + first.writeFrame({ payload: text("c"), timestamp: Timestamp.unscaled(first.sequence) }); j.finish(first); expect(first.closed.peek()).toBeNull(); }); diff --git a/js/net/src/ietf/object.ts b/js/net/src/ietf/object.ts index 0ccfcee6544..21aa91d5fbc 100644 --- a/js/net/src/ietf/object.ts +++ b/js/net/src/ietf/object.ts @@ -32,7 +32,8 @@ const FIRST_OBJECT_BIT = 0x40; /** * Encode an object's Properties block: its Timestamp in `timescale` units, or nothing. * - * Without units (`timescale` undefined) there is no Timestamp to write, even if the frame has one. + * Without units (`timescale` undefined) there is no Timestamp to write, even if the frame has one, + * and an unscaled one, a scale-less track's group key, is no time to write. * Synchronous, like the rest of an object's header: every object of a timed track pays for it. */ export function encodeObjectExtensions( @@ -40,7 +41,7 @@ export function encodeObjectExtensions( timescale: Timescale | undefined, version: IetfVersion, ): Uint8Array { - if (timestamp === undefined || timescale === undefined) { + if (timestamp?.scale === undefined || timescale === undefined) { return new Uint8Array(); } @@ -460,7 +461,7 @@ export class FetchFrame { timescale: Timescale | undefined, version: IetfVersion, ): Promise { - const stamped = this.timestamp !== undefined && timescale !== undefined; + const stamped = this.timestamp?.scale !== undefined && timescale !== undefined; const previous = position.previousGroup; // Everything before the payload goes out as one write, as in Frame.encode. const header: Uint8Array[] = []; diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index ef7703ae1c9..5c58361b6dd 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -2983,7 +2983,8 @@ test.each([false, true])("draft-22: exact terminal 5/5 survives group FIN=%s", a ); try { const group = track.createGroup(5n); - for (let i = 0; i < 5; i++) group.writeFrame({ payload: new Uint8Array([i]) }); + for (let i = 0; i < 5; i++) + group.writeFrame({ payload: new Uint8Array([i]), timestamp: Timestamp.unscaled(group.sequence) }); const uni = await fx.uni.read(); if (!uni.value) throw new Error("missing group stream"); const reader = new Reader(uni.value, undefined, V22); @@ -3050,7 +3051,8 @@ test("draft-22: an inline terminal reset after its FIN falls back to a standalon ); try { const group = track.createGroup(5n); - for (let i = 0; i < 5; i++) group.writeFrame({ payload: new Uint8Array([i]) }); + for (let i = 0; i < 5; i++) + group.writeFrame({ payload: new Uint8Array([i]), timestamp: Timestamp.unscaled(group.sequence) }); const uni = await fx.uni.read(); if (!uni.value) throw new Error("missing group stream"); const reader = new Reader(uni.value, undefined, V22); @@ -3740,14 +3742,14 @@ test("a paused relay subscription reacquires its upstream datagrams", async () = await RequestOk.decode(client.reader, V22); }; try { - datagrams.append(undefined, Uint8Array.of(1)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(1)); expect((await received.read()).done).toBe(false); await forward(2n, false); while (track.subscription.peek() !== undefined) await track.subscription.changed(); const before = upstream.client.sendStreams.bidi.length; await forward(4n, true); expect(upstream.client.sendStreams.bidi.length).toBe(before + 1); - datagrams.append(undefined, Uint8Array.of(2)); + datagrams.append(Timestamp.unscaled(1n), Uint8Array.of(2)); const next = await received.read(); if (!next.value) throw new Error("no resumed relay datagram"); expect((await ObjectDatagram.decode(next.value, V22)).body).toEqual({ payload: Uint8Array.of(2) }); @@ -3786,7 +3788,7 @@ test.each(["writable", "createWritable"] as const)( clients.push(client); } for (const [i, producer] of producers.entries()) { - producer.append(undefined, Uint8Array.of(i)); + producer.append(Timestamp.unscaled(0n), Uint8Array.of(i)); const next = await received.read(); if (!next.value) throw new Error("missing concurrent datagram"); const decoded = await ObjectDatagram.decode(next.value, V22); @@ -3827,7 +3829,7 @@ test("a paused IETF subscription leaves a blocked datagram write", async () => { }), ); try { - datagrams.append(undefined, Uint8Array.of(1)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(1)); await started.promise; const parameters = new Parameters(); parameters.forward = false; @@ -3870,14 +3872,14 @@ test("a datagram-only SUBSCRIBE releases demand on pause and resumes", async () }; try { if (!startPaused) { - datagrams.append(undefined, Uint8Array.of(1)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(1)); expect((await received.read()).done).toBe(false); await forward(2n, false); } while (datagrams.priority.peek() !== undefined) await datagrams.priority.changed(); - datagrams.append(undefined, Uint8Array.of(2)); + datagrams.append(Timestamp.unscaled(1n), Uint8Array.of(2)); await forward(4n, true); - datagrams.append(undefined, Uint8Array.of(3)); + datagrams.append(Timestamp.unscaled(2n), Uint8Array.of(3)); const next = await received.read(); if (!next.value) throw new Error("no resumed datagram"); const decoded = await ObjectDatagram.decode(next.value, V22); diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index cc03ccdba5d..04642c7107b 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -1207,7 +1207,7 @@ export class Publisher { for (;;) { const datagram = await datagrams.recv(); if (!datagram) return; - if (timescale !== undefined && datagram.timestamp === undefined) { + if (timescale !== undefined && datagram.timestamp.scale === undefined) { console.debug(`dropping a datagram without a timestamp: sequence=${datagram.sequence}`); continue; } diff --git a/js/net/src/ietf/range.ts b/js/net/src/ietf/range.ts index 4af91300e43..77e0f4ae7e7 100644 --- a/js/net/src/ietf/range.ts +++ b/js/net/src/ietf/range.ts @@ -4,7 +4,7 @@ import { type Fill, fillChanged, wantedRuns } from "../fill.ts"; import type * as group from "../group.ts"; import { receiveFrame, skipPayload } from "../payload.ts"; import type { Cursor, Reader } from "../stream.ts"; -import type { Timescale } from "../time.ts"; +import { received, type Timescale } from "../time.ts"; import type * as track from "../track.ts"; import { untilAborted } from "../util/abort.ts"; import { decodeFetchObject, EndOfRange } from "./object.ts"; @@ -184,7 +184,11 @@ export class RangeFetch { }); const producer = this.#current?.producer; if (producer) { - if (object.payload) producer.writeFrame({ payload: object.payload, timestamp: object.timestamp }); + if (object.payload) + producer.writeFrame({ + payload: object.payload, + timestamp: received(object.timestamp, producer.sequence), + }); else if (!(await untilAborted(receiveFrame(stream, producer, object), this.#stop.signal))) throw new Error("FETCH group was abandoned"); } else if (!object.payload) await untilAborted(skipPayload(stream, object.size), this.#stop.signal); diff --git a/js/net/src/ietf/subscriber.test.ts b/js/net/src/ietf/subscriber.test.ts index 6ac965f1f15..6de4c6862cb 100644 --- a/js/net/src/ietf/subscriber.test.ts +++ b/js/net/src/ietf/subscriber.test.ts @@ -10,7 +10,7 @@ import { createMockTransportPair, manual, settlesWithin } from "../mock.ts"; import * as Path from "../path.ts"; import { Reader, Stream } from "../stream.ts"; import { Tail } from "../tail.ts"; -import { Timescale } from "../time.ts"; +import { Timescale, Timestamp } from "../time.ts"; import * as track from "../track.ts"; import * as demandUtil from "../util/demand.ts"; import type * as Cluster from "./cluster.ts"; @@ -811,7 +811,8 @@ test("a track without TIMESCALE arrives untimed, even if an object carries a Tim const group = await track.ordered().nextGroup(); const frame = await group?.readFrame(); expect(frame?.payload).toEqual(new Uint8Array([42])); - expect(frame?.timestamp).toBeUndefined(); + // Its group's key, not the object's Timestamp. + expect(frame?.timestamp).toEqual(Timestamp.unscaled(0n)); track.close(); }); diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 8a4d822f75d..d634602ab8b 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -30,7 +30,7 @@ import * as Path from "../path.ts"; import { receiveFrame, skipPayload } from "../payload.ts"; import { type Cursor, Reader, type Stream } from "../stream.ts"; import { Tail } from "../tail.ts"; -import { Timescale, type Timestamp } from "../time.ts"; +import { received, Timescale, type Timestamp } from "../time.ts"; import type * as track from "../track.ts"; import { untilAborted } from "../util/abort.ts"; import { closeUnwanted } from "../util/announce.ts"; @@ -2032,7 +2032,11 @@ export class Subscriber { message: `object without a Timestamp on a track with TIMESCALE: group=${group.groupId}`, }); } - if (frame.payload) open().writeFrame({ payload: frame.payload, timestamp: frame.timestamp }); + if (frame.payload) + open().writeFrame({ + payload: frame.payload, + timestamp: received(frame.timestamp, open().sequence), + }); else if ( !(await receiveFrame(stream, open(), { size: frame.streamed ?? 0, timestamp: frame.timestamp })) ) @@ -2190,7 +2194,11 @@ export class Subscriber { continue; } } - if (object.payload) producer?.writeFrame({ payload: object.payload, timestamp: object.timestamp }); + if (object.payload) + producer?.writeFrame({ + payload: object.payload, + timestamp: received(object.timestamp, join.sequence), + }); else if (producer && !(await receiveFrame(stream, producer, object))) break; join.wrote(); next += 1; @@ -2316,7 +2324,7 @@ export class Subscriber { try { binding.producer.insert({ sequence: datagram.groupId, - timestamp, + timestamp: received(timestamp, datagram.groupId), payload: datagram.body.payload, }); } catch (err) { diff --git a/js/net/src/index.ts b/js/net/src/index.ts index 79fee8bad8d..e944b37bc44 100644 --- a/js/net/src/index.ts +++ b/js/net/src/index.ts @@ -39,7 +39,7 @@ export * as Path from "./path.ts"; export * as Socket from "./socket.ts"; /** Presentation timestamps and their timescales; durations live in {@link Clock}. */ export * as Time from "./time.ts"; -/** A value paired with the timestamp it was captured at, absent when untimed. */ +/** A value paired with its frame's timestamp: when it was captured, or a scale-less track's group key. */ export type { Timed } from "./timed.ts"; /** Track role handles. */ export * as Track from "./track.ts"; diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index cc5f138123e..b1ae87bc172 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -507,7 +507,7 @@ test("integration: lite-07 reports a relative start over a frameless newest grou const producer = broadcast.createTrack("video", { maxAge: null }); for (let sequence = 0; sequence < 5; sequence++) { const group = producer.appendGroup(); - group.writeFrame({ payload: new Uint8Array([sequence]) }); + group.writeFrame({ payload: new Uint8Array([sequence]), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } const newest = producer.appendGroup(); @@ -521,7 +521,7 @@ test("integration: lite-07 reports a relative start over a frameless newest grou await reader.info(); expect((await sibling.recvGroup())?.sequence).toBe(5n); expect(await reader.start()).toBe(5n); - newest.writeFrame({ payload: new Uint8Array([5]) }); + newest.writeFrame({ payload: new Uint8Array([5]), timestamp: Timestamp.unscaled(newest.sequence) }); newest.finish(); expect((await reader.recvGroup())?.sequence).toBe(5n); } finally { diff --git a/js/net/src/lite/compression.test.ts b/js/net/src/lite/compression.test.ts index 276a0b1ba87..32fe831e108 100644 --- a/js/net/src/lite/compression.test.ts +++ b/js/net/src/lite/compression.test.ts @@ -6,6 +6,7 @@ import { connect } from "../connection/connect.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer } from "../origin.ts"; import * as Path from "../path.ts"; +import { Timestamp } from "../time.ts"; import { ALPN_06, ALPN_07_WIP } from "./version.ts"; for (const downstream of [ALPN_06, ALPN_07_WIP]) { @@ -47,7 +48,8 @@ for (const downstream of [ALPN_06, ALPN_07_WIP]) { const reader = remote.subscribe({ delay: { max: Duration.millis(1000) } }); const group = track.appendGroup(); const payloads = [new Uint8Array(4096).fill(7), new Uint8Array(), new Uint8Array(4096).fill(7)]; - for (const payload of payloads) group.writeFrame({ payload }); + for (const payload of payloads) + group.writeFrame({ payload, timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); const received = await reader.recvGroup(); if (!received) throw new Error("missing group"); @@ -61,8 +63,8 @@ for (const downstream of [ALPN_06, ALPN_07_WIP]) { const unreliable = active.datagrams("data"); await unreliable.info(); while (datagrams.priority.peek() === undefined) await datagrams.priority.changed(); - for (const payload of [new Uint8Array(100).fill(4), new Uint8Array(100).fill(8)]) { - datagrams.append(undefined, payload); + for (const [i, payload] of [new Uint8Array(100).fill(4), new Uint8Array(100).fill(8)].entries()) { + datagrams.append(Timestamp.unscaled(i), payload); expect((await unreliable.recv())?.payload).toEqual(payload); } unreliable.close(); diff --git a/js/net/src/lite/datagram-tracks.test.ts b/js/net/src/lite/datagram-tracks.test.ts index 50e541af7bd..11a0851fdfd 100644 --- a/js/net/src/lite/datagram-tracks.test.ts +++ b/js/net/src/lite/datagram-tracks.test.ts @@ -135,10 +135,10 @@ test("lite-07 bounds and prioritizes independent createWritable datagram queues" while (datagrams.priority.peek() !== 19) await datagrams.priority.changed(); // The publisher opens a track's queue after writing its DATAGRAMS_INFO, so the reader can // see the info first: a datagram through each queue is what proves both are open. - datagrams.append(undefined, new Uint8Array([0])); + datagrams.append(Timestamp.unscaled(0n), new Uint8Array([0])); expect((await reader.recv())?.sequence).toBe(0n); await second.info(); - other.append(undefined, new Uint8Array([1])); + other.append(Timestamp.unscaled(0n), new Uint8Array([1])); await second.recv(); expect(pair.server.sendStreams.datagrams).toHaveLength(2); const queue = pair.server.sendStreams.datagrams.find( @@ -152,8 +152,8 @@ test("lite-07 bounds and prioritizes independent createWritable datagram queues" sendOrder({ priority: 31, publisher: 7 }), ); // Type, Subscribe ID, and Sequence each occupy one byte on this untimed track. - datagrams.append(undefined, new Uint8Array(898)); - datagrams.append(undefined, new Uint8Array(897)); + datagrams.append(Timestamp.unscaled(1n), new Uint8Array(898)); + datagrams.append(Timestamp.unscaled(2n), new Uint8Array(897)); expect((await reader.recv())?.sequence).toBe(2n); } finally { second.close(); @@ -183,7 +183,7 @@ for (const alpn of [ALPN_05, ALPN_06]) const grouped = remote.track("audio").subscribe(); try { await grouped.info(); - track.writeFrame({ payload: new Uint8Array([1]) }); + track.writeFrame({ payload: new Uint8Array([1]), timestamp: Timestamp.unscaled(0n) }); (await grouped.recvGroup())?.close(); reader ??= remote.datagrams("audio"); await reader.info(); @@ -196,7 +196,7 @@ for (const alpn of [ALPN_05, ALPN_06]) reader.setPriority(37); while (datagrams.priority.peek() !== 37) await datagrams.priority.changed(); grouped.close(); - datagrams.append(undefined, new Uint8Array([2])); + datagrams.append(Timestamp.unscaled(0n), new Uint8Array([2])); expect((await reader.recv())?.payload).toEqual(new Uint8Array([2])); datagrams.finish(); track.finish(); @@ -275,10 +275,10 @@ test("lite-06: a request cancelled behind the shared datagram writer sends nothi await Promise.all([a.info(), b.info()]); while (first.priority.peek() === undefined || second.priority.peek() === undefined) await Promise.race([first.priority.changed(), second.priority.changed()]); - first.append(undefined, Uint8Array.of(0xaa)); + first.append(Timestamp.unscaled(0n), Uint8Array.of(0xaa)); while (sent.length === 0) await settle(); // The second request now waits on the writer the first one holds. - second.append(undefined, Uint8Array.of(0xbb)); + second.append(Timestamp.unscaled(0n), Uint8Array.of(0xbb)); await settle(); b.close(); while (second.priority.peek() !== undefined) await second.priority.changed(); @@ -364,10 +364,10 @@ for (const alpn of [ALPN_05, ALPN_06, ALPN_07_WIP]) { reader.setPriority(5); try { while (datagrams.priority.peek() !== 5) await datagrams.priority.changed(); - datagrams.append(undefined, Uint8Array.of(1)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(1)); if (finish) datagrams.finish(); await reader.info(); - if (!finish) datagrams.append(undefined, Uint8Array.of(2)); + if (!finish) datagrams.append(Timestamp.unscaled(1n), Uint8Array.of(2)); expect((await reader.recv())?.payload).toEqual(Uint8Array.of(1)); if (finish) { // The end waits out the grace a late datagram gets. @@ -407,7 +407,7 @@ for (const alpn of [ALPN_05, ALPN_06, ALPN_07_WIP]) { try { await reader.info(); while (datagrams.priority.peek() !== 5) await datagrams.priority.changed(); - datagrams.append(undefined, Uint8Array.of(1)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(1)); datagrams.finish(); expect((await reader.recv())?.payload).toEqual(Uint8Array.of(1)); // The end waits out the grace a late datagram gets. @@ -450,7 +450,7 @@ for (const alpn of [ALPN_05, ALPN_06, ALPN_07_WIP]) { try { await reader.info(); while (datagrams.priority.peek() !== 5) await datagrams.priority.changed(); - for (let i = 0; i < BURST; i++) datagrams.append(undefined, Uint8Array.of(i)); + for (let i = 0; i < BURST; i++) datagrams.append(Timestamp.unscaled(i), Uint8Array.of(i)); datagrams.finish(); const counted = (async () => { let count = 0; @@ -669,10 +669,10 @@ test("lite-07: an unanswered track's datagrams don't delay another track's last" const request = await requested; if (!request) throw new Error("the request never reached the handler"); const a = request.accept({}); - for (let i = 0; i < 40; i++) a.append(undefined, Uint8Array.of(i)); + for (let i = 0; i < 40; i++) a.append(Timestamp.unscaled(i), Uint8Array.of(i)); // "a"'s datagrams reach the subscriber ahead of "b"'s. while (sends.peek() < 40) await sends.changed(); - b.append(undefined, Uint8Array.of(9)); + b.append(Timestamp.unscaled(0n), Uint8Array.of(9)); b.finish(); expect((await readerB.recv())?.payload).toEqual(Uint8Array.of(9)); expect(await readerB.recv()).toBeUndefined(); diff --git a/js/net/src/lite/fills.test.ts b/js/net/src/lite/fills.test.ts index 31d961047b1..d5e50374ff3 100644 --- a/js/net/src/lite/fills.test.ts +++ b/js/net/src/lite/fills.test.ts @@ -7,6 +7,7 @@ import { Producer as BroadcastProducer } from "../broadcast.ts"; import { StreamCode, StreamError } from "../error.ts"; import { hooks, type Run } from "../internal.ts"; import { Tail } from "../tail.ts"; +import { Timestamp } from "../time.ts"; import type { Fill, Producer } from "../track.ts"; import { dropped, passing, RangeFills, type RecoveryHandle, type Refused } from "./fills.ts"; @@ -18,7 +19,7 @@ async function missing(): Promise<{ broadcast: BroadcastProducer; track: Produce const broadcast = new BroadcastProducer(); const track = broadcast.createTrack("video", { maxAge: null }); const group = track.createGroup(4n); - group.writeFrame({ payload: enc.encode("live 4") }); + group.writeFrame({ payload: enc.encode("live 4"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); const requests = track.requests(); const reader = track.subscribe({ ranges: [{ start: { absolute: 0n } }], delay: { max: null }, order: "ascending" }); diff --git a/js/net/src/lite/group.test.ts b/js/net/src/lite/group.test.ts index 7c7d2009ee9..3edd11d2aac 100644 --- a/js/net/src/lite/group.test.ts +++ b/js/net/src/lite/group.test.ts @@ -3,6 +3,7 @@ import { ProtocolViolation, StreamCode, StreamError } from "../error.ts"; import { Producer } from "../group.ts"; import { hooks } from "../internal.ts"; import { concat, encodeVarint, Reader } from "../stream.ts"; +import { Timestamp } from "../time.ts"; import { SEGMENT_BYTES } from "../util/limits.ts"; import * as Varint from "../varint.ts"; import { frameDecoder, readFrames } from "./group.ts"; @@ -194,9 +195,9 @@ test.each([false, true])("a continuation's held-frame skip stops once the group const producer = new Producer(0n); let partial: ReturnType | undefined; if (cut) { - partial = producer.createFrame({ size }); + partial = producer.createFrame({ size, timestamp: Timestamp.unscaled(producer.sequence) }); partial.write(new Uint8Array(5)); - } else producer.writeFrame({ payload: new Uint8Array([1]) }); + } else producer.writeFrame({ payload: new Uint8Array([1]), timestamp: Timestamp.unscaled(producer.sequence) }); // The frame again, with fewer bytes than the skip needs, then nothing: the stream never ends. let controller!: ReadableStreamDefaultController; diff --git a/js/net/src/lite/group.ts b/js/net/src/lite/group.ts index 64e2fe380c2..d22e9a1de1c 100644 --- a/js/net/src/lite/group.ts +++ b/js/net/src/lite/group.ts @@ -178,9 +178,15 @@ export async function readFrames( continue; } } - if (frame.payload) hooks.writeGroupFrame(producer, frame as netGroup.Frame, frame.inflated); + if (frame.payload) + hooks.writeGroupFrame( + producer, + { payload: frame.payload, timestamp: Time.received(frame.timestamp, producer.sequence) }, + frame.inflated, + ); else if (ingest) { - const open = producer.createFrame({ size: frame.size, timestamp: frame.timestamp }); + const timestamp = Time.received(frame.timestamp, producer.sequence); + const open = producer.createFrame({ size: frame.size, timestamp }); // Kept until it completes, so a YIELDED reset cutting it short leaves it for the continuation. ingest.partial = open; if (!(await fillFrame(stream, producer, open))) return; diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index c129c3a723c..20ae1929221 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -3843,10 +3843,10 @@ test("lite-06: a datagram request failing behind the shared writer resets rather const a = await subscribe(0n, "a"); const b = await subscribe(1n, "b"); try { - first.append(undefined, Uint8Array.of(0xaa)); + first.append(Timestamp.unscaled(0n), Uint8Array.of(0xaa)); while (writes === 0) await settle(); // The second request now waits on the writer the first one holds. - second.append(undefined, Uint8Array.of(0xbb)); + second.append(Timestamp.unscaled(0n), Uint8Array.of(0xbb)); await settle(); // A SUBSCRIBE_UPDATE holding only its priority fails decoding, while the stream stays open. await b.writer.u53(1); @@ -3925,7 +3925,7 @@ for (const version of [Version.DRAFT_05, Version.DRAFT_06]) { const datagrams = legacy.broadcast.createDatagrams("events"); if (stream) legacy.broadcast.createTrack("events", { timescale: Timescale.MILLI }); await legacy.track(); - datagrams.append(undefined, Uint8Array.of(0xaa)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(0xaa)); await legacy.subscribe(-5); expect(legacy.sent.length).toBe(1); // Beside a stream track, only the SUBSCRIBE asks for the datagrams. @@ -3943,7 +3943,7 @@ for (const version of [Version.DRAFT_05, Version.DRAFT_06]) { try { const datagrams = legacy.broadcast.createDatagrams("events"); await legacy.track(); - datagrams.append(undefined, Uint8Array.of(0xaa)); + datagrams.append(Timestamp.unscaled(0n), Uint8Array.of(0xaa)); await legacy.track(); await legacy.subscribe(0); expect(legacy.sent.length).toBe(1); @@ -3959,11 +3959,11 @@ for (const version of [Version.DRAFT_05, Version.DRAFT_06]) { try { const first = legacy.broadcast.createDatagrams("events"); await legacy.track(); - first.append(undefined, Uint8Array.of(0xaa)); + first.append(Timestamp.unscaled(0n), Uint8Array.of(0xaa)); first.finish(); const current = legacy.broadcast.createDatagrams("events"); await legacy.track(); - current.append(undefined, Uint8Array.of(0xbb)); + current.append(Timestamp.unscaled(0n), Uint8Array.of(0xbb)); await legacy.subscribe(0); expect(legacy.sent.length).toBe(1); expect(legacy.sent[0]?.at(-1)).toBe(0xbb); diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index 769e87d1c30..847219ad977 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -146,8 +146,8 @@ interface Stamp { timescale: Timescale; /** * The timed track whose frames' own timestamps go out, at its timescale, which the wire's - * equals. Lite-05/06 cannot encode an untimed track, so an untimed one carries its send time - * instead. + * equals. Lite-05/06 cannot encode a scale-less track, so a scale-less one carries its send + * time instead. */ timed?: { timescale?: Timescale | null }; } @@ -160,10 +160,9 @@ function stamping(version: Version, info: { timescale?: Timescale | null }): Sta return timescale === undefined ? { timescale: wire } : { timescale: wire, timed: info }; } -/** A frame or datagram's `timestamp` as its raw wire value, or its send time when the track goes out untimed. */ -function wireTime(timestamp: Timestamp | undefined, stamp: Stamp): number { - const own = stamp.timed && rescale(stamp.timed, timestamp); - return own ? own.value : floorAt(Timestamp.now(), stamp.timescale); +/** A frame or datagram's `timestamp` as its raw wire value, or its send time when the track goes out scale-less. */ +function wireTime(timestamp: Timestamp, stamp: Stamp): number { + return stamp.timed ? rescale(stamp.timed, timestamp).value : floorAt(Timestamp.now(), stamp.timescale); } /** @@ -2448,7 +2447,7 @@ export class Publisher { const timestamp = stamp === undefined ? undefined - : datagram.timestamp === undefined + : datagram.timestamp.scale === undefined ? floorAt(Timestamp.now(), stamp.timescale) : floorAt(datagram.timestamp, stamp.timescale); const body = new DatagramMessage(options.sub, datagram.sequence, timestamp, datagram.payload).encode( diff --git a/js/net/src/lite/refresh.test.ts b/js/net/src/lite/refresh.test.ts index 12e2866d035..393d240e0c1 100644 --- a/js/net/src/lite/refresh.test.ts +++ b/js/net/src/lite/refresh.test.ts @@ -6,6 +6,7 @@ import { connect } from "../connection/connect.ts"; import { createMockTransportPair } from "../mock.ts"; import { Producer } from "../origin.ts"; import * as Path from "../path.ts"; +import { Timestamp } from "../time.ts"; import { SubscribeUpdate } from "./subscribe.ts"; import { ALPN_06, ALPN_07_WIP } from "./version.ts"; @@ -47,8 +48,8 @@ test.each([ALPN_06, ALPN_07_WIP])("refresh joins reach a browser publisher on %s expect((await remote.info()).refresh).toBe(alpn === ALPN_07_WIP ? 1 : null); const baseline = remote.subscribe(); const group = producer.appendGroup(); - group.writeFrame({ payload: new Uint8Array(3) }); - group.writeFrame({ payload: new Uint8Array(3) }); + group.writeFrame({ payload: new Uint8Array(3), timestamp: Timestamp.unscaled(group.sequence) }); + group.writeFrame({ payload: new Uint8Array(3), timestamp: Timestamp.unscaled(group.sequence) }); const received = await baseline.recvGroup(); if (!received) throw new Error("group missing"); expect((await received.readFrame())?.payload.length).toBe(3); @@ -57,7 +58,7 @@ test.each([ALPN_06, ALPN_07_WIP])("refresh joins reach a browser publisher on %s await clock.advance(Duration.nanos(0)); expect(producer.demand().wanted.peek()).toBe(0n); updates.length = 0; - group.writeFrame({ payload: new Uint8Array(1) }); + group.writeFrame({ payload: new Uint8Array(1), timestamp: Timestamp.unscaled(group.sequence) }); expect((await received.readFrame())?.payload.length).toBe(1); for (let i = 0; i < 8; i++) joins.push(remote.subscribe()); await clock.advance(Duration.nanos(0)); diff --git a/js/net/src/lite/subscriber.ts b/js/net/src/lite/subscriber.ts index 6214d765f23..a0a71b5a85a 100644 --- a/js/net/src/lite/subscriber.ts +++ b/js/net/src/lite/subscriber.ts @@ -3949,9 +3949,10 @@ export class Subscriber { if (!dg) return undefined; if (!entry || this.#datagrams.get(dg.subscribe) !== entry) return dg.subscribe; if (behindHeld && this.#isHeld(dg.subscribe)) return dg.subscribe; + // A scale-less datagram's timestamp is its sequence. const timestamp = dg.timestamp === undefined || entry.timescale === undefined - ? undefined + ? Time.Timestamp.unscaled(dg.sequence) : new Time.Timestamp(dg.timestamp, entry.timescale); entry.producer.insert({ sequence: dg.sequence, timestamp, payload: dg.payload }); return undefined; diff --git a/js/net/src/net-cleanups.test.ts b/js/net/src/net-cleanups.test.ts index 43ba5a1053e..223eec8e187 100644 --- a/js/net/src/net-cleanups.test.ts +++ b/js/net/src/net-cleanups.test.ts @@ -8,12 +8,13 @@ import { Producer as GroupProducer } from "./group.ts"; import { Connection as LiteConnection } from "./lite/connection.ts"; import { ALPN_03, ALPN_04, Version } from "./lite/version.ts"; import { createMockTransportPair } from "./mock.ts"; +import { Timestamp } from "./time.ts"; import { Producer as TrackProducer } from "./track.ts"; test("group read handles independently replay buffered and future frames", async () => { const group = new GroupProducer(0n); const first = group.consume(); - const payload = (text: string) => ({ payload: new TextEncoder().encode(text) }); + const payload = (text: string) => ({ payload: new TextEncoder().encode(text), timestamp: Timestamp.unscaled(0n) }); try { group.writeFrame(payload("a")); expect(await first.readString()).toBe("a"); @@ -42,9 +43,12 @@ test.each(["group", "track"] as const)("%s JSON rejects an empty frame without t const subscriber = kind === "track" ? producer.subscribe().ordered() : undefined; const group = kind === "track" ? producer.createGroup(0n) : new GroupProducer(0n); const reader = subscriber ?? group.consume(); - group.writeFrame({ payload: new Uint8Array() }); + group.writeFrame({ payload: new Uint8Array(), timestamp: Timestamp.unscaled(group.sequence) }); for (const value of [null, false, 0, ""]) - group.writeFrame({ payload: new TextEncoder().encode(JSON.stringify(value)) }); + group.writeFrame({ + payload: new TextEncoder().encode(JSON.stringify(value)), + timestamp: Timestamp.unscaled(group.sequence), + }); group.finish(); producer.finish(); producer.close(); diff --git a/js/net/src/origin-request.test.ts b/js/net/src/origin-request.test.ts index b11e2cc70bd..8959eaf7baf 100644 --- a/js/net/src/origin-request.test.ts +++ b/js/net/src/origin-request.test.ts @@ -5,6 +5,7 @@ import { WireRoute } from "./hop"; import * as Instance from "./instance.ts"; import { type Following, Producer, type Requesting } from "./origin"; import * as Path from "./path"; +import { Timestamp } from "./time.ts"; import { wireOf } from "./wire"; const settle = () => new Promise((resolve) => setImmediate(resolve)); @@ -402,7 +403,9 @@ test("an application serves held content without an exact route", async () => { const pending = (await requests.next()).value; if (!pending) throw new Error("expected a request"); const source = new BroadcastProducer(); - source.createTrack("data").writeFrame({ payload: new TextEncoder().encode("held") }); + source + .createTrack("data") + .writeFrame({ payload: new TextEncoder().encode("held"), timestamp: Timestamp.unscaled(0n) }); pending.serve(source.consume()); await settle(); const reader = request.active.peek()?.track("data").subscribe().ordered(); @@ -558,7 +561,9 @@ test("announcing a job route again keeps it answering under one instance", async const pending = (await requests.next()).value; if (!pending) throw new Error("expected a request"); const job = pending.route({ hidden: true }); - job.broadcast.createTrack("data").writeFrame({ payload: new TextEncoder().encode("published") }); + job.broadcast + .createTrack("data") + .writeFrame({ payload: new TextEncoder().encode("published"), timestamp: Timestamp.unscaled(0n) }); job.announce(); job.announce(); await settle(); @@ -595,7 +600,9 @@ for (const cleanup of ["close", "dispose", "broadcast", "announce"] as const) { await settle(); expect(later.closed.peek()).toBeUndefined(); using job = next.route({ hidden: true }); - job.broadcast.createTrack("data").writeFrame({ payload: new TextEncoder().encode(cleanup) }); + job.broadcast + .createTrack("data") + .writeFrame({ payload: new TextEncoder().encode(cleanup), timestamp: Timestamp.unscaled(0n) }); job.announce(); await settle(); expect(later.active.peek()?.instance).toBe(job.instance); @@ -990,7 +997,7 @@ test.each(["request", "follow"] as const)("a %s through a claim continues on its if (!pending) throw new Error("expected a request"); const job = pending.route({ hidden: true }); const data = job.broadcast.createTrack("data"); - data.writeFrame({ payload: new TextEncoder().encode("first") }); + data.writeFrame({ payload: new TextEncoder().encode("first"), timestamp: Timestamp.unscaled(0n) }); expect(job.instance).toBe(minted); job.announce(); await settle(); @@ -1000,7 +1007,7 @@ test.each(["request", "follow"] as const)("a %s through a claim continues on its expect(active?.instance).toBe(minted); const track = active?.track("data").subscribe().ordered(); expect(await track?.readString()).toBe("first"); - data.writeFrame({ payload: new TextEncoder().encode("second") }); + data.writeFrame({ payload: new TextEncoder().encode("second"), timestamp: Timestamp.unscaled(1n) }); expect(await track?.readString()).toBe("second"); expect(handed).toEqual([active]); expect(reader.closed.peek()).toBeUndefined(); @@ -1188,7 +1195,10 @@ test("a claim moving on keeps its held answers", async () => { const minted = claim.instance; const requests = claim.requested(); const held = new BroadcastProducer(); - held.createTrack("data").writeFrame({ payload: new TextEncoder().encode("held") }); + held.createTrack("data").writeFrame({ + payload: new TextEncoder().encode("held"), + timestamp: Timestamp.unscaled(0n), + }); const reader = origin.request(Path.from("room/held")); const asked = (await requests.next()).value; if (!asked) throw new Error("expected a request"); diff --git a/js/net/src/origin.test.ts b/js/net/src/origin.test.ts index 41e8aaead47..f9f8a384846 100644 --- a/js/net/src/origin.test.ts +++ b/js/net/src/origin.test.ts @@ -12,7 +12,7 @@ import type { Consumer, Table } from "./origin.ts"; import { Producer } from "./origin.ts"; import * as Path from "./path.ts"; import { readsFrom } from "./resume.test-util"; -import { Timescale } from "./time.ts"; +import { Timescale, Timestamp } from "./time.ts"; import { overrideBroadcastWire, wireOf } from "./wire.ts"; function publish(origin: Producer, path: Path.Valid) { @@ -64,11 +64,11 @@ for (const via of ["direct", "request", "follow"] as const) { if (!consumed) throw new Error("expected a broadcast"); const sub = consumed.track("video").subscribe(); const history = track.createGroup(0n); - history.writeFrame({ payload: new Uint8Array([1]) }); + history.writeFrame({ payload: new Uint8Array([1]), timestamp: Timestamp.unscaled(history.sequence) }); history.finish(); const finished = await sub.recvGroup(); const live = track.createGroup(1n); - live.writeFrame({ payload: new Uint8Array([2]) }); + live.writeFrame({ payload: new Uint8Array([2]), timestamp: Timestamp.unscaled(live.sequence) }); const group = await sub.recvGroup(); if (!finished || !group) throw new Error("expected both groups"); try { diff --git a/js/net/src/payload.test.ts b/js/net/src/payload.test.ts index ee9a6ab9139..ebe7d843738 100644 --- a/js/net/src/payload.test.ts +++ b/js/net/src/payload.test.ts @@ -2,12 +2,13 @@ import { expect, spyOn, test } from "bun:test"; import { type Dispose, Signal, type Subscriber } from "@moq/signals"; import { Producer } from "./group.ts"; import { sendPayload } from "./payload.ts"; +import { Timestamp } from "./time.ts"; // A stopped sender must release its wait even when the source frame stays open forever. // Otherwise each cancelled FETCH leaves a reader and its listeners on the shared frame. test("stopping a payload releases its pending chunk wait", async () => { const producer = new Producer(0n); - producer.createFrame({ size: 2 }); + producer.createFrame({ size: 2, timestamp: Timestamp.unscaled(producer.sequence) }); const group = producer.consume(); const frame = await group.nextFrame(); if (!frame) throw new Error("missing frame"); diff --git a/js/net/src/payload.ts b/js/net/src/payload.ts index 7a94dc0c3cb..6be11ce3552 100644 --- a/js/net/src/payload.ts +++ b/js/net/src/payload.ts @@ -9,7 +9,7 @@ import type * as frame from "./frame.ts"; import type * as group from "./group.ts"; import { hooks, type ReadGroupFrame } from "./internal.ts"; import type { Cursor, Reader } from "./stream.ts"; -import type { Timestamp } from "./time.ts"; +import { received, type Timestamp } from "./time.ts"; import { SEGMENT_BYTES } from "./util/limits.ts"; /** A decoded frame header, and its payload unless that streams in after it. */ @@ -18,6 +18,7 @@ export interface WirePayload { size: number; /** The payload, or `undefined` past one segment, where it follows the header on the stream. */ payload?: Uint8Array; + /** The timestamp the wire carried, which a scale-less track's frame lacks. */ timestamp?: Timestamp; } @@ -28,7 +29,8 @@ export function decodePayload(c: Cursor, size: number): Uint8Array | undefined { /** * Write a decoded frame into `producer`, streaming from `stream` a payload its decode left - * there. Resolves false if the group closed first, which already failed the frame. + * there. A frame the wire carried no timestamp for, on a scale-less track, takes its group's + * key. Resolves false if the group closed first, which already failed the frame. */ export async function receiveFrame( stream: Reader, @@ -36,11 +38,12 @@ export async function receiveFrame( frame: WirePayload, inflated?: number, ): Promise { + const timestamp = received(frame.timestamp, producer.sequence); if (frame.payload) { - hooks.writeGroupFrame(producer, frame as group.Frame, inflated); + hooks.writeGroupFrame(producer, { payload: frame.payload, timestamp }, inflated); return true; } - return fillFrame(stream, producer, producer.createFrame({ size: frame.size, timestamp: frame.timestamp })); + return fillFrame(stream, producer, producer.createFrame({ size: frame.size, timestamp })); } /** diff --git a/js/net/src/rescale.ts b/js/net/src/rescale.ts index 93190f95a38..0ad14ff4a73 100644 --- a/js/net/src/rescale.ts +++ b/js/net/src/rescale.ts @@ -1,25 +1,37 @@ // Shared by stream and datagram tracks, without a cycle through their model modules. import { TimestampMismatch } from "./error.ts"; -import { floorAt, type Timescale, Timestamp } from "./time.ts"; +import { floorAt, keyOf, type Timescale, Timestamp } from "./time.ts"; /** * `timestamp` at `info`'s timescale, rounded down, as Rust's `Timestamp::convert` does. * - * A track is all timed or all untimed, so a frame must match it: a timestamp on an untimed track, - * none on a timed one, or one the scale can't hold throws {@link TimestampMismatch}. A writer that - * changes state before its frame reaches the track, such as signing or encrypting it, rescales - * first so a refused timestamp changes nothing. Mirrors Rust's track and datagram `Info::rescale`. + * A track is all timed or all scale-less, so a frame must match it: an unscaled timestamp on a + * timed track, a scaled one on a scale-less track, or one the scale can't hold throws + * {@link TimestampMismatch}. A scale-less track's timestamp passes through unchanged. A writer + * that changes state before its frame reaches the track, such as signing or encrypting it, + * rescales first so a refused timestamp changes nothing. Mirrors Rust's track and datagram + * `Info::rescale`. */ -export function rescale( - info: { timescale?: Timescale | null }, - timestamp: Timestamp | undefined, -): Timestamp | undefined { +export function rescale(info: { timescale?: Timescale | null }, timestamp: Timestamp): Timestamp { const timescale = info.timescale ?? undefined; - if ((timescale !== undefined) !== (timestamp !== undefined)) throw new TimestampMismatch(); - if (timestamp === undefined || timescale === undefined) return undefined; + if ((timescale !== undefined) !== (timestamp.scale !== undefined)) throw new TimestampMismatch(); + if (timescale === undefined) return timestamp; try { return new Timestamp(floorAt(timestamp, timescale), timescale); } catch (cause) { throw new TimestampMismatch({ cause }); } } + +/** + * {@link rescale} for a frame of group `key` (or a datagram of sequence `key`): a scale-less + * track's frame carries its group's key, so any other throws {@link TimestampMismatch}. A writer + * that changes state before its frame reaches the group checks this first, as with + * {@link rescale}. Mirrors Rust's `Info::rescale_in`. + */ +export function rescaleIn(info: { timescale?: Timescale | null }, timestamp: Timestamp, key: bigint): Timestamp { + const rescaled = rescale(info, timestamp); + const unscaled = keyOf(rescaled); + if (unscaled !== undefined && unscaled !== key) throw new TimestampMismatch(); + return rescaled; +} diff --git a/js/net/src/resume.test-util.ts b/js/net/src/resume.test-util.ts index 41d2911028e..b6c7170bc6c 100644 --- a/js/net/src/resume.test-util.ts +++ b/js/net/src/resume.test-util.ts @@ -1,4 +1,5 @@ import type { Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast"; +import { Timestamp } from "./time.ts"; let probes = 0; @@ -8,7 +9,7 @@ export function readsFrom(active: BroadcastConsumer | undefined, source: Broadca const name = `probe-${probes++}`; const track = source.createTrack(name); const group = track.appendGroup(); - group.writeFrame({ payload: new Uint8Array([1]) }); + group.writeFrame({ payload: new Uint8Array([1]), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); const sub = active.track(name).subscribe(); try { diff --git a/js/net/src/resume.test.ts b/js/net/src/resume.test.ts index b479a39c6dd..d673571dbad 100644 --- a/js/net/src/resume.test.ts +++ b/js/net/src/resume.test.ts @@ -15,6 +15,7 @@ import type { Lister } from "./listing.ts"; import { Producer } from "./origin.ts"; import * as Path from "./path.ts"; import { Front, routeFailed } from "./resume.ts"; +import { Timestamp } from "./time.ts"; import type * as track from "./track.ts"; import { unwanted } from "./util/demand.ts"; import { overrideBroadcastWire, wireOf } from "./wire.ts"; @@ -61,7 +62,10 @@ function copy(source: BroadcastProducer) { write(sequence: bigint, frames = 1, close = true) { const group = producer.createGroup(sequence); for (let frame = 0; frame < frames; frame++) - group.writeFrame({ payload: enc.encode(`${sequence}.${frame}`) }); + group.writeFrame({ + payload: enc.encode(`${sequence}.${frame}`), + timestamp: Timestamp.unscaled(group.sequence), + }); if (close) group.finish(); return group; }, @@ -137,7 +141,7 @@ test("returning to a route reuses its subscription and held group", async () => expect(drain(sub)).toEqual([1n]); front.serve(handles[0]); await clock.advance(Duration.ZERO); - open.writeFrame({ payload: enc.encode("0.1") }); + open.writeFrame({ payload: enc.encode("0.1"), timestamp: Timestamp.unscaled(open.sequence) }); expect(dec.decode((await held.readFrame())?.payload)).toBe("0.1"); tracks[0].write(2n); expect(drain(sub)).toEqual([2n]); @@ -308,7 +312,7 @@ test("a backfill waits on a fill the finished route still owes", async () => { continue; } const group = fill.accept(sequence); - group.writeFrame({ payload: enc.encode("2.0") }); + group.writeFrame({ payload: enc.encode("2.0"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } fill.close(); @@ -527,7 +531,7 @@ test("a resumed fork shares an on-demand track request", async () => { const extra = wireOf(a).requested(); const fork = parent.fork(); try { - producer.writeFrame({ payload: enc.encode("shared") }); + producer.writeFrame({ payload: enc.encode("shared"), timestamp: Timestamp.unscaled(0n) }); for (const subscriber of [parent, fork]) { const group = await subscriber.recvGroup(); expect(dec.decode((await group?.readFrame())?.payload)).toBe("shared"); @@ -613,7 +617,8 @@ test("a lagging same-instance replica resumes from one publisher's bytes exactly if (!active) throw new Error("expected the incumbent route"); sub = active.track("seq").subscribe({ delay: { max: Duration.millis(60_000) } }); const first = source.appendGroup(); - for (let frame = 0; frame < 4; frame++) first.writeFrame({ payload: enc.encode(`0.${frame}`) }); + for (let frame = 0; frame < 4; frame++) + first.writeFrame({ payload: enc.encode(`0.${frame}`), timestamp: Timestamp.unscaled(first.sequence) }); first.finish(); const seen: string[] = []; const initial = await sub.recvGroup(); @@ -624,15 +629,18 @@ test("a lagging same-instance replica resumes from one publisher's bytes exactly initial.close(); const open = source.appendGroup(); - for (let frame = 0; frame < 2; frame++) open.writeFrame({ payload: enc.encode(`1.${frame}`) }); + for (let frame = 0; frame < 2; frame++) + open.writeFrame({ payload: enc.encode(`1.${frame}`), timestamp: Timestamp.unscaled(open.sequence) }); const interrupted = await sub.recvGroup(); if (!interrupted) throw new Error("expected group 1"); expect(interrupted.sequence).toBe(1n); for (let frame = 0; frame < 2; frame++) seen.push(dec.decode((await interrupted.readFrame())?.payload)); - for (let frame = 2; frame < 4; frame++) open.writeFrame({ payload: enc.encode(`1.${frame}`) }); + for (let frame = 2; frame < 4; frame++) + open.writeFrame({ payload: enc.encode(`1.${frame}`), timestamp: Timestamp.unscaled(open.sequence) }); open.finish(); const last = source.appendGroup(); - for (let frame = 0; frame < 4; frame++) last.writeFrame({ payload: enc.encode(`2.${frame}`) }); + for (let frame = 0; frame < 4; frame++) + last.writeFrame({ payload: enc.encode(`2.${frame}`), timestamp: Timestamp.unscaled(last.sequence) }); last.finish(); // The standby holds the complete original while the incumbent cannot pass 1.1. @@ -1146,7 +1154,7 @@ test("a fetched group failing as it is handed out releases its route once", asyn const fa = a.createTrack("fetch"); const failing = fa.createGroup(5n); - failing.writeFrame({ payload: enc.encode("5.0") }); + failing.writeFrame({ payload: enc.encode("5.0"), timestamp: Timestamp.unscaled(failing.sequence) }); const fetching = front.consumer.track("fetch").fetchGroup(5n); for (let turn = 0; turn < turns; turn++) await Promise.resolve(); failing.abort(new StreamError(StreamCode.SessionClosed)); @@ -1156,8 +1164,8 @@ test("a fetched group failing as it is handed out releases its route once", asyn // group the new route never has. const fb = b.createTrack("fetch"); const whole = fb.createGroup(5n); - whole.writeFrame({ payload: enc.encode("5.0") }); - whole.writeFrame({ payload: enc.encode("5.1") }); + whole.writeFrame({ payload: enc.encode("5.0"), timestamp: Timestamp.unscaled(whole.sequence) }); + whole.writeFrame({ payload: enc.encode("5.1"), timestamp: Timestamp.unscaled(whole.sequence) }); whole.finish(); b.createTrack("seq"); front.serve(b.consume()); @@ -1865,7 +1873,7 @@ test("a failed group on a route whose replacement was refused ends with the refu front.serve(a.consume()); const sub = front.consumer.track("seq").subscribe(); const stalled = ta.createGroup(0n); - stalled.writeFrame({ payload: enc.encode("0.0") }); + stalled.writeFrame({ payload: enc.encode("0.0"), timestamp: Timestamp.unscaled(stalled.sequence) }); const group = sub.tryRecvGroup(); if (!group) throw new Error("expected group 0"); await group.readFrame(); @@ -1941,7 +1949,7 @@ test("a fetch never reads a replacement refused for its track properties", async a.createTrack("seq", { priority: 1 }); const tb = b.createTrack("seq", { priority: 2 }); const group = tb.createGroup(0n); - group.writeFrame({ payload: enc.encode("b") }); + group.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); front.serve(a.consume()); await front.consumer.track("seq").info(); @@ -1963,7 +1971,7 @@ test("the first fetch of a track says what the track is for later routes", async ] as const) { const producer = source.createTrack("seq", { priority }); const group = producer.createGroup(0n); - group.writeFrame({ payload: enc.encode(`${priority}`) }); + group.writeFrame({ payload: enc.encode(`${priority}`), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } front.serve(a.consume()); @@ -2020,7 +2028,7 @@ test("a fetch never hands out a group from a route refused for its properties", a.createTrack("seq", { priority: 1 }); const tb = b.createTrack("seq", { priority: 2 }); const group = tb.createGroup(0n); - group.writeFrame({ payload: enc.encode("b") }); + group.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); front.serve(a.consume()); await front.consumer.track("seq").info(); @@ -2430,7 +2438,7 @@ test("a fetch waits for the copy being read to say what the track is", async () const requests = wireOf(a).requested(); const tb = b.createTrack("seq", { priority: 2 }); const group = tb.createGroup(0n); - group.writeFrame({ payload: enc.encode("b") }); + group.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); front.serve(a.consume()); const sub = front.consumer.track("seq").subscribe(); @@ -3459,7 +3467,8 @@ test("a fetch its route already answered continues the group past that route's a // The route answers, then aborts its track before recovery looks again. const rest = new GroupProducer(0n); - for (const frame of ["0.0", "0.1"]) rest.writeFrame({ payload: enc.encode(frame) }); + for (const frame of ["0.0", "0.1"]) + rest.writeFrame({ payload: enc.encode(frame), timestamp: Timestamp.unscaled(rest.sequence) }); rest.finish(); answer(rest.consume()); queueMicrotask(() => ta.producer.abort(new StreamError(StreamCode(70), { message: "cut" }))); @@ -3483,7 +3492,7 @@ for (const fetched of [false, true]) { const { route, asked } = counted(a, (sequence) => { if (!hostile) return undefined; const copy = new GroupProducer(sequence); - copy.writeFrame({ payload: enc.encode(`${sequence}.0`) }); + copy.writeFrame({ payload: enc.encode(`${sequence}.0`), timestamp: Timestamp.unscaled(copy.sequence) }); void yieldNow().then(() => copy.abort(malformed)); return copy.consume(); }); diff --git a/js/net/src/socket.test.ts b/js/net/src/socket.test.ts index edefcc7dbab..6ecc88bba72 100644 --- a/js/net/src/socket.test.ts +++ b/js/net/src/socket.test.ts @@ -10,6 +10,7 @@ import { Origin, Path, Socket } from "./index.ts"; import { hooks } from "./internal.ts"; import * as Lite from "./lite/index.ts"; import { createMockTransportPair } from "./mock.ts"; +import { Timestamp } from "./time.ts"; import type * as Track from "./track.ts"; import { wireOf } from "./wire.ts"; @@ -26,7 +27,9 @@ function time() { clocks.push(clock.install()); return clock; } -const bytes = (value: number) => ({ payload: new Uint8Array([value]) }); +// A scale-less frame of `value`, in the group `key` names: a TCP half's only group, or a track +// message's own group. +const bytes = (value: number, key = 0n) => ({ payload: new Uint8Array([value]), timestamp: Timestamp.unscaled(key) }); function setup() { const origin = new Origin.Producer(); origins.add(origin); @@ -144,7 +147,7 @@ test("track halves start reliable and ascending, preserve old messages and accep expect((await local.producer.info()).maxAge).toBeNull(); expect(remote.consumer.subscription.peek()?.order).toBe("ascending"); expect(remote.consumer.subscription.peek()?.delay?.max).toEqual(Duration.millis(Number.MAX_SAFE_INTEGER)); - for (let i = 0; i < 3; i++) local.producer.writeFrame(bytes(i)); + for (let i = 0; i < 3; i++) local.producer.writeFrame(bytes(i, BigInt(i))); local.producer.finish(); const reader = remote.consumer.ordered(); for (let i = 0; i < 3; i++) { @@ -165,7 +168,7 @@ test("a track session skips dropped groups and keeps the next message", async () const reading = reader.readFrame(); hooks.dropTrackGroups(local.producer, 0n, 1n, { error: new NotFound("message unavailable") }); const group = local.producer.createGroup(1n); - group.writeFrame(bytes(7)); + group.writeFrame(bytes(7, 1n)); group.finish(); local.producer.finish(); await clock.advance(Duration.ZERO); @@ -507,7 +510,7 @@ test(`a socket preserves an early answer burst over ${Lite.ALPN_07_WIP}`, async // SUBSCRIBE_OK carries the burst's last position, without a timing race. hooks.keepRequest(request); const staged = hooks.stageRequest(request, { maxAge: null }); - for (let i = 1; i <= 3; i++) staged.producer.writeFrame(bytes(i)); + for (let i = 1; i <= 3; i++) staged.producer.writeFrame(bytes(i, BigInt(i - 1))); staged.producer.finish(); staged.commit(); await clock.advance(Duration.ZERO); diff --git a/js/net/src/time.test.ts b/js/net/src/time.test.ts index a49ff802689..2d0ec0d0f9c 100644 --- a/js/net/src/time.test.ts +++ b/js/net/src/time.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { Timescale, Timestamp } from "./time.ts"; +import { floorAt, keyOf, Timescale, Timestamp } from "./time.ts"; test("Timescale accepts the safe integer range", () => { expect(Timescale(1)).toBe(Timescale.SECOND); @@ -66,3 +66,44 @@ test("Timestamp.isBefore orders across scales exactly", () => { expect(millis.isBefore(micros)).toBe(false); expect(new Timestamp(0.5, Timescale.SECOND).isBefore(Timestamp.fromMillis(501))).toBe(true); }); + +test("Timestamp.unscaled carries a group key with no scale", () => { + const key = Timestamp.unscaled(7n); + expect(key.value).toBe(7); + expect(key.scale).toBeUndefined(); + expect(Timestamp.unscaled(7)).toEqual(key); + expect(keyOf(key)).toBe(7n); + expect(keyOf(Timestamp.fromMillis(7))).toBeUndefined(); + for (const bad of [-1, 1.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1]) + expect(() => Timestamp.unscaled(bad)).toThrow(RangeError); + expect(() => Timestamp.unscaled(-1n)).toThrow(RangeError); + expect(() => Timestamp.unscaled(1n << 64n)).toThrow(RangeError); +}); + +test("Timestamp.unscaled takes any u64 key, refusing only to read one past 2^53 as a number", () => { + // A received group can name any key, so building its timestamp never throws. + const max = Timestamp.unscaled((1n << 64n) - 1n); + expect(keyOf(max)).toBe((1n << 64n) - 1n); + expect(() => max.value).toThrow(RangeError); + const below = Timestamp.unscaled((1n << 64n) - 2n); + expect(below.isBefore(max)).toBe(true); + expect(max.isBefore(below)).toBe(false); +}); + +test("Timestamp conversions refuse an unscaled timestamp", () => { + const key = Timestamp.unscaled(1000n); + expect(() => key.as(Timescale.MILLI)).toThrow(RangeError); + expect(() => key.asMillis()).toThrow(RangeError); + expect(() => key.asMicros()).toThrow(RangeError); + expect(() => floorAt(key, Timescale.MILLI)).toThrow(RangeError); +}); + +test("Timestamp.isBefore orders unscaled keys, and a scaled against an unscaled not at all", () => { + expect(Timestamp.unscaled(1n).isBefore(Timestamp.unscaled(2n))).toBe(true); + expect(Timestamp.unscaled(2n).isBefore(Timestamp.unscaled(1n))).toBe(false); + expect(Timestamp.unscaled(2n).isBefore(Timestamp.unscaled(2n))).toBe(false); + // Rust's `partial_cmp` is `None` here, so neither is before the other. + expect(Timestamp.unscaled(1n).isBefore(Timestamp.fromMillis(2))).toBe(false); + expect(Timestamp.fromMillis(2).isBefore(Timestamp.unscaled(1n))).toBe(false); + expect(Timestamp.fromMillis(0).isBefore(Timestamp.unscaled(5n))).toBe(false); +}); diff --git a/js/net/src/time.ts b/js/net/src/time.ts index 57a73aa3edb..cf1c314854d 100644 --- a/js/net/src/time.ts +++ b/js/net/src/time.ts @@ -7,6 +7,7 @@ */ import { Instant, now } from "@moq/clock"; import { toNumber } from "./util/bigint.ts"; +import { checkSequence } from "./util/sequence.ts"; /** Units per second for a {@link Timestamp}'s value, e.g. `1000` for milliseconds. */ export type Timescale = number & { readonly _brand: "timescale" }; @@ -43,32 +44,90 @@ function assertTimestampValue(value: number): void { } } +const MAX_SAFE_KEY = BigInt(Number.MAX_SAFE_INTEGER); + +/** + * The group key of an unscaled timestamp, exact at any u64, or `undefined` when it is scaled. + * Unlike {@link Timestamp.value}, it never throws on a key past `Number.MAX_SAFE_INTEGER`. + */ +export let keyOf: (timestamp: Timestamp) => bigint | undefined; + +// The key the constructor is building an unscaled timestamp for, set only inside `unscaled`. +let building: bigint | undefined; + /** - * A presentation timestamp: a raw value in a given {@link Timescale}. + * A presentation timestamp: a raw value in a given {@link Timescale}, or a scale-less track's + * group key. * * Mirrors the Rust `Timestamp`. Unlike a `@moq/clock` `Duration` it carries its * own scale, so a track can pick its units and conversions can't silently mix them up. * - * The value must be finite and in `[0, Number.MAX_SAFE_INTEGER]`. Fractions are allowed + * A scaled value must be finite and in `[0, Number.MAX_SAFE_INTEGER]`. Fractions are allowed * ({@link Timestamp.now}, scale conversion). Encoding rounds down to an integer at the * track timescale, as the Rust `Timestamp::convert` does. Integer timestamps are exact through `2^53 - 1` units; this package * does not use BigInt for timestamps. {@link Timestamp.as} also refuses a converted * value that leaves that range, so a legal source timestamp cannot become an unencodable * wire timestamp after a scale change. + * + * A scale-less track has no time: each frame's timestamp is {@link Timestamp.unscaled}, whose + * {@link value} is its group's key. Anything that needs a scale ({@link as}, {@link asMillis}, + * {@link asMicros}) throws a `RangeError` on one, and {@link isBefore} orders it against no + * scaled timestamp. */ export class Timestamp { - /** The raw value, in `scale` units. */ - readonly value: number; - /** Units per second the {@link value} is measured in. */ - readonly scale: Timescale; + /** + * The raw value, in `scale` units, or the group key when unscaled. + * + * Reading a key past `Number.MAX_SAFE_INTEGER` throws a `RangeError`, since a number can't + * hold it exactly. + */ + readonly value!: number; + /** Units per second the {@link value} is measured in, or `undefined` when unscaled. */ + readonly scale: Timescale | undefined; + // The group key of an unscaled timestamp, exact at any u64. + #key: bigint | undefined; /** Build a timestamp of `value` units at `scale`. */ constructor(value: number, scale: Timescale) { + const key = building; + building = undefined; + if (key !== undefined) { + this.scale = undefined; + this.#key = key; + if (key <= MAX_SAFE_KEY) this.value = Number(key); + // Built from any received key, so only reading it as a number can fail. + else + Object.defineProperty(this, "value", { + enumerable: true, + get: () => { + throw new RangeError(`timestamp key outside the safe number range: ${key}`); + }, + }); + return; + } assertTimestampValue(value); this.value = value; this.scale = Timescale(scale); } + /** + * A scale-less track's timestamp: `key` is its group's key (or a datagram's sequence), with no + * time. Any u64 key is accepted, including one past `Number.MAX_SAFE_INTEGER`. + */ + static unscaled(key: bigint | number): Timestamp { + if (typeof key === "number") { + if (!Number.isSafeInteger(key) || key < 0) throw new RangeError(`invalid timestamp key: ${key}`); + key = BigInt(key); + } + building = checkSequence(key); + // The constructor reads `building` first, so its arguments go unused. + return new Timestamp(0, Timescale.SECOND); + } + + static { + keyOf = (timestamp) => timestamp.#key; + } + /** The installed clock's monotonic instant in milliseconds, not wall-clock time. */ static now(): Timestamp { return new Timestamp(now().since(Instant.ZERO).asMillis(), Timescale.MILLI); @@ -84,49 +143,76 @@ export class Timestamp { return new Timestamp(us, Timescale.MICRO); } - /** This timestamp's value re-expressed at `scale`; throws if the result is not in the safe range. */ + /** + * This timestamp's value re-expressed at `scale`; throws a `RangeError` if it is unscaled or + * the result is not in the safe range. + */ as(scale: Timescale): number { const dest = Timescale(scale); - const converted = dest === this.scale ? this.value : (this.value * dest) / this.scale; + const from = scaled(this); + const converted = dest === from ? this.value : (this.value * dest) / from; assertTimestampValue(converted); return converted; } - /** The value in milliseconds. */ + /** The value in milliseconds; throws a `RangeError` if it is unscaled. */ asMillis(): number { return this.as(Timescale.MILLI); } - /** The value in microseconds. */ + /** The value in microseconds; throws a `RangeError` if it is unscaled. */ asMicros(): number { return this.as(Timescale.MICRO); } - /** Whether this presents before `other`, whatever their scales. */ + /** + * Whether this presents before `other`, whatever their scales. Two unscaled timestamps order + * by key; a scaled and an unscaled one don't order, so neither is before the other. + */ isBefore(other: Timestamp): boolean { + const [a, b] = [this.#key, other.#key]; + if (a !== undefined || b !== undefined) return a !== undefined && b !== undefined && a < b; if (this.scale === other.scale) return this.value < other.value; // Cross-scale products can pass 2^53, so they are compared exactly in BigInt, as Rust does in u128. const [ma, ea] = binary(this.value); const [mb, eb] = binary(other.value); const e = Math.min(ea, eb); - return (ma * BigInt(other.scale)) << BigInt(ea - e) < (mb * BigInt(this.scale)) << BigInt(eb - e); + return (ma * BigInt(scaled(other))) << BigInt(ea - e) < (mb * BigInt(scaled(this))) << BigInt(eb - e); } } +/** + * A received frame's timestamp: the one its wire carried, or on a scale-less track, which + * carries none, {@link Timestamp.unscaled} at its group's `key` (a datagram's sequence). + * Mirrors Rust's `Timestamp::received`. + * + * @internal + */ +export function received(timestamp: Timestamp | undefined, key: bigint): Timestamp { + return timestamp ?? Timestamp.unscaled(key); +} + +// `timestamp`'s scale, or a `RangeError` when unscaled: a group key has no time to convert. +function scaled(timestamp: Timestamp): Timescale { + if (timestamp.scale === undefined) throw new RangeError("an unscaled timestamp has no time"); + return timestamp.scale; +} + /** * `timestamp`'s value at `scale`, rounded down exactly, as Rust's integer `Timestamp::convert`. * * A double is a dyadic rational, so the conversion runs in BigInt with no rounding before the * floor: an aligned value keeps it (4611686021000000 ns is 4611686021 ms, where `as` gives * 4611686020.999999), and one instant gives one result whatever scale it came from. Throws a - * `RangeError` past the safe integer range. + * `RangeError` past the safe integer range, or for an unscaled timestamp. * * @internal */ export function floorAt(timestamp: Timestamp, scale: Timescale): number { - if (timestamp.scale === scale) return Math.floor(timestamp.value); + const from = scaled(timestamp); + if (from === scale) return Math.floor(timestamp.value); const [mantissa, exponent] = binary(timestamp.value); - return toNumber((mantissa * BigInt(scale)) / (BigInt(timestamp.scale) << BigInt(-exponent))); + return toNumber((mantissa * BigInt(scale)) / (BigInt(from) << BigInt(-exponent))); } // A finite, non-negative `value` exactly as `mantissa * 2^exponent`. Doubling is exact, so diff --git a/js/net/src/timed.ts b/js/net/src/timed.ts index 60e1a38c919..c3adaf1d947 100644 --- a/js/net/src/timed.ts +++ b/js/net/src/timed.ts @@ -1,14 +1,20 @@ /** - * A value paired with when it was captured. + * A value paired with its frame's timestamp. * * @module */ import type { Timestamp } from "./time.ts"; -/** A value, and optionally when it was captured. An absent `at` means untimed, never now. */ +/** + * A value to publish or read, and its frame's timestamp. + * + * On a timed track `at` is when the value was captured. A scale-less track has no time, so + * there it is {@link Timestamp.unscaled} at a group key: a write that opens a group starts it + * at that key, which must pass every group the track holds. + */ export interface Timed { /** The value. */ value: T; - /** When the value was captured, written as its frame's timestamp. Absent for an untimed track. */ - at?: Timestamp; + /** The frame's timestamp: its capture time, or its group key on a scale-less track. */ + at: Timestamp; } diff --git a/js/net/src/track.test.ts b/js/net/src/track.test.ts index 553411fb530..8cb4f06f984 100644 --- a/js/net/src/track.test.ts +++ b/js/net/src/track.test.ts @@ -8,7 +8,7 @@ import type { Consumer as GroupConsumer, Producer as GroupProducer } from "./gro import { Version as IetfVersion } from "./ietf/version.ts"; import { hooks, skipHole, staleHole } from "./internal.ts"; import { textFrame } from "./mock.ts"; -import { rescale } from "./rescale.ts"; +import { rescale, rescaleIn } from "./rescale.ts"; import { Timescale, Timestamp } from "./time.ts"; import { authoredInfo, Subscription, Producer as TrackProducer } from "./track.ts"; import { unwanted } from "./util/demand.ts"; @@ -25,7 +25,7 @@ test("a cached transport tail covers only positions it actually holds", async () const producer = new TrackProducer("test"); const subscriber = producer.subscribe(); const tail = hooks.receiveGroup(producer, { sequence: 0n, start: 5 }); - tail.writeFrame({ payload: new Uint8Array([7]) }); + tail.writeFrame({ payload: new Uint8Array([7]), timestamp: Timestamp.unscaled(tail.sequence) }); tail.finish(); expect(hooks.cachedGroup(subscriber, 0n)).toBeUndefined(); const covered = hooks.cachedGroup(subscriber, 0n, 5); @@ -334,8 +334,8 @@ test("wanted stays set across churn while at least one subscriber remains", asyn // Append a group of a 3 byte first frame and a `delta` byte delta to an untimed track. function refreshGroup(producer: TrackProducer, delta: number): void { const group = producer.appendGroup(); - group.writeFrame({ payload: enc.encode("key") }); - if (delta > 0) group.writeFrame({ payload: new Uint8Array(delta) }); + group.writeFrame({ payload: enc.encode("key"), timestamp: Timestamp.unscaled(group.sequence) }); + if (delta > 0) group.writeFrame({ payload: new Uint8Array(delta), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } @@ -422,13 +422,18 @@ test("rescale matches the track's timeline, rounded down as Rust's convert does" // The same binary value at two scales floors alike, below 0.117 s: 116 ms, as Rust converts it. expect(rescale(timed, new Timestamp(0.11699999999999999, Timescale.SECOND))?.value).toBe(116); expect(rescale(timed, new Timestamp(131730289100587, Timescale(1125899906842624)))?.value).toBe(116); - expect(() => rescale(timed, undefined)).toThrow(TimestampMismatch); + expect(() => rescale(timed, Timestamp.unscaled(2n))).toThrow(TimestampMismatch); const huge = new Timestamp(Number.MAX_SAFE_INTEGER, Timescale.MILLI); expect(() => rescale({ timescale: Timescale.NANO }, huge)).toThrow(TimestampMismatch); - expect(rescale({}, undefined)).toBeUndefined(); - expect(rescale({ timescale: null }, undefined)).toBeUndefined(); + const key = Timestamp.unscaled(2n); + expect(rescale({}, key)).toBe(key); + expect(rescale({ timescale: null }, key)).toBe(key); expect(() => rescale({}, Timestamp.fromMillis(1))).toThrow(TimestampMismatch); + // A scale-less frame carries its group's key, so another key is a mismatch. + expect(rescaleIn({ timescale: null }, key, 2n)).toBe(key); + expect(() => rescaleIn({ timescale: null }, key, 3n)).toThrow(TimestampMismatch); + expect(rescaleIn(timed, Timestamp.fromMillis(1), 3n).value).toBe(1); }); // The in-range datagrams, sent between the dropped ones, show the reader is live, so the @@ -1653,7 +1658,10 @@ test("a parked read wakes at its wall deadline", async () => { try { const track = producer.subscribe({ delay: { max: Duration.millis(1000) } }); const held = producer.appendGroup(); - held.writeFrame({ payload: enc.encode("x"), timestamp: timescale && Timestamp.fromMillis(0) }); + held.writeFrame({ + payload: enc.encode("x"), + timestamp: timescale ? Timestamp.fromMillis(0) : Timestamp.unscaled(held.sequence), + }); const group = track.tryRecvGroup(); if (!group) throw new Error("missing group"); expect(await group.readString()).toBe("x"); @@ -2639,7 +2647,8 @@ test("a recreated largest group is reported before the end", async () => { /** Write untimed group `sequence` with `frames` frames, left open. */ function framedGroup(producer: TrackProducer, sequence: bigint, frames: number): GroupProducer { const group = producer.createGroup(sequence); - for (let i = 0; i < frames; i++) group.writeFrame({ payload: enc.encode("x") }); + for (let i = 0; i < frames; i++) + group.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.unscaled(group.sequence) }); return group; } @@ -3246,17 +3255,20 @@ function drain(track: { tryRecvGroup(): GroupConsumer | undefined }): bigint[] { return sequences; } -/** An untimed track holding `count` single-frame groups. */ +/** A scale-less track holding `count` single-frame groups. */ function untimedTrack(count: number): TrackProducer { const producer = new TrackProducer("test").accept({ maxAge: null }); - for (let i = 0; i < count; i++) producer.writeFrame({ payload: enc.encode(`${i}`) }); + for (let i = 0; i < count; i++) + producer.writeFrame({ payload: enc.encode(`${i}`), timestamp: Timestamp.unscaled(i) }); return producer; } test("a frame whose timedness disagrees with its track is refused", async () => { const timed = new TrackProducer("timed").accept({ timescale: Timescale.MILLI }); const group = timed.appendGroup(); - expect(() => group.writeFrame({ payload: enc.encode("x") })).toThrow(TimestampMismatch); + expect(() => group.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.unscaled(group.sequence) })).toThrow( + TimestampMismatch, + ); group.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(1) }); const untimed = new TrackProducer("untimed").accept({}); @@ -3265,13 +3277,13 @@ test("a frame whose timedness disagrees with its track is refused", async () => expect(() => other.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(1) })).toThrow( TimestampMismatch, ); - other.writeFrame({ payload: enc.encode("y") }); + other.writeFrame({ payload: enc.encode("y"), timestamp: Timestamp.unscaled(other.sequence) }); other.finish(); - // The untimed frame reaches the subscriber as it was written: no time filled in. + // The scale-less frame reaches the subscriber as it was written: its key, no time filled in. const frame = await (await subscriber.recvGroup())?.readFrame(); expect(frame && dec.decode(frame.payload)).toBe("y"); - expect(frame?.timestamp).toBeUndefined(); + expect(frame?.timestamp).toEqual(Timestamp.unscaled(other.sequence)); }); test("unknown track metadata keeps presentation-time expiry until accepted", () => { @@ -3302,7 +3314,7 @@ test("an untimed subscriber starts at the oldest fresh group", () => { try { // Group N's successor arrives N + 1 seconds in. for (let i = 0; i < 5; i++) { - producer.writeFrame({ payload: enc.encode(`${i}`) }); + producer.writeFrame({ payload: enc.encode(`${i}`), timestamp: Timestamp.unscaled(i) }); clock.set(10_000 + (i + 1) * 1000); } @@ -3314,7 +3326,7 @@ test("an untimed subscriber starts at the oldest fresh group", () => { expect(drain(live)).toEqual([4n]); expect(drain(producer.subscribe({ delay: { max: Duration.millis(60_000) } }))).toEqual([0n, 1n, 2n, 3n, 4n]); - producer.writeFrame({ payload: enc.encode("5") }); + producer.writeFrame({ payload: enc.encode("5"), timestamp: Timestamp.unscaled(5n) }); expect(drain(live)).toEqual([5n]); } finally { producer.finish(); @@ -3332,19 +3344,40 @@ test("an untimed subscriber without a start begins at the latest group under its }); test("a single-frame write that throws aborts its group", () => { - const producer = new TrackProducer("test").accept({ maxAge: null }); + const producer = new TrackProducer("test").accept({ timescale: Timescale.MILLI, maxAge: null }); const subscriber = producer.subscribe({ delay: { max: Duration.millis(60_000) } }); - // A timestamped frame is refused by an untimed track. - expect(() => producer.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.fromMillis(1) })).toThrow( + // A mismatched timestamp is refused before any group exists. + expect(() => producer.writeFrame({ payload: enc.encode("x"), timestamp: Timestamp.unscaled(0n) })).toThrow( + TimestampMismatch, + ); + expect(producer.largest()).toBeUndefined(); + producer.writeFrame({ payload: enc.encode("y"), timestamp: Timestamp.fromMillis(10) }); + // A rewind is only known once the group exists, so the write ends it. + expect(() => producer.writeFrame({ payload: enc.encode("z"), timestamp: Timestamp.fromMillis(5) })).toThrow( TimestampMismatch, ); - producer.writeFrame({ payload: enc.encode("y") }); - // The failed group ends in that error instead of leaving a reader waiting on it. + expect(subscriber.tryRecvGroup()?.sequence).toBe(0n); const failed = subscriber.tryRecvGroup(); - expect(failed?.sequence).toBe(0n); + expect(failed?.sequence).toBe(1n); expect(failed?.closed.peek()).toBeInstanceOf(TimestampMismatch); - expect(drain(subscriber)).toEqual([1n]); +}); + +test("a scale-less single-frame write lands at its key", () => { + const producer = new TrackProducer("test").accept({ maxAge: null }); + const subscriber = producer.subscribe({ delay: { max: Duration.millis(60_000) } }); + producer.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(3n) }); + // A key at or below one held is a rewind, refused before any group exists. + expect(() => producer.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(3n) })).toThrow( + TimestampMismatch, + ); + expect(() => producer.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(1n) })).toThrow( + TimestampMismatch, + ); + producer.writeFrame({ payload: enc.encode("c"), timestamp: Timestamp.unscaled(5n) }); + expect(drain(subscriber)).toEqual([3n, 5n]); + // The keys below each write are skipped, so creating one is refused. + expect(() => producer.createGroup(4n)).toThrow(Skipped); }); // An explicit start is a floor, not a request: the budget still skips what it finds stale @@ -3407,9 +3440,9 @@ test("an upstream drop ends a group still arriving with its error", async () => const producer = new TrackProducer("video").accept({ maxAge: null }); const reader = producer.subscribe({ delay: { max: Duration.millis(60_000) } }); const arriving = producer.createGroup(3n); - arriving.writeFrame({ payload: enc.encode("3") }); + arriving.writeFrame({ payload: enc.encode("3"), timestamp: Timestamp.unscaled(arriving.sequence) }); const finished = producer.createGroup(4n); - finished.writeFrame({ payload: enc.encode("4") }); + finished.writeFrame({ payload: enc.encode("4"), timestamp: Timestamp.unscaled(finished.sequence) }); finished.finish(); const group = await reader.recvGroup(); @@ -3450,7 +3483,7 @@ test.each([ const producer = new TrackProducer("video").accept({ maxAge: null }); hooks.receiveTrack(producer); const first = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); - first.writeFrame({ payload: enc.encode("a") }); + first.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(first.sequence) }); const reader = first.consume(); expect(await reader.readString()).toBe("a"); @@ -3459,7 +3492,7 @@ test.each([ const failed = new Error("reset"); if (kind === "closed") { // A frame still buffered when the copy fails stays readable after a close. - first.writeFrame({ payload: enc.encode("c") }); + first.writeFrame({ payload: enc.encode("c"), timestamp: Timestamp.unscaled(first.sequence) }); first.abort(failed); reader.close(); expect(await reader.readString()).toBe("c"); @@ -3481,8 +3514,8 @@ test.each([ expect(closed).toBe(failed); // The failure it reported stands: a re-offer arriving later does not take over. const late = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); - late.writeFrame({ payload: enc.encode("a") }); - late.writeFrame({ payload: enc.encode("b") }); + late.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(late.sequence) }); + late.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(late.sequence) }); await expect(reader.readString()).rejects.toBe(failed); producer.close(); return; @@ -3497,8 +3530,8 @@ test.each([ expect(reader.closed.peek()).toBeUndefined(); const third = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); held?.release(); - third.writeFrame({ payload: enc.encode("a") }); - third.writeFrame({ payload: enc.encode("b") }); + third.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(third.sequence) }); + third.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(third.sequence) }); third.finish(); expect(await reader.readString()).toBe("b"); expect(await reader.readString()).toBeUndefined(); @@ -3510,7 +3543,8 @@ test.each([ // A re-offer that took over and then shed the reader's frame settles `closed` then. const again = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); arriving?.release(); - for (const payload of ["a", "b", "c"]) again.writeFrame({ payload: enc.encode(payload) }); + for (const payload of ["a", "b", "c"]) + again.writeFrame({ payload: enc.encode(payload), timestamp: Timestamp.unscaled(again.sequence) }); let closed: Error | null | undefined; reader.closed.then((value) => { closed = value; @@ -3572,9 +3606,9 @@ test.each([ arriving?.release(); if (kind === "shed") { // The re-offer drops frames 0 and 1 to cache pressure before the reader reaches it. - again.writeFrame({ payload: enc.encode("a") }); - again.writeFrame({ payload: enc.encode("b") }); - again.writeFrame({ payload: enc.encode("c") }); + again.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(again.sequence) }); + again.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(again.sequence) }); + again.writeFrame({ payload: enc.encode("c"), timestamp: Timestamp.unscaled(again.sequence) }); hooks.shedGroup(again, Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, []); await reading; expect(read).toMatchObject({ code: StreamCode.Evicted }); @@ -3589,8 +3623,8 @@ test.each([ } await settle(); expect(read).toBeUndefined(); - again.writeFrame({ payload: enc.encode("a") }); - again.writeFrame({ payload: enc.encode("b") }); + again.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(again.sequence) }); + again.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(again.sequence) }); again.finish(); await reading; expect(read).toEqual(kind === "readable" ? enc.encode("b") : "b"); @@ -3609,7 +3643,7 @@ test("a chain of takeovers keeps a stalled reader within the tail", async () => let copy = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); const reader = copy.consume(); for (let index = 0; index < 8; index++) { - copy.writeFrame({ payload }); + copy.writeFrame({ payload, timestamp: Timestamp.unscaled(copy.sequence) }); const held = hooks.arrivingGroup(producer, 3n); copy.abort(failed); copy = hooks.receiveGroup(producer, { sequence: 3n, start: index + 1 }); @@ -3631,13 +3665,14 @@ test.each(["buffered", "handed", "partial", "replay", "positions", "once"])( hooks.receiveTrack(producer); const track = producer.subscribe(); const first = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); - first.writeFrame({ payload: enc.encode("a") }); + first.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(first.sequence) }); const arriving = hooks.arrivingGroup(producer, 3n); const failed = new Error("reset"); const takeover = () => { const again = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); arriving?.release(); - for (const payload of ["a", "b", "c"]) again.writeFrame({ payload: enc.encode(payload) }); + for (const payload of ["a", "b", "c"]) + again.writeFrame({ payload: enc.encode(payload), timestamp: Timestamp.unscaled(again.sequence) }); again.finish(); }; @@ -3658,7 +3693,7 @@ test.each(["buffered", "handed", "partial", "replay", "positions", "once"])( const ahead = first.consume(); const behind = first.consume(); expect(await ahead.readString()).toBe("a"); - const cut = first.createFrame({ size: 2 }); + const cut = first.createFrame({ size: 2, timestamp: Timestamp.unscaled(first.sequence) }); cut.write(enc.encode("b")); // `ahead` joins the frame as it arrives; `behind` has yet to reach it. const joining = ahead.readString(); @@ -3677,18 +3712,19 @@ test.each(["buffered", "handed", "partial", "replay", "positions", "once"])( const second = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); arriving?.release(); const held = hooks.arrivingGroup(producer, 3n); - second.createFrame({ size: 2 }).write(enc.encode("a")); + second.createFrame({ size: 2, timestamp: Timestamp.unscaled(second.sequence) }).write(enc.encode("a")); second.abort(failed); await settle(); expect(reader.closed.peek()).toBeUndefined(); const third = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); held?.release(); - for (const payload of ["a", "b"]) third.writeFrame({ payload: enc.encode(payload) }); + for (const payload of ["a", "b"]) + third.writeFrame({ payload: enc.encode(payload), timestamp: Timestamp.unscaled(third.sequence) }); third.finish(); expect(await reader.readString()).toBe("b"); expect(await reader.readString()).toBeUndefined(); } else if (kind === "positions") { - first.writeFrame({ payload: enc.encode("b") }); + first.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(first.sequence) }); const readers = [first.consume(), first.consume(), first.consume()]; expect(await readers[1].readString()).toBe("a"); expect(await readers[2].readString()).toBe("a"); @@ -3704,7 +3740,7 @@ test.each(["buffered", "handed", "partial", "replay", "positions", "once"])( } else { const group = await track.recvGroup(); const reader = first.consume(); - first.writeFrame({ payload: enc.encode("b") }); + first.writeFrame({ payload: enc.encode("b"), timestamp: Timestamp.unscaled(first.sequence) }); first.abort(failed); await settle(); expect(group?.closed.peek()).toBeUndefined(); @@ -3720,7 +3756,7 @@ test.each(["buffered", "handed", "partial", "replay", "positions", "once"])( } // A re-offer arriving later serves new readers only. const late = hooks.receiveGroup(producer, { sequence: 3n, start: 0 }); - late.writeFrame({ payload: enc.encode("a") }); + late.writeFrame({ payload: enc.encode("a"), timestamp: Timestamp.unscaled(late.sequence) }); await expect(reader.readString()).rejects.toBe(failed); expect(await late.consume().readString()).toBe("a"); } @@ -4029,7 +4065,8 @@ test("a mid-group end survives eviction and a filtered late reader", async () => try { producer.finishAt({ group: 5n, frame: 5 }); const group = producer.createGroup(5n); - for (let i = 0; i < 5; i++) group.writeFrame({ payload: new Uint8Array([i]) }); + for (let i = 0; i < 5; i++) + group.writeFrame({ payload: new Uint8Array([i]), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); producer.finish(); producer.close(); @@ -4070,7 +4107,8 @@ test("a wire end repeats or refines but never moves", () => { test("a mid-group end refuses to cut existing frames", () => { const producer = new TrackProducer("video").accept({}); const group = producer.createGroup(5n); - for (let i = 0; i < 6; i++) group.writeFrame({ payload: new Uint8Array([i]) }); + for (let i = 0; i < 6; i++) + group.writeFrame({ payload: new Uint8Array([i]), timestamp: Timestamp.unscaled(group.sequence) }); expect(() => producer.finishAt({ group: 5n, frame: 5 })).toThrow(ProtocolViolation); expect(hooks.subscriberEnd(producer.subscribe())).toBeUndefined(); producer.finishAt({ group: 5n, frame: 6 }); @@ -4085,10 +4123,12 @@ test.each([true, false])("a mid-group end caps later writes (declared first: %s) if (early) producer.finishAt(end); const group = producer.createGroup(5n); - group.writeFrame({ payload: new Uint8Array([0]) }); + group.writeFrame({ payload: new Uint8Array([0]), timestamp: Timestamp.unscaled(group.sequence) }); if (!early) producer.finishAt(end); - group.writeFrame({ payload: new Uint8Array([1]) }); - expect(() => group.writeFrame({ payload: new Uint8Array([2]) })).toThrow("past the track's end"); + group.writeFrame({ payload: new Uint8Array([1]), timestamp: Timestamp.unscaled(group.sequence) }); + expect(() => + group.writeFrame({ payload: new Uint8Array([2]), timestamp: Timestamp.unscaled(group.sequence) }), + ).toThrow("past the track's end"); expect(group.consume().frameCount).toBe(2); expect(() => producer.createGroup(6n)).toThrow(); producer.finish(); @@ -4318,7 +4358,7 @@ test("an untimed track takes groups in any order", () => { const producer = new TrackProducer("test").accept({}); for (const sequence of [5n, 3n, 4n]) { const group = producer.createGroup(sequence); - group.writeFrame({ payload: enc.encode(`${sequence}`) }); + group.writeFrame({ payload: enc.encode(`${sequence}`), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); } }); @@ -4328,7 +4368,7 @@ test("replay and backlogs deliver sparse out-of-order arrivals ascending without const all = { ranges: [{ start: { absolute: 0n } }], delay: { max: null } }; const write = (sequence: bigint) => { const group = producer.createGroup(sequence); - group.writeFrame({ payload: enc.encode(`${sequence}`) }); + group.writeFrame({ payload: enc.encode(`${sequence}`), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); }; const drain = (reader: ReturnType) => { @@ -4472,7 +4512,7 @@ test("an end refined after the subscription ended owes no tail", async () => { delay: { max: Duration.millis(60_000) }, }); const group = producer.createGroup(2n); - group.writeFrame({ payload: new Uint8Array([1]) }); + group.writeFrame({ payload: new Uint8Array([1]), timestamp: Timestamp.unscaled(group.sequence) }); group.finish(); hooks.endTrack(producer, { group: 3n, frame: 0 }); hooks.cacheTrack(producer); @@ -4871,7 +4911,8 @@ for (const refresh of [undefined, 0]) { const group = producer.appendGroup(); const consumer = await ordered.nextGroup(); if (!consumer) throw new Error("missing group"); - for (let index = 0; index < 40; index++) group.writeFrame({ payload: new Uint8Array(1024 * 1024) }); + for (let index = 0; index < 40; index++) + group.writeFrame({ payload: new Uint8Array(1024 * 1024), timestamp: Timestamp.unscaled(group.sequence) }); await expect(consumer.readFrame()).rejects.toMatchObject({ code: StreamCode.Evicted }); expect(producer.demand().wanted.peek()).toBe(refresh === undefined ? 0n : 1n); const joiner = producer.subscribe(); @@ -4893,7 +4934,7 @@ test("reading a shed frame head requests refresh", async () => { const consumer = await ordered.nextGroup(); if (!consumer) throw new Error("missing group"); const MiB = 1024 * 1024; - const frame = group.createFrame({ size: 40 * MiB }); + const frame = group.createFrame({ size: 40 * MiB, timestamp: Timestamp.unscaled(group.sequence) }); for (let index = 0; index < 40; index++) frame.write(new Uint8Array(MiB)); frame.finish(); await expect(consumer.readFrame()).rejects.toMatchObject({ code: StreamCode.Evicted }); diff --git a/js/net/src/track.ts b/js/net/src/track.ts index c219735a0e7..bb48b4d51a3 100644 --- a/js/net/src/track.ts +++ b/js/net/src/track.ts @@ -51,7 +51,8 @@ import { type TrackRequestOptions, } from "./internal.ts"; import { RUN_OVERHEAD as ARRIVAL_OVERHEAD, grace, Ledger } from "./ledger.ts"; -import { Timescale, type Timestamp } from "./time.ts"; +import { rescale } from "./rescale.ts"; +import { keyOf, Timescale, type Timestamp } from "./time.ts"; import { checkEnd, checkSequence, max, min, UNBOUNDED } from "./util/sequence.ts"; import { bounded, @@ -142,8 +143,9 @@ export interface Info { /** How frame payloads are compressed per group on lite-07; other sessions receive plain frames. */ compression: Compression; /** - * Units per second for this track's frame timestamps, or `null` for an untimed track, whose - * frames carry no timestamp. + * Units per second for this track's frame timestamps, or `null` for a scale-less (untimed) + * track, whose frames carry no time: each one's timestamp is {@link Timestamp.unscaled} at + * its group's sequence. * * A timed track declares its scale, such as {@link Timescale | Timescale.MILLI}, or * {@link Timescale | Timescale.MICRO} for media that needs sub-millisecond timing. Reported in @@ -2140,12 +2142,17 @@ export class Producer { * or instance. Reusing a sequence only corrupts this broadcast for its readers. Datagram * sequences name no cached content and are exempt. * - * Its frames must match the track's {@link Info.timescale}: a timestamp on every frame of - * a timed track and none on an untimed one, or the write throws {@link TimestampMismatch}. + * Its frames must match the track's {@link Info.timescale}: a scaled timestamp on every frame + * of a timed track, and on a scale-less one {@link Timestamp.unscaled} at the group's + * sequence, or the write throws {@link TimestampMismatch}. * So does a first frame that starts before the group below it or after the one above it. */ appendGroup(): GroupProducer { - const sequence = this.#nextSequence(); + return this.#appendAt(this.#nextSequence()); + } + + // Append a group at `sequence`, at or past the next sequence, skipping what it leaves below. + #appendAt(sequence: bigint): GroupProducer { this.#refuseSkipped(sequence); const group = new GroupProducer(sequence); this.#bind(group); @@ -2561,9 +2568,21 @@ export class Producer { return true; } - /** Append a frame as its own single-frame group. */ + /** + * Append a frame as its own single-frame group. + * + * On a timed track the group takes the next sequence, as {@link appendGroup}. A scale-less + * track has no time, so its frame's {@link Timestamp.unscaled} names the group instead: the + * key must pass every group this track holds, and the sequences it leaves below are skipped, + * as {@link appendGroup} skips them. A key at or below one held, or a timestamp that doesn't + * match the track, throws {@link TimestampMismatch} before any group exists. + */ writeFrame(frame: Frame) { - const group = this.appendGroup(); + const info = this.#state.info.peek(); + const key = keyOf(info ? rescale(info, frame.timestamp) : frame.timestamp); + if (key !== undefined && key < this.#received) throw new TimestampMismatch(); + if (key !== undefined) this.#writable(key); + const group = key === undefined ? this.appendGroup() : this.#appendAt(key); try { group.writeFrame(frame); } catch (err) { diff --git a/js/net/src/untimed.test.ts b/js/net/src/untimed.test.ts index 874ee4403f3..ad256feafe7 100644 --- a/js/net/src/untimed.test.ts +++ b/js/net/src/untimed.test.ts @@ -1,6 +1,6 @@ // A track keeps its timedness end to end: no receive path fills in arrival time, a wire that -// can't declare a timeline delivers frames untimed, and lite-05/06, which can't declare an -// untimed one, deliver its send times. Mirrors rs/moq-net/tests/integration/untimed.rs. +// can't declare a timeline delivers frames scale-less, stamped with their group's key, and +// lite-05/06, which can't declare a scale-less one, deliver its send times. Mirrors rs/moq-net/tests/integration/untimed.rs. import { expect, test } from "bun:test"; import { type Consumer as BroadcastConsumer, Producer as BroadcastProducer } from "./broadcast.ts"; import { accept, connect } from "./connection/index.ts"; @@ -36,7 +36,7 @@ const WIRES: Wire[] = [ ]; /** Serve one group of `timestamps` on a track with `timescale`, and read it back over `wire`. */ -async function roundTrip(wire: Wire, timescale: Timescale | undefined, timestamps: (Timestamp | undefined)[]) { +async function roundTrip(wire: Wire, timescale: Timescale | undefined, timestamps: Timestamp[]) { const pair = createMockTransportPair(wire.protocol); const origin = new OriginProducer(); const [client, server] = await Promise.all([ @@ -83,7 +83,8 @@ async function republish(input: BroadcastConsumer): Promise { try { const groups = subscription.ordered(); for (let group = await groups.nextGroup(); group; group = await groups.nextGroup()) { - const out = video.appendGroup(); + // The same group, so a scale-less frame's key still names it. + const out = video.createGroup(group.sequence); for (let frame = await group.readFrame(); frame; frame = await group.readFrame()) out.writeFrame(frame); out.finish(); } @@ -143,7 +144,7 @@ test("a forwarded untimed track stays untimed", async () => { expect((await track.info()).timescale).toBeNull(); const frame = await (await track.ordered().nextGroup())?.readFrame(); expect(frame?.payload).toEqual(new Uint8Array([7])); - expect(frame?.timestamp).toBeUndefined(); + expect(frame?.timestamp).toEqual(Timestamp.unscaled(0n)); broadcast.close(); await serving; @@ -156,10 +157,12 @@ test("a forwarded untimed track stays untimed", async () => { for (const wire of WIRES) { test(`${wire.name}: an untimed track arrives ${wire.sends ? "with its send times" : "untimed"}`, async () => { - const { info, frames } = await roundTrip(wire, undefined, [undefined, undefined]); + // The track's first group is 0, which each scale-less frame's key names. + const key = Timestamp.unscaled(0n); + const { info, frames } = await roundTrip(wire, undefined, [key, key]); expect(frames.map((frame) => frame.payload[0])).toEqual([0, 1]); expect(info.timescale).toBe(wire.sends ? Timescale.MILLI : null); - for (const frame of frames) expect(frame.timestamp !== undefined).toBe(wire.sends); + for (const frame of frames) expect(frame.timestamp.scale !== undefined).toBe(wire.sends); }); test(`${wire.name}: a timed track keeps its timestamps only where the wire delivers them`, async () => { @@ -173,9 +176,10 @@ for (const wire of WIRES) { expect(info.timescale).toBe(Timescale.MICRO); expect(frames.map((frame) => frame.timestamp?.asMicros())).toEqual([1_000, 1_234]); } else { - // No trusted units on the wire, and no arrival time made up in their place. + // No trusted units on the wire, and no arrival time made up in their place: each frame + // carries only its group's key. expect(info.timescale).toBeNull(); - for (const frame of frames) expect(frame.timestamp).toBeUndefined(); + for (const frame of frames) expect(frame.timestamp).toEqual(Timestamp.unscaled(0n)); } }); } diff --git a/js/net/src/window.ts b/js/net/src/window.ts index b6aa712384e..7d8ef6be169 100644 --- a/js/net/src/window.ts +++ b/js/net/src/window.ts @@ -9,7 +9,7 @@ * @module */ -import { Timestamp } from "./time.ts"; +import { Timescale, Timestamp } from "./time.ts"; import type { Anchor, Range } from "./track.ts"; import { max, min, saturatingSub, UNBOUNDED } from "./util/sequence.ts"; @@ -29,11 +29,12 @@ export const LIVE: Range = Object.freeze({ start: Object.freeze({ relative: 0n } function anchorOf(field: string, anchor: Anchor): Anchor { const keys = typeof anchor === "object" && anchor !== null ? Object.keys(anchor) : []; const value = keys.length === 1 ? (anchor as Record)[keys[0]] : undefined; - if (keys[0] === "time" && value instanceof Timestamp) return { time: value }; + // A time anchor names a presentation time, which an unscaled timestamp lacks. + if (keys[0] === "time" && value instanceof Timestamp && value.scale !== undefined) return { time: value }; if (keys[0] === "absolute" && typeof value === "bigint" && value >= 0n) return { absolute: value }; if (keys[0] === "relative" && typeof value === "bigint" && value >= 0n) return { relative: value }; throw new TypeError( - `${field} must be {absolute}, a non-negative bigint, {relative}, a non-negative bigint, or {time}, a Timestamp`, + `${field} must be {absolute}, a non-negative bigint, {relative}, a non-negative bigint, or {time}, a scaled Timestamp`, ); } @@ -90,7 +91,8 @@ function anchorKey(anchor: Anchor | undefined, none: [number, number]): [number, if (anchor === undefined) return none; if (anchor.absolute !== undefined) return [0, anchor.absolute]; if (anchor.relative !== undefined) return [1, anchor.relative]; - return [2, anchor.time.value / anchor.time.scale]; + // `rangesOf` refuses an unscaled time anchor, so this has a scale. + return [2, anchor.time.as(Timescale.SECOND)]; } /** diff --git a/js/net/src/zod.test.ts b/js/net/src/zod.test.ts index b3adce4dc95..63c7bce2dd1 100644 --- a/js/net/src/zod.test.ts +++ b/js/net/src/zod.test.ts @@ -2,12 +2,13 @@ import { expect, test } from "bun:test"; import * as z from "zod/mini"; import { Producer } from "./group.ts"; import { textFrame } from "./mock.ts"; +import { Timestamp } from "./time.ts"; import * as Zod from "./zod.ts"; test("read validates each frame and ends only at the end of the group", async () => { const producer = new Producer(0n); const consumer = producer.consume(); - producer.writeFrame({ payload: new Uint8Array() }); + producer.writeFrame({ payload: new Uint8Array(), timestamp: Timestamp.unscaled(producer.sequence) }); // Falsy values are frames like any other, not the end of the stream. for (const json of ["0", "false", "null", '"ok"']) producer.writeFrame(textFrame(json)); producer.finish(); diff --git a/js/stats/src/stats.test.ts b/js/stats/src/stats.test.ts index 522cd2359dd..5e326da9ac6 100644 --- a/js/stats/src/stats.test.ts +++ b/js/stats/src/stats.test.ts @@ -1,6 +1,6 @@ import { expect, test } from "bun:test"; import * as Json from "@moq/json"; -import { type Broadcast, Origin, Path } from "@moq/net"; +import { type Broadcast, Origin, Path, Time } from "@moq/net"; import { Consumer, Nodes, @@ -12,6 +12,10 @@ import { TrafficSchema, } from "./index.ts"; +// A scale-less snapshot write names the group it opens; keys only have to increase. +let keys = 0n; +const key = () => Time.Timestamp.unscaled(keys++); + function publish(broadcast: Broadcast.Producer, name: string) { return new Json.Snapshot.Producer({ track: broadcast.createTrack(name) }); } @@ -41,7 +45,7 @@ test("a bucket follows the listing to each new run", async () => { const bucket = new Consumer(broadcast.consume()).bucket(""); const first = publish(broadcast, "bucket.e1.json"); - first.update({ value: { epoch: "e1", tiers: { "": { paths: { cam: { publisher: { bytes: 5 } } } } } } }); + first.update({ value: { epoch: "e1", tiers: { "": { paths: { cam: { publisher: { bytes: 5 } } } } } }, at: key() }); const update = await bucket.next(); if (update?.kind !== "frame") throw new Error("expected a frame"); expect(update.frame.epoch).toBe("e1"); @@ -49,7 +53,7 @@ test("a bucket follows the listing to each new run", async () => { expect(update.frame.tiers[""]?.paths.cam?.publisher).toMatchObject({ bytes: 5, frames: 0, stale: { bytes: 0 } }); // A merge-patch delta lands on the snapshot. - first.update({ value: { epoch: "e1", tiers: { "": { paths: { cam: { publisher: { bytes: 9 } } } } } } }); + first.update({ value: { epoch: "e1", tiers: { "": { paths: { cam: { publisher: { bytes: 9 } } } } } }, at: key() }); expect(await bucket.next()).toMatchObject({ kind: "frame", frame: { tiers: { "": { paths: { cam: { publisher: { bytes: 9 } } } } } }, @@ -58,11 +62,11 @@ test("a bucket follows the listing to each new run", async () => { // A new run replaces the old one, whose counters start over. const second = publish(broadcast, "bucket.e2.json"); broadcast.removeTrack("bucket.e1.json"); - second.update({ value: { epoch: "e2", tiers: {} } }); + second.update({ value: { epoch: "e2", tiers: {} }, at: key() }); expect(await bucket.next()).toMatchObject({ kind: "frame", frame: { epoch: "e2", tiers: {} } }); // Another bucket's run is not this one's. - publish(broadcast, "acme/bucket.e3.json").update({ value: { epoch: "e3" } }); + publish(broadcast, "acme/bucket.e3.json").update({ value: { epoch: "e3" }, at: key() }); broadcast.removeTrack("bucket.e2.json"); expect(await bucket.next()).toEqual({ kind: "ended" }); @@ -76,9 +80,12 @@ test("totals, presence, and prefix tracks decode with defaults", async () => { const broadcast = origin.route(Path.from(".stats/node")).broadcast; const consumer = new Consumer(broadcast.consume()); - publish(broadcast, "totals.json").update({ value: { "": { sessions: { sessions_started: 2 } } } }); - publish(broadcast, presenceTrack("acme")).update({ value: { gold: {} } }); - publish(broadcast, prefixTrack("acme/live")).update({ value: { "": { self: { publisher: { bytes: 3 } } } } }); + publish(broadcast, "totals.json").update({ value: { "": { sessions: { sessions_started: 2 } } }, at: key() }); + publish(broadcast, presenceTrack("acme")).update({ value: { gold: {} }, at: key() }); + publish(broadcast, prefixTrack("acme/live")).update({ + value: { "": { self: { publisher: { bytes: 3 } } } }, + at: key(), + }); expect((await consumer.totals().next())?.[""]?.sessions.sessions_started).toBe(2); expect((await consumer.presence("acme").next())?.gold?.sessions_failed.transport).toBe(0); @@ -93,12 +100,12 @@ test("concurrent bucket reads each get a frame", async () => { const broadcast = origin.route(Path.from(".stats/node/a")).broadcast; const bucket = new Consumer(broadcast.consume()).bucket(""); const run = publish(broadcast, "bucket.e1.json"); - run.update({ value: { epoch: "e1" } }); + run.update({ value: { epoch: "e1" }, at: key() }); const first = bucket.next(); const second = bucket.next(); expect((await first)?.kind).toBe("frame"); - run.update({ value: { epoch: "e1", tiers: { "": {} } } }); + run.update({ value: { epoch: "e1", tiers: { "": {} } }, at: key() }); expect(await second).toMatchObject({ kind: "frame", frame: { tiers: { "": {} } } }); bucket.close(); origin.close(); diff --git a/js/watch/src/broadcast.test.ts b/js/watch/src/broadcast.test.ts index dd3a8720cc8..8a4e10e2ef1 100644 --- a/js/watch/src/broadcast.test.ts +++ b/js/watch/src/broadcast.test.ts @@ -6,6 +6,7 @@ import * as Msf from "@moq/msf"; import * as Moq from "@moq/net"; import { Origin, Path } from "@moq/net"; import { Effect, Signal } from "@moq/signals"; +import { key } from "../../json/src/key.test-util.ts"; import * as Mint from "../../net/src/mint.ts"; import { wireOf } from "../../net/src/wire.ts"; import { Broadcast } from "./broadcast"; @@ -116,13 +117,13 @@ describe("unsupported catalog", () => { const source = new Broadcast({ origin: owner, name, catalogFormat: "hang", enabled }); try { const ready = source.out.catalog.changed(); - producer.update({ value: {} }); + producer.update({ value: {}, at: key() }); await ready; expect(source.out.status.peek()).toBe("live"); const rejected = source.out.status.changed(); const cleared = source.out.catalog.changed(); - producer.update({ value: { version: Catalog.VERSION + 1 } }); + producer.update({ value: { version: Catalog.VERSION + 1 }, at: key() }); await rejected; await cleared; expect(source.out.status.peek()).toBe("unsupported"); @@ -168,7 +169,7 @@ for (const driven of [true, false]) { }); try { const ready = source.out.catalog.changed(); - producer.update({ value: { video: { renditions: { hd: video("avc1.640028") } } } }); + producer.update({ value: { video: { renditions: { hd: video("avc1.640028") } } }, at: key() }); await ready; expect(source.out.status.peek()).toBe("live"); @@ -1170,8 +1171,12 @@ it("subscribes a cross-broadcast MSF rendition on its namespace", async () => { const owner = new Origin.Producer(); const catalogBroadcast = publish(owner, Path.from("room/catalog.msf")); const referenced = publish(owner, Path.from("room/left")); - catalogBroadcast.createTrack("video").writeFrame({ payload: new Uint8Array([1]) }); - referenced.createTrack("video").writeFrame({ payload: new Uint8Array([2]) }); + catalogBroadcast + .createTrack("video") + .writeFrame({ payload: new Uint8Array([1]), timestamp: Moq.Time.Timestamp.unscaled(0n) }); + referenced + .createTrack("video") + .writeFrame({ payload: new Uint8Array([2]), timestamp: Moq.Time.Timestamp.unscaled(0n) }); catalogBroadcast.createTrack("catalog").writeFrame({ payload: Msf.encode({ tracks: [ @@ -1184,6 +1189,7 @@ it("subscribes a cross-broadcast MSF rendition on its namespace", async () => { }, ], }), + timestamp: Moq.Time.Timestamp.unscaled(0n), }); const source = new Broadcast({ origin: owner, diff --git a/js/watch/src/include.test.ts b/js/watch/src/include.test.ts index 3c1462cf818..79d5d1d666d 100644 --- a/js/watch/src/include.test.ts +++ b/js/watch/src/include.test.ts @@ -2,8 +2,9 @@ import { expect, it } from "bun:test"; import { yieldNow } from "@moq/clock"; import * as Catalog from "@moq/hang/catalog"; import * as Json from "@moq/json"; -import { Instance, Origin, Path, type Track } from "@moq/net"; +import { Instance, Origin, Path, Time, type Track } from "@moq/net"; import { Effect, Signal } from "@moq/signals"; +import { key } from "../../json/src/key.test-util.ts"; import * as Mint from "../../net/src/mint.ts"; import { wireOf } from "../../net/src/wire.ts"; import { Broadcast } from "./broadcast"; @@ -42,9 +43,14 @@ it("reads a listed include's catalog but no media, keeping its rendition IDs its const include = publish(origin, "room/extra.hang"); const includeCatalog = include.createTrack(Catalog.TRACK); const catalog = new Json.Snapshot.Producer({ track: includeCatalog, deltas: false }); - parent.createTrack("video").writeFrame({ payload: new TextEncoder().encode("source frames") }); + parent + .createTrack("video") + .writeFrame({ payload: new TextEncoder().encode("source frames"), timestamp: Time.Timestamp.unscaled(0n) }); const includeVideo = include.createTrack("video"); - includeVideo.writeFrame({ payload: new TextEncoder().encode("include frames") }); + includeVideo.writeFrame({ + payload: new TextEncoder().encode("include frames"), + timestamp: Time.Timestamp.unscaled(0n), + }); // The include and a source rendition share the ID "video". const root = Catalog.RootSchema.parse({ @@ -59,7 +65,7 @@ it("reads a listed include's catalog but no media, keeping its rendition IDs its }); const selection = new Effect(); try { - catalog.update({ value: { video: { renditions: { video: video("vp8") } } } }); + catalog.update({ value: { video: { renditions: { video: video("vp8") } } }, at: key() }); await settle(); expect(source.out.catalog.peek()?.include).toEqual(root.include); // The root selects every include it lists: it reads the catalog, but subscribes no media. @@ -422,7 +428,7 @@ it("offers every include's renditions to selection, subscribing only the selecte const ladderCatalog = new Json.Snapshot.Producer({ track: ladderTrack, deltas: false }); const rung = (name: string) => { const track = ladder.createTrack(name); - track.writeFrame({ payload: new TextEncoder().encode(name) }); + track.writeFrame({ payload: new TextEncoder().encode(name), timestamp: key() }); return track; }; const [high, low, ladderHd] = [rung("720p"), rung("360p"), rung("hd")]; @@ -455,6 +461,7 @@ it("offers every include's renditions to selection, subscribing only the selecte // An include's own include needs a selection of its own, so it is never expanded. include: { nested: { broadcast: "./nested.hang" } }, }), + at: key(), }); const root = (own: Record) => @@ -541,7 +548,7 @@ it("dedupes an include's rendition only against a usable one", async () => { track: broadcast.createTrack(Catalog.TRACK), deltas: false, }); - catalog.update({ value: Catalog.RootSchema.parse({ video: { renditions: { v: video("vp8") } } }) }); + catalog.update({ value: Catalog.RootSchema.parse({ video: { renditions: { v: video("vp8") } } }), at: key() }); return broadcast; }; const old = serve(oldInstance); @@ -581,6 +588,7 @@ it("takes a missing section's display settings from the include whose renditions new Json.Snapshot.Producer({ track: broadcast.createTrack(Catalog.TRACK), deltas: false }).update( { value, + at: key(), }, ); return broadcast; diff --git a/js/watch/src/offline.test.ts b/js/watch/src/offline.test.ts index 42032c618b2..de0839ca551 100644 --- a/js/watch/src/offline.test.ts +++ b/js/watch/src/offline.test.ts @@ -6,6 +6,7 @@ import * as Catalog from "@moq/hang/catalog"; import * as Json from "@moq/json"; import { Origin, Path } from "@moq/net"; import { Effect, Signal } from "@moq/signals"; +import { key } from "../../json/src/key.test-util.ts"; import { accept, connect } from "../../net/src/connection/index.ts"; import * as Lite from "../../net/src/lite/index.ts"; import { createMockTransportPair, waitFor } from "../../net/src/mock.ts"; @@ -25,7 +26,7 @@ test("a live broadcast reads connecting until connected and again once the conne track: route.broadcast.createTrack(Catalog.TRACK), deltas: false, }); - producer.update({ value: {} }); + producer.update({ value: {}, at: key() }); const viewer = new Origin.Producer(); const source = new Broadcast({ origin: viewer, name, catalogFormat: "hang", enabled: true }); diff --git a/kt/README.md b/kt/README.md index d38d5befeda..f15067e8d7f 100644 --- a/kt/README.md +++ b/kt/README.md @@ -59,9 +59,9 @@ The `dev.moq` package is intentionally thin: Kotlin has extension functions, so - **Flow extensions** (`Flows.kt`): `updates()`, `groups()`, and `frames()` turn the pull-based consumers into coroutine `Flow`s with cancellation wired through. - **Media** (`dev.moq.media`): catalog, importers, and container consumers, with `catalog(broadcast)` for a one-shot catalog read and `frames()` for container-decoded media. - **Fetched media**: `dev.moq.media.GroupConsumer.fetch(...).frames()` streams the decoded frames of one retained group, then completes. `dev.moq.media.GroupConsumer(group, container)` decodes a raw group instead, such as one from `recvGroup()` for arrival-order reads. -- **Timestamp conversions** (`Timestamp.kt`): media timestamps are exact `Timestamp(value, scale)` records. `Duration.toTimestamp()` builds one exactly at nanosecond scale, and `Timestamp.toDuration()` reads one back as a `kotlin.time.Duration`, rounded down to a whole nanosecond. Durations such as `stats.rtt` are already `java.time.Duration`. +- **Timestamp conversions** (`Timestamp.kt`): media timestamps are exact `Timestamp(value, scale)` records. A scale-less track has no time, so its frames carry `Timestamp.unscaled(groupKey)` (a datagram, its sequence), whose `scale` is null. `Duration.toTimestamp()` builds one exactly at nanosecond scale, and `Timestamp.toDuration()` reads one back as a `kotlin.time.Duration`, rounded down to a whole nanosecond, throwing on an unscaled one. Durations such as `stats.rtt` are already `java.time.Duration`. - **`logLevel(...)`**: configures native Rust tracing without importing the raw bindings package. -- **Datagram tracks**: create with `BroadcastProducer.createDatagrams(name, info)` and subscribe with `BroadcastConsumer.datagrams(name)`. The separate producer uses `append(timestamp, payload)` / `insert(Datagram)`, and its consumer uses `recv()` / the `datagrams()` flow. Aggregate demand is `priority()` / `priorityChanged(last)`; on-demand requests come from `BroadcastRequests.datagrams()`. Keep payloads below the path MTU; delivery needs a datagram-capable transport and lite-05 or newer, with no stream fallback. +- **Datagram tracks**: create with `BroadcastProducer.createDatagrams(name, info)` and subscribe with `BroadcastConsumer.datagrams(name)`. The separate producer uses `append(timestamp, payload)` / `insert(Datagram)`, where an unscaled timestamp names the sequence, and its consumer uses `recv()` / the `datagrams()` flow. Aggregate demand is `priority()` / `priorityChanged(last)`; on-demand requests come from `BroadcastRequests.datagrams()`. Keep payloads below the path MTU; delivery needs a datagram-capable transport and lite-05 or newer, with no stream fallback. - **`MoqException.isShutdown`** (`Errors.kt`): true for the graceful `Cancelled`/`Closed` cases. ## Versioning diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 6b86506ca0b..740f510eb8d 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -172,13 +172,22 @@ typealias GroupProducer = uniffi.moq.MoqGroupProducer typealias GroupConsumer = uniffi.moq.MoqGroupConsumer // Data types. -/** A datagram payload with its own sequence and optional timestamp. */ +/** + * A datagram payload with its own sequence and timestamp, the [unscaled] sequence on a scale-less + * track. + */ typealias Datagram = uniffi.moq.MoqDatagram -/** A payload plus the timestamp it should be presented at. */ +/** + * A payload plus the timestamp it should be presented at, or its group's [unscaled] key on a + * scale-less track. + */ typealias Frame = uniffi.moq.MoqFrame -/** An exact media timestamp: `value` units at `scale` units per second. */ +/** + * An exact timestamp: `value` units at `scale` units per second, or on a scale-less track a null + * `scale` and the group key (a datagram's sequence) as `value`; see [unscaled]. + */ typealias Timestamp = uniffi.moq.MoqTimestamp /** @@ -238,7 +247,10 @@ typealias AnchorAbsolute = uniffi.moq.MoqAnchor.Absolute /** The group `groups` before the latest one, clamped to group 0: 0 is the latest. */ typealias AnchorRelative = uniffi.moq.MoqAnchor.Relative -/** The group containing `timestamp`, a presentation time. */ +/** + * The group containing `timestamp`, an unscaled time in the track's timescale; a scaled one is + * refused. + */ typealias AnchorTime = uniffi.moq.MoqAnchor.Time /** Which pending group the publisher sends first: `ASCENDING` or `DESCENDING`. */ diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Timestamp.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Timestamp.kt index e6675856781..414a26c9e84 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Timestamp.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Timestamp.kt @@ -5,13 +5,24 @@ import kotlin.time.Duration import kotlin.time.Duration.Companion.nanoseconds import kotlin.time.Duration.Companion.seconds +/** + * The scale-less timestamp at group key [key] (a datagram's sequence). + * + * A scale-less track has no time, so each frame carries its group's key instead; any other key is + * refused at the write with a `TimestampMismatch`. + */ +fun uniffi.moq.MoqTimestamp.Companion.unscaled(key: ULong): Timestamp = + Timestamp(value = key, scale = null) + /** * This timestamp as a [Duration], rounded down to a whole nanosecond. * * The [Timestamp] itself is exact; keep it, not the duration, to republish a tick such as 90 kHz. - * Throws [IllegalArgumentException] past [Duration]'s range, about 146 million years. + * Throws [IllegalArgumentException] when [unscaled], since a group key has no time, or past + * [Duration]'s range, about 146 million years. */ fun Timestamp.toDuration(): Duration { + val scale = requireNotNull(scale) { "unscaled timestamp #$value has no duration" } // The sub-second remainder times a billion can exceed 64 bits. val remainder = (value % scale).toString().toBigInteger() val divisor = scale.toString().toBigInteger() diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/json/Tracks.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/json/Tracks.kt index 6f73ff9242f..0c91326308b 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/json/Tracks.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/json/Tracks.kt @@ -67,8 +67,11 @@ typealias WindowEventSkip = uniffi.moq.MoqJsonWindowEvent.Skip /** A [WindowEvent] and the timestamp of the frame that carried it. */ typealias WindowTimed = uniffi.moq.MoqJsonWindowTimed -/** A decoded JSON value and its frame timestamp on the broadcast clock, or null when untimed. */ -data class Timed(val value: T, val timestamp: Timestamp?) +/** + * A decoded JSON value and its frame timestamp on the broadcast clock, or its group's [unscaled] + * key on a scale-less track. + */ +data class Timed(val value: T, val timestamp: Timestamp) /** Decode [Value.value] as [T], keeping the timestamp. */ inline fun Value.decode(): Timed = diff --git a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt index 6f51b27bd2f..2ac3e0e2f2e 100644 --- a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt +++ b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/SmokeTest.kt @@ -88,10 +88,13 @@ class SmokeTest { val producer = request.accept(null) assertEquals(12.toByte(), producer.priority()) assertNull(stream.demand().wanted()) - producer.insert(Datagram(7uL, null, "seven".encodeToByteArray())) + producer.insert(Datagram(7uL, Timestamp.unscaled(7uL), "seven".encodeToByteArray())) val datagram = assertNotNull(consumer.recv()) assertEquals(7uL, datagram.sequence) assertEquals("seven", datagram.payload.decodeToString()) + // An unscaled append writes at the sequence it names. + assertEquals(9uL, producer.append(Timestamp.unscaled(9uL), "nine".encodeToByteArray())) + assertEquals(Timestamp.unscaled(9uL), assertNotNull(consumer.recv()).timestamp) consumer.cancel() assertNull(producer.priorityChanged(12)) producer.finish() @@ -399,14 +402,17 @@ class SmokeTest { } @Test - fun `raw writes without timestamps stay untimed`() = runTest { + fun `raw writes on a scale-less track carry the group key`() = runTest { BroadcastProducer().use { broadcast -> val track = broadcast.publishTrack("untimed", null) val consumer = track.consume(null) - track.writeFrame(Frame(payload = "payload".encodeToByteArray())) + track.writeFrame( + Frame(payload = "payload".encodeToByteArray(), timestamp = Timestamp.unscaled(0uL)) + ) val frame = consumer.readFrame() assertEquals("payload", frame?.payload?.decodeToString()) - assertNull(frame?.timestamp) + assertEquals(Timestamp.unscaled(0uL), frame?.timestamp) + assertNull(frame?.timestamp?.scale) assertNull(consumer.info().timescale) track.finish() } @@ -460,13 +466,14 @@ class SmokeTest { ) // Fetch while the track is still published: finishing the media producer - // unpublishes it, and the fetch would then miss with NotFound. + // unpublishes it, and the fetch would then miss with NotFound. A group is named by + // its start, its first packet's timestamp in microseconds. val fetched = dev.moq.media.GroupConsumer.fetch( consumer, dev.moq.media.GroupConfig( rendition = dev.moq.media.AudioRendition(name, audio), - sequence = 0uL, + sequence = 5_000_000uL, options = FetchGroupOptions(priority = 3), ), ) @@ -475,7 +482,7 @@ class SmokeTest { media.finish() fetched.use { - assertEquals(0uL, it.sequence()) + assertEquals(5_000_000uL, it.sequence()) val frames = it.frames().toList() assertEquals(1, frames.size) val frame = frames.single() @@ -502,16 +509,17 @@ class SmokeTest { assertNotNull(audio) } - /** A 90 kHz tick, a present zero, and an untimed frame survive a write and read exactly. */ + /** A 90 kHz tick, a present zero, and an unscaled key survive a write and read exactly. */ @Test fun `timestamps round-trip exactly`() = runTest { - for (timestamp in listOf(Timestamp(1uL, 90_000uL), Timestamp(0uL, 90_000uL), null)) { + for (timestamp in + listOf(Timestamp(1uL, 90_000uL), Timestamp(0uL, 90_000uL), Timestamp.unscaled(0uL))) { BroadcastProducer().use { broadcast -> - // A timed track refuses an untimed frame, so null goes to an untimed one. + // A timed track refuses an unscaled frame, so it goes to a scale-less one. val track = broadcast.publishTrack( "ticks", - timestamp?.let { TrackInfo(timescale = 90_000uL) }, + timestamp.scale?.let { TrackInfo(timescale = it) }, ) val consumer = track.consume(null) track.writeFrame(Frame(payload = ByteArray(0), timestamp = timestamp)) @@ -547,6 +555,7 @@ class SmokeTest { assertEquals(Timestamp(20_000_000uL, 1_000_000_000uL), 20.milliseconds.toTimestamp()) assertFailsWith { (-1).nanoseconds.toTimestamp() } assertEquals(11_111.nanoseconds, Timestamp(1uL, 90_000uL).toDuration()) + assertFailsWith { Timestamp.unscaled(1uL).toDuration() } assertEquals(666_666_666.nanoseconds, Timestamp(2uL, 3uL).toDuration()) assertEquals(0.nanoseconds, Timestamp(1uL, 2_000_000_000uL).toDuration()) // The sub-second remainder times a billion overflows 64 bits at the largest scale, and the @@ -612,11 +621,11 @@ class SmokeTest { val push = assertIs(timed.event) assertEquals(n.toULong(), push.index) assertEquals(Status(state = "live${n + 1}"), push.decode()) - assertEquals((n + 1).milliseconds, timed.timestamp?.toDuration()) + assertEquals((n + 1).milliseconds, timed.timestamp.toDuration()) } val pop = checkNotNull(consumer.next()) assertEquals(WindowEventPop(0uL, 2uL), pop.event) - assertEquals(4.milliseconds, pop.timestamp?.toDuration()) + assertEquals(4.milliseconds, pop.timestamp.toDuration()) // The cut's new group restates record 2, which this reader already has. producer.cut() diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index a1c87923159..5a577bbc148 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -178,7 +178,7 @@ client = moq.Client( - `await .requested_track() → TrackRequest`. Call `.accept()` on it for a `TrackProducer`, or `.reject(code)` to fail the subscriber. - Async iterator yielding `TrackRequest` - **`TrackProducer` / `GroupProducer`**. Write raw payloads with no codec parsing. - - `.write_frame(payload, timestamp=None)` writes a payload with optional capture time. + - `.write_frame(payload, timestamp)` writes a payload with its capture time; on a scale-less track (no `timescale`) the timestamp is `Timestamp.unscaled(key)` at the frame's group key. - `await .ready(size)` (`GroupProducer`) waits until a frame of `size` bytes fits a window ahead of the fastest reader. Await it before each write for backpressure. A writer that doesn't wait sheds the front of the group instead. - `.create_group(sequence)` creates a sparse or replayed group at an explicit sequence. - `.skip_groups(start, end)` (`TrackProducer`) declares groups `start` up to the exclusive `end` will never be produced, so readers move past them; creating one there then fails. @@ -230,8 +230,8 @@ on the broadcast clock, read with `media.CatalogProducer.now()`, so their track A window keeps a sliding run of records a reader can join at any point, and `.cut()` ends its open group so the next edit opens one restating the window; the catalog has no window mode, so `WindowProducer` takes only the track. -A raw frame may omit its timestamp on an untimed track, but every JSON write takes one on the -broadcast clock. Publish JSON tracks with +A raw frame on a scale-less track carries `Timestamp.unscaled` at its group key, but every JSON +write takes a time on the broadcast clock. Publish JSON tracks with `TrackInfo(compression=moq.Compression.DEFLATE)`: the track compresses each group on the wire, and consumers read it the same either way. Opaque bytes go on a plain `TrackProducer`. @@ -296,10 +296,10 @@ moq-transport on every relay hop; this API does not add pre-07 capability detect ### Types - **`media.Catalog`**. `.audio: dict[str, Audio]`, `.video: dict[str, Video]`, `.display`, `.rotation`, `.flip`. -- **`Timestamp`**. `.value: int` units at `.scale: int` units per second, exact at any scale such as 90 kHz. `Timestamp.from_timedelta(duration)` is exact at microsecond scale; `.to_timedelta()` rounds down to a whole microsecond. A write with a scale of 0 or past `2**62 - 1`, or a value past `2**61 - 1`, raises `Error.TimeOverflow`. -- **`Frame`**. `.payload: bytes`, `.timestamp: Timestamp | None`. The unit of every write and every raw read; `None` means untimed when publishing or reading. +- **`Timestamp`**. `.value: int` units at `.scale: int | None` units per second, exact at any scale such as 90 kHz. `Timestamp.unscaled(key)` has no scale: on a scale-less track it is a frame's group key or a datagram's sequence. `Timestamp.from_timedelta(duration)` is exact at microsecond scale; `.to_timedelta()` rounds down to a whole microsecond, and raises `ValueError` when unscaled. A write with a scale of 0 or past `2**62 - 1`, or a value past `2**61 - 1`, raises `Error.TimeOverflow`. +- **`Frame`**. `.payload: bytes`, `.timestamp: Timestamp`. The unit of every write and every raw read; on a scale-less track the timestamp is `Timestamp.unscaled` at the group key. - **`media.MediaFrame`**. `.payload: bytes`, `.timestamp: Timestamp`, `.keyframe: bool`. The frame variant of a media read. `keyframe` marks a group start or video keyframe; for audio it is true only at a group start. -- **`Datagram`**. `.sequence: int`, `.timestamp: Timestamp | None`, `.payload: bytes`. Delivered only on datagram-capable transports with lite-05 or newer moq-lite. +- **`Datagram`**. `.sequence: int`, `.timestamp: Timestamp`, `.payload: bytes`; on a scale-less track the timestamp is `Timestamp.unscaled(sequence)`. Delivered only on datagram-capable transports with lite-05 or newer moq-lite. - **`media.Audio`**. `.codec`, `.sample_rate`, `.channel_count`, `.bitrate`, `.enabled`, `.description`. - **`media.Video`**. `.codec`, `.coded: Dimensions`, `.display_aspect`, `.bitrate`, `.enabled`, `.framerate`, `.description`. A false `.enabled` means no frames are coming, so don't select the rendition. - **`Subscription`**. Subscriber delivery preferences: priority, staleness, and optional group range. @@ -325,7 +325,8 @@ moq-transport on every relay hop; this API does not add pre-07 capability detect Create a separate live track with `broadcast.create_datagrams(name, info)` and subscribe with `broadcast.consume().datagrams(name)`. The `DatagramProducer` writes with -`append(payload, timestamp=...)` or `insert(Datagram(...))`, and the `DatagramConsumer` +`append(payload, timestamp=...)` (on a scale-less track, `Timestamp.unscaled(sequence)` writes at that +sequence) or `insert(Datagram(...))`, and the `DatagramConsumer` reads with `await recv()` or async iteration. Demand is aggregate subscriber `priority()` / `await priority_changed(last)`; `broadcast.requests().datagrams()` handles on-demand tracks. Keep payloads below the path MTU. Delivery needs a datagram-capable transport and lite-05 or diff --git a/py/moq-rs/benches/ffi.py b/py/moq-rs/benches/ffi.py index 9c96b2ef0d9..207fc8ebbdf 100644 --- a/py/moq-rs/benches/ffi.py +++ b/py/moq-rs/benches/ffi.py @@ -129,7 +129,7 @@ async def accept(server: moq.socket.Endpoint) -> None: async def dial(server: moq.socket.Endpoint) -> None: stream = await client.dial_track(server.id) - stream.producer.write_frame(b"echo") + stream.producer.write_frame(b"echo", moq.Timestamp.unscaled(0)) # Scale-less: the first group is key 0. stream.producer.finish() assert [item async for item in stream.consumer] == [] diff --git a/py/moq-rs/moq/_records.py b/py/moq-rs/moq/_records.py index 3627fd47f08..f7f94c9ea75 100644 --- a/py/moq-rs/moq/_records.py +++ b/py/moq-rs/moq/_records.py @@ -64,10 +64,18 @@ class Timestamp: ``value`` is at most ``2**61 - 1`` and ``scale`` is non-zero and at most ``2**62 - 1``; a write past either bound raises ``moq.Error.TimeOverflow``. A read carries the track's timescale, and a write at another scale is floored to it. + + A scale-less track has no time, so there ``scale`` is ``None`` and ``value`` is the + frame's group key (a datagram's sequence); build one with :meth:`unscaled`. """ value: int - scale: int + scale: int | None + + @staticmethod + def unscaled(key: int) -> Timestamp: + """The scale-less timestamp at group key (or datagram sequence) ``key``.""" + return Timestamp(key, None) @staticmethod def from_timedelta(duration: timedelta) -> Timestamp: @@ -80,7 +88,10 @@ def to_timedelta(self) -> timedelta: """This timestamp as a ``timedelta``, rounded down to a whole microsecond. The timestamp itself is exact; keep it, not the ``timedelta``, to republish a tick such as 90 kHz. + Raises ``ValueError`` on an :meth:`unscaled` timestamp, which has no time. """ + if self.scale is None: + raise ValueError(f"an unscaled timestamp (key {self.value}) has no time") return timedelta(microseconds=self.value * 1_000_000 // self.scale) def _ffi(self) -> MoqTimestamp: @@ -91,51 +102,44 @@ def _from_ffi(value: MoqTimestamp) -> Timestamp: return Timestamp(value.value, value.scale) -def _opt_ts(value: Timestamp | None) -> MoqTimestamp | None: - return None if value is None else value._ffi() - - -def _opt_from_ts(value: MoqTimestamp | None) -> Timestamp | None: - return None if value is None else Timestamp._from_ffi(value) - - @dataclass(frozen=True) class Frame: """A raw track frame: a payload and its presentation timestamp. - ``timestamp`` is ``None`` on an untimed track, on both publish and receive. + On a scale-less track ``timestamp`` is :meth:`Timestamp.unscaled` at the frame's + group key, on both publish and receive. """ payload: bytes - timestamp: Timestamp | None = None + timestamp: Timestamp def _ffi(self) -> MoqFrame: - return MoqFrame(payload=self.payload, timestamp=_opt_ts(self.timestamp)) + return MoqFrame(payload=self.payload, timestamp=self.timestamp._ffi()) @staticmethod def _from_ffi(frame: MoqFrame) -> Frame: - return Frame(payload=frame.payload, timestamp=_opt_from_ts(frame.timestamp)) + return Frame(payload=frame.payload, timestamp=Timestamp._from_ffi(frame.timestamp)) @dataclass(frozen=True) class Datagram: - """A datagram payload with its own sequence and optional timestamp. + """A datagram payload with its own sequence and timestamp. - ``timestamp`` is ``None`` on a datagram read from an untimed track. + On a scale-less track ``timestamp`` is :meth:`Timestamp.unscaled` at ``sequence``. """ sequence: int - timestamp: Timestamp | None + timestamp: Timestamp payload: bytes def _ffi(self) -> MoqDatagram: - return MoqDatagram(sequence=self.sequence, timestamp=_opt_ts(self.timestamp), payload=self.payload) + return MoqDatagram(sequence=self.sequence, timestamp=self.timestamp._ffi(), payload=self.payload) @staticmethod def _from_ffi(datagram: MoqDatagram) -> Datagram: return Datagram( sequence=datagram.sequence, - timestamp=_opt_from_ts(datagram.timestamp), + timestamp=Timestamp._from_ffi(datagram.timestamp), payload=datagram.payload, ) @@ -187,7 +191,10 @@ def _ffi(self) -> MoqAnchor: @dataclass(frozen=True) class _AnchorTime(Anchor): - """The group containing this presentation time. An untimed track refuses it.""" + """The group containing this time: :meth:`Timestamp.unscaled` in the track's timescale. + + A scaled timestamp is refused, since the subscription can't check its scale against the track. + """ timestamp: Timestamp diff --git a/py/moq-rs/moq/datagram.py b/py/moq-rs/moq/datagram.py index 884ce4364ee..c845c9dd070 100644 --- a/py/moq-rs/moq/datagram.py +++ b/py/moq-rs/moq/datagram.py @@ -4,7 +4,6 @@ from moq_ffi import MoqDatagramConsumer, MoqDatagramProducer, MoqDatagramRequest, MoqDatagramRequests -from ._records import _opt_ts from ._stream import _Stream from .types import Datagram, DatagramInfo, Timestamp @@ -24,9 +23,13 @@ def info(self) -> DatagramInfo: """The publisher properties fixed when this track was created.""" return self._inner.info() - def append(self, payload: bytes, *, timestamp: Timestamp | None = None) -> int: - """Send a datagram and return its sequence; keep the payload below the path MTU.""" - return self._inner.append(_opt_ts(timestamp), payload) + def append(self, payload: bytes, *, timestamp: Timestamp) -> int: + """Send a datagram and return its sequence; keep the payload below the path MTU. + + On a scale-less track ``timestamp`` is :meth:`Timestamp.unscaled` at the sequence to + write; a scaled one takes the next sequence. + """ + return self._inner.append(timestamp._ffi(), payload) def insert(self, datagram: Datagram) -> None: """Send a datagram at its explicit sequence; appends continue past the highest one.""" diff --git a/py/moq-rs/moq/json.py b/py/moq-rs/moq/json.py index a3113eed85e..bf7ae383a87 100644 --- a/py/moq-rs/moq/json.py +++ b/py/moq-rs/moq/json.py @@ -36,7 +36,7 @@ ) from . import publish, subscribe -from ._records import Timestamp, _opt_from_ts +from ._records import Timestamp from ._stream import _Stream from .publish import BroadcastProducer, TrackDemand @@ -59,17 +59,18 @@ class Timed: """A parsed JSON value and the timestamp of the frame that carried it. - ``timestamp`` is on the broadcast clock, and ``None`` on an untimed track. + ``timestamp`` is on the broadcast clock, or :meth:`moq.Timestamp.unscaled` at the + frame's group key on a scale-less track. """ value: Any - timestamp: Timestamp | None + timestamp: Timestamp @staticmethod def _from_ffi(value: MoqJsonValue | None) -> Timed | None: if value is None: return None - return Timed(value=json.loads(value.value), timestamp=_opt_from_ts(value.timestamp)) + return Timed(value=json.loads(value.value), timestamp=Timestamp._from_ffi(value.timestamp)) class SnapshotProducer: @@ -265,7 +266,7 @@ def _window_event(timed: MoqJsonWindowTimed | None) -> Timed | None: value = WindowEventSkip(range(event.start, event.end)) else: raise TypeError(f"unknown window event: {event!r}") - return Timed(value=value, timestamp=_opt_from_ts(timed.timestamp)) + return Timed(value=value, timestamp=Timestamp._from_ffi(timed.timestamp)) class WindowConsumer(_Stream[Timed]): diff --git a/py/moq-rs/moq/publish.py b/py/moq-rs/moq/publish.py index e184ff2e2f8..aaa4655fb85 100644 --- a/py/moq-rs/moq/publish.py +++ b/py/moq-rs/moq/publish.py @@ -62,9 +62,11 @@ async def ready(self, size: int) -> None: """ await self._inner.ready(size) - def write_frame(self, payload: bytes, timestamp: Timestamp | None = None) -> None: + def write_frame(self, payload: bytes, timestamp: Timestamp) -> None: """Write a frame with its presentation timestamp. + On a scale-less track the timestamp is ``Timestamp.unscaled(self.sequence)``. + Never blocks: without :meth:`ready`, a writer that outpaces its readers sheds the front of the group instead. """ @@ -143,8 +145,12 @@ def skip_groups(self, start: int, end: int) -> None: """ self._inner.skip_groups(start, end) - def write_frame(self, payload: bytes, timestamp: Timestamp | None = None) -> None: - """Write a single-frame group with its presentation timestamp.""" + def write_frame(self, payload: bytes, timestamp: Timestamp) -> None: + """Write a single-frame group with its presentation timestamp. + + The group starts at the frame's time; on a scale-less track the timestamp is + :meth:`Timestamp.unscaled` at the new group's key instead. + """ self._inner.write_frame(Frame(payload, timestamp)._ffi()) def largest(self) -> Position | None: diff --git a/py/moq-rs/tests/test_data_timestamps.py b/py/moq-rs/tests/test_data_timestamps.py index 2995a0c518b..4472176160c 100644 --- a/py/moq-rs/tests/test_data_timestamps.py +++ b/py/moq-rs/tests/test_data_timestamps.py @@ -1,4 +1,4 @@ -"""Publishers preserve the caller's exact capture time, including absence.""" +"""Publishers preserve the caller's exact capture time, and a scale-less track's group key.""" import asyncio import json @@ -9,7 +9,7 @@ import moq_ffi import pytest -TIMESTAMPS = [None, moq.Timestamp(0, 90_000), moq.Timestamp(1, 90_000)] +TIMESTAMPS = [moq.Timestamp(0, 90_000), moq.Timestamp(1, 90_000)] async def test_raw_timestamps_are_exact(): @@ -23,12 +23,19 @@ async def test_raw_timestamps_are_exact(): assert frame is not None assert frame.timestamp == moq.Timestamp(value, 90_000) - untimed = broadcast.publish_track("events", moq.TrackInfo(timescale=None)) - untimed_consumer = untimed.consume() - untimed.write_frame(b"event") - frame = await asyncio.wait_for(untimed_consumer.read_frame(), timeout=5.0) + unscaled = broadcast.publish_track("events", moq.TrackInfo(timescale=None)) + unscaled_consumer = unscaled.consume() + # A scale-less track's frame carries its group key, and refuses time. + with pytest.raises(moq.Error.Protocol): + unscaled.write_frame(b"event", moq.Timestamp(0, 90_000)) + unscaled.write_frame(b"event", moq.Timestamp.unscaled(0)) + frame = await asyncio.wait_for(unscaled_consumer.read_frame(), timeout=5.0) assert frame is not None - assert frame.timestamp is None + assert frame.timestamp == moq.Timestamp.unscaled(0) + assert frame.timestamp.scale is None + # A timed track refuses a group key. + with pytest.raises(moq.Error.Protocol): + track.write_frame(b"frame", moq.Timestamp.unscaled(2)) for invalid in [moq.Timestamp(0, 0), moq.Timestamp(0, 1 << 62), moq.Timestamp(1 << 61, 90_000)]: with pytest.raises(moq.Error.TimeOverflow): @@ -44,10 +51,14 @@ def test_timedelta_conversions(): assert moq.Timestamp(1, 2_000_000).to_timedelta() == timedelta(0) assert moq.Timestamp(2, 3_000_000).to_timedelta() == timedelta(0) assert moq.Timestamp(1, 90_000).to_timedelta() == timedelta(microseconds=11) + # A group key has no time. + assert moq.Timestamp.unscaled(3) == moq.Timestamp(3, None) + with pytest.raises(ValueError): + moq.Timestamp.unscaled(3).to_timedelta() @pytest.mark.parametrize("mode", ["snapshot", "stream"]) -@pytest.mark.parametrize("timestamp", [timestamp for timestamp in TIMESTAMPS if timestamp is not None]) +@pytest.mark.parametrize("timestamp", TIMESTAMPS) async def test_json_timestamp_round_trip(mode: str, timestamp: moq.Timestamp): broadcast = moq.BroadcastProducer() track = broadcast.publish_track("value", moq.TrackInfo(timescale=timestamp.scale)) @@ -91,7 +102,9 @@ def test_whole_media_frames_require_the_callers_timestamp(): broadcast, moq.media.AudioInit(format=moq.media.AudioFormat.OPUS, data=head) ) with pytest.raises(moq.Error.Codec, match="timestamp"): - producer._inner.write_frame(moq_ffi.MoqFrame(payload=b"opus frame", timestamp=None)) + producer._inner.write_frame( + moq_ffi.MoqFrame(payload=b"opus frame", timestamp=moq_ffi.MoqTimestamp(value=0, scale=None)) + ) # Exercise the runtime refusal when a caller omits the required timestamp. with pytest.raises(TypeError): producer.write_frame(b"opus frame") # type: ignore[call-arg] diff --git a/py/moq-rs/tests/test_local.py b/py/moq-rs/tests/test_local.py index 1c31db4dd0c..b180b9cde21 100644 --- a/py/moq-rs/tests/test_local.py +++ b/py/moq-rs/tests/test_local.py @@ -32,7 +32,7 @@ async def polled() -> None: await asyncio.sleep(0) track = moq.BroadcastProducer().publish_track("polled") consumer = track.consume() - track.write_frame(b"ready") + track.write_frame(b"ready", moq.Timestamp.unscaled(0)) assert await consumer.read_frame() is not None @@ -135,11 +135,11 @@ async def test_datagram_tracks_have_independent_sequences_and_priority(): consumer.set_priority(7) assert track.priority() == 7 assert stream.demand().wanted() is None - track.insert(moq.Datagram(sequence=7, payload=b"seven", timestamp=None)) + track.insert(moq.Datagram(sequence=7, payload=b"seven", timestamp=moq.Timestamp.unscaled(7))) datagram = await asyncio.wait_for(consumer.recv(), 5) assert datagram is not None assert (datagram.sequence, datagram.payload) == (7, b"seven") - assert track.append(b"next") == 8 + assert track.append(b"next", timestamp=moq.Timestamp.unscaled(8)) == 8 # Datagrams are numbered apart from groups, so they never move `largest`. assert stream.largest() is None consumer.cancel() @@ -211,9 +211,9 @@ def test_publish_media_cut_and_seek(): media.write_frame(b"opus frame", moq.Timestamp(i * 20_000, 1_000_000)) media.cut() - # The same boundary, with the next group explicitly numbered. + # The same boundary, with the next group's start named explicitly. media.write_frame(b"opus frame", moq.Timestamp(60_000, 1_000_000)) - media.seek(42) + media.seek(80_000) media.discontinuity() media.write_frame(b"resumed opus frame", moq.Timestamp(100_000, 1_000_000)) @@ -476,7 +476,7 @@ async def test_finish_serves_until_closed(): assert isinstance(await next_route(announced), moq.AnnounceEventPresent) track = broadcast.publish_track("status") - track.write_frame(b"tail") + track.write_frame(b"tail", moq.Timestamp.unscaled(0)) track.finish() moq.media.CatalogProducer(broadcast).finish() @@ -493,7 +493,7 @@ async def test_finish_serves_until_closed(): def test_publish_lifecycle(): broadcast = moq.BroadcastProducer() track = broadcast.publish_track("status") - track.write_frame(b'{"cmd": "ready"}') + track.write_frame(b'{"cmd": "ready"}', moq.Timestamp.unscaled(0)) track.finish() broadcast.close() @@ -505,7 +505,7 @@ async def test_publish_track_info_and_subscription(): track = broadcast.publish_track("status", info) consumer = track.consume(moq.Subscription(priority=3)) - track.write_frame(b"ready") + track.write_frame(b"ready", moq.Timestamp.unscaled(0)) frame = await asyncio.wait_for(consumer.read_frame(), timeout=5.0) assert frame is not None @@ -749,7 +749,7 @@ async def test_dynamic_track_request(): # Accept the request as a raw track (which unblocks the subscribe), then write. track = request.accept() payload = b"hello dynamic track" - track.write_frame(payload) + track.write_frame(payload, moq.Timestamp.unscaled(0)) track_consumer = await asyncio.wait_for(subscribe, timeout=5.0) frame = await asyncio.wait_for(track_consumer.read_frame(), timeout=5.0) @@ -930,7 +930,7 @@ def test_raw_group_empty_payload(): track = broadcast.publish_track("empty") group = track.append_group() - group.write_frame(b"") + group.write_frame(b"", moq.Timestamp.unscaled(group.sequence)) group.finish() @@ -941,7 +941,7 @@ def test_raw_group_write_after_finish_fails(): group.finish() with pytest.raises(Exception): - group.write_frame(b"too late") + group.write_frame(b"too late", moq.Timestamp.unscaled(group.sequence)) def test_raw_group_abort_after_finish(): @@ -965,7 +965,7 @@ def test_raw_track_write_after_finish_fails(): track.finish() with pytest.raises(Exception): - track.write_frame(b"late") + track.write_frame(b"late", moq.Timestamp.unscaled(0)) def test_raw_sparse_groups_and_known_end(): @@ -988,18 +988,20 @@ def test_raw_sparse_groups_and_known_end(): def test_raw_parallel_groups(): """Appending a new group before finishing the previous is allowed; - both groups carry distinct sequences and can be written independently.""" + both groups carry distinct starts and can be written independently.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("parallel", moq.TrackInfo(timescale=1_000_000)) + # Without a start, each append takes the next start past the largest. g0 = track.append_group() g1 = track.append_group() assert g0.sequence == 0 assert g1.sequence == 1 + # A frame lands at or past its group's start and before the next group's. g0.write_frame(b"a0", moq.Timestamp(0, 1_000_000)) - g1.write_frame(b"b0", moq.Timestamp(0, 1_000_000)) - g0.write_frame(b"a1", moq.Timestamp(1, 1_000_000)) + g1.write_frame(b"b0", moq.Timestamp(1, 1_000_000)) + g0.write_frame(b"a1", moq.Timestamp(0, 1_000_000)) g0.finish() g1.finish() @@ -1073,7 +1075,7 @@ async def test_raw_publish_consume(): raw_consumer = await broadcast_consumer.subscribe_track("events") payload = b'{"cmd": "button_changed", "arm": "left", "button": "THUMB", "state": "PRESSED"}' - raw.write_frame(payload) + raw.write_frame(payload, moq.Timestamp.unscaled(0)) async for group in raw_consumer: async for frame in group: @@ -1102,8 +1104,8 @@ async def test_raw_multiple_frames(): b'{"cmd": "tone", "arm": "right", "freq": 440}', b'{"cmd": "tone_stop", "arm": "right"}', ] - for msg in messages: - raw.write_frame(msg) + for key, msg in enumerate(messages): + raw.write_frame(msg, moq.Timestamp.unscaled(key)) received = [] async for group in raw_consumer: @@ -1122,8 +1124,8 @@ async def test_raw_producer_consume_direct(): track = broadcast.publish_track("direct", moq.TrackInfo(max_age=None)) consumer = track.consume(moq.Subscription(delay=moq.Delay(max=timedelta(seconds=1)), ranges=FROM_GROUP_0)) - track.write_frame(b"hello") - track.write_frame(b"world") + track.write_frame(b"hello", moq.Timestamp.unscaled(0)) + track.write_frame(b"world", moq.Timestamp.unscaled(1)) received = [] async for group in consumer: @@ -1160,15 +1162,16 @@ async def test_ascending_subscription_reads_a_cached_backlog_oldest_first(): async def test_time_anchor_starts_at_the_group_containing_it(): - """A time anchor takes a wrapper Timestamp and starts at the group containing that time.""" + """A time anchor takes an unscaled time in the track's timescale and starts at the group containing it.""" broadcast = moq.BroadcastProducer() track = broadcast.publish_track("timed", moq.TrackInfo(max_age=None, timescale=1_000)) for second in range(3): - group = track.append_group() + # Each group starts at its frame's time, in the track's milliseconds. + group = track.create_group(second * 1_000) group.write_frame(b"x", moq.Timestamp(second, 1)) group.finish() - start = moq.Anchor.TIME(timestamp=moq.Timestamp(1_500, 1_000)) + start = moq.Anchor.TIME(timestamp=moq.Timestamp.unscaled(1_500)) consumer = track.consume( moq.Subscription( delay=moq.Delay(max=timedelta(seconds=10)), @@ -1178,7 +1181,7 @@ async def test_time_anchor_starts_at_the_group_containing_it(): ) group = await asyncio.wait_for(consumer.recv_group(), timeout=5.0) assert group is not None - assert group.sequence == 1 + assert group.sequence == 1_000 async def test_raw_group_producer_consume_direct(): @@ -1189,8 +1192,8 @@ async def test_raw_group_producer_consume_direct(): group_consumer = group.consume() assert group_consumer.sequence == group.sequence - group.write_frame(b"a") - group.write_frame(b"b") + group.write_frame(b"a", moq.Timestamp.unscaled(group.sequence)) + group.write_frame(b"b", moq.Timestamp.unscaled(group.sequence)) group.finish() received = [frame.payload async for frame in group_consumer] @@ -1204,7 +1207,7 @@ async def test_broadcast_producer_consume_direct(): consumer = broadcast.consume() raw_consumer = await consumer.subscribe_track("events") - raw.write_frame(b"event-0") + raw.write_frame(b"event-0", moq.Timestamp.unscaled(0)) async for group in raw_consumer: async for frame in group: @@ -1264,7 +1267,7 @@ async def test_default_iteration_is_sequence_order(): for sequence in (5, 3): group = raw.create_group(sequence) - group.write_frame(f"group-{sequence}".encode()) + group.write_frame(f"group-{sequence}".encode(), moq.Timestamp.unscaled(sequence)) group.finish() # Arrival order sees them as produced, newest sequence first. @@ -1393,7 +1396,7 @@ async def test_raw_multi_frame_group(): group_producer = raw.append_group() chunks = [b"chunk-0", b"chunk-1", b"chunk-2"] for chunk in chunks: - group_producer.write_frame(chunk) + group_producer.write_frame(chunk, moq.Timestamp.unscaled(group_producer.sequence)) group_producer.finish() async for group in raw_consumer: @@ -1410,9 +1413,9 @@ async def test_read_frame_one_per_group(): track = broadcast.publish_track("status", moq.TrackInfo(max_age=None)) consumer = track.consume(moq.Subscription(delay=moq.Delay(max=timedelta(seconds=1)), ranges=FROM_GROUP_0)) - track.write_frame(b"ready") - track.write_frame(b"running") - track.write_frame(b"done") + track.write_frame(b"ready", moq.Timestamp.unscaled(0)) + track.write_frame(b"running", moq.Timestamp.unscaled(1)) + track.write_frame(b"done", moq.Timestamp.unscaled(2)) frame = await consumer.read_frame() assert frame is not None @@ -1455,11 +1458,11 @@ async def test_read_frame_skips_remaining_frames_in_group(): consumer = track.consume(moq.Subscription(delay=moq.Delay(max=timedelta(seconds=1)), ranges=FROM_GROUP_0)) group = track.append_group() - group.write_frame(b"first") - group.write_frame(b"second-ignored") + group.write_frame(b"first", moq.Timestamp.unscaled(group.sequence)) + group.write_frame(b"second-ignored", moq.Timestamp.unscaled(group.sequence)) group.finish() - track.write_frame(b"next-group-first") + track.write_frame(b"next-group-first", moq.Timestamp.unscaled(group.sequence + 1)) frame = await consumer.read_frame() assert frame is not None @@ -1475,7 +1478,7 @@ async def test_read_frame_returns_none_when_track_finished(): track = broadcast.publish_track("done") consumer = track.consume() - track.write_frame(b"only") + track.write_frame(b"only", moq.Timestamp.unscaled(0)) track.finish() frame = await consumer.read_frame() @@ -1974,7 +1977,7 @@ async def test_datagram_only_request_reaches_its_handler(): assert request.name == "events" track = request.accept() assert (await consumer.info()).timescale is None - track.append(b"payload") + track.append(b"payload", timestamp=moq.Timestamp.unscaled(0)) datagram = await consumer.recv() assert datagram is not None assert datagram.payload == b"payload" diff --git a/py/moq-rs/tests/test_socket.py b/py/moq-rs/tests/test_socket.py index e7d065941d0..3b789213b3f 100644 --- a/py/moq-rs/tests/test_socket.py +++ b/py/moq-rs/tests/test_socket.py @@ -103,7 +103,7 @@ async def test_track_default_retains_ascending_messages_and_subscription(): assert consumer.info().max_age is None for sequence, payload in ((1, b"later"), (0, b"first")): group = producer.create_group(sequence) - group.write_frame(payload) + group.write_frame(payload, moq.Timestamp.unscaled(sequence)) group.finish() producer.finish() assert [(group.sequence, [frame.payload async for frame in group]) async for group in consumer] == [ diff --git a/py/moq-rs/tests/test_testing_clock.py b/py/moq-rs/tests/test_testing_clock.py index 94de7dc9925..866f8c870ff 100644 --- a/py/moq-rs/tests/test_testing_clock.py +++ b/py/moq-rs/tests/test_testing_clock.py @@ -35,7 +35,7 @@ async def main(): # group 1 has been out for delay.max: a native timer is pending. _stalled = track.append_group() later = track.append_group() - later.write_frame(b"later") + later.write_frame(b"later", moq.Timestamp.unscaled(later.sequence)) later.finish() pending = asyncio.ensure_future(consumer.read_frame()) diff --git a/quest/m2/time-addressing/model.md b/quest/m2/time-addressing/model.md index 704e010bed9..c4c112c0b73 100644 --- a/quest/m2/time-addressing/model.md +++ b/quest/m2/time-addressing/model.md @@ -166,8 +166,8 @@ recorded only sequencing. historical start; lite-01/02 may not. On any that can't, an `Oldest` or earlier start gets the live feed, and the PR documents that exception rather than changing the old format. -- **Open, for whoever starts this, with the maintainer:** how lite-05/06, - which carry both a sequence and frame timestamps, map on the way out. +- **lite-05/06 egress.** Decided 2026-11-13 (maintainer, /quest-complete), + below. Decision prompts: @@ -180,6 +180,86 @@ Decision prompts: - A scale-less frame's timestamp: âś… its group key; any increasing number. - Max-age on a scale-less track: âś… bare units; doesn't apply. +### The unscaled timestamp and the PR's open calls + +Decided 2026-11-13 (maintainer, /quest-complete on +[moq-dev/moq#8638](https://github.com/moq-dev/moq/pull/8638)). Why: a +scale-less frame's timestamp is its group key (2026-11-03, above), so the +type that carries it should say so instead of leaving it absent. Asked why +JSON was special, the maintainer was told it isn't: only moq-json picks group +starts for the caller. + +- **A scale-less key is a `Timestamp`.** âś… `Timestamp { value: u64, scale: + Option }`, with `Timestamp::unscaled(key)` for a scale-less + track, whose value is the group key (a datagram's sequence). The scale-less + case is a typed `None`, never a scale of 0, and `Timescale` stays + `NonZero`. Every write takes a required `Timestamp`: moq-net's group and + track `write_frame` (which now works on a scale-less track, at the key), + `Frame::new`, datagram `append`, moq-json's `Timed::at`, and the JS, moq-ffi, + moq-c, and binding counterparts, with names mirrored. A timestamp whose + scale doesn't match its track, or an unscaled one naming another group's + key, is `TimestampMismatch`. Rejected: a bare `u64` key with an + always-scaled `Timestamp`; `Option`. +- **Conversions that need a scale are fallible.** Picked in the PR: + `convert`, `as_scale`, `as_secs`, `as_millis`, `as_micros`, and `as_nanos` + return `Result<_, TimeOverflow>`, erring on an unscaled timestamp, and + `Duration` is `TryFrom`. There is no `Ord`: `PartialOrd` is + temporal across scales, compares keys when both are unscaled, and is `None` + between a scaled and an unscaled timestamp. `unscaled` takes any `u64`, + since a wide legacy key passes 2^61. In JS, where keys are `bigint`, an + unscaled key past 2^53 throws only when `value` is read. Confining + conversions to a second, always-scaled type was the alternative; it adds a + type to every binding for what one `Option` says. +- **`append_group(start)` stays.** A multi-frame group still opens before its + first frame is written, so it needs its start up front. +- **moq-json `Timed::at`: âś… required, with the unscaled key on a scale-less + track.** A snapshot or header write opens its group at `at`, which must pass + every group the track holds, and a delta, op, or stream frame inside the + open group carries that group's key. Rejected: `Option`; + refusing scale-less JSON tracks. +- **lite-05/06 egress: âś… as built.** The group ID is the key. A scale-less + track declares milliseconds in TRACK_INFO, because those published drafts + treat a `Timescale` of 0 as invalid, and its frames carry the group key. + The code and the draft say this is forced by the published wire. Rejected: + main's send-time stamps ([moq-dev/moq#5461](https://github.com/moq-dev/moq/pull/5461)); + refusing scale-less tracks on lite-05/06. +- **lite-01..04 lag by the hole budget: âś… accepted.** Those wires can't say + the starts between two groups were skipped, so an in-order reader waits its + budget at each gap. Rejected: extending the legacy-ingest rule so keys below + the newest arrival count as never arriving. +- **Released `@moq/hang` players reading new Legacy audio over lite-05/06: âś… + accept the gaps.** The packages release independently, and old players + report gaps until they upgrade. Rejected: shipping `@moq/hang` before or + with the Rust publishers. +- **A resume after a pause over wires without declared skips: âś… accepted.** + It waits up to the playout delay and reports a gap. Rejected: a heuristic + for older wires. + +Taken by the agent under /quest-afk on 2026-11-13, from the PR's local +reviews, and accepted by the maintainer on 2026-11-14 (/quest-complete): + +- **Own-end reach ships before the wire carries the end: âś… accepted.** An origin whose + producer calls `group::Producer::finish(end)` ages a paused track's last + group out after its span. A relay receives the group with a bare FIN + (`rs/moq-net/src/recv.rs` closes it), so it still bounds that group by its + successor, as main does on every hop. Relays serve nothing main wouldn't, and + the gap closes when lite-07's GROUP_END carries the end + ([lite07.md](/quest/m2/time-addressing/lite07.md)). Rejected: carrying the end on the wire in this PR, + which is lite07's wire, draft, and JS change; holding own-end reach back + until then, which reverts the 2026-10-26 decision above. +- **A transcode fill locates a gap through one source read: âś… accepted.** A miss reads + the source once, in key order, for the groups around it, and drops the span + between them at once. A source holding no group at all can't place the + read until it holds one, so a live rung waits for the source's next group, + as any time-anchored reader of the source would, holding no encoder slot. + A retired rung's drain drops the miss and refuses the rest of the fill + instead, since nothing says the source lost it. Rejected: a fetch per + microsecond, which costs millions of requests per GOP gap; dropping the rest + when the read can't answer at once, which loses groups a remote or + handler-served source has. The long-term answer is moq-net serving a fill + from another track's range (or an at-or-after fetch), as a relay serves one + upstream: new API in every language, so a follow-up rather than this PR. + ### Spike evidence The spike published groups at their start ticks (2 s GOPs at 1 kHz and at diff --git a/quest/m3/layered-runs.md b/quest/m3/layered-runs.md new file mode 100644 index 00000000000..1b220b6e873 --- /dev/null +++ b/quest/m3/layered-runs.md @@ -0,0 +1,31 @@ +# [S] Collapse a window's layered runs to one shared set + +## Goal + +`rs/moq-net`'s subscription window holds its groups as plain shared runs, +without the moving layer that time ends once walked, and every path that +diffed or searched that layer is gone, with no change in what readers get. + +## Plan + +Since [the model](/quest/m2/time-addressing/model.md), a time end settles at +once on keys, so `Window::rebuild` always builds `Layered` with an empty +layer. What still assumes one: + +- `model/subscription/runs.rs`: `Layered`, its `gained` and `minus`, and the + property tests over a moving layer. +- `model/track.rs`: `find_servable` leapfrogging the layer after the fixed + runs, `Admitted::gained`, and `Servable::layered` in its property test. +- `model/track/miss.rs`: the misses view that judges only what a moved layer + covered, and `shared_layer` in its property test. + +Replace `Layered` with `Arc`, comparing windows by pointer where they +compared fixed runs. A window that moves now rebuilds, so the incremental +paths become a full recompute of what is owed; keep the property tests that +check owed groups and servable finds against a scan. + +Public API: none. Wire: none. + +## Required + +- [Rust groups by start](/quest/m2/time-addressing/model.md) - deletes the time-end walks that fed the layer diff --git a/rs/hang/examples/video.rs b/rs/hang/examples/video.rs index 7910196fd66..f037c7f5958 100644 --- a/rs/hang/examples/video.rs +++ b/rs/hang/examples/video.rs @@ -66,9 +66,10 @@ fn create_track(broadcast: &mut moq_net::broadcast::Producer) -> anyhow::Result< // Publish the catalog as a "catalog.json" track in the broadcast. let catalog_track = broadcast.create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info())?; - let mut group = catalog_track.append_group()?; - group.write_frame(moq_net::Timestamp::now(), catalog.to_json()?)?; - group.finish()?; + // This catalog never changes, so its one group starts at zero, where the media's timeline does. + let mut group = catalog_track.append_group(0)?; + group.write_frame(moq_net::Timestamp::ZERO, catalog.to_json()?)?; + group.close()?; // Actually create the media track now. // track_info() pins the microsecond timescale that the legacy container encodes with, and diff --git a/rs/hang/src/catalog/clock.rs b/rs/hang/src/catalog/clock.rs index 7a5df5c44ff..71305ff3b60 100644 --- a/rs/hang/src/catalog/clock.rs +++ b/rs/hang/src/catalog/clock.rs @@ -54,8 +54,12 @@ impl Clock { if wall.value() > MAX_SAFE_INTEGER { return Err(crate::Error::InvalidWall(wall.value())); } - if wall.scale().as_u64() > u32::MAX as u64 { - return Err(crate::Error::InvalidTimescale(wall.scale().as_u64())); + // A wall clock needs a scale: an unscaled timestamp has no time. + let scale = wall + .scale() + .ok_or(crate::Error::TimestampOverflow(moq_net::TimeOverflow))?; + if scale.as_u64() > u32::MAX as u64 { + return Err(crate::Error::InvalidTimescale(scale.as_u64())); } Ok(Self { wall }) } @@ -63,11 +67,12 @@ impl Clock { /// The wall-clock time of `pts` under this fixed mapping. /// /// Exact down to the nanosecond; an instant finer than that (a 90 kHz tick) is floored. - /// Refuses a result past the JSON-safe range or the wall clock's 1970-2554 range. + /// Refuses a result past the JSON-safe range or the wall clock's 1970-2554 range, and an + /// [unscaled](moq_net::Timestamp::unscaled) `pts` or wall, which has no time. pub fn wall_clock(&self, pts: moq_net::Timestamp) -> Result { let overflow = || crate::Error::TimestampOverflow(moq_net::TimeOverflow); - let wall_scale = self.wall.scale().as_u64() as u128; - let pts_scale = pts.scale().as_u64() as u128; + let wall_scale = self.wall.scale().ok_or_else(overflow)?.as_u64() as u128; + let pts_scale = pts.scale().ok_or_else(overflow)?.as_u64() as u128; // Sum as one exact fraction of a second, `num / den`, so neither side is rounded into the // other's units first. @@ -98,9 +103,13 @@ impl Serialize for Clock { where S: serde::Serializer, { + let scale = self + .wall + .scale() + .ok_or_else(|| serde::ser::Error::custom("an unscaled wall clock has no time"))?; Wire { wall: self.wall.value(), - timescale: self.wall.scale().as_u64() as u32, + timescale: u32::try_from(scale.as_u64()).map_err(serde::ser::Error::custom)?, } .serialize(serializer) } diff --git a/rs/hang/src/catalog/consumer.rs b/rs/hang/src/catalog/consumer.rs index 1a37fdf3d27..c9de8cb7abf 100644 --- a/rs/hang/src/catalog/consumer.rs +++ b/rs/hang/src/catalog/consumer.rs @@ -136,9 +136,9 @@ mod test { .unwrap(); let mut consumer = Catalog::<()>::subscribe(&broadcast.consume()).await.unwrap(); let published = serde_json::json!({ "echo": { "path": reference } }).to_string(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(moq_net::Timestamp::ZERO, published).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let result = consumer.next().await; if accepted { assert!(result.unwrap().unwrap().echo.is_some()); @@ -160,11 +160,11 @@ mod test { "audio".into(), crate::catalog::AudioConfig::new(crate::catalog::AudioCodec::Opus, 48_000, 2), ); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group .write_frame(moq_net::Timestamp::ZERO, published.to_json().unwrap()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(consumer.next().await.unwrap(), Some(published)); } @@ -189,11 +189,11 @@ mod test { .create_track(Catalog::DEFAULT_NAME, Catalog::default_track_info()) .unwrap(); let mut consumer = Catalog::<()>::subscribe(&broadcast.consume()).await.unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group .write_frame(moq_net::Timestamp::ZERO, r#"{"version":5,"video":[]}"#) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert!(matches!( consumer.next().await, Err(crate::Error::UnsupportedVersion(5)) diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 0384d4fd77a..155e2753cd0 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -1221,7 +1221,7 @@ mod test { let catalog = Catalog::<()>::from_str(json).expect("the packaged fixture must decode"); let clock = catalog.media.clock.expect("the fixture carries a clock"); - assert_eq!(clock.wall.scale().as_u64(), 1_000_000); + assert_eq!(clock.wall.scale().map(moq_net::Timescale::as_u64), Some(1_000_000)); assert_eq!(catalog.media.interval(), Some(std::time::Duration::from_secs(2))); // PTS 2000 ms lands 2s after the wall epoch, whatever the track timescale. diff --git a/rs/hang/src/snapshot.rs b/rs/hang/src/snapshot.rs index 20ea67d4ef5..546da791f42 100644 --- a/rs/hang/src/snapshot.rs +++ b/rs/hang/src/snapshot.rs @@ -39,7 +39,8 @@ impl<'de> DeserializeAs<'de, Duration> for Millis { } impl SerializeAs for Millis { fn serialize_as(value: &moq_net::Timestamp, serializer: S) -> Result { - let value = u64::try_from(value.as_millis()).map_err(serde::ser::Error::custom)?; + let millis = value.as_millis().map_err(serde::ser::Error::custom)?; + let value = u64::try_from(millis).map_err(serde::ser::Error::custom)?; Number::serialize_as(&value, serializer) } } diff --git a/rs/moq-archive/benches/producer.rs b/rs/moq-archive/benches/producer.rs index e16c6cb6051..229dfc11ce5 100644 --- a/rs/moq-archive/benches/producer.rs +++ b/rs/moq-archive/benches/producer.rs @@ -29,12 +29,12 @@ fn setup(tracks: usize, groups: u64) -> (broadcast::Producer, Vec, Path); +/// One listed object: where it starts, and its key. +pub(super) type Listing = (Start, Path); /// What one listing of a track's directory found. #[derive(Default)] @@ -30,13 +29,8 @@ pub(super) struct Listed { pub(super) struct Index { pub dir: Path, pub timescale: Option, - /// Each object's first frame timestamp, on a timed track, and key. - objects: BTreeMap, Path)>, - /// Each timed object starting a group, by that group's start timestamp with where it - /// starts, so a time finds the newest such object at or before it in one lookup. Group - /// starts never rewind, so they sort as the objects do. An object continuing a group is - /// left out: its first frame says nothing about where the groups after it start. - times: BTreeSet<(u64, Start)>, + /// Each object's key. A group's sequence is its start, so these sort by time too. + objects: BTreeMap, /// The newest key listed, which the next listing starts after. cursor: Path, } @@ -48,7 +42,6 @@ impl Index { dir, timescale: super::track::timescale(info)?, objects: BTreeMap::new(), - times: BTreeSet::new(), cursor, }) } @@ -64,13 +57,8 @@ impl Index { /// Take in the objects a listing after [`Self::cursor`] found. pub fn extend(&mut self, objects: &[Listing]) { - for (start, pts, path) in objects { - self.objects.insert(*start, (*pts, path.clone())); - if let Some(pts) = pts - && start.1 == 0 - { - self.times.insert((*pts, *start)); - } + for (start, path) in objects { + self.objects.insert(*start, path.clone()); self.cursor = path.clone(); } } @@ -79,7 +67,7 @@ impl Index { pub fn newest(&self) -> Option<(Start, Path)> { self.objects .last_key_value() - .map(|(start, (_, path))| (*start, path.clone())) + .map(|(start, path)| (*start, path.clone())) } /// The objects holding group `sequence` from frame `frame` on, oldest first: the last one @@ -87,11 +75,11 @@ impl Index { /// object after them, which bounds a gap. pub fn holding(&self, sequence: u64, frame: u64) -> (Vec<(Start, Path)>, Option) { let mut holding = Vec::new(); - if let Some((start, (_, path))) = self.objects.range(..=(sequence, frame)).next_back() { + if let Some((start, path)) = self.objects.range(..=(sequence, frame)).next_back() { holding.push((*start, path.clone())); } let mut after = None; - for (start, (_, path)) in self.objects.range((sequence, frame.saturating_add(1))..) { + for (start, path) in self.objects.range((sequence, frame.saturating_add(1))..) { if start.0 != sequence { after = Some(*start); break; @@ -101,30 +89,23 @@ impl Index { (holding, after) } - /// The objects that may hold the newest group starting at or before `pts`, newest first: - /// from the newest object starting a group at or before it, or the oldest object when none - /// does, up to the next object starting a group after it, which holds only later groups. - /// One continuing a group the next object still continues starts none, so it is skipped. - /// Empty on an untimed track. - pub fn around(&self, pts: Timestamp) -> Vec<(Start, Path)> { - let Some(timescale) = self.timescale else { - return Vec::new(); - }; - // A stored timestamp is whole, so it is at or before `pts` exactly when it is at or - // before `pts` rounded down to the track's timescale. - let limit = u64::try_from(pts.as_scale(timescale)).unwrap_or(u64::MAX); + /// The objects that may hold the newest group starting at or before `limit`, in the track's + /// timescale, newest first: from the newest object starting a group at or before it, or the + /// oldest object when none does, through the last object starting at or before it. Every later + /// object holds only later groups. One continuing a group the next object still continues + /// starts none, so it is skipped. + pub fn around(&self, limit: u64) -> Vec<(Start, Path)> { + let last = (limit, u64::MAX); let from = self - .times - .range(..=(limit, (u64::MAX, u64::MAX))) - .next_back() - .map_or((0, 0), |(_, start)| *start); + .objects + .range(..=last) + .rev() + .find(|(start, _)| start.1 == 0) + .map_or((0, 0), |(start, _)| *start); let mut around = Vec::new(); - let mut objects = self.objects.range(from..).peekable(); - while let Some((start, (first, path))) = objects.next() { - if start.1 == 0 && first.is_some_and(|first| first > limit) { - break; - } + let mut objects = self.objects.range(from..=last).peekable(); + while let Some((start, path)) = objects.next() { let continued = start.1 > 0 && objects.peek().is_some_and(|(next, _)| next.0 == start.0); if !continued { around.push((*start, path.clone())); @@ -136,7 +117,7 @@ impl Index { /// The object just after the one starting at `start`, or the oldest for `None`. pub fn after(&self, start: Option) -> Option<(Start, Path)> { - let (start, (_, path)) = match start { + let (start, path) = match start { Some(start) => self.objects.range((start.0, start.1.saturating_add(1))..).next()?, None => self.objects.first_key_value()?, }; @@ -145,9 +126,7 @@ impl Index { /// Forget an object the store no longer has, such as one a DVR pruned. pub fn forget(&mut self, start: Start) { - if let Some((Some(pts), _)) = self.objects.remove(&start) { - self.times.remove(&(pts, start)); - } + self.objects.remove(&start); } } @@ -177,9 +156,7 @@ pub(super) async fn list( match Name::parse(filename, timed)? { Name::Info => {} Name::End => found = Some(meta.location), - Name::Object(start) => listed - .objects - .push(((start.group, start.frame), start.pts, meta.location)), + Name::Object(start) => listed.objects.push(((start.group, start.frame), meta.location)), } } listed.objects.sort(); diff --git a/rs/moq-archive/src/consumer/mod.rs b/rs/moq-archive/src/consumer/mod.rs index d45206bc2c6..a14fccfc9ed 100644 --- a/rs/moq-archive/src/consumer/mod.rs +++ b/rs/moq-archive/src/consumer/mod.rs @@ -7,8 +7,8 @@ //! demand from the store through [`track::Requests`](moq_net::track::Requests) fills. //! //! A reader's time start older than the newest group is looked up from the object keys, each of -//! which carries its first frame's timestamp, so the range starts at the stored group containing -//! it however old, and a time end stops just past the group containing it. +//! which carries where its group starts, so the range starts at the stored group containing it +//! however old, and a time end stops just past the group containing it. //! //! A growing archive is followed: every [`Config::refresh`], one delimiter LIST of the prefix //! finds added tracks, and one LIST per track after its newest key finds new objects. Once the @@ -333,14 +333,7 @@ async fn refresh(shared: Arc>, mut tracks: Vec ( - listed - .objects - .into_iter() - .map(|(start, _, path)| (start, path)) - .collect(), - 0, - ), + true => (listed.objects, 0), // Only the newest sequence: fills serve the rest. false => { let Some((start, path)) = track.index.lock().unwrap().newest() else { diff --git a/rs/moq-archive/src/consumer/tests.rs b/rs/moq-archive/src/consumer/tests.rs index 46d97ec159e..cb374c1cf77 100644 --- a/rs/moq-archive/src/consumer/tests.rs +++ b/rs/moq-archive/src/consumer/tests.rs @@ -96,9 +96,9 @@ fn describe(frame: &Frame) -> String { } } -fn read(timestamp: Option, payload: Bytes) -> String { +fn read(timestamp: Timestamp, payload: Bytes) -> String { describe(&Frame { - timestamp: timestamp.map(Timestamp::value), + timestamp: timestamp.scale().map(|_| timestamp.value()), payload, }) } @@ -130,19 +130,27 @@ async fn fetch(broadcast: &broadcast::Producer, track: &str, sequence: u64) -> m async fn subscribe(broadcast: &broadcast::Producer, track: &str) -> track::Ordered { let track = broadcast.consume().track(track).unwrap(); let subscription = Subscription::default() - .with_range(0..) + .with_ranges([from(0)]) .with_order(Order::Ascending) .with_delay(Delay::default().with_max(None)); track.subscribe(subscription).await.unwrap().ordered() } -/// The next group's sequence, without waiting for it to end. +/// Every group from the one starting at `start` on. +fn from(start: u64) -> track::Range { + track::Range { + start: track::Anchor::Time(start), + end: None, + } +} + +/// The next group's start, without waiting for it to end. async fn sequence(reader: &mut track::Ordered) -> Option { timeout(Duration::from_secs(10), reader.next_group()) .await .expect("a group arrives") .unwrap() - .map(|group| group.sequence) + .map(|group| group.start) } /// The next group and every frame in it, once the group ends. @@ -159,7 +167,7 @@ async fn next(reader: &mut track::Ordered) -> Option<(u64, Vec)> { { frames.push(read(frame.timestamp, frame.payload)); } - Some((group.sequence, frames)) + Some((group.start, frames)) } fn not_found(result: moq_net::Result>) { @@ -189,19 +197,19 @@ async fn a_recording_round_trips() { sleep(Duration::from_millis(1)).await; } for sequence in 0..6 { - let mut group = video.create_group(group::Info { sequence }).unwrap(); + let mut group = video.create_group(group::Info { start: sequence }).unwrap(); for index in 0..3 { let pts = Timestamp::from_millis(sequence * 10_000 + index).unwrap(); group .write_frame(pts, Bytes::from(vec![sequence as u8, index as u8, 0xff])) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); sleep(Duration::from_millis(10)).await; } - let mut group = log.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(None, "a").unwrap(); - group.finish().unwrap(); + let mut group = log.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), "a").unwrap(); + group.close().unwrap(); sleep(Duration::from_millis(10)).await; let _video = video.finish().unwrap(); let _log = log.finish().unwrap(); @@ -219,10 +227,10 @@ async fn a_recording_round_trips() { let mut reader = subscribe(&broadcast, "video").await; for sequence in 0..6 { let mut group = reader.next_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, sequence); + assert_eq!(group.start, sequence); for index in 0..3 { let frame = group.read_frame().await.unwrap().unwrap(); - assert_eq!(frame.timestamp.unwrap().value(), sequence * 10_000 + index); + assert_eq!(frame.timestamp.value(), sequence * 10_000 + index); assert_eq!(&frame.payload[..], &[sequence as u8, index as u8, 0xff]); } assert!(group.read_frame().await.unwrap().is_none()); @@ -282,35 +290,35 @@ async fn a_narrowed_read_skips_the_group_it_left() { let archive = Archive::new(); archive.track("video", 0, true).await; for sequence in 0..4 { - archive.object("video", 0, vec![frames(true, sequence, 0..1)]).await; + archive.object("video", 0, vec![timed(sequence, 0..1)]).await; } let (broadcast, _run) = serve(&archive.store).await; let mock = archive.store.inner().clone(); mock.take(); let gate = mock.stall_gets("rec/video/"); - let ascending = |groups| { + let ascending = |start| { Subscription::default() .with_delay(Delay::default().with_max(None)) - .with_range(groups) + .with_ranges([from(start)]) .with_order(Order::Ascending) }; let mut subscriber = broadcast .consume() .track("video") .unwrap() - .subscribe(ascending(0..)) + .subscribe(ascending(0)) .await .unwrap(); let waiter = kio::Waiter::noop(); let _ = subscriber.poll_recv_group(&waiter); assert!(object_read(&mock, 0).await, "group 0 is read from its object"); - subscriber.update(ascending(1..)).unwrap(); + subscriber.update(ascending(1_000)).unwrap(); let _ = subscriber.poll_recv_group(&waiter); assert!( - object_read(&mock, 1).await, - "group 1 waited behind group 0, which nobody reads" + object_read(&mock, 1_000).await, + "group 1000 waited behind group 0, which nobody reads" ); gate.add_permits(16); } @@ -541,10 +549,7 @@ async fn a_completed_archive_ends_as_recorded() { archive.complete().await; let (broadcast, _run) = serve(&archive.store).await; - for (track, end) in [ - ("video", Position::group(2)), - ("audio", Position { group: 1, frame: 2 }), - ] { + for (track, end) in [("video", Position::group(2)), ("audio", Position::new(1, 2))] { let mut reader = subscribe(&broadcast, track).await; assert_eq!(next(&mut reader).await.unwrap(), (0, expected(true, 0, 0..2))); assert_eq!(next(&mut reader).await.unwrap(), (1, expected(true, 1, 0..2))); @@ -559,7 +564,7 @@ async fn a_completed_archive_ends_as_recorded() { let end = timeout(Duration::from_secs(10), reader.next_group()) .await .expect("the track ends"); - let end = end.map(|group| group.map(|group| group.sequence)); + let end = end.map(|group| group.map(|group| group.start)); assert!( matches!(end, Err(moq_net::Error::Stream(moq_net::StreamError::App(7)))), "{end:?}" @@ -607,10 +612,10 @@ async fn a_re_added_track_replays_both_runs() { sleep(Duration::from_millis(1)).await; } for sequence in sequences { - let mut group = track.create_group(group::Info { sequence }).unwrap(); + let mut group = track.create_group(group::Info { start: sequence }).unwrap(); let pts = Timestamp::from_millis(sequence * 1000).unwrap(); group.write_frame(pts, format!("run{run}/{sequence}")).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let track = track.finish().unwrap(); sleep(Duration::from_secs(2)).await; @@ -679,7 +684,7 @@ async fn an_unordered_listing_is_sorted() { assert_eq!(super::tests::sequence(&mut reader).await, Some(3)); } -/// Read `range` in order with no age limit: the sequences of the groups it delivers before +/// Read `range` in order with no age limit: the starts of the groups it delivers before /// it stops delivering. A range ending before the live edge parks once read, as on any live /// track. async fn read_range(broadcast: &broadcast::Producer, track: &str, range: track::Range) -> Vec { @@ -692,15 +697,25 @@ async fn read_range(broadcast: &broadcast::Producer, track: &str, range: track:: let mut read = Vec::new(); while let Ok(group) = timeout(5 * REFRESH, reader.next_group()).await { match group.unwrap() { - Some(group) => read.push(group.sequence), + Some(group) => read.push(group.start), None => break, } } read } +/// A time on a millisecond track. fn at(millis: u64) -> track::Anchor { - track::Anchor::Time(Timestamp::from_millis(millis).unwrap()) + track::Anchor::Time(millis) +} + +/// Group `sequence`'s frames `frames` on a timed track, keyed by its first frame's time: +/// `sequence` seconds. +fn timed(sequence: u64, frames: std::ops::Range) -> Entry { + Entry { + sequence: sequence * 1000, + ..self::frames(true, sequence, frames) + } } /// Groups `sequences` of a timed track, two to an object, each group starting at its @@ -708,7 +723,7 @@ fn at(millis: u64) -> track::Anchor { async fn stored(archive: &Archive, sequences: impl IntoIterator) { let sequences: Vec = sequences.into_iter().collect(); for pair in sequences.chunks(2) { - let entries = pair.iter().map(|sequence| frames(true, *sequence, 0..3)).collect(); + let entries = pair.iter().map(|sequence| timed(*sequence, 0..3)).collect(); archive.object("video", 0, entries).await; } } @@ -726,22 +741,22 @@ async fn a_time_range_resolves_from_the_archive() { let range = |start, end| track::Range { start, end }; assert_eq!( read_range(&broadcast, "video", range(at(4_500), Some(at(6_500)))).await, - [4, 5, 6] + [4_000, 5_000, 6_000] ); // A start on the first frame of its group, in the second object of a pair. assert_eq!( read_range(&broadcast, "video", range(at(3_000), Some(at(3_001)))).await, - [3] + [3_000] ); // An end alone, from the oldest group. assert_eq!( - read_range(&broadcast, "video", range(track::Anchor::Absolute(0), Some(at(1_200)))).await, - [0, 1] + read_range(&broadcast, "video", range(at(0), Some(at(1_200)))).await, + [0, 1_000] ); // The cache now holds groups 6 and 9, but not what lies between: still looked up. assert_eq!( read_range(&broadcast, "video", range(at(8_500), Some(at(8_600)))).await, - [8] + [8_000] ); // A start alone reads on through the live edge. @@ -752,7 +767,7 @@ async fn a_time_range_resolves_from_the_archive() { .with_delay(moq_net::track::Delay::default().with_max(None)); let mut reader = track.subscribe(subscription).await.unwrap().ordered(); for expected in 7..10 { - assert_eq!(sequence(&mut reader).await, Some(expected)); + assert_eq!(sequence(&mut reader).await, Some(expected * 1000)); } } @@ -763,7 +778,7 @@ async fn a_completed_archive_reads_a_time_range() { let archive = Archive::new(); archive.track("video", 0, true).await; stored(&archive, 0..6).await; - archive.end("video", End::Finish(At { group: 6, frame: 0 })).await; + archive.end("video", End::Finish(At { group: 6_000, frame: 0 })).await; archive.complete().await; let (broadcast, _run) = serve(&archive.store).await; @@ -771,7 +786,10 @@ async fn a_completed_archive_reads_a_time_range() { start: at(2_500), end: None, }; - assert_eq!(read_range(&broadcast, "video", range).await, [2, 3, 4, 5]); + assert_eq!( + read_range(&broadcast, "video", range).await, + [2_000, 3_000, 4_000, 5_000] + ); } /// A time older than every stored object reaches back to the oldest stored group. @@ -787,7 +805,7 @@ async fn a_time_before_the_archive_starts_at_its_oldest_group() { start: at(500), end: Some(at(4_500)), }; - assert_eq!(read_range(&broadcast, "video", range).await, [3, 4]); + assert_eq!(read_range(&broadcast, "video", range).await, [3_000, 4_000]); } /// A time older than an archive whose oldest object continues a pruned group reaches back to @@ -796,18 +814,16 @@ async fn a_time_before_the_archive_starts_at_its_oldest_group() { async fn a_time_before_a_pruned_head_starts_at_the_oldest_whole_group() { let archive = Archive::new(); archive.track("video", 0, true).await; - archive - .object("video", 2, vec![frames(true, 5, 2..4), frames(true, 6, 0..2)]) - .await; - archive.object("video", 0, vec![frames(true, 7, 0..2)]).await; - archive.object("video", 0, vec![frames(true, 8, 0..1)]).await; + archive.object("video", 2, vec![timed(5, 2..4), timed(6, 0..2)]).await; + archive.object("video", 0, vec![timed(7, 0..2)]).await; + archive.object("video", 0, vec![timed(8, 0..1)]).await; let (broadcast, _run) = serve(&archive.store).await; let range = track::Range { start: at(500), end: Some(at(7_500)), }; - assert_eq!(read_range(&broadcast, "video", range).await, [6, 7]); + assert_eq!(read_range(&broadcast, "video", range).await, [6_000, 7_000]); } /// A time in a gap the archive never stored starts at the group before the gap, the newest @@ -823,12 +839,12 @@ async fn a_time_in_a_gap_starts_before_it() { start: at(4_500), end: Some(at(6_500)), }; - assert_eq!(read_range(&broadcast, "video", range).await, [3, 6]); + assert_eq!(read_range(&broadcast, "video", range).await, [3_000, 6_000]); } /// A continuation's first frame bounds no group start: an object continuing group 0 at a -/// later timestamp than the group 1 it then starts still holds the group containing a time -/// between the two. +/// later timestamp than the group at 500 it then starts still holds the group containing a +/// time between the two. #[tokio::test(start_paused = true)] async fn a_continuation_does_not_bound_a_group_start() { let archive = Archive::new(); @@ -857,7 +873,7 @@ async fn a_continuation_does_not_bound_a_group_start() { frames: vec![frame(0, 1, 1_000)], }, Entry { - sequence: 1, + sequence: 500, frames: vec![frame(1, 0, 500)], }, ], @@ -870,32 +886,48 @@ async fn a_continuation_does_not_bound_a_group_start() { start: at(600), end: Some(at(700)), }; - assert_eq!(read_range(&broadcast, "video", range).await, [1]); + assert_eq!(read_range(&broadcast, "video", range).await, [500]); } -/// An untimed track refuses a time range rather than looking it up. +/// A group starts at its key even when its first frame comes later, as one a seek opened, so +/// the archive resolves a time to the group the live cache would. #[tokio::test(start_paused = true)] -async fn an_untimed_track_refuses_a_time_range() { +async fn a_time_resolves_by_key_not_first_frame() { + let archive = Archive::new(); + archive.track("video", 0, true).await; + let frame = |sequence: u64, pts: u64| Frame { + timestamp: Some(pts), + payload: Bytes::from(format!("{sequence}")), + }; + for (sequence, pts) in [(1_000, 1_000), (2_000, 2_500), (3_000, 3_000)] { + let entry = Entry { + sequence, + frames: vec![frame(sequence, pts)], + }; + archive.object("video", 0, vec![entry]).await; + } + + let (broadcast, _run) = serve(&archive.store).await; + let range = track::Range { + start: at(2_200), + end: Some(at(2_300)), + }; + assert_eq!(read_range(&broadcast, "video", range).await, [2_000]); +} + +/// A scale-less track's times are bare starts, which the archive looks up by key. +#[tokio::test(start_paused = true)] +async fn a_scale_less_track_looks_up_a_time_by_key() { let archive = Archive::new(); archive.track("log", 0, false).await; - for sequence in 0..4 { + for sequence in [0, 2, 5, 7] { archive.object("log", 0, vec![frames(false, sequence, 0..1)]).await; } let (broadcast, _run) = serve(&archive.store).await; - let track = broadcast.consume().track("log").unwrap(); - let subscription = Subscription::default() - .with_ranges([track::Range { - start: at(1_000), - end: None, - }]) - .with_delay(moq_net::track::Delay::default().with_max(None)); - let result = match track.subscribe(subscription).await { - Ok(subscriber) => timeout(Duration::from_secs(10), subscriber.ordered().next_group()) - .await - .expect("the read is refused") - .map(|group| group.map(|group| group.sequence)), - Err(err) => Err(err), + let range = track::Range { + start: at(3), + end: Some(at(5)), }; - assert!(matches!(result, Err(moq_net::Error::TimestampMismatch)), "{result:?}"); + assert_eq!(read_range(&broadcast, "log", range).await, [2, 5]); } diff --git a/rs/moq-archive/src/consumer/track.rs b/rs/moq-archive/src/consumer/track.rs index 3af6bebbae5..8c2c5b4a94f 100644 --- a/rs/moq-archive/src/consumer/track.rs +++ b/rs/moq-archive/src/consumer/track.rs @@ -82,7 +82,7 @@ impl Live { open } None => { - let group = self.producer.create_group(group::Info { sequence })?; + let group = self.producer.create_group(group::Info { start: sequence })?; let open = self.open.insert(Open { group, sequence, @@ -96,7 +96,7 @@ impl Live { } }; for frame in &entry.frames { - let timestamp = timestamp(frame.timestamp, self.timescale)?; + let timestamp = timestamp(frame.timestamp, self.timescale, entry.sequence)?; open.group.write_frame(timestamp, frame.payload.clone())?; open.next += 1; } @@ -105,7 +105,7 @@ impl Live { fn close(&mut self) -> Result<()> { if let Some(open) = self.open.take() { - open.group.finish()?; + open.group.close()?; } Ok(()) } @@ -124,13 +124,7 @@ impl Live { self.close()?; match end { End::Finish(at) => { - self.producer.finish_at( - Position { - group: at.group, - frame: at.frame, - }, - None, - )?; + self.producer.finish_at(Position::new(at.group, at.frame), None)?; // An empty group stores no object, so the replay declares it never produces what // the archive holds nothing for up to the end, and the end settles over it. let fin = at.group + u64::from(at.frame != 0); @@ -159,22 +153,18 @@ pub(super) async fn serve(shared: Arc>, index: Arc(shared: &Shared, index: &Mutex, fill: &Fill, pts: Timestamp) { - let (around, timed, timescale) = { +/// Answer `fill`'s lookup of `pts`, in the track's timescale, with the group containing it: the +/// newest group starting at or before it, found among the objects whose keys say they may hold +/// it, or the oldest stored group when every one starts after it. A group starts at its key, +/// as the live cache resolves it. Left unanswered when the archive holds no group, or the store +/// fails. +async fn look_up(shared: &Shared, index: &Mutex, fill: &Fill, pts: u64) { + let (around, timed) = { let index = index.lock().unwrap(); - (index.around(pts), index.timed(), index.timescale) + (index.around(pts), index.timed()) }; - let Some(timescale) = timescale else { - // A reader's time start on an untimed track is refused before it is looked up. - return; - }; - let limit = pts.as_scale(timescale); - - // Newest first: only a group whose first frame an object holds says where it starts. + // Newest first, skipping a group whose head was pruned, which can't be served from its first + // frame. for (start, path) in around { let Some(object) = load(shared, index, start, &path, timed).await else { return; @@ -185,13 +175,8 @@ async fn look_up(shared: &Shared, index: &Mutex, fill: .enumerate() .rev() .filter(|(position, _)| *position > 0 || object.frame_start == 0) - .find_map(|(_, entry)| { - let first = entry.frames.first()?; - first - .timestamp - .is_some_and(|at| u128::from(at) <= limit) - .then_some(entry.sequence) - }); + .find(|(_, entry)| !entry.frames.is_empty() && entry.sequence <= pts) + .map(|(_, entry)| entry.sequence); if let Some(sequence) = found { return serve_group(shared, index, fill, sequence).await; } @@ -343,7 +328,7 @@ async fn serve_group(shared: &Shared, index: &Mutex, f if index < frame { continue; } - let written = timestamp(stored.timestamp, timescale) + let written = timestamp(stored.timestamp, timescale, sequence) .map_err(|_| moq_net::Error::ProtocolViolation) .and_then(|timestamp| producer.write_frame(timestamp, stored.payload.clone())); if let Err(err) = written { @@ -352,7 +337,7 @@ async fn serve_group(shared: &Shared, index: &Mutex, f return; } } - let _ = producer.finish(); + let _ = producer.close(); } /// The served properties of a track stored with `info`: no max age, since the archive's @@ -372,10 +357,11 @@ pub(super) fn timescale(info: &Info) -> Result> { } /// A stored timestamp on its track, which must be timed exactly when the frame is. -fn timestamp(value: Option, timescale: Option) -> Result> { +fn timestamp(value: Option, timescale: Option, sequence: u64) -> Result { match (value, timescale) { - (Some(value), Some(timescale)) => Ok(Some(Timestamp::new(value, timescale).map_err(|_| Error::Id(value))?)), - (None, None) => Ok(None), + (Some(value), Some(timescale)) => Timestamp::new(value, timescale).map_err(|_| Error::Id(value)), + // A scale-less frame's timestamp is its group's key. + (None, None) => Ok(Timestamp::unscaled(sequence)), _ => Err(Error::Timing), } } diff --git a/rs/moq-archive/src/layout.rs b/rs/moq-archive/src/layout.rs index e4a5e3f864c..f3030f11247 100644 --- a/rs/moq-archive/src/layout.rs +++ b/rs/moq-archive/src/layout.rs @@ -194,10 +194,10 @@ impl Object { 1 => self.frame_start, _ => 0, }; - Ok(Position { - group: last.sequence, - frame: first.checked_add(last.frames.len() as u64).ok_or(Error::Overflow)?, - }) + Ok(Position::new( + last.sequence, + first.checked_add(last.frames.len() as u64).ok_or(Error::Overflow)?, + )) } /// The version, the first frame index, then each entry's sequence (the first absolute, the @@ -468,13 +468,13 @@ mod tests { pts: Some(10) } ); - assert_eq!(object.end().unwrap(), Position { group: 7, frame: 1 }); + assert_eq!(object.end().unwrap(), Position::new(7, 1)); let split = Object { frame_start: 4, entries: vec![object.entries[0].clone()], }; - assert_eq!(split.end().unwrap(), Position { group: 3, frame: 6 }); + assert_eq!(split.end().unwrap(), Position::new(3, 6)); } #[test] diff --git a/rs/moq-archive/src/producer/backends.rs b/rs/moq-archive/src/producer/backends.rs index e90e0105694..50430993924 100644 --- a/rs/moq-archive/src/producer/backends.rs +++ b/rs/moq-archive/src/producer/backends.rs @@ -32,22 +32,24 @@ fn source(groups: u64, chat: &[(u64, &[&str])]) -> (broadcast::Producer, [track: ) .unwrap(); for sequence in 0..groups { - let mut group = video.create_group(group::Info { sequence }).unwrap(); + let mut group = video.create_group(group::Info { start: sequence }).unwrap(); for frame in 0..2 { let pts = Timestamp::from_millis(sequence * 1_000 + frame * 500).unwrap(); group.write_frame(pts, format!("v{sequence}.{frame}")).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let text = source .create_track(CHAT, track::Info::default().with_max_age(None)) .unwrap(); for (sequence, frames) in chat { - let mut group = text.create_group(group::Info { sequence: *sequence }).unwrap(); + let mut group = text.create_group(group::Info { start: *sequence }).unwrap(); for frame in *frames { - group.write_frame(None, frame.to_string()).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(*sequence), frame.to_string()) + .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } (source, [video, text]) } diff --git a/rs/moq-archive/src/producer/mod.rs b/rs/moq-archive/src/producer/mod.rs index 13bc888f5a8..ed2c2f91e68 100644 --- a/rs/moq-archive/src/producer/mod.rs +++ b/rs/moq-archive/src/producer/mod.rs @@ -21,7 +21,6 @@ //! # } //! ``` -mod ordered; mod resume; mod track; @@ -58,9 +57,9 @@ pub struct Config { /// The object is cut there, mid-group if need be. Keep it above [`Self::duration`], or /// objects are cut by it instead. pub flush: Duration, - /// The longest a missing group is waited for, capping the publisher's max age. A group still - /// missing then is a gap, and the frames behind it wait for that verdict. A track with nothing - /// stored waits this long from its first arrival for a lower sequence, unless that one is 0. + /// How far newer content may run ahead of a group the track still owes before it is given up + /// as a gap, capping the publisher's max age. The frames behind it wait for that verdict. A + /// group the publisher skipped or dropped is never waited for. pub max_delay: Duration, /// Prune old objects (a DVR), or keep everything when `None` (an archive). pub dvr: Option, diff --git a/rs/moq-archive/src/producer/ordered.rs b/rs/moq-archive/src/producer/ordered.rs deleted file mode 100644 index bab5d08e3bb..00000000000 --- a/rs/moq-archive/src/producer/ordered.rs +++ /dev/null @@ -1,252 +0,0 @@ -//! A track's groups in sequence order, waiting a bounded time for a late one. -//! -//! `moq_net::track::Ordered` seeks past a missing group at once, so it fits a player, not an -//! archive that must keep every group it can in key order. - -use std::collections::{BTreeMap, VecDeque}; -use std::pin::Pin; -use std::task::{Poll, ready}; -use std::time::Duration; - -use kio::clock::Instant; -use moq_net::group; -use moq_net::track::{Position, Subscriber}; - -/// Hands out each sequence once every lower one was handed out or given up: at once when it is -/// the next, otherwise once `max_delay` passes from the oldest arrival still waiting, which is -/// when the gap below it was first seen. While no start is known, anything but 0 waits the same -/// way, since arrival is not sequence order. Once the groups end, everything received is handed -/// out without waiting. -pub(super) struct Ordered { - subscriber: Subscriber, - max_delay: Duration, - /// The next sequence to hand out; `None` until a start is known or the first wait ends. - next: Option, - /// Received sequences waiting on a lower one. - /// Boxed, since a group consumer is large and moves as the map rebalances. - pending: BTreeMap>, - /// Each pending sequence with when it arrived, oldest first. The front is always pending, - /// and the ones handed out behind it go once it does, so this holds at most what arrived - /// within a max delay. The oldest dates the gap, so a later, lower arrival never extends it. - arrivals: VecDeque<(u64, Instant)>, - timer: Option>>, - groups_done: bool, - /// The error the groups ended with: an end declared with an error, or an abort. - error: Option, -} - -impl Ordered { - /// Order `subscriber` from `next`, where a resume continues; otherwise from the subscription's - /// floor when it names one. - pub(super) fn new(subscriber: Subscriber, max_delay: Duration, next: Option) -> Self { - let next = next.or_else(|| subscriber.floor()); - Self { - subscriber, - max_delay, - next, - pending: BTreeMap::new(), - arrivals: VecDeque::new(), - timer: None, - groups_done: false, - error: None, - } - } - - /// Take every arrival so far, registering `waiter` for the next. - pub(super) fn poll_arrivals(&mut self, waiter: &kio::Waiter) { - let now = kio::clock::now(); - while !self.groups_done { - match self.subscriber.poll_recv_group(waiter) { - Poll::Ready(Ok(Some(group))) => self.arrive(Box::new(group), now), - // How the track ended is read from `poll_finished`. - Poll::Ready(Ok(None)) => self.groups_done = true, - Poll::Ready(Err(err)) => { - self.error.get_or_insert(err); - self.groups_done = true; - } - Poll::Pending => break, - } - } - } - - fn arrive(&mut self, group: Box, now: Instant) { - let sequence = group.sequence; - // A sequence names one group, so a copy of one already here or handed out is dropped. - if self.next.is_none_or(|next| sequence >= next) - && let std::collections::btree_map::Entry::Vacant(entry) = self.pending.entry(sequence) - { - entry.insert(group); - self.arrivals.push_back((sequence, now)); - } - } - - /// The next group in order, or `None` once the groups ended and nothing is left. - pub(super) fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>> { - self.poll_arrivals(waiter); - let draining = self.groups_done; - loop { - let Some(entry) = self.pending.first_entry() else { - return match draining { - true => Poll::Ready(None), - false => Poll::Pending, - }; - }; - let sequence = *entry.key(); - let gap = match self.next { - Some(next) if sequence < next => { - entry.remove(); - self.trim(); - continue; - } - Some(next) => sequence > next, - // Nothing is below 0. - None => sequence > 0, - }; - if gap && !draining { - // The oldest arrival still pending is when this gap was first seen. - let (_, since) = self.arrivals.front().expect("a pending sequence arrived"); - let deadline = since.saturating_add(self.max_delay); - if kio::clock::now() < deadline { - let timer = self - .timer - .get_or_insert_with(|| Box::pin(kio::clock::sleep_until(deadline))); - timer.as_mut().reset(deadline); - if waiter.poll_future(timer.as_mut()).is_pending() { - return Poll::Pending; - } - } - } - let group = self.pending.remove(&sequence).expect("the lowest pending sequence"); - self.trim(); - self.next = Some(sequence.saturating_add(1)); - return Poll::Ready(Some(group)); - } - } - - /// Drop arrivals handed out, up to the oldest one still pending. - fn trim(&mut self) { - while self - .arrivals - .front() - .is_some_and(|(sequence, _)| !self.pending.contains_key(sequence)) - { - self.arrivals.pop_front(); - } - } - - /// How the track ended, once it has: the error the groups ended with, which an end declared - /// with an error reads as while keeping its history, or else its declared end. - pub(super) fn poll_finished(&mut self, waiter: &kio::Waiter) -> Poll> { - let end = ready!(self.subscriber.poll_finished(waiter))?; - Poll::Ready(match self.error.clone() { - Some(err) => Err(err), - None => Ok(end), - }) - } -} - -#[cfg(test)] -mod tests { - use moq_net::track::{Delay, Order, Range, Subscription}; - use moq_net::{broadcast, track}; - - use super::*; - - const MAX_DELAY: Duration = Duration::from_secs(10); - - /// A source track with no publisher max age, so a superseded group still reaches the reader - /// and only [`MAX_DELAY`] decides how long a gap is waited for. - fn source(broadcast: &broadcast::Producer) -> track::Producer { - broadcast - .create_track("t", track::Info::default().with_max_age(None)) - .unwrap() - } - - fn group(track: &track::Producer, sequence: u64) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, "x").unwrap(); - group.finish().unwrap(); - } - - fn next(ordered: &mut Ordered) -> Option { - match ordered.poll_next(&kio::Waiter::noop()) { - Poll::Ready(Some(group)) => Some(group.sequence), - _ => None, - } - } - - async fn ordered(broadcast: &broadcast::Producer, next: Option) -> Ordered { - let subscription = Subscription::default() - .with_order(Order::Ascending) - .with_delay(Delay::default().with_max(MAX_DELAY)) - .with_ranges([Range::oldest()]); - let subscriber = broadcast - .consume() - .track("t") - .unwrap() - .subscribe(subscription) - .await - .unwrap(); - Ordered::new(subscriber, MAX_DELAY, next) - } - - /// The next sequence is handed out at once, and one past a missing sequence only once the max - /// delay from its arrival passes. - #[tokio::test(start_paused = true)] - async fn only_a_missing_sequence_waits_the_max_delay() { - let broadcast = broadcast::Info::new().produce(); - let track = source(&broadcast); - let mut ordered = ordered(&broadcast, None).await; - - group(&track, 0); - assert_eq!(next(&mut ordered), Some(0)); - - group(&track, 2); - assert_eq!(next(&mut ordered), None); - group(&track, 1); - assert_eq!(next(&mut ordered), Some(1)); - assert_eq!(next(&mut ordered), Some(2)); - - group(&track, 4); - assert_eq!(next(&mut ordered), None); - kio::clock::sleep(MAX_DELAY - Duration::from_millis(1)).await; - assert_eq!(next(&mut ordered), None); - kio::clock::sleep(Duration::from_millis(1)).await; - assert_eq!(next(&mut ordered), Some(4)); - - // Too late for its turn. - group(&track, 3); - let _track = track.finish().unwrap(); - assert!(matches!(ordered.poll_next(&kio::Waiter::noop()), Poll::Ready(None))); - } - - /// A gap's wait runs from when it was first seen, so ever lower arrivals above it can't extend it. - #[tokio::test(start_paused = true)] - async fn lower_arrivals_never_extend_a_gap() { - let broadcast = broadcast::Info::new().produce(); - let track = source(&broadcast); - let mut ordered = ordered(&broadcast, Some(0)).await; - - group(&track, 10); - assert_eq!(next(&mut ordered), None); - kio::clock::sleep(MAX_DELAY - Duration::from_millis(1)).await; - group(&track, 9); - assert_eq!(next(&mut ordered), None); - kio::clock::sleep(Duration::from_millis(1)).await; - assert_eq!(next(&mut ordered), Some(9)); - assert_eq!(next(&mut ordered), Some(10)); - } - - /// A gap-free track keeps no arrival it handed out. - #[tokio::test(start_paused = true)] - async fn handed_out_arrivals_are_dropped() { - let broadcast = broadcast::Info::new().produce(); - let track = source(&broadcast); - let mut ordered = ordered(&broadcast, None).await; - for sequence in 0..100 { - group(&track, sequence); - assert_eq!(next(&mut ordered), Some(sequence)); - } - assert!(ordered.arrivals.is_empty()); - } -} diff --git a/rs/moq-archive/src/producer/tests.rs b/rs/moq-archive/src/producer/tests.rs index f9c1ddac07e..c0d44718551 100644 --- a/rs/moq-archive/src/producer/tests.rs +++ b/rs/moq-archive/src/producer/tests.rs @@ -45,11 +45,11 @@ fn ms(ms: u64) -> Timestamp { /// Write a whole group of `(pts, payload)` frames. fn group(track: &track::Producer, sequence: u64, frames: &[(u64, &str)]) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); + let mut group = track.create_group(group::Info { start: sequence }).unwrap(); for (pts, payload) in frames { group.write_frame(ms(*pts), payload.to_string()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } /// Let the producer catch up without moving time far. @@ -191,8 +191,10 @@ async fn timed_objects_cut_at_group_boundaries_by_duration() { assert!(complete(&store).await); } +/// A group created below one already stored is a gap, and never stalls the groups after it: the +/// recorder reads in key order, and a local track owes nothing between the groups it holds. #[tokio::test(start_paused = true)] -async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { +async fn a_late_group_is_a_gap_that_never_stalls_later_groups() { let store = store(); let broadcast = broadcast::Info::new().produce(); let video = broadcast.create_track("video", timed()).unwrap(); @@ -204,15 +206,8 @@ async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { group(&video, 2, &[(2000, "c")]); group(&video, 3, &[(3000, "d")]); settle().await; - // Group 0 is flushed alone; 2 and 3 wait on the missing 1. + // Nothing waits on 1, so every group is stored within the flush. sleep(SECOND + SECOND / 2).await; - assert_eq!( - objects(&store, "video", true).await.keys().cloned().collect::>(), - [key(0, 0, 0)] - ); - - // Once the max delay passes, the gap is given up and the rest follow within the flush. - sleep(2 * SECOND).await; let stored = objects(&store, "video", true).await; assert_eq!( stored.keys().cloned().collect::>(), @@ -223,7 +218,7 @@ async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { [(2, vec!["2000/c".to_string()]), (3, vec!["3000/d".to_string()]),] ); - // The lost group arriving too late stays a gap. + // A group below the stored ones arrives too late and stays a gap. group(&video, 1, &[(1000, "b")]); let _video = video.finish().unwrap(); broadcast.close(); @@ -233,40 +228,6 @@ async fn a_lost_group_is_a_gap_that_never_stalls_later_groups() { assert_eq!(end(&store, "video").await, Some(End::Finish(At { group: 4, frame: 0 }))); } -/// Arrival is not sequence order, so a fresh track's first arrival waits for a lower sequence. -#[tokio::test(start_paused = true)] -async fn a_fresh_track_waits_for_a_group_below_its_first_arrival() { - let store = store(); - let broadcast = broadcast::Info::new().produce(); - let video = broadcast.create_track("video", timed()).unwrap(); - let config = config().with_flush(SECOND).with_max_delay(2 * SECOND); - let task = spawn(&store, &broadcast, config); - subscribed(&video).await; - - group(&video, 2, &[(2000, "c")]); - settle().await; - group(&video, 1, &[(1000, "b")]); - settle().await; - // Nothing is stored until the wait from the first arrival gives up on group 0. - sleep(SECOND + SECOND / 2).await; - assert!(objects(&store, "video", true).await.is_empty()); - - sleep(2 * SECOND).await; - let stored = objects(&store, "video", true).await; - assert_eq!(stored.keys().cloned().collect::>(), [key(1, 0, 1000)]); - assert_eq!( - contents(&stored)[0], - [(1, vec!["1000/b".to_string()]), (2, vec!["2000/c".to_string()]),] - ); - - // Group 0 arriving after that is too late. - group(&video, 0, &[(0, "a")]); - let _video = video.finish().unwrap(); - broadcast.close(); - finished(task).await; - assert_eq!(objects(&store, "video", true).await.len(), 1); -} - #[tokio::test(start_paused = true)] async fn an_oversized_timed_group_splits_into_ordered_unique_keys() { let store = store(); @@ -315,14 +276,14 @@ async fn an_open_untimed_group_is_cut_at_each_flush() { let task = spawn(&store, &broadcast, config().with_flush(SECOND)); subscribed(&data).await; - let mut open = data.create_group(group::Info { sequence: 0 }).unwrap(); - open.write_frame(None, "a").unwrap(); + let mut open = data.create_group(group::Info { start: 0 }).unwrap(); + open.write_frame(moq_net::Timestamp::unscaled(0), "a").unwrap(); settle().await; sleep(SECOND).await; let head = untimed(0, 0); let first = get(&store, &format!("data/{head}")).await.unwrap(); - open.write_frame(None, "b").unwrap(); + open.write_frame(moq_net::Timestamp::unscaled(0), "b").unwrap(); settle().await; sleep(SECOND).await; assert_eq!(get(&store, &format!("data/{head}")).await.unwrap(), first); @@ -333,7 +294,7 @@ async fn an_open_untimed_group_is_cut_at_each_flush() { [[(0, vec!["a".to_string()])], [(0, vec!["b".to_string()])]] ); - open.finish().unwrap(); + open.close().unwrap(); let _data = data.finish().unwrap(); broadcast.close(); finished(task).await; @@ -352,18 +313,18 @@ async fn an_untimed_group_splits_by_wall_clock() { let task = spawn(&store, &broadcast, config()); subscribed(&data).await; - let mut open = data.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = data.create_group(group::Info { start: 0 }).unwrap(); for payload in ["a", "b", "c", "d", "e"] { - open.write_frame(None, payload).unwrap(); + open.write_frame(moq_net::Timestamp::unscaled(0), payload).unwrap(); settle().await; sleep(SECOND / 2).await; } - open.finish().unwrap(); + open.close().unwrap(); settle().await; sleep(2 * SECOND).await; - let mut next = data.create_group(group::Info { sequence: 1 }).unwrap(); - next.write_frame(None, "f").unwrap(); - next.finish().unwrap(); + let mut next = data.create_group(group::Info { start: 1 }).unwrap(); + next.write_frame(moq_net::Timestamp::unscaled(1), "f").unwrap(); + next.close().unwrap(); settle().await; let _data = data.finish().unwrap(); broadcast.close(); @@ -393,7 +354,7 @@ async fn an_idle_track_is_flushed_within_the_bound() { let task = spawn(&store, &broadcast, config().with_flush(SECOND)); subscribed(&video).await; - let mut open = video.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = video.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ms(0), "a").unwrap(); settle().await; sleep(SECOND / 2).await; @@ -406,7 +367,7 @@ async fn an_idle_track_is_flushed_within_the_bound() { // The open group continues in the next object. open.write_frame(ms(100), "b").unwrap(); - open.finish().unwrap(); + open.close().unwrap(); settle().await; let _video = video.finish().unwrap(); broadcast.close(); @@ -428,11 +389,13 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { subscribed(&video).await; subscribed(&data).await; - let mut timed_group = video.create_group(group::Info { sequence: 0 }).unwrap(); - let mut untimed_group = data.create_group(group::Info { sequence: 0 }).unwrap(); + let mut timed_group = video.create_group(group::Info { start: 0 }).unwrap(); + let mut untimed_group = data.create_group(group::Info { start: 0 }).unwrap(); for (pts, payload) in [(0, "a"), (100, "b")] { timed_group.write_frame(ms(pts), payload).unwrap(); - untimed_group.write_frame(None, payload).unwrap(); + untimed_group + .write_frame(moq_net::Timestamp::unscaled(0), payload) + .unwrap(); } settle().await; sleep(SECOND).await; @@ -441,13 +404,13 @@ async fn a_restart_mid_group_loses_and_duplicates_nothing() { // Both groups grow while nothing records them. timed_group.write_frame(ms(200), "c").unwrap(); - untimed_group.write_frame(None, "c").unwrap(); + untimed_group.write_frame(moq_net::Timestamp::unscaled(0), "c").unwrap(); let task = spawn(&store, &broadcast, config); settle().await; timed_group.write_frame(ms(300), "d").unwrap(); - untimed_group.write_frame(None, "d").unwrap(); - timed_group.finish().unwrap(); - untimed_group.finish().unwrap(); + untimed_group.write_frame(moq_net::Timestamp::unscaled(0), "d").unwrap(); + timed_group.close().unwrap(); + untimed_group.close().unwrap(); settle().await; let _video = video.finish().unwrap(); let _data = data.finish().unwrap(); @@ -493,7 +456,7 @@ async fn a_partial_group_resumes_after_its_last_stored_frame() { subscribed(&video).await; group(&video, 0, &[(0, "a")]); - let mut partial = video.create_group(group::Info { sequence: 1 }).unwrap(); + let mut partial = video.create_group(group::Info { start: 1 }).unwrap(); partial.start_at(2).unwrap(); partial.write_frame(ms(500), "c").unwrap(); settle().await; @@ -504,7 +467,7 @@ async fn a_partial_group_resumes_after_its_last_stored_frame() { let task = spawn(&store, &broadcast, config); settle().await; partial.write_frame(ms(600), "d").unwrap(); - partial.finish().unwrap(); + partial.close().unwrap(); settle().await; let _video = video.finish().unwrap(); broadcast.close(); @@ -587,9 +550,9 @@ async fn track_names_stay_one_segment_through_a_resume() { let task = spawn(&store, &broadcast, config().with_flush(SECOND)); for track in &tracks { subscribed(track).await; - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(None, "x").unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), "x").unwrap(); + group.close().unwrap(); } settle().await; sleep(SECOND).await; @@ -713,12 +676,12 @@ async fn markers_wait_for_the_listing_to_end() { subscribed(&audio).await; group(&video, 0, &[(0, "a")]); - let mut last = video.create_group(group::Info { sequence: 1 }).unwrap(); + let mut last = video.create_group(group::Info { start: 1 }).unwrap(); last.write_frame(ms(1000), "b").unwrap(); last.write_frame(ms(1100), "c").unwrap(); // A mid-group end: frames 0 and 1 of group 1. - video.finish_at(Position { group: 1, frame: 2 }, None).unwrap(); - last.finish().unwrap(); + video.finish_at(Position::new(1, 2), None).unwrap(); + last.close().unwrap(); settle().await; sleep(SECOND).await; // Ended, but a track re-added under the name would continue it. @@ -795,6 +758,40 @@ async fn a_re_added_track_continues_and_its_marker_records_the_last_run() { assert_eq!(end(&store, "video").await, Some(End::Error(64 + 3))); } +/// A re-added track that still holds the last group stored whole hands it back, since the next +/// run asks from one past it and a time start names the group containing that: it is passed +/// over rather than stored again. +#[tokio::test(start_paused = true)] +async fn a_re_added_track_holding_the_last_stored_group_stores_it_once() { + let store = store(); + let broadcast = broadcast::Info::new().produce(); + let first = broadcast.create_track("video", timed()).unwrap(); + let task = spawn(&store, &broadcast, config()); + subscribed(&first).await; + group(&first, 0, &[(0, "a")]); + group(&first, 1000, &[(1000, "b")]); + let first = first.finish().unwrap(); + settle().await; + drop(first); + settle().await; + + let second = broadcast.create_track("video", timed()).unwrap(); + group(&second, 1000, &[(1000, "b")]); + subscribed(&second).await; + group(&second, 2000, &[(2000, "c")]); + settle().await; + let _second = second.finish().unwrap(); + broadcast.close(); + finished(task).await; + + let frames: Vec = contents(&objects(&store, "video", true).await) + .into_iter() + .flatten() + .flat_map(|(_, frames)| frames) + .collect(); + assert_eq!(frames, ["0/a", "1000/b", "2000/c"]); +} + /// A track re-added with other properties after it was listed is refused before any of its /// frames is stored under the listed properties. #[tokio::test(start_paused = true)] diff --git a/rs/moq-archive/src/producer/track.rs b/rs/moq-archive/src/producer/track.rs index 65066905195..204d0a362ce 100644 --- a/rs/moq-archive/src/producer/track.rs +++ b/rs/moq-archive/src/producer/track.rs @@ -14,7 +14,6 @@ use object_store::path::Path; use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload}; use super::Shared; -use super::ordered::Ordered; use super::resume::Stored; use crate::layout::{self, End, Entry, Frame, Object}; use crate::{Error, Info, Result}; @@ -71,7 +70,7 @@ struct Building { enum Event { Frame(u64, moq_net::Result>), - Next(Option>), + Next(moq_net::Result>), Timer, } @@ -162,13 +161,13 @@ impl Track { let config = &shared.config; let range = match self.next { Some(next) => Range { - start: Anchor::Absolute(next.group), + start: Anchor::Time(next.group), end: None, }, None => Range::oldest(), }; // The cap, since `info` may describe an older instance than the one subscribed, and a - // backlog a smaller budget skipped is not sent again. The wait for a gap is narrowed below. + // backlog a smaller budget skipped is not sent again. The budget is narrowed below. let subscription = Subscription::default() .with_order(Order::Ascending) .with_delay(Delay::default().with_max(config.max_delay)) @@ -191,9 +190,15 @@ impl Track { self.properties(subscriber.info()).await?; let max_delay = delay(config, subscriber.info()); - let mut ordered = Ordered::new(subscriber, max_delay, self.next.map(|next| next.group)); + // Groups in key order: one the track still owes is waited for until the budget gives up on + // it, and one it skipped or dropped costs nothing. + let mut ordered = subscriber.ordered(); + let narrowed = ordered.subscription().with_delay(Delay::default().with_max(max_delay)); + ordered.update(narrowed)?; // The group being stored, frame by frame. let mut current: Option> = None; + // The error the groups ended with, if they did. + let mut failed = None; loop { self.flush_due(kio::clock::now())?; self.write().await?; @@ -211,11 +216,9 @@ impl Track { if let Poll::Ready(frame) = current.poll_read_frame(waiter) { return Poll::Ready(Event::Frame(index, frame)); } - // Arrival times decide the waits, so arrivals are taken meanwhile. - ordered.poll_arrivals(waiter); } None => { - if let Poll::Ready(next) = ordered.poll_next(waiter) { + if let Poll::Ready(next) = ordered.poll_next_group(waiter) { return Poll::Ready(Event::Next(next)); } } @@ -227,8 +230,11 @@ impl Track { let now = kio::clock::now(); match event { Event::Frame(index, read) => self.frame(&mut current, index, read, now)?, - Event::Next(Some(group)) => { - let sequence = group.sequence; + // A time start names the containing group, so a resumed read can hand back the group + // it last stored whole, keyed below where it resumes: that group is already stored. + Event::Next(Ok(Some(group))) if self.next.is_some_and(|next| group.start < next.group) => {} + Event::Next(Ok(Some(group))) => { + let sequence = group.start; // A resumed group continues after its stored frames. The clamp also starts a // group whose head is held elsewhere at its first frame, rather than lagging // behind it. @@ -237,20 +243,25 @@ impl Track { .filter(|next| next.group == sequence) .map_or(0, |next| next.frame); let group = Box::new(group.with_range(stored..)); - self.next = Some(Position { - group: sequence, - frame: group.index(), - }); + self.next = Some(Position::new(sequence, group.index())); current = Some(group); } - Event::Next(None) => break, + Event::Next(Ok(None)) => break, + // An end declared with an error reads as that error once its groups are read. + Event::Next(Err(err)) => { + failed = Some(err); + break; + } Event::Timer => {} } } - let end = match kio::wait(|waiter| ordered.poll_finished(waiter)).await { - Ok(end) => End::Finish(end.into()), - Err(err) => error(&err), + let end = match failed { + Some(err) => error(&err), + None => match kio::wait(|waiter| ordered.poll_finished(waiter)).await { + Ok(end) => End::Finish(end.into()), + Err(err) => error(&err), + }, }; self.close()?; self.write().await?; @@ -266,7 +277,7 @@ impl Track { read: moq_net::Result>, now: Instant, ) -> Result<()> { - let sequence = current.as_ref().expect("reading a group").sequence; + let sequence = current.as_ref().expect("reading a group").start; match read { Ok(Some(frame)) => { let frame = Frame { @@ -274,10 +285,7 @@ impl Track { payload: frame.payload, }; self.push(sequence, index, frame, now)?; - self.next = Some(Position { - group: sequence, - frame: index + 1, - }); + self.next = Some(Position::new(sequence, index + 1)); } ended => { if let Err(err) = ended { @@ -291,11 +299,10 @@ impl Track { } /// A frame's timestamp in the track's timescale, which it carries exactly when timed. - fn stamp(&self, timestamp: Option) -> Result> { - match (timestamp, self.timescale) { - (Some(timestamp), Some(timescale)) => { - Ok(Some(timestamp.convert(timescale).map_err(|_| Error::Timing)?.value())) - } + fn stamp(&self, timestamp: moq_net::Timestamp) -> Result> { + match (timestamp.scale(), self.timescale) { + (Some(_), Some(timescale)) => Ok(Some(timestamp.convert(timescale).map_err(|_| Error::Timing)?.value())), + // A scale-less frame's timestamp is its group's key, which the entry already stores. (None, None) => Ok(None), _ => Err(Error::Timing), } diff --git a/rs/moq-audio/src/decode/backpressure.rs b/rs/moq-audio/src/decode/backpressure.rs index 13b81a68e45..4b8b6fbc642 100644 --- a/rs/moq-audio/src/decode/backpressure.rs +++ b/rs/moq-audio/src/decode/backpressure.rs @@ -52,8 +52,11 @@ impl Backpressure { } /// Note that media at `timestamp`, on the consumer's timeline, sounds at `at`. + /// An unscaled timestamp carries no time, so it is ignored. pub fn observe(&self, timestamp: moq_net::Timestamp, at: Instant) { - self.update(|state| state.playhead = Some((Duration::from(timestamp), at))); + if let Ok(timestamp) = Duration::try_from(timestamp) { + self.update(|state| state.playhead = Some((timestamp, at))); + } } fn update(&self, f: impl FnOnce(&mut State)) { diff --git a/rs/moq-audio/src/decode/conformance.rs b/rs/moq-audio/src/decode/conformance.rs index f367a02fb17..4e5c0ab1c97 100644 --- a/rs/moq-audio/src/decode/conformance.rs +++ b/rs/moq-audio/src/decode/conformance.rs @@ -232,8 +232,9 @@ async fn consumer_preserves_reference_samples_and_timeline() { if let decode::Read::Frame(frame) = read { let start = *first.get_or_insert(frame.timestamp); let elapsed = Duration::from_secs_f64(pcm.len() as f64 / vector.channels as f64 / vector.rate as f64); - let actual = Duration::from(frame.timestamp) - .checked_sub(Duration::from(start)) + let actual = Duration::try_from(frame.timestamp) + .unwrap() + .checked_sub(Duration::try_from(start).unwrap()) .expect("monotonic PCM"); assert!( actual.abs_diff(elapsed) <= Duration::from_micros(1), diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index ff9062dfb2a..f3d0da8fe1f 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -394,14 +394,16 @@ impl Consumer { if self.end.is_some() && group != self.end_group { self.restart()?; } - // Nothing precedes group 0, and a group delivers from its frame 0 or not at - // all, so reading it first means decoding from the encoder's first packet. - // A finished group's end marker may come first; it carries no media. - (frame, first && group == 0) + // A group with nothing below it, which its publisher declares by skipping + // every start before it, is the track's first, and a group delivers from its + // frame 0 or not at all, so reading it first means decoding from the + // encoder's first packet. A finished group's end marker may come first; it + // carries no media. + (frame, first && self.track.is_skipped(0..group)) } // Behind whatever the resampler held from before the loss. Some(Encoded::Gap { timestamp, duration }) => { - self.boundary = Some(timestamp.into()); + self.boundary = Some(Duration::try_from(timestamp)?); self.started = true; self.restart()?; self.warmup_pending = !self.warmup.is_zero(); @@ -413,7 +415,7 @@ impl Consumer { // after the end, starting before it, once a fresh encoder or the track's end // drains it. Some(Encoded::End { group, end }) => { - self.boundary = Some(end.into()); + self.boundary = Some(Duration::try_from(end)?); self.end = Some(end); self.end_group = group; self.frames_decoded = 0; @@ -435,14 +437,17 @@ impl Consumer { // reading it off the packet, so there is nothing left to compare. if self.end.is_none() && let Some(next) = self.next_start - && discontinuous(next, timestamp) + && discontinuous(next, timestamp)? { if let Some(frame) = self.gap()? { self.ready.push_back(Read::Frame(frame)); } // From where the timeline should have continued to where it did, which an // overlap leaves empty. - let start = next.convert(timestamp.scale())?.min(timestamp); + let start = earlier( + next.convert(timestamp.scale().ok_or(moq_net::TimeOverflow)?)?, + timestamp, + ); self.ready.push_back(Read::Gap { timestamp: start, duration: timestamp.checked_sub(start)?, @@ -470,7 +475,8 @@ impl Consumer { // first packet has no history to warm up from: the codec's pre-skip, trimmed by // the decoder, is all the priming its encoder declared. if std::mem::take(&mut self.warmup_pending) && !track_start { - let warmup = moq_net::Timestamp::try_from(self.warmup)?.convert(timestamp.scale())?; + let warmup = moq_net::Timestamp::try_from(self.warmup)? + .convert(timestamp.scale().ok_or(moq_net::TimeOverflow)?)?; self.warmup_until = Some(timestamp.checked_add(warmup)?); } // Codec delay trimmed off the front is media this packet covered even @@ -489,7 +495,7 @@ impl Consumer { if let Some(end) = self.end { let terminal_start = *self .terminal_start - .get_or_insert(rewind(mux_frame.timestamp, self.delay_trimmed, rate)?.max(epoch)); + .get_or_insert(later(rewind(mux_frame.timestamp, self.delay_trimmed, rate)?, epoch)); let total = frames_between(terminal_start, end, rate)?; let remaining = total.saturating_sub(self.frames_decoded); decoded.truncate(remaining.saturating_mul(self.decoder.layout().channels() as usize)); @@ -502,7 +508,7 @@ impl Consumer { // The codec delay is padding before the epoch, not a hole after the // first short frame. Keep later output contiguous by moving it back over // everything trimmed since this decoder epoch began. - rewind(mux_frame.timestamp, self.delay_trimmed, rate)?.max(epoch) + later(rewind(mux_frame.timestamp, self.delay_trimmed, rate)?, epoch) }; if self.end.is_some() { self.frames_decoded += frames; @@ -593,10 +599,12 @@ impl Consumer { // Read before judging the head, so any update after that wakes the wait. let seen = self.gate.as_ref().map_or(0, |gate| gate.read().version); let deadline = match self.pending.front() { - Some((Encoded::Frame { frame, .. }, arrived)) => match self.admit(frame.timestamp.into(), *arrived) { - None => return Ok(self.pop()), - due => due, - }, + Some((Encoded::Frame { frame, .. }, arrived)) => { + match self.admit(Duration::try_from(frame.timestamp)?, *arrived) { + None => return Ok(self.pop()), + due => due, + } + } Some(_) => return Ok(self.pop()), None => match self.ended.as_mut() { Some(failed) => return failed.take().map_or(Ok(None), Err), @@ -632,6 +640,10 @@ impl Consumer { .await; match read { + // A scale-less track carries group keys, not times, so there is nothing to place. + Some(Ok(Some(Encoded::Frame { frame, .. }))) if frame.timestamp.scale().is_none() => { + self.ended = Some(Some(moq_net::Error::TimestampMismatch.into())); + } Some(Ok(Some(read))) => { // The estimator times arrivals, so it learns of a declared silence as the // packets around it arrive rather than as they decode. The end's own group @@ -658,10 +670,11 @@ impl Consumer { // decoder will refuse, so it is observed before anything can fail. match &read { Encoded::Frame { frame, .. } => { - let arrival = jitter::millis(now().duration_since(self.origin)); - let media = - frame.timestamp.value() as f64 * 1000.0 / frame.timestamp.scale().as_u64() as f64; - self.target.observe(arrival, media); + if let Some(scale) = frame.timestamp.scale() { + let arrival = jitter::millis(now().duration_since(self.origin)); + let media = frame.timestamp.value() as f64 * 1000.0 / scale.as_u64() as f64; + self.target.observe(arrival, media); + } } _ => self.breaks += 1, } @@ -695,7 +708,7 @@ impl Consumer { // Media queued behind the packet covers the hold, or nothing more continues it. let covered = self.breaks > 0 || matches!(self.pending.back(), Some((Encoded::Frame { frame, .. }, _)) - if Duration::from(frame.timestamp) >= timestamp + state.hold); + if Duration::try_from(frame.timestamp).is_ok_and(|end| end >= timestamp + state.hold)); if covered { return None; } @@ -828,7 +841,7 @@ impl Consumer { { let channels = self.decoder.layout().channels() as usize; let frames = pcm.len() / channels; - let duration = Duration::from(until.checked_sub(timestamp)?); + let duration = Duration::try_from(until.checked_sub(timestamp)?)?; let trim = (duration.as_nanos() * u128::from(self.resolved_sample_rate)).div_ceil(1_000_000_000); let trim = usize::try_from(trim).unwrap_or(usize::MAX).min(frames); pcm.drain(..trim * channels); @@ -888,11 +901,25 @@ fn now() -> Instant { /// finest quantization a packet can be assumed to have kept. That floor stays under /// the shortest packet anything here can send, 2.5 ms of Opus, so it never /// swallows a lost one. -fn discontinuous(expected: moq_net::Timestamp, timestamp: moq_net::Timestamp) -> bool { - let scale = expected.scale().max(timestamp.scale()); +fn discontinuous(expected: moq_net::Timestamp, timestamp: moq_net::Timestamp) -> Result { + let unscaled = moq_net::TimeOverflow; + let scale = expected + .scale() + .ok_or(unscaled)? + .max(timestamp.scale().ok_or(unscaled)?); let quantum = scale.min(moq_net::Timescale::MILLI); let tolerance = (scale.as_u64() as u128).div_ceil(quantum.as_u64() as u128) + 1; - expected.as_scale(scale).abs_diff(timestamp.as_scale(scale)) > tolerance + Ok(expected.as_scale(scale)?.abs_diff(timestamp.as_scale(scale)?) > tolerance) +} + +/// The later of two timestamps on one timeline, `a` when they don't compare. +fn later(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { + if b > a { b } else { a } +} + +/// The earlier of two timestamps on one timeline, `a` when they don't compare. +fn earlier(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { + if b < a { b } else { a } } /// `timestamp` moved forward by `frames` at `sample_rate`, in its own timescale. @@ -901,14 +928,16 @@ fn advance(timestamp: moq_net::Timestamp, frames: usize, sample_rate: u32) -> Re return Ok(timestamp); } - let offset = moq_net::Timestamp::from_scale(frames as u64, sample_rate as u64)?.convert(timestamp.scale())?; + let offset = moq_net::Timestamp::from_scale(frames as u64, sample_rate as u64)? + .convert(timestamp.scale().ok_or(moq_net::TimeOverflow)?)?; Ok(timestamp.checked_add(offset)?) } /// Codec-rate frames in the interval, rounding a microsecond marker to the nearest frame. fn frames_between(start: moq_net::Timestamp, end: moq_net::Timestamp, sample_rate: u32) -> Result { let duration = end.checked_sub(start)?; - let frames = (std::time::Duration::from(duration).as_nanos() * sample_rate as u128 + 500_000_000) / 1_000_000_000; + let frames = + (std::time::Duration::try_from(duration)?.as_nanos() * sample_rate as u128 + 500_000_000) / 1_000_000_000; usize::try_from(frames).map_err(|_| Error::Unsupported("audio duration does not fit in memory".into())) } @@ -921,10 +950,11 @@ fn rewind(timestamp: moq_net::Timestamp, frames: usize, sample_rate: u32) -> Res return Ok(timestamp); } - let offset = moq_net::Timestamp::from_scale(frames as u64, sample_rate as u64)?.convert(timestamp.scale())?; + let scale = timestamp.scale().ok_or(moq_net::TimeOverflow)?; + let offset = moq_net::Timestamp::from_scale(frames as u64, sample_rate as u64)?.convert(scale)?; Ok(timestamp .checked_sub(offset) - .unwrap_or(moq_net::Timestamp::new(0, timestamp.scale())?)) + .unwrap_or(moq_net::Timestamp::new(0, scale)?)) } #[cfg(test)] @@ -1158,7 +1188,7 @@ mod tests { let mut stamps = Vec::new(); let mut frames = 0; while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - stamps.push(frame.timestamp.as_micros()); + stamps.push(frame.timestamp.as_micros().unwrap()); frames += frame.data.len() / size_of::(); } (stamps, frames) @@ -1286,7 +1316,7 @@ mod tests { .await .unwrap(); let frame = next_frame(&mut consumer).await.unwrap().unwrap(); - assert_eq!(frame.timestamp.as_micros(), 80_000, "the backlog was replayed"); + assert_eq!(frame.timestamp.as_micros().unwrap(), 80_000, "the backlog was replayed"); } /// A latest join with a warmup loses no group the publisher bursts before its first @@ -1362,7 +1392,7 @@ mod tests { let mut stamps = Vec::new(); while let Some(read) = consumer.read().await.unwrap() { match read { - Read::Frame(frame) => stamps.push(frame.timestamp.as_micros()), + Read::Frame(frame) => stamps.push(frame.timestamp.as_micros().unwrap()), Read::Gap { timestamp, duration } => panic!("a gap of {duration:?} after {timestamp:?}"), Read::End { .. } => {} } @@ -1410,7 +1440,7 @@ mod tests { .unwrap(); let mut stamps = Vec::new(); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - stamps.push(frame.timestamp.as_micros()); + stamps.push(frame.timestamp.as_micros().unwrap()); } assert_eq!( stamps, @@ -1491,7 +1521,7 @@ mod tests { let mut stamps = Vec::new(); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - stamps.push(frame.timestamp.as_micros()); + stamps.push(frame.timestamp.as_micros().unwrap()); } assert_eq!(stamps, [1_080_000, 1_100_000]); } @@ -1538,7 +1568,7 @@ mod tests { .await .unwrap(); let frame = next_frame(&mut consumer).await.unwrap().unwrap(); - assert!(frame.timestamp.as_micros() >= 1_080_000); + assert!(frame.timestamp.as_micros().unwrap() >= 1_080_000); let first = f32::from_le_bytes(frame.data[..4].try_into().unwrap()); assert!( (first - 0.25).abs() < 0.0001, @@ -1592,7 +1622,7 @@ mod tests { .unwrap(); let (mut first, mut frames) = (None, 0); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - first.get_or_insert(frame.timestamp.as_micros()); + first.get_or_insert(frame.timestamp.as_micros().unwrap()); frames += frame.data.len() / size_of::(); } (first, frames) @@ -1658,7 +1688,7 @@ mod tests { assert!(next_frame(&mut consumer).await.is_err()); let mut stamps = Vec::new(); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - stamps.push(frame.timestamp.as_micros()); + stamps.push(frame.timestamp.as_micros().unwrap()); } assert_eq!(stamps, [100_000, 120_000]); } @@ -1711,7 +1741,7 @@ mod tests { let _finished = producer.finish().unwrap(); let mut stamps = Vec::new(); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - stamps.push(frame.timestamp.as_micros()); + stamps.push(frame.timestamp.as_micros().unwrap()); } assert_eq!(stamps.len(), 4, "each span's body and tail: {stamps:?}"); assert!( @@ -1809,7 +1839,7 @@ mod tests { // 312 samples at 48 kHz is 6.5 ms. for (micros, frames) in [(0, 960 - 312), (13_500, 960), (33_500, 960)] { let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); - assert_eq!(frame.timestamp.as_micros(), micros); + assert_eq!(frame.timestamp.as_micros().unwrap(), micros); assert_eq!(frame.data.len() / size_of::(), frames); } } @@ -1851,9 +1881,13 @@ mod tests { let mut reads = Vec::new(); while let Some(event) = consumer.read().await.unwrap() { reads.push(match event { - Read::Frame(frame) => format!("frame {}", frame.timestamp.as_micros()), - Read::Gap { timestamp, duration } => format!("gap {}+{}", timestamp.as_micros(), duration.as_micros()), - Read::End { end } => format!("end {}", end.as_micros()), + Read::Frame(frame) => format!("frame {}", frame.timestamp.as_micros().unwrap()), + Read::Gap { timestamp, duration } => format!( + "gap {}+{}", + timestamp.as_micros().unwrap(), + duration.as_micros().unwrap() + ), + Read::End { end } => format!("end {}", end.as_micros().unwrap()), }); } assert_eq!(reads, ["frame 0", "frame 20000", "gap 40000+460000", "frame 500000"]); @@ -1984,7 +2018,7 @@ mod tests { } let first = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); - assert_eq!(first.timestamp.as_micros(), 0); + assert_eq!(first.timestamp.as_micros().unwrap(), 0); // Continuity, not a fixed number: the second frame starts where the first // one's samples end, whatever they came to. Within a few frames rather than @@ -1994,7 +2028,7 @@ mod tests { let second = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); let first_frames = (first.data.len() / size_of::()) as u128; let ends_at = first_frames * 1_000_000 / 48_000; - let gap = second.timestamp.as_micros().abs_diff(ends_at); + let gap = second.timestamp.as_micros().unwrap().abs_diff(ends_at); assert!(gap < 100, "expected the frames to meet, got a {gap} us gap"); } @@ -2056,7 +2090,7 @@ mod tests { // It picks up where the first frame's samples ended, within the same few // frames of whole-frame rounding as above. let ends_at = (first_frames as u128) * 1_000_000 / 48_000; - let gap = tail.timestamp.as_micros().abs_diff(ends_at); + let gap = tail.timestamp.as_micros().unwrap().abs_diff(ends_at); assert!(gap < 100, "expected the tail to meet the body, got a {gap} us gap"); // Together they cover the packet and no more: 1024 frames at 44.1 kHz is @@ -2148,7 +2182,7 @@ mod tests { // what labelling by the packet most recently submitted does, since the // resampler is handing back audio from before that packet. It puts the // boundary a chunk early instead of a fraction of a chunk late. - let delay = actual.as_micros() as i128 - expected.as_micros() as i128; + let delay = actual.as_micros().unwrap() as i128 - expected.as_micros().unwrap() as i128; let chunk_us = 20_000i128; assert!( (0..chunk_us).contains(&delay), @@ -2207,7 +2241,10 @@ mod tests { let mut read = Vec::new(); while let Some(frame) = next_frame(&mut consumer).await.unwrap() { - read.push((frame.timestamp.as_micros(), frame.data.len() / size_of::())); + read.push(( + frame.timestamp.as_micros().unwrap(), + frame.data.len() / size_of::(), + )); } read } @@ -2260,7 +2297,7 @@ mod tests { kio::clock::timeout(wait, next_frame(consumer)) .await .ok() - .map(|frame| Duration::from(frame.unwrap().unwrap().timestamp)) + .map(|frame| Duration::try_from(frame.unwrap().unwrap().timestamp).unwrap()) }; // Ten packets cover 180 ms behind the first, short of the hold. @@ -2322,7 +2359,7 @@ mod tests { // The audio after the hole is stamped by its own packet. Rewinding over the // resampler's buffer instead would put it ~3 ms early, in the middle of the // hole, and splice the two sides together through the filter. - assert_eq!(read[2].0, stamps[1].as_micros()); + assert_eq!(read[2].0, stamps[1].as_micros().unwrap()); // And the hole is the packet that never arrived: 1024 frames at 44.1 kHz. let ends_at = read[1].0 + (read[1].1 as u128) * 1_000_000 / 48_000; @@ -2365,7 +2402,7 @@ mod tests { let read = pcm_gaps(44_100, 48_000, 882, &stamps).await; let mut r = crate::resample::Resampler::new(44_100, 48_000, 1, 882).unwrap(); r.process(&[0.25; 882], stamps[0]).unwrap(); - let expected = r.output_at(stamps[1]).unwrap().as_micros(); + let expected = r.output_at(stamps[1]).unwrap().as_micros().unwrap(); assert_eq!(read[1].0, expected); } @@ -2444,7 +2481,11 @@ mod tests { // The output begins with the first packet's samples, so it is stamped and // labelled from that packet. Rewinding from the terminal one instead drops it // most of a second into the future, carrying the DTX label with it. - assert_eq!(frame.timestamp.as_micros(), 0, "held samples moved with the jump"); + assert_eq!( + frame.timestamp.as_micros().unwrap(), + 0, + "held samples moved with the jump" + ); assert!( frame.activity.is_active(), "held samples took the terminal packet's label" @@ -2555,7 +2596,7 @@ mod tests { // The hole is real, so codec prediction starts over but stream-level pre-skip // does not. The audio is stamped where the packet says rather than 2.5 ms early. let second = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); - assert_eq!(second.timestamp.as_micros(), 22_500); + assert_eq!(second.timestamp.as_micros().unwrap(), 22_500); assert_eq!(second.data.len() / size_of::(), 960, "pre-skip was reapplied"); let third = next_frame(&mut consumer) @@ -2583,8 +2624,9 @@ mod tests { /// sample count either way, and an unoptimized Opus codec takes minutes over /// the whole corpus. [`opus_states_its_own_frame_duration`] covers Opus. The /// lowest common rate keeps each packet small, since only its duration counts. - /// With `hole`, the group after each end marker never arrives, so the consumer reads a - /// gap between the marker and what resumes. + /// With `hole`, the publisher leaves the starts between each end marker and what resumes + /// undeclared, so whatever lay there may have been lost: a group after the marker that never + /// arrived. async fn replay(case: &crate::jitter::tests::Case, hole: bool) -> Vec { use crate::jitter::tests::duration; @@ -2623,28 +2665,27 @@ mod tests { let payload: Bytes = vec![0u8; samples * size_of::()].into(); let mut series = Vec::with_capacity(case.arrival.len()); - // The sequence of the next group, one per packet and one per marker. - let mut sequence = 0; for (index, (&arrival, &media)) in case.arrival.iter().zip(&case.media).enumerate() { + // The track's timescale is microseconds, so this is also the packet group's start. + let micros = media as u64 * 1000; // The marker goes out as the utterance stops, so it is read during the silence // rather than alongside what resumes it. if case.ends.contains(&index) { producer.discontinuity().unwrap(); - sequence += 1; match consumer.read().await.unwrap() { Some(Read::End { .. }) => {} other => panic!("packet {index} is preceded by {other:?}, not its end"), } + // A seek creates the group without appending it, so nothing declares the starts + // before it skipped. if hole { - sequence += 1; - producer.seek(sequence).unwrap(); + producer.seek(micros).unwrap(); } } - sequence += 1; CLOCK.set(Some(origin.saturating_add(duration(arrival)))); producer .write(moq_mux::container::Frame { - timestamp: Timestamp::from_micros(media as u64 * 1000).unwrap(), + timestamp: Timestamp::from_micros(micros).unwrap(), duration: None, payload: payload.clone(), keyframe: true, @@ -2724,7 +2765,7 @@ mod tests { let start = kio::clock::now(); for &(at, sequence, media) in events { tokio::time::advance(Duration::from_millis(at).saturating_sub(start.elapsed())).await; - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); let frame = moq_mux::container::Frame { timestamp: Timestamp::from_micros(media * 1000).unwrap(), duration: None, @@ -2732,7 +2773,7 @@ mod tests { keyframe: true, }; container.write(&mut group, &[frame]).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); if read(sequence) { next_frame(&mut consumer).await.unwrap().expect("one frame per packet"); } @@ -3178,7 +3219,7 @@ mod tests { let frame = next_frame(&mut consumer).await.unwrap().expect("decoded frame"); // `as_micros`, not `==`: the CMAF path carries the fmp4 timescale and // `Timestamp`'s equality is structural, so the scales would have to match too. - assert_eq!(frame.timestamp.as_micros(), 0); + assert_eq!(frame.timestamp.as_micros().unwrap(), 0); let samples = Format::F32.as_interleaved_f32(&frame.data, 2).unwrap(); assert_eq!(samples.len(), (960 - 312) * 2); } diff --git a/rs/moq-audio/src/encode/backlog.rs b/rs/moq-audio/src/encode/backlog.rs index f840671ee8d..b59e27a20df 100644 --- a/rs/moq-audio/src/encode/backlog.rs +++ b/rs/moq-audio/src/encode/backlog.rs @@ -3,9 +3,17 @@ use std::time::Duration; use moq_net::Timestamp; +/// Where one codec frame's packet starts and ends, its codec priming folded in. +#[derive(Clone, Copy, Debug)] +pub(super) struct Stamp { + pub start: Timestamp, + pub end: Timestamp, +} + pub(super) struct Raw { pub samples: Vec, - pub timestamp: Timestamp, + pub stamp: Stamp, + /// Where the frame's input ends, which retention ages from. pub end: Timestamp, pub reset: bool, pub discontinuity: bool, @@ -56,7 +64,7 @@ impl Backlog { while self .frames .front() - .is_some_and(|first| Duration::from(first.end).saturating_add(age) <= Duration::from(now)) + .is_some_and(|first| time(first.end).saturating_add(age) <= time(now)) { self.evict(); } @@ -65,12 +73,12 @@ impl Backlog { /// Whether audio ending at `end` is past retention at `now`. #[cfg(any(feature = "capture", test))] pub fn expired(&self, end: Timestamp, now: Timestamp) -> bool { - Duration::from(end).saturating_add(self.retention) <= Duration::from(now) + time(end).saturating_add(self.retention) <= time(now) } #[cfg(any(feature = "capture", test))] pub fn deadline(&self) -> Option { - Some(Duration::from(self.frames.front()?.end).saturating_add(self.retention)) + Some(time(self.frames.front()?.end).saturating_add(self.retention)) } fn evict(&mut self) { @@ -98,6 +106,11 @@ impl Backlog { } } +/// A stamp the producer made itself, always in microseconds, or its broadcast clock's now. +fn time(timestamp: Timestamp) -> Duration { + Duration::try_from(timestamp).expect("the producer stamps at a scale") +} + #[cfg(test)] mod tests { use super::*; @@ -108,7 +121,10 @@ mod tests { for index in 0..200 { backlog.push(Raw { samples: vec![index as f32; 1920], - timestamp: Timestamp::ZERO, + stamp: Stamp { + start: Timestamp::ZERO, + end: Timestamp::ZERO, + }, end: Timestamp::from_micros(20_000).unwrap(), reset: false, discontinuity: false, diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index acfaff02e84..9001f4a8f85 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -415,7 +415,7 @@ fn stamp(clock: &moq_mux::Clock, captured: Instant) -> Result { Err(moq_mux::Error::InvalidCapture) => { let now = clock.now(); let ago = Instant::now().saturating_duration_since(captured).as_micros(); - if ago > now.as_micros() { + if ago > now.as_micros().expect("the broadcast clock is scaled") { moq_net::Timestamp::ZERO } else { now @@ -423,7 +423,7 @@ fn stamp(clock: &moq_mux::Clock, captured: Instant) -> Result { } timestamp => timestamp?, }; - Ok(timestamp.as_micros() as u64) + Ok(timestamp.as_micros()? as u64) } /// Encodes retained history while a subscriber wants it, and expires it otherwise. @@ -462,7 +462,8 @@ impl Retention<'_> { } let delay = producer .backlog_deadline() - .map(|at| at.saturating_sub(self.clock.now().into())); + .map(|at| at.saturating_sub(self.clock.now().try_into().expect("the broadcast clock is scaled"))); + tokio::select! { changed = self.demand.wanted_changed() => { changed?; } _ = async { @@ -728,7 +729,7 @@ mod tests { ); let encoded = packet(&mut track).await; assert_eq!( - encoded.timestamp.as_micros(), + encoded.timestamp.as_micros().unwrap(), captured.duration_since(epoch).as_micros() ); assert_eq!( @@ -751,7 +752,7 @@ mod tests { block.captured = Some(captured); samples.try_push(Ok(block)).unwrap(); observer.read().await.unwrap(); - assert_eq!(packet(&mut track).await.timestamp.as_micros(), 240_000); + assert_eq!(packet(&mut track).await.timestamp.as_micros().unwrap(), 240_000); encode_task.abort(); assert!(encode_task.await.unwrap_err().is_cancelled()); // The capture and independent observer remain live after the encoder goes away too. @@ -831,9 +832,9 @@ mod tests { // Only fresh capture, resuming after the discontinuity, reaches the subscriber. samples.try_push(Ok(Samples::plain(vec![0.5; 1920], true))).unwrap(); observer.read().await.unwrap(); - assert!(packet(&mut track).await.timestamp.as_micros() > 30_000_000); + assert!(packet(&mut track).await.timestamp.as_micros().unwrap() > 30_000_000); } else { - assert_eq!(packet(&mut track).await.timestamp.as_micros(), 100_000); + assert_eq!(packet(&mut track).await.timestamp.as_micros().unwrap(), 100_000); assert!(matches!(publication.state().status(), Status::Live)); } encode_task.abort(); @@ -877,7 +878,7 @@ mod tests { assert!(!publication.is_finished()); let track = if subscribe_late { let mut track = subscribe(&broadcast, Duration::from_secs(30)).await; - assert_eq!(packet(&mut track).await.timestamp.as_micros(), 100_000); + assert_eq!(packet(&mut track).await.timestamp.as_micros().unwrap(), 100_000); Some(track) } else { None @@ -922,6 +923,8 @@ mod tests { assert!(publication.is_finished()); } + /// A gap resumes on the shared capture clock, unless the block before it ran past that: a + /// timeline only moves forward, so it then resumes where that block's packet ends. #[tokio::test(start_paused = true)] async fn opus_uses_the_shared_capture_clock_including_gaps_and_clamping() { tokio::task::LocalSet::new() @@ -952,13 +955,14 @@ mod tests { samples.try_push(Ok(block)).unwrap(); let observed = observer.read().await.unwrap(); assert_eq!(observed.captured, captured.unwrap_or(now)); - assert_eq!(packet(&mut track).await.timestamp.as_micros(), expected); + assert_eq!(packet(&mut track).await.timestamp.as_micros().unwrap(), expected); let mut block = Samples::plain(vec![0.5; 1920], true); block.captured = Some(now); samples.try_push(Ok(block)).unwrap(); assert!(observer.read().await.unwrap().gap); let next = packet(&mut track).await; - assert_eq!(next.timestamp.as_micros(), 1_000_000); + // The first block is one 20 ms packet. + assert_eq!(next.timestamp.as_micros().unwrap(), (expected + 20_000).max(1_000_000)); encode_task.abort(); assert!(encode_task.await.unwrap_err().is_cancelled()); capture_task.abort(); diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index dbda8f6fd56..1d3fb4c6f89 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -8,7 +8,7 @@ use moq_mux::catalog::hang::CatalogExt; use moq_mux::container::Frame as MuxFrame; use moq_net::Timestamp; -use super::backlog::{Backlog, Raw}; +use super::backlog::{Backlog, Raw, Stamp}; use super::encoded::Encoded; use super::encoder::{Config, Encoder, Input}; use crate::resample::{Remix, Resampler}; @@ -100,11 +100,13 @@ pub struct Producer { frames_produced: u64, /// Stamps of the frames the codec has been given but has not yet returned a packet for, /// oldest first: it returns one packet per frame in order, however late. - in_flight: VecDeque, - /// Wall-clock anchor in microseconds, taken from the first frame after each - /// (re)start. Emitted PTS = `anchor + frames_produced / codec_rate`. `None` - /// until the first write so the next frame re-anchors to its timestamp. - anchor_us: Option, + in_flight: VecDeque, + /// Taken from the first frame after each (re)start. `None` until the first write so the + /// next frame re-anchors to its timestamp. + anchor: Option, + /// Where the latest stamped packet ends, in microseconds: the next anchor's floor, since + /// group starts strictly increase. + stamped: u64, /// An encoder reset that still needs a marker group before its next packet. pending_discontinuity: bool, /// Whether no packet has been published since the last marker group, or the track began, so a @@ -114,6 +116,16 @@ pub struct Producer { activity: Activity, } +/// Where a timeline begins: emitted PTS = `at + frames_produced / codec_rate`. +#[derive(Clone, Copy, Debug)] +struct Anchor { + /// The first frame's timestamp in microseconds, raised to `floor`. + at: u64, + /// Where the packets stamped before this timeline end, in microseconds. Codec priming folded + /// below `at` stops here. + floor: u64, +} + struct Terminal { packets: Vec, end: Timestamp, @@ -234,7 +246,8 @@ impl Reserved { grouped: 0, frames_produced: 0, in_flight: VecDeque::new(), - anchor_us: None, + anchor: None, + stamped: 0, pending_discontinuity: false, decoder_boundary: true, activity: Activity::Active, @@ -358,7 +371,7 @@ impl Producer { } fn reset_state(&mut self) { - self.anchor_us = None; + self.anchor = None; self.activity = Activity::Active; self.frames_produced = 0; self.pending.clear(); @@ -383,18 +396,29 @@ impl Producer { /// The first frame after construction (or [`reanchor`](Self::reanchor)) /// anchors the timeline: its timestamp becomes the anchor, and emitted PTS /// then advances purely by the running sample count, so subsequent frames' - /// timestamps are ignored. An idle gap is only reflected in the PTS if you - /// call [`reanchor`](Self::reanchor) on resume (which re-anchors from - /// the next frame's wall-clock stamp); writing straight across a gap without - /// resetting compresses it out. + /// timestamps are ignored. A timeline only moves forward, so an anchor before + /// where the packets already stamped end is raised to there. An idle gap is + /// only reflected in the PTS if you call [`reanchor`](Self::reanchor) on + /// resume (which re-anchors from the next frame's wall-clock stamp); writing + /// straight across a gap without resetting compresses it out. /// /// [`Frame::activity`] is ignored: the encoder classifies what it actually /// produced, which [`activity`](Self::activity) reports. pub fn write(&mut self, frame: &Frame) -> Result<(), Error> { self.check()?; - let timestamp_us = u64::try_from(frame.timestamp.as_micros()) - .map_err(|_| Error::Unsupported(format!("frame timestamp {:?} out of range", frame.timestamp)))?; - let anchor_us = *self.anchor_us.get_or_insert(timestamp_us); + let timestamp_us = (frame.timestamp.as_micros().ok()) + .and_then(|micros| u64::try_from(micros).ok()) + .ok_or_else(|| { + Error::Unsupported(format!( + "frame timestamp {:?} is unscaled or out of range", + frame.timestamp + )) + })?; + let floor = self.stamped; + let anchor = *self.anchor.get_or_insert(Anchor { + at: timestamp_us.max(floor), + floor, + }); let input = &self.input; let (format, channels) = (input.format, input.layout.channels()); @@ -410,7 +434,7 @@ impl Producer { self.pending.extend(pcm); - self.publish_full_frames(anchor_us) + self.publish_full_frames(anchor) } /// Encode and publish every full frame in `pending`, keeping any partial @@ -418,7 +442,7 @@ impl Producer { /// /// Frames are read in place and the remainder moves to the front once, so a /// large batch costs linear rather than quadratic copying. - fn publish_full_frames(&mut self, anchor_us: u64) -> Result<(), Error> { + fn publish_full_frames(&mut self, anchor: Anchor) -> Result<(), Error> { let frame_samples = self.encoder.frame_size() * self.encoder.codec_channels() as usize; // Taken so each frame can be borrowed while `self` encodes it; restored below on every path. let mut pending = std::mem::take(&mut self.pending); @@ -428,7 +452,7 @@ impl Producer { let frame = &pending[consumed..consumed + frame_samples]; // A frame that fails is consumed too, so it is never encoded twice. consumed += frame_samples; - result = self.publish_frame(anchor_us, frame); + result = self.publish_frame(anchor, frame); } pending.drain(..consumed); self.pending = pending; @@ -436,24 +460,18 @@ impl Producer { } /// Stamp one codec frame, then encode it now or keep it in the backlog. - fn publish_frame(&mut self, anchor_us: u64, samples: &[f32]) -> Result<(), Error> { - let timestamp = Self::timestamp( - anchor_us, - self.frames_produced, - self.encoder.folded_delay(), - self.encoder.codec_rate(), - )?; - self.frames_produced += self.encoder.frame_size() as u64; - let end = Self::timestamp(anchor_us, self.frames_produced, 0, self.encoder.codec_rate())?; + fn publish_frame(&mut self, anchor: Anchor, samples: &[f32]) -> Result<(), Error> { + let stamp = self.stamp(anchor)?; + let end = Self::timestamp(anchor, self.frames_produced, 0, self.encoder.codec_rate())?; let reset = std::mem::take(&mut self.reset_pending); let discontinuity = std::mem::take(&mut self.pending_discontinuity); if self.active && self.backlog.is_empty() { - return self.encode_frame(samples, timestamp, reset, discontinuity); + return self.encode_frame(samples, stamp, reset, discontinuity); } // Only a frame that waits needs its own copy. self.backlog.push(Raw { samples: samples.to_vec(), - timestamp, + stamp, end, reset, discontinuity, @@ -461,13 +479,17 @@ impl Producer { Ok(()) } - fn encode_frame( - &mut self, - samples: &[f32], - timestamp: Timestamp, - reset: bool, - discontinuity: bool, - ) -> Result<(), Error> { + /// Stamp the next codec frame on `anchor`, and count it produced. + fn stamp(&mut self, anchor: Anchor) -> Result { + let (delay, codec_rate) = (self.encoder.folded_delay(), self.encoder.codec_rate()); + let start = Self::timestamp(anchor, self.frames_produced, delay, codec_rate)?; + self.frames_produced += self.encoder.frame_size() as u64; + let end = Self::timestamp(anchor, self.frames_produced, delay, codec_rate)?; + self.stamped = end.as_micros()? as u64; + Ok(Stamp { start, end }) + } + + fn encode_frame(&mut self, samples: &[f32], stamp: Stamp, reset: bool, discontinuity: bool) -> Result<(), Error> { self.check()?; // A break before anything was published, such as history expiring before the first // subscriber, has nothing to separate. @@ -481,15 +503,15 @@ impl Producer { self.in_flight.clear(); } encoder.encode(samples)?; - self.publish_ready(Some(timestamp)) + self.publish_ready(Some(stamp)) } /// Publish every packet the codec has ready, each stamped with the frame it pairs with: /// the oldest held stamp, then `submitted`, the frame just encoded, which is queued only /// when the codec holds its packet, so a codec that holds nothing never fills the queue. - fn publish_ready(&mut self, mut submitted: Option) -> Result<(), Error> { + fn publish_ready(&mut self, mut submitted: Option) -> Result<(), Error> { while let Some(packet) = self.encoder.receive()? { - let Some(timestamp) = self.in_flight.pop_front().or_else(|| submitted.take()) else { + let Some(stamp) = self.in_flight.pop_front().or_else(|| submitted.take()) else { return Err(Error::Codec(format!( "{} returned more packets than it was given frames", self.encoder.name() @@ -498,7 +520,7 @@ impl Producer { self.activity = packet.activity; // The codec and its timestamp queue already advanced past this packet, so no later // write could stamp what follows: end the producer with the cause. - if let Err(err) = self.publish(packet, timestamp) { + if let Err(err) = self.publish(packet, stamp) { self.fail_track(); self.failed = Some(err.clone()); return Err(err); @@ -529,8 +551,8 @@ impl Producer { self.active = active; // One codec frame per turn lets capture keep draining while catching up. if active && let Some(raw) = self.backlog.pop() { - self.track.history(raw.timestamp); - self.encode_frame(&raw.samples, raw.timestamp, raw.reset, raw.discontinuity)?; + self.track.history(raw.stamp.start); + self.encode_frame(&raw.samples, raw.stamp, raw.reset, raw.discontinuity)?; } Ok(()) } @@ -539,13 +561,13 @@ impl Producer { /// are gone they expire on the same window, rather than reaching the track stale when it finishes. #[cfg(any(feature = "capture", test))] fn expire_partial(&mut self, now: Timestamp) -> Result<(), Error> { - let Some(anchor_us) = self.anchor_us else { return Ok(()) }; + let Some(anchor) = self.anchor else { return Ok(()) }; if !self.backlog.is_empty() { return Ok(()); } let channels = self.encoder.codec_channels() as usize; let frames = self.frames_produced + (self.pending.len() / channels) as u64; - let end = Self::timestamp(anchor_us, frames, 0, self.encoder.codec_rate())?; + let end = Self::timestamp(anchor, frames, 0, self.encoder.codec_rate())?; if self.backlog.expired(end, now) { self.reanchor(); } @@ -571,18 +593,18 @@ impl Producer { /// PTS of the frame `frames` samples past the anchor, stamped `delay` samples /// earlier to fold in codec priming the catalog can't signal. /// - /// Priming that would land before a zero anchor is stamped at zero instead: it - /// decodes to the codec's warm-up rather than to input, so only its spacing is - /// lost. - fn timestamp(anchor_us: u64, frames: u64, delay: usize, codec_rate: u32) -> Result { + /// Priming that would land before the anchor's floor, zero or where the previous + /// timeline's packets end, is stamped at the floor instead: it decodes to the + /// codec's warm-up rather than to input, so only its spacing is lost. + fn timestamp(anchor: Anchor, frames: u64, delay: usize, codec_rate: u32) -> Result { let frames = i128::from(frames) - delay as i128; let offset_us = (frames * 1_000_000).div_euclid(i128::from(codec_rate)); - let micros = (i128::from(anchor_us) + offset_us).max(0); + let micros = (i128::from(anchor.at) + offset_us).max(i128::from(anchor.floor)); let micros = u64::try_from(micros).map_err(|_| moq_net::TimeOverflow)?; Ok(Timestamp::from_micros(micros)?) } - fn publish(&mut self, encoded: Encoded, timestamp: Timestamp) -> Result<(), Error> { + fn publish(&mut self, encoded: Encoded, stamp: Stamp) -> Result<(), Error> { // Every packet decodes on its own, so a group may start at any of them: the first packet // after a cut opens one. Codecs recover independently after a dropped group. let keyframe = self.track.needs_keyframe(); @@ -590,7 +612,7 @@ impl Producer { self.grouped = 0; } let mux_frame = MuxFrame { - timestamp, + timestamp: stamp.start, payload: encoded.payload, keyframe, duration: None, @@ -598,10 +620,12 @@ impl Producer { self.track.write(mux_frame)?; self.grouped += self.encoder.frame_size() as u64; // Cut as soon as the group is long enough rather than when the next packet arrives, so - // the relay sees it finish without waiting. No boundary to give: the next packet bounds - // this one, and codec frames have a deterministic duration anyway. - if self.grouped >= self.group_samples { - self.track.cut(None)?; + // the relay sees it finish without waiting. The group ends whole where its last packet + // ends, which is where the next one starts, so a reader can tell the group starting there + // continues it. Priming stamped at the floor ends where it starts, as the packet after it + // does: group starts strictly increase, so the group stays open for the packet that moves on. + if self.grouped >= self.group_samples && stamp.end > stamp.start { + self.track.cut(Some(stamp.end))?; } Ok(()) } @@ -620,7 +644,8 @@ impl Producer { for (index, packet) in terminal.packets.into_iter().enumerate() { let offset = Timestamp::from_scale((index * terminal.frame_size) as u64, terminal.codec_rate as u64)? - .convert(terminal.start.scale())?; + .convert(terminal.start.scale().ok_or(moq_net::TimeOverflow)?)?; + let timestamp = terminal.start.checked_add(offset)?; track.write(MuxFrame { timestamp, @@ -687,8 +712,8 @@ impl Producer { // Preserve the final retained history after capture ends, even without live demand. self.active = true; while let Some(raw) = self.backlog.pop() { - self.track.history(raw.timestamp); - self.encode_frame(&raw.samples, raw.timestamp, raw.reset, raw.discontinuity)?; + self.track.history(raw.stamp.start); + self.encode_frame(&raw.samples, raw.stamp, raw.reset, raw.discontinuity)?; } if self.reset_pending { self.encoder.reset(); @@ -703,8 +728,12 @@ impl Producer { // The drained resampler tail can span multiple frames. Publish those first // so only the final partial frame reaches the encoder's terminal drain. - let anchor_us = self.anchor_us.unwrap_or(0); - self.publish_full_frames(anchor_us)?; + // Nothing written since a reanchor: whatever is left continues where the stamps end. + let anchor = self.anchor.unwrap_or(Anchor { + at: self.stamped, + floor: self.stamped, + }); + self.publish_full_frames(anchor)?; // What the codec still holds came from full frames, so it goes out at their stamps, leaving // the terminal drain's packets to start where the source's partial tail does. self.encoder.flush()?; @@ -728,9 +757,9 @@ impl Producer { let channels = self.encoder.codec_channels() as usize; let source_frames = self.pending.len() / channels; let delay = self.encoder.folded_delay(); - let start = Self::timestamp(anchor_us, self.frames_produced, delay, codec_rate)?; + let start = Self::timestamp(anchor, self.frames_produced, delay, codec_rate)?; // The source ends where it ends: priming only moves the packets carrying it. - let end = Self::timestamp(anchor_us, self.frames_produced + source_frames as u64, 0, codec_rate)?; + let end = Self::timestamp(anchor, self.frames_produced + source_frames as u64, 0, codec_rate)?; let finish = self.encoder.drain(&self.pending)?; let discard_padding = finish.discard_padding(); let packets = finish.into_packets(); @@ -748,10 +777,9 @@ impl Producer { )?; } else { for packet in packets { - let timestamp = Self::timestamp(anchor_us, self.frames_produced, delay, codec_rate)?; + let stamp = self.stamp(anchor)?; self.activity = packet.activity; - self.publish(packet, timestamp)?; - self.frames_produced += frame_size as u64; + self.publish(packet, stamp)?; } } Ok(()) @@ -926,12 +954,15 @@ mod tests { for index in 0..150 { // Capture buffers need not coincide with a codec frame, even across a demand gap. producer - .write(&pcm_frame(&vec![index as f32; 274], at(index).as_micros() as u64)) + .write(&pcm_frame( + &vec![index as f32; 274], + at(index).as_micros().unwrap() as u64, + )) .unwrap(); producer .write(&pcm_frame( &vec![index as f32; 1646], - at(index).as_micros() as u64 + 2854, + at(index).as_micros().unwrap() as u64 + 2854, )) .unwrap(); } @@ -967,12 +998,12 @@ mod tests { } producer.set_active(false, at(150)).unwrap(); producer - .write(&pcm_frame(&vec![0.5; 1920], at(150).as_micros() as u64)) + .write(&pcm_frame(&vec![0.5; 1920], at(150).as_micros().unwrap() as u64)) .unwrap(); producer.set_active(true, at(151)).unwrap(); assert_eq!(packet(&mut reader).await.unwrap().timestamp, at(150)); producer - .write(&pcm_frame(&vec![0.5; 1920], at(151).as_micros() as u64)) + .write(&pcm_frame(&vec![0.5; 1920], at(151).as_micros().unwrap() as u64)) .unwrap(); assert_eq!(packet(&mut reader).await.unwrap().timestamp, at(151)); let _finished = producer.finish().unwrap(); @@ -996,7 +1027,10 @@ mod tests { producer.set_active(false, at(0)).unwrap(); for index in 0..150 { producer - .write(&pcm_frame(&vec![index as f32; 1920], at(index).as_micros() as u64)) + .write(&pcm_frame( + &vec![index as f32; 1920], + at(index).as_micros().unwrap() as u64, + )) .unwrap(); } let track = consumer @@ -1037,13 +1071,13 @@ mod tests { producer.set_active(false, at(0)).unwrap(); for index in 0..50 { producer - .write(&pcm_frame(&vec![0.1; 1920], at(index).as_micros() as u64)) + .write(&pcm_frame(&vec![0.1; 1920], at(index).as_micros().unwrap() as u64)) .unwrap(); } // Another track already reached the live edge. for index in 45..50 { other - .write(&pcm_frame(&vec![0.1; 1920], at(index).as_micros() as u64)) + .write(&pcm_frame(&vec![0.1; 1920], at(index).as_micros().unwrap() as u64)) .unwrap(); } loop { @@ -1054,7 +1088,7 @@ mod tests { } for index in 50..55 { producer - .write(&pcm_frame(&vec![0.1; 1920], at(index).as_micros() as u64)) + .write(&pcm_frame(&vec![0.1; 1920], at(index).as_micros().unwrap() as u64)) .unwrap(); } let producer = producer.finish().unwrap(); @@ -1108,7 +1142,7 @@ mod tests { // The terminal packet carries its own end; the marker ends the epoch before the gap, ahead of it. let next = reader.read().await.unwrap().unwrap(); assert!( - matches!(next, moq_mux::container::Read::End { end, .. } if end.as_micros() < 1_040_000), + matches!(next, moq_mux::container::Read::End { end, .. } if end.as_micros().unwrap() < 1_040_000), "the final partial frame joined the epoch before the gap: {next:?}" ); } @@ -1150,7 +1184,7 @@ mod tests { let first = reader.read().await.unwrap().unwrap(); assert!( - matches!(first, moq_mux::container::Read::Frame { ref frame, .. } if frame.timestamp.as_micros() == 1_000_000), + matches!(first, moq_mux::container::Read::Frame { ref frame, .. } if frame.timestamp.as_micros().unwrap() == 1_000_000), "{first:?}" ); // Nothing retained reaches the track after it expired. @@ -1590,7 +1624,7 @@ mod tests { let mut resumed_frames = 0; while let Some(frame) = decoded_frame(&mut audio).await.unwrap() { - if frame.timestamp.as_micros() >= 1_000_000 { + if frame.timestamp.as_micros().unwrap() >= 1_000_000 { resumed_frames += frame.data.len() / size_of::(); } } @@ -1694,7 +1728,7 @@ mod tests { } producer.write(frame).unwrap(); let read = packet(&mut reader).await.expect("a packet per full frame"); - pts.push(read.timestamp.as_micros()); + pts.push(read.timestamp.as_micros().unwrap()); } pts } @@ -1854,9 +1888,8 @@ mod tests { assert!(partial.read_frame().await.unwrap().is_none()); let mut resumed = track.recv_group().await.unwrap().expect("resumed group"); assert_eq!( - resumed.sequence, - partial.sequence + 2, - "one marker separates the anchors" + resumed.start, 1_000_000, + "the resumed group starts at its first packet, in the track's microseconds" ); for _ in 0..5 { assert!(resumed.read_frame().await.unwrap().is_some()); @@ -1909,7 +1942,14 @@ mod tests { producer.write(&pcm_frame(&[0.1; 4 * 1024], anchor_us)).unwrap(); let mut pts = Vec::new(); for _ in 0..4 { - pts.push(packet(&mut reader).await.expect("a packet").timestamp.as_micros()); + pts.push( + packet(&mut reader) + .await + .expect("a packet") + .timestamp + .as_micros() + .unwrap(), + ); } pts } @@ -1920,6 +1960,63 @@ mod tests { assert_eq!(pts(0).await, vec![0, 0, 0, 20_000]); } + /// Group starts strictly increase, so a reanchor never stamps below where the packets before + /// it end: an AAC resume closer than its encoder delay stamps the priming there, and a resume + /// that rewinds starts there. + #[tokio::test] + async fn a_reanchor_resumes_no_earlier_than_the_packets_before_it_end() { + async fn pts(resume_us: u64) -> Vec { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let consumer = broadcast.consume(); + + let input = Input::new(48_000, Layout::Mono); + let options = Options { + track: Some("audio".to_string()), + config: Config::from_input(crate::encode::Codec::Aac, &input), + ..Options::default() + }; + let stub = crate::encode::backend::stub::install(); + let mut producer = Producer::new(&mut broadcast, catalog, input, &options).unwrap(); + drop(stub); + + let track = consumer + .track("audio") + .unwrap() + .subscribe( + moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(1))), + ) + .await + .unwrap(); + let mut reader = moq_mux::container::Consumer::new( + track, + moq_mux::container::legacy::Wire(moq_mux::container::Kind::Audio), + ); + + // Stamped 956_000 to 1_020_000, the last packet ending at 1_041_333. + producer.write(&pcm_frame(&[0.1; 4 * 1024], 1_000_000)).unwrap(); + producer.reanchor(); + producer.write(&pcm_frame(&[0.1; 4 * 1024], resume_us)).unwrap(); + let mut pts = Vec::new(); + for _ in 0..8 { + pts.push( + packet(&mut reader) + .await + .expect("a packet") + .timestamp + .as_micros() + .unwrap(), + ); + } + pts.split_off(4) + } + + // Priming would start at 1_006_000. + assert_eq!(pts(1_050_000).await, vec![1_041_333, 1_041_333, 1_048_666, 1_070_000]); + assert_eq!(pts(1_000_000).await, vec![1_041_333, 1_041_333, 1_041_333, 1_061_333]); + } + /// A codec that holds frames returns their packets late, and each still goes out at the stamp /// of the frame it encoded, including the one only `finish` flushes out ahead of the drain. #[tokio::test] @@ -1957,7 +2054,7 @@ mod tests { let mut published = Vec::new(); while let Some(frame) = packet(&mut reader).await { let index = u64::from_be_bytes(frame.payload[..].try_into().unwrap()); - published.push((index, frame.timestamp.as_micros())); + published.push((index, frame.timestamp.as_micros().unwrap())); } // Each frame is stamped 2112 frames of priming early, the drain's three silent ones too. assert_eq!( @@ -2015,7 +2112,7 @@ mod tests { producer.write(&pcm_frame(&vec![0.1; 44_100], 1_000_000)).unwrap(); let first = packet(&mut reader).await.expect("a packet"); - assert_eq!(first.timestamp.as_micros(), 1_000_000); + assert_eq!(first.timestamp.as_micros().unwrap(), 1_000_000); } #[tokio::test] diff --git a/rs/moq-audio/src/resample.rs b/rs/moq-audio/src/resample.rs index da6120245da..f95b76f73ff 100644 --- a/rs/moq-audio/src/resample.rs +++ b/rs/moq-audio/src/resample.rs @@ -141,11 +141,10 @@ impl Resampler { .checked_sub(self.output_frames as u128 * self.input_rate as u128) .ok_or_else(|| Error::Resample("output exceeds input duration".into()))?; let held_frames = u64::try_from(held_frames).map_err(|_| Error::Resample("held duration overflow".into()))?; + let scale = held.scale().ok_or(moq_net::TimeOverflow)?; let offset = moq_net::Timestamp::from_scale(held_frames, self.input_rate as u64 * self.output_rate as u64)? - .convert(held.scale())?; - Ok(held - .checked_sub(offset) - .unwrap_or(moq_net::Timestamp::new(0, held.scale())?)) + .convert(scale)?; + Ok(held.checked_sub(offset).unwrap_or(moq_net::Timestamp::new(0, scale)?)) } /// Drop everything held, buffered input and filter state alike, returning to @@ -266,8 +265,9 @@ impl Resampler { // Convert its total consumed duration once to preserve fractional progress. if self.pending.len() < buffered { let consumed = (samples.len() - self.pending.len()) / self.channels; - let elapsed = - moq_net::Timestamp::from_scale(consumed as u64, self.input_rate as u64)?.convert(at.scale())?; + let elapsed = moq_net::Timestamp::from_scale(consumed as u64, self.input_rate as u64)? + .convert(at.scale().ok_or(moq_net::TimeOverflow)?)?; + self.held = Some(at.checked_add(elapsed)?); } diff --git a/rs/moq-audio/tests/integration/roundtrip.rs b/rs/moq-audio/tests/integration/roundtrip.rs index 3baf277fd47..0999b604b16 100644 --- a/rs/moq-audio/tests/integration/roundtrip.rs +++ b/rs/moq-audio/tests/integration/roundtrip.rs @@ -101,7 +101,7 @@ async fn opus_roundtrip(case: Case) { let expected = 123_000 + pcm.len() as u128 / channels as u128 * 1_000_000 / case.rate as u128; // Microsecond stamping rounds each chunk independently. assert!( - frame.timestamp.as_micros().abs_diff(expected) <= 1, + frame.timestamp.as_micros().unwrap().abs_diff(expected) <= 1, "{:?} versus {expected}us", frame.timestamp ); diff --git a/rs/moq-bench/src/connection.rs b/rs/moq-bench/src/connection.rs index 5a413ac6b8e..7ae5d84b385 100644 --- a/rs/moq-bench/src/connection.rs +++ b/rs/moq-bench/src/connection.rs @@ -50,6 +50,8 @@ struct GroupHeader<'a> { /// broadcast it didn't publish. Extra fields on the wire are ignored. #[derive(Deserialize)] struct RecvHeader { + /// The publisher's count of its groups, which group starts, being times, don't give. + group: u64, fps: u64, frame_size: u64, group_size: u64, @@ -226,6 +228,12 @@ pub async fn run(ctx: Connection) { } } +/// Milliseconds on this process's monotonic clock since its origin, what frames are stamped with. +fn millis() -> u64 { + let elapsed = kio::clock::now().saturating_duration_since(kio::clock::origin().0); + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) +} + /// Produce frames for one track at `fps`, opening a new group every `group_size` /// frames. Each group starts with a JSON keyframe; the rest are zeroed. async fn produce( @@ -250,11 +258,15 @@ async fn produce( ticker.set_missed_tick_behavior(kio::clock::MissedTickBehavior::Delay); let mut sequence = 0u64; + let mut previous = None; loop { - let mut group = track.append_group()?; - // Keyframe: the JSON header describing this connection's rolled parameters. ticker.tick().await; + // A group starts at its keyframe, in the track's milliseconds. Starts strictly increase, + // so one landing in the same millisecond as the last starts a millisecond later. + let start = previous.map_or(millis(), |previous: u64| millis().max(previous + 1)); + previous = Some(start); + let mut group = track.append_group(start)?; let mut header = GroupHeader { connection, broadcast: &path, @@ -280,17 +292,17 @@ async fn produce( payload = serde_json::to_vec(&header)?; } let header = Bytes::from(payload); - group.write_frame(moq_net::Timestamp::now(), header.clone())?; + group.write_frame(moq_net::Timestamp::from_millis(start)?, header.clone())?; stats.frame_sent(header.len()); // The remaining frames in the group are zeroed payload. for _ in 0..rolled.group_size { ticker.tick().await; - group.write_frame(moq_net::Timestamp::now(), zeros.clone())?; + group.write_frame(moq_net::Timestamp::from_millis(millis().max(start))?, zeros.clone())?; stats.frame_sent(zeros.len()); } - group.finish()?; + group.close()?; if sequence == 0 { published.establish(); } @@ -462,8 +474,9 @@ async fn drain(broadcast: broadcast::Consumer, max_delay: Duration, stats: &Stat // `recv_group` yields groups in arrival order, including out of sequence, so we // can spot holes. `next_group` would silently drop late arrivals and hide them. while let Some(mut group) = track.recv_group().await? { - let sequence = group.sequence; - match read_group(&mut group, &mut learned_shape, stats).await { + // Group starts are times, so holes are counted by the number the keyframe header carries. + let mut sequence = None; + match read_group(&mut group, &mut learned_shape, &mut sequence, stats).await { // Only a group read end to end counts as delivered. Ok(()) => { if established.is_none() { @@ -474,20 +487,27 @@ async fn drain(broadcast: broadcast::Consumer, max_delay: Duration, stats: &Stat // The relay failed this one group: `Error::Lagged` once we fall behind. Err(err) => { gaps.fail(sequence); - tracing::debug!(sequence, %err, "group ended early"); + tracing::debug!(start = group.start, ?sequence, %err, "group ended early"); } } } Ok(()) } -/// Read one group to its end, counting every frame and sampling the keyframe header. -async fn read_group(group: &mut group::Consumer, learned_shape: &mut bool, stats: &Stats) -> moq_net::Result<()> { +/// Read one group to its end, counting every frame and sampling the keyframe header, whose +/// group number lands in `sequence`. +async fn read_group( + group: &mut group::Consumer, + learned_shape: &mut bool, + sequence: &mut Option, + stats: &Stats, +) -> moq_net::Result<()> { let mut first = true; while let Some(frame) = group.read_frame().await? { // The first frame of every group is the JSON keyframe. Parse it once to // learn the publisher's shape (we may be watching a peer, not ourselves). if first && let Ok(header) = serde_json::from_slice::(&frame.payload) { + *sequence = Some(header.group); stats.latency(header.timestamp_ms); if !*learned_shape { tracing::debug!( @@ -565,13 +585,20 @@ impl<'a> GapTracker<'a> { } } - fn complete(&mut self, sequence: u64) { + /// A group read end to end, numbered `sequence` when its header said. + fn complete(&mut self, sequence: Option) { self.stats.groups_recv.fetch_add(1, Ordering::Relaxed); - self.record(sequence, true); + if let Some(sequence) = sequence { + self.record(sequence, true); + } } - fn fail(&mut self, sequence: u64) { - self.record(sequence, false); + /// A group that failed. One that failed before its header named it is counted once the + /// frontier passes its number instead. + fn fail(&mut self, sequence: Option) { + if let Some(sequence) = sequence { + self.record(sequence, false); + } } fn record(&mut self, sequence: u64, complete: bool) { @@ -707,22 +734,22 @@ mod tests { let stats = stats.clone(); tokio::spawn(async move { drain(consumer, Duration::ZERO, &stats).await }) }; - let mut failed = track.append_group().unwrap(); + let mut failed = track.append_group(0).unwrap(); failed - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(b"{}")) + .write_frame(moq_net::Timestamp::ZERO, Bytes::from_static(b"{}")) .unwrap(); wait_for(&stats.frames_recv, 1).await; assert!(!stats.ready()); assert_eq!(stats.subscriptions.load(Ordering::Relaxed), 1); assert_eq!(stats.subscriptions_established.load(Ordering::Relaxed), 0); failed.abort(moq_net::Error::Lagged).unwrap(); - let mut complete = track.append_group().unwrap(); + let mut complete = track.append_group(1).unwrap(); complete - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(b"{}")) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), Bytes::from_static(b"{}")) .unwrap(); wait_for(&stats.frames_recv, 2).await; assert!(!stats.ready()); - complete.finish().unwrap(); + complete.close().unwrap(); wait_for(&stats.subscriptions_established, 1).await; assert!(stats.ready()); let _track = track.finish().unwrap(); @@ -807,6 +834,18 @@ mod tests { .with_max_age(None) } + /// A keyframe header numbering its group `group`. + fn header(group: u64) -> Bytes { + let header = serde_json::json!({ + "group": group, + "fps": 1, + "frame_size": 0, + "group_size": 0, + "timestamp_ms": 0, + }); + Bytes::from(serde_json::to_vec(&header).unwrap()) + } + async fn wait_for(counter: &AtomicU64, value: u64) { while counter.load(Ordering::Relaxed) < value { tokio::task::yield_now().await; @@ -1011,11 +1050,11 @@ mod tests { .route("bench/current/0/0", moq_net::origin::RouteOptions::default()) .unwrap(); let track = peer.broadcast().create_track(TRACK, timed()).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(b"{}")) + .write_frame(moq_net::Timestamp::ZERO, Bytes::from_static(b"{}")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let announced = discover(&origin, "bench/current"); @@ -1045,7 +1084,7 @@ mod tests { .unwrap(); let track = broadcast.broadcast().create_track(TRACK, timed()).unwrap(); tokio::task::yield_now().await; - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); let header = serde_json::json!({ "fps": 1, "frame_size": 200, @@ -1053,9 +1092,9 @@ mod tests { "timestamp_ms": 0, }); group - .write_frame(moq_net::Timestamp::now(), serde_json::to_vec(&header).unwrap()) + .write_frame(moq_net::Timestamp::ZERO, serde_json::to_vec(&header).unwrap()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); task.await.unwrap().unwrap(); @@ -1070,12 +1109,13 @@ mod tests { /// measured under fewer subscribers than it was asked to serve. #[tokio::test] async fn drain_survives_a_failed_group() { - fn write_group(track: &mut track::Producer) { - let mut group = track.append_group().unwrap(); + // Group `n` starts at `n` milliseconds. + fn write_group(track: &mut track::Producer, n: u64) { + let mut group = track.append_group(n).unwrap(); group - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(b"{}")) + .write_frame(moq_net::Timestamp::from_millis(n).unwrap(), header(n)) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let stats = Arc::new(Stats::default()); @@ -1084,7 +1124,7 @@ mod tests { let consumer = broadcast.consume(); // Group 0 lands intact. - write_group(&mut track); + write_group(&mut track, 0); let task = { let stats = stats.clone(); @@ -1098,17 +1138,17 @@ mod tests { // Group 1 opens and is picked up by the drain, then the relay gives up on it. // Aborting a group nobody is reading drops its cached frames, so wait for the // drain to be parked inside it before failing it. - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(1).unwrap(); group - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(b"{}")) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), header(1)) .unwrap(); wait_for(&stats.frames_recv, 2).await; group.abort(moq_net::Error::Lagged).unwrap(); // Groups 2 and 3 land intact, then the publisher is done. - write_group(&mut track); + write_group(&mut track, 2); wait_for(&stats.groups_recv, 2).await; - write_group(&mut track); + write_group(&mut track, 3); wait_for(&stats.groups_recv, 3).await; let _track = track.finish().unwrap(); broadcast.close(); @@ -1127,13 +1167,15 @@ mod tests { /// asks for the next group, group 1 already trails group 2 by 500ms of media /// time. Timestamps are explicit, so no wall clock is involved. async fn lost_to_an_overtaken_group(max_delay: Duration) -> u64 { - fn write_group(track: &mut track::Producer, millis: &[u64]) { - let mut group = track.append_group().unwrap(); - for &millis in millis { + // Group `n` starts at its first frame, which carries the header numbering it. + fn write_group(track: &mut track::Producer, n: u64, millis: &[u64]) { + let mut group = track.append_group(millis[0]).unwrap(); + for (i, &millis) in millis.iter().enumerate() { let at = moq_net::Timestamp::from_millis(millis).unwrap(); - group.write_frame(at, Bytes::from_static(b"{}")).unwrap(); + let payload = if i == 0 { header(n) } else { Bytes::from_static(b"{}") }; + group.write_frame(at, payload).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let stats = Arc::new(Stats::default()); @@ -1146,18 +1188,18 @@ mod tests { tokio::spawn(async move { drain(consumer, max_delay, &stats).await }) }; - let mut first = track.append_group().unwrap(); + let mut first = track.append_group(1_000).unwrap(); let at = moq_net::Timestamp::from_millis(1_000).unwrap(); - first.write_frame(at, Bytes::from_static(b"{}")).unwrap(); + first.write_frame(at, header(0)).unwrap(); wait_for(&stats.frames_recv, 1).await; - write_group(&mut track, &[1_100]); - write_group(&mut track, &[1_500, 1_600]); - first.finish().unwrap(); + write_group(&mut track, 1, &[1_100]); + write_group(&mut track, 2, &[1_500, 1_600]); + first.close().unwrap(); wait_for(&stats.groups_recv, 2).await; // A last group moves the live frontier past group 2, so 1 and 2 are accounted. - write_group(&mut track, &[2_000]); + write_group(&mut track, 3, &[2_000]); let _track = track.finish().unwrap(); broadcast.close(); @@ -1213,7 +1255,7 @@ mod tests { let stats = Stats::default(); let mut gaps = GapTracker::new(&stats); for seq in [0, 1, 3, 4] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!(stats.groups_recv.load(Ordering::Relaxed), 4); assert_eq!(lost(&stats), 1); @@ -1227,17 +1269,17 @@ mod tests { let mut gaps = GapTracker::new(&stats); for seq in [0, 1, 3, 4] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!((lost(&stats), outstanding(&stats)), (1, 1)); - gaps.complete(2); + gaps.complete(Some(2)); assert_eq!(stats.groups_recv.load(Ordering::Relaxed), 5); assert_eq!((lost(&stats), outstanding(&stats)), (1, 0), "the late group stays lost"); // 6 is skipped; 8 settles 7, confirming it. for seq in [5, 7, 8] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!((lost(&stats), outstanding(&stats)), (2, 1)); } @@ -1248,7 +1290,7 @@ mod tests { let stats = Stats::default(); let mut gaps = GapTracker::new(&stats); for seq in [0, 3, 1, 2, 4] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!(stats.groups_recv.load(Ordering::Relaxed), 5); assert_eq!((lost(&stats), outstanding(&stats)), (0, 0)); @@ -1264,13 +1306,13 @@ mod tests { // Joins live at 10; 11 and 12 settle the span from 10. for seq in [10, 11, 12] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!(stats.groups_expected.load(Ordering::Relaxed), 2); // Cached history arrives behind the live frontier, leaving 0..=4 and 6 unsent. for seq in [5, 7, 8, 9] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!(stats.groups_recv.load(Ordering::Relaxed), 7); assert_eq!(history(&stats), 4); @@ -1283,12 +1325,12 @@ mod tests { // A skip after the start is still loss: 13 never arrives, 15 settles 14. for seq in [14, 15] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!((lost(&stats), outstanding(&stats)), (1, 1)); // A failed history group is lost wherever it falls. - gaps.fail(3); + gaps.fail(Some(3)); assert_eq!(history(&stats), 4, "a failed group is not history"); assert_eq!((lost(&stats), outstanding(&stats)), (2, 2)); } @@ -1302,12 +1344,12 @@ mod tests { // 3 is missing, 4 is the live frontier: nothing is settled past 2 yet. for seq in [0, 1, 2, 4] { - gaps.complete(seq); + gaps.complete(Some(seq)); } assert_eq!(lost(&stats), 0); // 5 advances the frontier, settling 4 and confirming 3 was skipped. - gaps.complete(5); + gaps.complete(Some(5)); assert_eq!(lost(&stats), 1); } @@ -1317,11 +1359,11 @@ mod tests { let stats = Stats::default(); let mut gaps = GapTracker::new(&stats); - gaps.fail(0); + gaps.fail(Some(0)); assert_eq!(stats.groups_recv.load(Ordering::Relaxed), 0); assert_eq!(lost(&stats), 1); - gaps.complete(1); + gaps.complete(Some(1)); assert_eq!( lost(&stats), 1, @@ -1335,12 +1377,12 @@ mod tests { let stats = Stats::default(); let mut gaps = GapTracker::new(&stats); - gaps.complete(0); - gaps.fail(1); + gaps.complete(Some(0)); + gaps.fail(Some(1)); assert_eq!(stats.groups_recv.load(Ordering::Relaxed), 1); assert_eq!(lost(&stats), 1); - gaps.complete(2); + gaps.complete(Some(2)); assert_eq!( lost(&stats), 1, diff --git a/rs/moq-boy/src/input.rs b/rs/moq-boy/src/input.rs index b925d443010..bb4d739b789 100644 --- a/rs/moq-boy/src/input.rs +++ b/rs/moq-boy/src/input.rs @@ -60,12 +60,23 @@ impl Replies { /// Send the viewer its latency breakdown. pub fn latency(&mut self, latency: &[LatencyEntry]) { let reply = serde_json::to_vec(&Reply { latency }).expect("a reply always serializes"); - if let Err(err) = self.0.write_frame(None, reply) { + if let Err(err) = send(&self.0, reply) { tracing::debug!(%err, "viewer reply dropped"); } } } +/// Send `message` on a session track as a group of its own. +/// +/// A session track carries no timescale, so its groups have no time to start at: each starts one +/// past the newest the track holds. +fn send(track: &moq_net::track::Producer, message: Vec) -> moq_net::Result<()> { + let mut group = track.append_group(track.largest().map_or(0, |largest| largest.group + 1))?; + group.write_frame(moq_net::Timestamp::unscaled(group.start), message)?; + group.close()?; + Ok(()) +} + /// A viewer's session event, in the order the emulator applies them. pub enum Command { /// A viewer's session opened. @@ -232,7 +243,7 @@ mod tests { } fn send(session: &mut TrackStream, command: &str) { - session.producer.write_frame(None, command.as_bytes().to_vec()).unwrap(); + super::send(&session.producer, command.as_bytes().to_vec()).unwrap(); } async fn recv(rx: &mut tokio::sync::mpsc::Receiver) -> Command { @@ -284,12 +295,12 @@ mod tests { }; // The reader parks on this group's first frame, which never arrives. - let _stalled = session.producer.append_group().unwrap(); - tokio::task::yield_now().await; - session + let _stalled = session .producer - .write_frame(None, br#"{"type":"buttons","buttons":["a"]}"#.to_vec()) + .append_group(session.producer.largest().map_or(0, |largest| largest.group + 1)) .unwrap(); + tokio::task::yield_now().await; + send(&mut session, r#"{"type":"buttons","buttons":["a"]}"#); let Command::Buttons { viewer: from, buttons, .. diff --git a/rs/moq-boy/src/status.rs b/rs/moq-boy/src/status.rs index 339785ad1cb..8f39dad0968 100644 --- a/rs/moq-boy/src/status.rs +++ b/rs/moq-boy/src/status.rs @@ -60,11 +60,17 @@ impl StatusPublisher { if let std::task::Poll::Ready(Err(err)) = self.producer.poll_refresh(&kio::Waiter::noop()) { tracing::warn!(%err, "failed to refresh status"); } - if let Err(err) = self - .producer - .update(moq_net::Timed::from(status).at(moq_net::Timestamp::now())) - { + if let Err(err) = self.producer.update(moq_json::Timed::new(status, now())) { tracing::warn!(%err, "failed to publish status"); } } } + +/// Milliseconds on this process's monotonic clock since its origin: the status track's timescale. +fn now() -> moq_net::Timestamp { + let elapsed = kio::clock::now().saturating_duration_since(kio::clock::origin().0); + u64::try_from(elapsed.as_millis()) + .ok() + .and_then(|millis| moq_net::Timestamp::from_millis(millis).ok()) + .expect("a process's uptime fits a timestamp") +} diff --git a/rs/moq-c/README.md b/rs/moq-c/README.md index fb814007a7b..e38561df8ff 100644 --- a/rs/moq-c/README.md +++ b/rs/moq-c/README.md @@ -127,6 +127,7 @@ int32_t moq_publish_track(uint32_t broadcast, const char *name, uintptr_t name_l int32_t moq_publish_track_group(uint32_t track); int32_t moq_publish_track_frame(uint32_t track, const uint8_t *payload, uintptr_t payload_size, const moq_timestamp *timestamp); int32_t moq_publish_group_frame(uint32_t group, const uint8_t *payload, uintptr_t payload_size, const moq_timestamp *timestamp); +int32_t moq_publish_group_sequence(uint32_t group, uint64_t *dst); int32_t moq_publish_group_finish(uint32_t group); int32_t moq_publish_group_ready(uint32_t group, uint64_t size, moq_status_callback on_ready, void *user_data); int32_t moq_publish_track_finish(uint32_t track); @@ -212,10 +213,10 @@ int32_t moq_audio_encoder_frame_size(uint32_t encoder); int32_t moq_audio_encoder_init(uint32_t encoder, moq_audio_init *dst); int32_t moq_audio_encoder_bitrate(uint32_t encoder, uint64_t bitrate); int32_t moq_audio_encoder_encode(uint32_t encoder, const float *pcm, uintptr_t pcm_len); -int32_t moq_audio_encoder_receive(uint32_t encoder, moq_frame *dst); +int32_t moq_audio_encoder_receive(uint32_t encoder, moq_audio_packet *dst); int32_t moq_audio_encoder_flush(uint32_t encoder); int32_t moq_audio_encoder_finish(uint32_t encoder, const float *pcm, uintptr_t pcm_len, uint64_t *discard_padding); -int32_t moq_audio_encoder_packet(uint32_t encoder, uint32_t index, moq_frame *dst); +int32_t moq_audio_encoder_packet(uint32_t encoder, uint32_t index, moq_audio_packet *dst); int32_t moq_audio_encoder_free(uint32_t encoder); int32_t moq_video_encoder_create(const moq_video_encoder_input *input, const moq_video_encoder_output *output); int32_t moq_video_encoder_encode(uint32_t encoder, const moq_video_encoder_frame *frame); @@ -228,10 +229,14 @@ int32_t moq_video_encoder_free(uint32_t encoder); Raw track frames use the same `moq_frame` record as media frames. Timestamps are exact `moq_timestamp` records, `value` units at `scale` units per second. -`moq_publish_track_frame` and `moq_publish_group_frame` convert theirs to the -track's timescale, a microsecond one unless `moq_track_info.timescale` sets -another, and `moq_consume_track_frame` points `moq_frame.timestamp` at the -timestamp read, valid until the frame is freed, or NULL on an untimed track. +A scale-less track has no time, so there `has_scale` is false and `value` is +the frame's group key (a datagram's sequence). Every write takes a non-NULL +timestamp: `moq_publish_track_frame` and `moq_publish_group_frame` convert +theirs to the track's timescale, a microsecond one unless +`moq_track_info.timescale` sets another, and `moq_publish_group_sequence` reads +the key a scale-less group's frames carry. `moq_consume_track_frame` points +`moq_frame.timestamp` at the timestamp read, never NULL, valid until the frame +is freed. A zeroed `moq_track_info`, like a NULL one, advertises a zero max age, how long a superseded group stays fresh, so only the live edge is served. Set @@ -240,9 +245,10 @@ which ignores `max_age_us`. Zero-initialize the struct (`moq_track_info info = {0};`) before assigning members, so every flag starts false. -JSON window pushes and pops accept an optional `const moq_timestamp *timestamp`, +JSON window pushes and pops take a required `const moq_timestamp *timestamp`, like snapshot updates and stream appends. It is when the edit occurred, -converted to the track's milliseconds; pass NULL to stamp the broadcast clock. +converted to the track's milliseconds; stamp it from `moq_publish_catalog_now`. +A NULL timestamp fails with `MOQ_ERROR_INVALID_POINTER`. ```c int32_t moq_publish_json_window_push(uint32_t window, const char *value, uintptr_t value_len, const moq_timestamp *timestamp); diff --git a/rs/moq-c/c-tests/audio-decode-output.c b/rs/moq-c/c-tests/audio-decode-output.c index 1952443eabb..01244fecb35 100644 --- a/rs/moq-c/c-tests/audio-decode-output.c +++ b/rs/moq-c/c-tests/audio-decode-output.c @@ -51,7 +51,7 @@ static void on_catalog(void *data, int32_t id) { } static void check_frame(Context *c, const moq_audio_frame *frame) { - CHECK(frame->timestamp.scale == 1000000); + CHECK(frame->timestamp.has_scale && frame->timestamp.scale == 1000000); CHECK(frame->timestamp.value == 123000 + c->frames * 1000000 / 48000); size_t bytes = c->format == MOQ_AUDIO_SAMPLE_FORMAT_S16 ? 2 : 4; CHECK(frame->data_size % (bytes * c->channels) == 0); @@ -115,7 +115,7 @@ int main(void) { CHECK(producer > 0); float pcm[960 * 2]; for (size_t sample = 0; sample < 960; sample++) { pcm[sample * 2] = 0.25f; pcm[sample * 2 + 1] = -0.5f; } - moq_audio_frame frame = {{123000, 1000000}, (const uint8_t *)pcm, sizeof(pcm)}; + moq_audio_frame frame = {{123000, 1000000, true}, (const uint8_t *)pcm, sizeof(pcm)}; CHECK(moq_encode_audio_frame((uint32_t)producer, &frame) == 0); CHECK(moq_encode_audio_finish((uint32_t)producer) == 0); diff --git a/rs/moq-c/c-tests/decode-output.c b/rs/moq-c/c-tests/decode-output.c index a3cec57b985..cf88cb29795 100644 --- a/rs/moq-c/c-tests/decode-output.c +++ b/rs/moq-c/c-tests/decode-output.c @@ -189,7 +189,7 @@ static uint64_t next_ts; static void publish_frames(int32_t producer, int count) { for (int i = 0; i < count; i++) { - moq_video_encoder_frame frame = {{next_ts, 1000000}, gray_rgba, sizeof(gray_rgba)}; + moq_video_encoder_frame frame = {{next_ts, 1000000, true}, gray_rgba, sizeof(gray_rgba)}; next_ts += 33333; int32_t rc = moq_encode_video_frame((uint32_t)producer, &frame); if (rc < 0) fail("error: moq_encode_video_frame failed: %d (%s)\n", rc, moq_error()); diff --git a/rs/moq-c/c-tests/timestamps.c b/rs/moq-c/c-tests/timestamps.c index 97d905b3749..4f5c2ea1cf9 100644 --- a/rs/moq-c/c-tests/timestamps.c +++ b/rs/moq-c/c-tests/timestamps.c @@ -1,8 +1,8 @@ // Exact timestamps across the installed C ABI: a 90 kHz tick and a present zero // read back unchanged from raw frames and datagrams, a JSON value carries its -// frame's timestamp, and a write refuses an out-of-range timestamp or an -// untimed frame on a timed track. Stream tracks default to timed; -// the untimed frame read path is covered by the Rust tests. +// frame's timestamp, and a write refuses an out-of-range timestamp, a NULL one, +// or an unscaled one on a timed track. Stream tracks default to timed; the +// scale-less read path is covered by the Rust tests. #include #include @@ -70,7 +70,8 @@ static void on_value(void *data, int32_t id) { } static bool same(const moq_timestamp *got, moq_timestamp want) { - return got != NULL && got->value == want.value && got->scale == want.scale; + return got != NULL && got->has_scale == want.has_scale && got->value == want.value && + (!want.has_scale || got->scale == want.scale); } int main(void) { @@ -96,14 +97,19 @@ int main(void) { const uint8_t payload[] = {1, 2, 3}; const char *doc = "{\"n\":1}"; - const moq_timestamp invalid[] = {{0, 0}, {0, 1ull << 62}, {1ull << 61, 1000000}}; + const moq_timestamp invalid[] = {{0, 0, true}, {0, 1ull << 62, true}, {1ull << 61, 1000000, true}}; for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++) { CHECK(moq_publish_track_frame((uint32_t)track, payload, sizeof(payload), &invalid[i]) == MOQ_ERROR_TIMESTAMP_OVERFLOW); CHECK(moq_publish_json_stream_append((uint32_t)json, doc, strlen(doc), &invalid[i]) == MOQ_ERROR_TIMESTAMP_OVERFLOW); } - CHECK(moq_publish_track_frame((uint32_t)track, payload, sizeof(payload), NULL) == MOQ_ERROR_MOQ); + CHECK(moq_publish_track_frame((uint32_t)track, payload, sizeof(payload), NULL) == MOQ_ERROR_INVALID_POINTER); + CHECK(moq_publish_datagrams_append((uint32_t)datagram_track, payload, sizeof(payload), NULL, NULL) == + MOQ_ERROR_INVALID_POINTER); + // Zeroed, a timestamp is the unscaled key 0, which only a scale-less track takes. + const moq_timestamp unscaled = {0}; + CHECK(moq_publish_track_frame((uint32_t)track, payload, sizeof(payload), &unscaled) == MOQ_ERROR_MOQ); // A JSON write is never stamped for you. CHECK(moq_publish_json_stream_append((uint32_t)json, doc, strlen(doc), NULL) == MOQ_ERROR_INVALID_POINTER); @@ -118,7 +124,7 @@ int main(void) { // One group at a time, so the default subscription never skips the older one, and in // time order, since a group may not start before the one preceding it. - const moq_timestamp zero = {0, 90000}, tick = {1, 90000}; + const moq_timestamp zero = {0, 90000, true}, tick = {1, 90000, true}; CHECK(moq_publish_track_frame((uint32_t)track, payload, sizeof(payload), &zero) == 0); WAIT_FOR(&c, c.frame_count == 1); CHECK(moq_publish_track_frame((uint32_t)track, payload, sizeof(payload), &tick) == 0); @@ -127,7 +133,7 @@ int main(void) { CHECK(moq_publish_datagrams_append((uint32_t)datagram_track, payload, sizeof(payload), &tick, NULL) == 0); WAIT_FOR(&c, c.datagram); // JSON tracks moq-c creates count milliseconds. - const moq_timestamp stamped = {1500, 1000}; + const moq_timestamp stamped = {1500, 1000, true}; CHECK(moq_publish_json_stream_append((uint32_t)json, doc, strlen(doc), &stamped) == 0); WAIT_FOR(&c, c.value); diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 1829cf930e1..44fa8216b40 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -551,25 +551,44 @@ pub struct moq_json_snapshot_config { pub deltas: bool, } -/// An exact media timestamp: `value` units at `scale` units per second. +/// An exact timestamp: `value` units at `scale` units per second. /// /// Reads carry the scale the publisher wrote, so a tick such as 90 kHz survives a read and /// republish exactly. A write with a zero scale, a scale above `2^62 - 1`, or a value above /// `2^61 - 1` fails with `MOQ_ERROR_TIMESTAMP_OVERFLOW`. +/// +/// A scale-less track has no time, so there `has_scale` is false and `value` is the frame's +/// group key, or a datagram's sequence. Zero-initialized, it is the unscaled key 0. #[repr(C)] #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct moq_timestamp { - /// The time in `scale` units, at most `2^61 - 1`. + /// The time in `scale` units, at most `2^61 - 1`, or the group key when unscaled. pub value: u64, - /// Units per second, non-zero and at most `2^62 - 1`. + /// Units per second, non-zero and at most `2^62 - 1`. Ignored unless `has_scale`. pub scale: u64, + /// Whether `scale` is set; false on a scale-less track. + pub has_scale: bool, +} + +impl moq_timestamp { + /// The scale-less timestamp at group key (or datagram sequence) `key`. + #[cfg(test)] + pub(crate) const fn unscaled(key: u64) -> Self { + Self { + value: key, + scale: 0, + has_scale: false, + } + } } impl From for moq_timestamp { fn from(timestamp: moq_net::Timestamp) -> Self { + let scale = timestamp.scale(); Self { value: timestamp.value(), - scale: timestamp.scale().as_u64(), + scale: scale.map_or(0, moq_net::Timescale::as_u64), + has_scale: scale.is_some(), } } } @@ -578,7 +597,10 @@ impl TryFrom for moq_net::Timestamp { type Error = moq_net::TimeOverflow; fn try_from(timestamp: moq_timestamp) -> Result { - Self::from_scale(timestamp.value, timestamp.scale) + match timestamp.has_scale { + true => Self::from_scale(timestamp.value, timestamp.scale), + false => Ok(Self::unscaled(timestamp.value)), + } } } @@ -599,8 +621,7 @@ pub struct moq_json_value { /// The length in bytes of `json`. pub json_len: usize, - /// The timestamp of the frame that carried the value, or NULL on an untimed track. - /// Valid as long as `json`. + /// The timestamp of the frame that carried the value, never NULL. Valid as long as `json`. pub timestamp: *const moq_timestamp, } @@ -637,8 +658,8 @@ pub struct moq_json_window_event { /// The length in bytes of `json`. pub json_len: usize, - /// The timestamp of the frame that carried the change, or NULL on an untimed track. Valid - /// until the event is freed. A record first learned of from a group's restatement (after a + /// The timestamp of the frame that carried the change, never NULL. Valid until the event + /// is freed. A record first learned of from a group's restatement (after a /// join, or while behind) carries that frame's time, not its original push time. pub timestamp: *const moq_timestamp, } @@ -651,8 +672,8 @@ pub struct moq_frame { /// The length in bytes of `payload`. pub payload_size: usize, - /// The presentation timestamp of the frame, or NULL on an untimed track. Media frames - /// always have one. Valid as long as `payload`. + /// The presentation timestamp of the frame, or on a scale-less track its group's unscaled + /// key (`has_scale` false). Never NULL on a frame; valid as long as `payload`. pub timestamp: *const moq_timestamp, /// Whether this frame opens a group or is a video keyframe; audio is true only at a group start. @@ -681,8 +702,9 @@ pub struct moq_datagram { /// The length in bytes of `payload`. pub payload_size: usize, - /// The presentation timestamp of the datagram, or NULL on an untimed track. Valid as - /// long as `payload`. + /// The presentation timestamp of the datagram, or on a scale-less track its unscaled + /// sequence (`has_scale` false). Never NULL; valid as long as `payload`. A write requires + /// one too. pub timestamp: *const moq_timestamp, /// The datagram track's own sequence number. @@ -875,10 +897,12 @@ pub struct moq_range { pub struct moq_anchor { /// `moq_anchor_kind` discriminant. pub kind: u32, - /// The group, or with `MOQ_ANCHOR_RELATIVE` how many groups before the latest one - /// (0 is the latest, and an offset past the first group clamps to it). + /// The group's start, in its track's timescale, or with `MOQ_ANCHOR_RELATIVE` how many + /// groups before the latest one (0 is the latest, and an offset past every group held + /// reaches back to the oldest). pub group: u64, - /// The presentation time, read only with `MOQ_ANCHOR_TIME`. + /// The time, read only with `MOQ_ANCHOR_TIME`: unscaled (`has_scale` false), with `value` in + /// the track's timescale. A scaled one is refused with a timestamp mismatch. pub timestamp: moq_timestamp, } @@ -887,11 +911,11 @@ pub struct moq_anchor { #[expect(non_camel_case_types, reason = "C ABI type names match moq.h")] #[derive(Clone, Copy, Debug)] pub enum moq_anchor_kind { - /// A fixed group, delivered from its first frame. + /// The group starting at `group`, delivered from its first frame. MOQ_ANCHOR_ABSOLUTE = 0, /// A count of groups back from the latest one. MOQ_ANCHOR_RELATIVE = 1, - /// The group containing a presentation time. An untimed track refuses it. + /// The group containing a time: the newest one starting at or before it. MOQ_ANCHOR_TIME = 2, /// The oldest group still around, as far back as `delay.max_us` allows. A start only. MOQ_ANCHOR_OLDEST = 3, @@ -908,16 +932,33 @@ pub enum moq_order { MOQ_ORDER_ASCENDING = 1, } +/// `anchor` as a start. A group is named by its start, so an absolute group names that time. +/// A time must be unscaled, in the track's timescale: the timescale is unknown here, so a scaled +/// one is refused rather than read in the wrong units. fn parse_anchor(anchor: &moq_anchor) -> Result { Ok(match anchor.kind { - v if v == moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32 => moq_net::track::Anchor::Absolute(anchor.group), + v if v == moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32 => moq_net::track::Anchor::Time(anchor.group), v if v == moq_anchor_kind::MOQ_ANCHOR_RELATIVE as u32 => moq_net::track::Anchor::Relative(anchor.group), - v if v == moq_anchor_kind::MOQ_ANCHOR_TIME as u32 => moq_net::track::Anchor::Time(anchor.timestamp.try_into()?), + v if v == moq_anchor_kind::MOQ_ANCHOR_TIME as u32 => match anchor.timestamp.has_scale { + false => moq_net::track::Anchor::Time(anchor.timestamp.value), + true => return Err(moq_net::Error::TimestampMismatch.into()), + }, v if v == moq_anchor_kind::MOQ_ANCHOR_OLDEST as u32 => moq_net::track::Anchor::Oldest, _ => return Err(Error::InvalidCode), }) } +/// `anchor` as an end, or `None` for an absolute end at group 0, which holds nothing. An +/// absolute end is exclusive, so it ends with the group before it. +fn parse_end(anchor: &moq_anchor) -> Result, Error> { + match anchor.kind { + v if v == moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32 => { + Ok(anchor.group.checked_sub(1).map(moq_net::track::Anchor::Time)) + } + _ => parse_anchor(anchor).map(Some), + } +} + /// # Safety /// - `subscription->ranges` must be NULL or point to `subscription->ranges_len` ranges. pub(crate) unsafe fn parse_subscription( @@ -937,13 +978,19 @@ pub(crate) unsafe fn parse_subscription( let ranges = ranges .iter() .map(|range| { - Ok(moq_net::track::Range { - start: parse_anchor(&range.start)?, - end: range.end_present.then(|| parse_anchor(&range.end)).transpose()?, - }) + let start = parse_anchor(&range.start)?; + let end = match range.end_present { + // One ending before group 0 holds nothing, so it is dropped. + true => match parse_end(&range.end)? { + Some(end) => Some(end), + None => return Ok(None), + }, + false => None, + }; + Ok(Some(moq_net::track::Range { start, end })) }) .collect::, Error>>()?; - out = out.with_ranges(ranges); + out = out.with_ranges(ranges.into_iter().flatten()); } Ok(out) } @@ -3098,11 +3145,11 @@ pub extern "C" fn moq_publish_media_cut(media: u32) -> i32 { }) } -/// Draw a group boundary and number the next group `sequence`. +/// Draw a group boundary and start the next group at `sequence`, in the track's timescale. /// -/// [moq_publish_media_cut] with an explicit sequence, for a caller whose group numbers have to be -/// deterministic: two encoders publishing the same content align per GOP so a consumer can fail -/// over between them. +/// [moq_publish_media_cut] keyed explicitly rather than at the next frame's time, which has to come +/// after every group the track holds. Groups are keyed by their first frame's time, so two encoders +/// of the same content align without it. /// /// Returns a zero on success, or a negative code on failure. #[unsafe(no_mangle)] @@ -3233,7 +3280,8 @@ pub extern "C" fn moq_publish_container_cut(container: u32) -> i32 { }) } -/// Start a new segment and number its groups `sequence`. +/// Start a new segment whose first group starts at `sequence`, in its track's timescale, after +/// every group the track holds. /// /// Returns zero on success, or a negative code on failure. #[unsafe(no_mangle)] @@ -3620,7 +3668,8 @@ unsafe fn parse_track_info(info: *const moq_track_info) -> Result i3 /// /// Convenience for the common one-frame-per-group pattern. Equivalent to /// appending a group, writing one frame, and finishing it. The timestamp is -/// converted to the track's timescale; NULL writes an untimed frame, which only -/// an untimed track accepts. +/// converted to the track's timescale and starts the group. A scale-less track has +/// no time, so there it is the unscaled key of the group to write (`has_scale` +/// false). NULL fails with `MOQ_ERROR_INVALID_POINTER`. /// /// Returns a zero on success, or a negative code on failure. /// /// # Safety /// - The caller must ensure that payload is a valid pointer to payload_size bytes of data. -/// - `timestamp` must be NULL or a valid pointer to a [moq_timestamp]. +/// - `timestamp` must be a valid pointer to a [moq_timestamp]. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_track_frame( track: u32, @@ -3685,10 +3735,13 @@ pub unsafe extern "C" fn moq_publish_track_frame( /// Append a datagram to a producer from [crate::moq_publish_datagrams], returning its sequence. /// -/// Keep the payload well below the path MTU. No stream fallback is available. +/// Keep the payload well below the path MTU. No stream fallback is available. The +/// timestamp must match the track; a scale-less track has no time, so there it is the +/// unscaled sequence to write (`has_scale` false). NULL fails with +/// `MOQ_ERROR_INVALID_POINTER`. /// /// # Safety -/// `payload` must point to `payload_size` bytes, `timestamp` must be NULL or valid, +/// `payload` must point to `payload_size` bytes, `timestamp` must be valid, /// and `out_sequence` must be NULL or writable. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_datagrams_append( @@ -4075,14 +4128,15 @@ pub extern "C" fn moq_group_request_free(request: u32) -> i32 { /// Write a frame into a raw group created by [moq_publish_track_group]. /// -/// The timestamp is converted to the track's timescale; NULL writes an untimed -/// frame, which only an untimed track accepts. +/// The timestamp is converted to the track's timescale. A scale-less track has no +/// time, so there it is the group's unscaled key (`has_scale` false). NULL fails with +/// `MOQ_ERROR_INVALID_POINTER`. /// /// Returns a zero on success, or a negative code on failure. /// /// # Safety /// - The caller must ensure that payload is a valid pointer to payload_size bytes of data. -/// - `timestamp` must be NULL or a valid pointer to a [moq_timestamp]. +/// - `timestamp` must be a valid pointer to a [moq_timestamp]. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_group_frame( group: u32, @@ -4098,6 +4152,25 @@ pub unsafe extern "C" fn moq_publish_group_frame( }) } +/// Read the sequence of a raw group, which names it within its track. +/// +/// On a scale-less track this is the key every [moq_publish_group_frame] timestamp carries +/// (unscaled, `has_scale` false), including for a group [moq_publish_track_group] numbered. +/// +/// Returns a zero on success, or a negative code on failure. +/// +/// # Safety +/// - `dst` must be a valid pointer to a writable `uint64_t`. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn moq_publish_group_sequence(group: u32, dst: *mut u64) -> i32 { + ffi::enter(move || { + let group = ffi::parse_id(group)?; + let dst = unsafe { dst.as_mut() }.ok_or(Error::InvalidPointer)?; + *dst = State::lock().publish.group_sequence(group)?; + Ok(()) + }) +} + /// Finish a raw group. No more frames can be written. /// /// Returns a zero on success, or a negative code on failure. @@ -4152,10 +4225,13 @@ pub unsafe extern "C" fn moq_publish_group_ready( /// `size` bytes, or its handle ends. pub(crate) fn group_ready(group: crate::Id, size: u64) -> Result + Send + 'static, Error> { let (group, mut ended) = State::lock().publish.group(group)?; - let frame = moq_net::frame::Info { size, timestamp: None }; Ok(async move { let ready = kio::wait(|waiter| match group.lock().as_ref() { - Some(group) => group.poll_ready(waiter, &frame), + Some(group) => { + // Only the size paces the write; the timestamp is a placeholder at the group's key. + let timestamp = moq_net::Timestamp::unscaled(group.start); + group.poll_ready(waiter, &moq_net::frame::Info { size, timestamp }) + } // Finished or aborted: the next write reports why. None => std::task::Poll::Ready(()), }); @@ -4217,7 +4293,7 @@ pub unsafe extern "C" fn moq_publish_json_snapshot( /// /// # Safety /// - The caller must ensure `value` is a valid pointer to `value_len` bytes. -/// - `timestamp` must be NULL or a valid pointer to a [moq_timestamp]. +/// - `timestamp` must be a valid pointer to a [moq_timestamp]. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_json_snapshot_update( json: u32, @@ -4229,7 +4305,7 @@ pub unsafe extern "C" fn moq_publish_json_snapshot_update( let json = ffi::parse_id(json)?; let value = unsafe { ffi::parse_slice(value.cast::(), value_len)? }; let value = serde_json::from_slice(value)?; - let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }.ok_or(Error::InvalidPointer)?; + let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }; State::lock().publish.json_snapshot_update(json, value, timestamp) }) } @@ -4278,7 +4354,7 @@ pub unsafe extern "C" fn moq_publish_json_stream(broadcast: u32, name: *const c_ /// /// # Safety /// - The caller must ensure `value` is a valid pointer to `value_len` bytes. -/// - `timestamp` must be NULL or a valid pointer to a [moq_timestamp]. +/// - `timestamp` must be a valid pointer to a [moq_timestamp]. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_json_stream_append( stream: u32, @@ -4290,7 +4366,7 @@ pub unsafe extern "C" fn moq_publish_json_stream_append( let stream = ffi::parse_id(stream)?; let value = unsafe { ffi::parse_slice(value.cast::(), value_len)? }; let value = serde_json::from_slice(value)?; - let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }.ok_or(Error::InvalidPointer)?; + let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }; State::lock().publish.json_stream_append(stream, value, timestamp) }) } @@ -4353,7 +4429,7 @@ pub unsafe extern "C" fn moq_publish_json_window( /// /// # Safety /// - The caller must ensure `value` is a valid pointer to `value_len` bytes. -/// - `timestamp` must be NULL or a valid pointer to a [moq_timestamp]. +/// - `timestamp` must be a valid pointer to a [moq_timestamp]. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_json_window_push( window: u32, @@ -4365,7 +4441,7 @@ pub unsafe extern "C" fn moq_publish_json_window_push( let window = ffi::parse_id(window)?; let value = unsafe { ffi::parse_slice(value.cast::(), value_len)? }; let value = serde_json::from_slice(value)?; - let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }.ok_or(Error::InvalidPointer)?; + let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }; State::lock().publish.json_window_push(window, value, timestamp) }) } @@ -4379,12 +4455,12 @@ pub unsafe extern "C" fn moq_publish_json_window_push( /// Returns a zero on success, or a negative code on failure. /// /// # Safety -/// - `timestamp` must be NULL or a valid pointer to a [moq_timestamp]. +/// - `timestamp` must be a valid pointer to a [moq_timestamp]. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_json_window_pop(window: u32, count: u64, timestamp: *const moq_timestamp) -> i32 { ffi::enter(move || { let window = ffi::parse_id(window)?; - let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }.ok_or(Error::InvalidPointer)?; + let timestamp = unsafe { ffi::parse_timestamp(timestamp)? }; State::lock().publish.json_window_pop(window, count, timestamp) }) } @@ -4881,7 +4957,7 @@ pub unsafe extern "C" fn moq_consume_track_update(track: u32, subscription: *con /// /// Fills `dst.payload` / `dst.payload_size`; the pointer is valid until the /// frame is released with [moq_consume_track_frame_free], and so is `dst.timestamp`, -/// NULL on an untimed track. `dst.keyframe` is reported as false because raw tracks +/// the group's unscaled key on a scale-less track. `dst.keyframe` is reported as false because raw tracks /// do not parse codec metadata. /// /// Returns a zero on success, or a negative code on failure. @@ -4962,8 +5038,8 @@ pub unsafe extern "C" fn moq_consume_datagrams( /// Read a datagram delivered via the [moq_consume_datagrams] callback. /// /// Fills `dst.payload` / `dst.payload_size` (valid until the datagram is released with -/// [moq_consume_datagram_free]), plus `dst.timestamp` (NULL on an untimed track, and valid -/// as long as the payload) and `dst.sequence`. +/// [moq_consume_datagram_free]), plus `dst.timestamp` (the unscaled sequence on a scale-less +/// track, and valid as long as the payload) and `dst.sequence`. /// /// Returns a zero on success, or a negative code on failure. /// diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 98fec57641d..0ba8738f4ef 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -226,31 +226,31 @@ enum RawStep { Stop, } -/// A buffered read plus the timestamp C borrows through a pointer, `None` when untimed. +/// A buffered read plus the timestamp C borrows through a pointer. /// /// Boxed because the slab moves its entries as it grows, and the pointer must stay valid /// until the read is freed. struct Timed { item: T, - timestamp: Option>, + timestamp: Box, } impl Timed { - fn new(item: T, timestamp: Option) -> Self { + fn new(item: T, timestamp: moq_net::Timestamp) -> Self { Self { item, - timestamp: timestamp.map(|timestamp| Box::new(timestamp.into())), + timestamp: Box::new(timestamp.into()), } } - /// The timestamp pointer handed to C, NULL when untimed. + /// The timestamp pointer handed to C, never NULL. fn timestamp(&self) -> *const moq_timestamp { - self.timestamp.as_deref().map_or(std::ptr::null(), std::ptr::from_ref) + std::ptr::from_ref(&*self.timestamp) } } /// A JSON value pre-serialized to a string, with its frame timestamp. -fn json_value(value: moq_net::Timed) -> Result, Error> { +fn json_value(value: moq_json::Timed) -> Result, Error> { Ok(Timed::new(serde_json::to_string(&value.value)?, value.at)) } @@ -264,7 +264,7 @@ struct JsonWindowEvent { impl JsonWindowEvent { /// Pre-serialize an event, keeping its frame timestamp. - fn new(event: moq_net::Timed>) -> Result, Error> { + fn new(event: moq_json::Timed>) -> Result, Error> { use moq_json::window::Event; let (kind, range, json) = match event.value { @@ -712,7 +712,7 @@ impl Consume { read = track.read() => match read? { Some(moq_mux::container::Read::Frame { frame, .. }) => { let timestamp = frame.timestamp; - Read::Frame(Timed::new(frame, Some(timestamp))) + Read::Frame(Timed::new(frame, timestamp)) } Some(moq_mux::container::Read::Gap { timestamp, duration }) => Read::gap(timestamp, duration)?, Some(moq_mux::container::Read::End { end, .. }) => Read::end(end), @@ -1472,7 +1472,7 @@ mod read_tests { ); let timestamp = |us| moq_net::Timestamp::from_micros(us).unwrap(); let write = |sequence, us, payload: bytes::Bytes| { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); format() .write( &mut group, @@ -1484,7 +1484,7 @@ mod read_tests { }], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; let mut ids = Vec::::new(); let callback = unsafe { OnStatus::new((&mut ids as *mut Vec).cast(), Some(received)) }.unwrap(); @@ -1504,6 +1504,7 @@ mod read_tests { let micros = |value| moq_timestamp { value, scale: 1_000_000, + has_scale: true, }; let mut read = std::mem::MaybeUninit::::uninit(); for (index, id) in ids.into_iter().enumerate() { @@ -1621,14 +1622,17 @@ mod tests { } fn write(&mut self, sequence: u64, micros: u64) { - let mut group = self.track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = self + .track + .create_group(moq_net::group::Info { start: sequence }) + .unwrap(); hang::container::Frame { timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), payload: bytes::Bytes::from_static(b"frame"), } .write_to(&mut group) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } async fn frame(&self) -> Option { @@ -1642,7 +1646,7 @@ mod tests { let Read::Frame(frame) = read else { panic!("expected a frame"); }; - Some(frame.item.timestamp.as_micros()) + Some(frame.item.timestamp.as_micros().unwrap()) } async fn cancel(self) { diff --git a/rs/moq-c/src/datagram.rs b/rs/moq-c/src/datagram.rs index 953610641b5..af5312db889 100644 --- a/rs/moq-c/src/datagram.rs +++ b/rs/moq-c/src/datagram.rs @@ -54,7 +54,8 @@ pub unsafe extern "C" fn moq_publish_datagrams( /// Insert a datagram at its explicit sequence; later appends continue past the highest one. /// /// # Safety -/// `datagram` must point to a record whose payload and optional timestamp are valid. +/// `datagram` must point to a record whose payload and timestamp are valid. A NULL timestamp +/// fails with `MOQ_ERROR_INVALID_POINTER`; on a scale-less track it is the unscaled sequence. #[unsafe(no_mangle)] pub unsafe extern "C" fn moq_publish_datagrams_insert(producer: u32, datagram: *const moq_datagram) -> i32 { ffi::enter(|| { diff --git a/rs/moq-c/src/decode.rs b/rs/moq-c/src/decode.rs index a7bedc6f41f..5b198009150 100644 --- a/rs/moq-c/src/decode.rs +++ b/rs/moq-c/src/decode.rs @@ -71,7 +71,7 @@ impl Read { Ok(Self::Gap(moq_decode_gap { timestamp: timestamp.into(), duration_us: duration - .as_micros() + .as_micros()? .try_into() .map_err(|_| Error::TimestampOverflow(moq_net::TimeOverflow))?, })) diff --git a/rs/moq-c/src/encoder.rs b/rs/moq-c/src/encoder.rs index 5fad0a8ea12..bfb30f83cbb 100644 --- a/rs/moq-c/src/encoder.rs +++ b/rs/moq-c/src/encoder.rs @@ -24,33 +24,49 @@ pub struct Encoders { video: NonZeroSlab>, } -/// One packet the last call produced, owning what [moq_frame] borrows. -struct Packet { +/// One packet emitted by a codec-only audio encoder. +/// +/// The codec stamps nothing, so unlike a [moq_frame] it carries no timestamp or keyframe. +#[repr(C)] +pub struct moq_audio_packet { + /// The encoded codec payload, borrowed from the encoder until its next call or + /// [moq_audio_encoder_free]. + pub payload: *const u8, + /// The length in bytes of `payload`. + pub payload_size: usize, +} + +/// Read packet `index` of the last call into `dst`. +fn packet(packets: &[Bytes], index: u32, dst: &mut moq_audio_packet) -> Result<(), Error> { + let payload = packets.get(index as usize).ok_or(Error::NoIndex)?; + *dst = moq_audio_packet { + payload: payload.as_ptr(), + payload_size: payload.len(), + }; + Ok(()) +} + +/// One access unit the last call produced, owning what [moq_frame] borrows. +struct VideoPacket { payload: Bytes, - /// Boxed so the pointer handed to C survives the packet list moving. - timestamp: Option>, + /// Boxed so the pointer handed to C stays put while the encoder moves. + timestamp: Box, keyframe: bool, } -impl Packet { +impl VideoPacket { fn info(&self) -> moq_frame { moq_frame { payload: self.payload.as_ptr(), payload_size: self.payload.len(), - timestamp: self.timestamp.as_deref().map_or(std::ptr::null(), std::ptr::from_ref), + timestamp: std::ptr::from_ref(&*self.timestamp), keyframe: self.keyframe, } } } -/// Read packet `index` of the last call into `dst`. -fn packet(packets: &[Packet], index: u32, dst: &mut moq_frame) -> Result<(), Error> { - *dst = packets.get(index as usize).ok_or(Error::NoIndex)?.info(); - Ok(()) -} - /// The packet count a call returns. -fn count(packets: &[Packet]) -> Result { +fn count(packets: &[Bytes]) -> Result { i32::try_from(packets.len()).map_err(|_| Error::OutOfRange("too many packets".into())) } @@ -59,7 +75,7 @@ struct AudioEncoder { encoder: Option, /// The init that publishes the packets, read once at creation. init: Option<(crate::moq_audio_format, Bytes)>, - packets: Vec, + packets: Vec, } impl AudioEncoder { @@ -72,11 +88,7 @@ impl AudioEncoder { fn keep(&mut self, packets: impl IntoIterator) -> Result { // Reuses the buffer, since every received packet passes through it. self.packets.clear(); - self.packets.extend(packets.into_iter().map(|packet| Packet { - payload: packet.payload, - timestamp: None, - keyframe: false, - })); + self.packets.extend(packets.into_iter().map(|packet| packet.payload)); count(&self.packets) } } @@ -86,7 +98,7 @@ struct VideoEncoder { format: moq_video_pixel_format, size: moq_video::Size, /// The last received access unit, owning what [moq_video_encoder_receive] lent out. - packet: Option, + packet: Option, } /// Run `f` on an encoder resolved under the global lock, then locked on its own. @@ -226,13 +238,12 @@ pub unsafe extern "C" fn moq_audio_encoder_encode(encoder: u32, pcm: *const f32, /// [moq_audio_encoder_encode] or [moq_audio_encoder_flush]. /// /// Returns one when it wrote a packet, zero while the codec holds the rest, or a negative code. -/// `payload` borrows the encoder until its next call or [moq_audio_encoder_free]; `timestamp` -/// is NULL and `keyframe` false, since the codec stamps nothing. +/// `payload` borrows the encoder until its next call or [moq_audio_encoder_free]. /// /// # Safety -/// - `dst` must point at a writable [moq_frame]. +/// - `dst` must point at a writable [moq_audio_packet]. #[unsafe(no_mangle)] -pub unsafe extern "C" fn moq_audio_encoder_receive(encoder: u32, dst: *mut moq_frame) -> i32 { +pub unsafe extern "C" fn moq_audio_encoder_receive(encoder: u32, dst: *mut moq_audio_packet) -> i32 { ffi::enter(move || { let dst = unsafe { dst.as_mut() }.ok_or(Error::InvalidPointer)?; with(encoder, audio, |audio| { @@ -294,14 +305,13 @@ pub unsafe extern "C" fn moq_audio_encoder_finish( /// Read packet `index` from the last [moq_audio_encoder_finish]. /// -/// `payload` borrows the encoder until its next call or [moq_audio_encoder_free]; `timestamp` -/// is NULL and `keyframe` false, since the codec stamps nothing. Returns zero, or a negative -/// code, `MOQ_ERROR_NO_INDEX` past the last packet. +/// `payload` borrows the encoder until its next call or [moq_audio_encoder_free]. Returns zero, +/// or a negative code, `MOQ_ERROR_NO_INDEX` past the last packet. /// /// # Safety -/// - `dst` must point at a writable [moq_frame]. +/// - `dst` must point at a writable [moq_audio_packet]. #[unsafe(no_mangle)] -pub unsafe extern "C" fn moq_audio_encoder_packet(encoder: u32, index: u32, dst: *mut moq_frame) -> i32 { +pub unsafe extern "C" fn moq_audio_encoder_packet(encoder: u32, index: u32, dst: *mut moq_audio_packet) -> i32 { ffi::enter(move || { let dst = unsafe { dst.as_mut() }.ok_or(Error::InvalidPointer)?; with(encoder, audio, |audio| packet(&audio.packets, index, dst)) @@ -414,9 +424,9 @@ pub unsafe extern "C" fn moq_video_encoder_receive(encoder: u32, dst: *mut moq_f let Some(ready) = video.encoder.receive()? else { return Ok(0); }; - let packet = video.packet.insert(Packet { + let packet = video.packet.insert(VideoPacket { payload: ready.payload, - timestamp: Some(Box::new(ready.timestamp.into())), + timestamp: Box::new(ready.timestamp.into()), keyframe: ready.keyframe, }); *dst = packet.info(); diff --git a/rs/moq-c/src/ffi.rs b/rs/moq-c/src/ffi.rs index a64c00a63ce..a6f3a256e0c 100644 --- a/rs/moq-c/src/ffi.rs +++ b/rs/moq-c/src/ffi.rs @@ -597,14 +597,13 @@ pub unsafe fn parse_slice<'a, T>(data: *const T, size: usize) -> Result<&'a [T], Ok(data) } -/// Parse an optional timestamp, NULL meaning untimed. +/// Parse a required timestamp, refusing NULL with [Error::InvalidPointer]. /// /// # Safety /// `timestamp` must be NULL or a valid pointer to a [crate::moq_timestamp]. -pub unsafe fn parse_timestamp(timestamp: *const crate::moq_timestamp) -> Result, Error> { - Ok(unsafe { timestamp.as_ref() } - .map(|×tamp| timestamp.try_into()) - .transpose()?) +pub unsafe fn parse_timestamp(timestamp: *const crate::moq_timestamp) -> Result { + let timestamp = *unsafe { timestamp.as_ref() }.ok_or(Error::InvalidPointer)?; + Ok(timestamp.try_into()?) } #[cfg(test)] diff --git a/rs/moq-c/src/publish.rs b/rs/moq-c/src/publish.rs index 9e8d294f3c1..2436e473864 100644 --- a/rs/moq-c/src/publish.rs +++ b/rs/moq-c/src/publish.rs @@ -112,6 +112,12 @@ enum Family { #[derive(Clone)] pub struct Children(Arc>); +/// One past the newest group's start, or 0 on an empty track: where a group the caller gave +/// no start for begins, which stands in until every write carries a timestamp. +fn next_start(track: &moq_net::track::Producer) -> u64 { + track.largest().map_or(0, |largest| largest.group.saturating_add(1)) +} + /// The family of a standalone broadcast of its own. #[cfg(test)] impl Default for Children { @@ -664,11 +670,10 @@ impl Publish { Ok(()) } - /// Draw a group boundary and number the next group `sequence`. + /// Draw a group boundary and start the next group at `sequence`, in the track's timescale. /// - /// [`media_cut`](Self::media_cut) with an explicit sequence, for a caller whose group numbers - /// have to be deterministic: two encoders publishing the same content align per GOP so a - /// consumer can fail over between them. + /// [`media_cut`](Self::media_cut) keyed explicitly rather than at the next frame's time, which + /// has to come after every group the track holds. pub fn media_seek(&mut self, media: Id, sequence: u64) -> Result<(), Error> { let track = self.media.get_mut(media).ok_or(Error::MediaNotFound)?; track.seek(sequence)?; @@ -707,7 +712,7 @@ impl Publish { Ok(result?) } - /// Start a new segment and number its groups `sequence`. + /// Start a new segment whose first group starts at `sequence`, in its track's timescale. pub fn container_seek(&mut self, container: Id, sequence: u64) -> Result<(), Error> { let container = self.containers.get_mut(container).ok_or(Error::MediaNotFound)?; container.driver.check()?; @@ -848,7 +853,7 @@ impl Publish { } /// The raw track producer behind a handle, for tests that publish what the C API - /// can't express (an untimed frame). + /// can't express (a scale-less track). #[cfg(test)] pub(crate) fn track_producer(&mut self, track: Id) -> Option<&mut moq_net::track::Producer> { self.tracks.get_mut(track).map(|raw| &mut raw.inner) @@ -1039,6 +1044,11 @@ impl Publish { /// Hand `fill` to C a group at a time, each once the one before it reached a reader or /// nobody reads it any more, until every group is answered or nobody waits on the fill. async fn serve_fill(callback: OnStatus, fill: Arc) { + // A lookup of a time owes no group, and a request names one: drop it unanswered, as a + // handler that can't resolve times does, so the reader falls back to the groups held. + if fill.time().is_some() { + return; + } let mut changes = (*fill).clone(); while let Some(sequence) = changes.wanted_changed().await { let (pending, answered) = oneshot::channel(); @@ -1210,17 +1220,18 @@ impl Publish { } } - /// Append a new group to a raw track, returning a group producer. + /// Append a new group to a raw track, one past the newest group's start, returning a + /// group producer. pub fn track_group(&mut self, track: Id) -> Result { let raw = self.tracks.get_mut(track).ok_or(Error::TrackNotFound)?; - let group = raw.inner.append_group()?; + let group = raw.inner.append_group(next_start(&raw.inner))?; self.insert_group(group) } - /// Create a raw group with an explicit sequence number. + /// Create a raw group starting at `sequence`, in the track's timescale, which names it. pub fn track_group_at(&mut self, track: Id, sequence: u64) -> Result { let raw = self.tracks.get_mut(track).ok_or(Error::TrackNotFound)?; - let group = raw.inner.create_group(moq_net::group::Info { sequence })?; + let group = raw.inner.create_group(moq_net::group::Info { start: sequence })?; self.insert_group(group) } @@ -1231,13 +1242,9 @@ impl Publish { Ok(()) } - /// Write a single-frame group to a raw track, `None` when untimed. - pub fn track_frame( - &mut self, - track: Id, - timestamp: Option, - payload: &[u8], - ) -> Result<(), Error> { + /// Write a single-frame group to a raw track, starting at its frame's timestamp: the + /// unscaled group key on a scale-less track. + pub fn track_frame(&mut self, track: Id, timestamp: moq_net::Timestamp, payload: &[u8]) -> Result<(), Error> { let raw = self.tracks.get_mut(track).ok_or(Error::TrackNotFound)?; raw.inner .write_frame(timestamp, bytes::Bytes::copy_from_slice(payload))?; @@ -1466,7 +1473,7 @@ impl Publish { timestamp: moq_net::Timestamp, ) -> Result<(), Error> { let window = &mut self.json_window.get_mut(window).ok_or(Error::TrackNotFound)?.data; - window.producer.push(moq_net::Timed::from(&value).at(timestamp))?; + window.producer.push(moq_json::Timed::new(&value, timestamp))?; Ok(()) } @@ -1474,7 +1481,7 @@ impl Publish { /// `timestamp` on the broadcast clock. pub fn json_window_pop(&mut self, window: Id, count: u64, timestamp: moq_net::Timestamp) -> Result<(), Error> { let window = &mut self.json_window.get_mut(window).ok_or(Error::TrackNotFound)?.data; - window.producer.pop(moq_net::Timed::from(count).at(timestamp))?; + window.producer.pop(moq_json::Timed::new(count, timestamp))?; Ok(()) } @@ -1502,13 +1509,8 @@ impl Publish { } } - /// Write a frame into a raw group, `None` when untimed. - pub fn group_frame( - &mut self, - group: Id, - timestamp: Option, - payload: &[u8], - ) -> Result<(), Error> { + /// Write a frame into a raw group, at the group's unscaled key on a scale-less track. + pub fn group_frame(&mut self, group: Id, timestamp: moq_net::Timestamp, payload: &[u8]) -> Result<(), Error> { let group = self.groups.get(group).ok_or(Error::GroupNotFound)?; let mut group = group.inner.lock(); group @@ -1541,6 +1543,14 @@ impl Publish { Ok((entry.inner.clone(), entry.ended.subscribe())) } + /// The sequence (start) of a raw group, which names it: on a scale-less track, the + /// unscaled key its frames carry. + pub fn group_sequence(&self, group: Id) -> Result { + let group = self.groups.get(group).ok_or(Error::GroupNotFound)?; + let group = group.inner.lock(); + Ok(group.as_ref().ok_or(Error::GroupNotFound)?.start) + } + /// Finish a raw group. No more frames can be written. pub fn group_finish(&mut self, group: Id) -> Result<(), Error> { let group = self @@ -1548,7 +1558,7 @@ impl Publish { .remove(group) .and_then(|group| group.inner.take()) .ok_or(Error::GroupNotFound)?; - group.finish()?; + group.close()?; Ok(()) } @@ -1861,12 +1871,7 @@ impl Publish { children.insert(&mut self.datagrams, entry, Child::Datagrams) } - pub fn datagrams_append( - &mut self, - track: Id, - timestamp: Option, - payload: &[u8], - ) -> Result { + pub fn datagrams_append(&mut self, track: Id, timestamp: moq_net::Timestamp, payload: &[u8]) -> Result { Ok(self .datagrams .get_mut(track) diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index e03af2485d5..addf41784ea 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -14,12 +14,13 @@ fn micros(value: u64) -> moq_timestamp { moq_timestamp { value, scale: 1_000_000, + has_scale: true, } } -/// The timestamp a read points at, `None` when untimed. -fn read_timestamp(timestamp: *const moq_timestamp) -> Option { - unsafe { timestamp.as_ref() }.copied() +/// The timestamp a read points at, which is never NULL. +fn read_timestamp(timestamp: *const moq_timestamp) -> moq_timestamp { + *unsafe { timestamp.as_ref() }.expect("a read always carries a timestamp") } /// The broadcast clock's current time, which every write on the broadcast is stamped from. @@ -170,6 +171,21 @@ impl Callback { self.rx.recv().expect("callback channel closed") } + /// Wait for the next request while `reader` keeps reading: a time start the cache can't + /// place is looked up first, which the request handler drops, and the reader asks for its + /// groups only once it reads again. + fn recv_reading(&self, reader: &mut moq_net::track::Subscriber) -> i32 { + let waiter = kio::Waiter::noop(); + loop { + let _ = reader.poll_recv_group(&waiter); + match self.rx.recv_timeout(std::time::Duration::from_millis(10)) { + Ok(code) => return code, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => panic!("callback channel closed"), + } + } + } + /// Wait for the terminal callback (code <= 0) the task delivers after close /// or stream end. Must be drained before the Callback (user_data) drops, /// since user_data must outlive the final callback. @@ -1676,7 +1692,7 @@ fn raw_loc_video_uses_the_declared_catalog_container() { assert_eq!(unsafe { encoded_frame(frame_id, &mut frame) }, 0); assert_eq!( read_timestamp(frame.timestamp), - Some(micros(timestamp_us)), + micros(timestamp_us), "a media frame is always timed" ); assert!(frame.keyframe); @@ -2095,8 +2111,9 @@ fn track_info_max_age() { } /// Raw frames, datagrams, and JSON values read back exactly the timestamp written: a 90 kHz -/// tick that microseconds cannot express, a present zero, and untimed. The C API only creates -/// timed tracks, so the tracks and the JSON producer come from moq-net. +/// tick that microseconds cannot express, a present zero, and a scale-less track's unscaled +/// key. The C API only creates timed tracks, so the tracks and the JSON producer come from +/// moq-net. #[test] fn timestamps_round_trip_exactly() { let origin = id(moq_origin_create()); @@ -2107,14 +2124,17 @@ fn timestamps_round_trip_exactly() { let tick = moq_timestamp { value: 1, scale: 90_000, + has_scale: true, }; let zero = moq_timestamp { value: 0, scale: 90_000, + has_scale: true, }; - for (name, written) in [("tick", Some(tick)), ("zero", Some(zero)), ("untimed", None)] { + for (name, written) in [("tick", tick), ("zero", zero), ("unscaled", moq_timestamp::unscaled(0))] { + let scale = written.has_scale.then_some(written.scale); let info = - moq_net::track::Info::default().with_timescale(written.map(|_| moq_net::Timescale::new(90_000).unwrap())); + moq_net::track::Info::default().with_timescale(scale.map(|scale| moq_net::Timescale::new(scale).unwrap())); let create = |name: &str| -> u32 { let mut state = State::lock(); state @@ -2131,7 +2151,7 @@ fn timestamps_round_trip_exactly() { name.len(), &moq_datagram_info { priority: 0, - timescale: written.map(|_| 90_000).unwrap_or_default(), + timescale: scale.unwrap_or_default(), }, ) }); @@ -2171,7 +2191,7 @@ fn timestamps_round_trip_exactly() { ) }); - let timestamp = written.as_ref().map_or(std::ptr::null(), std::ptr::from_ref); + let timestamp = std::ptr::from_ref(&written); let payload = b"payload"; assert_eq!( unsafe { moq_publish_track_frame(track, payload.as_ptr(), payload.len(), timestamp) }, @@ -2219,9 +2239,9 @@ fn timestamps_round_trip_exactly() { .unwrap() .clone(); let mut json = moq_json::stream::Producer::::new(producer, Default::default()); - json.append(moq_net::Timed { + json.append(moq_json::Timed { value: &serde_json::json!(1), - at: written.map(|written| written.try_into().unwrap()), + at: written.try_into().unwrap(), }) .unwrap(); let value_id = id(values.recv()); @@ -2247,8 +2267,8 @@ fn timestamps_round_trip_exactly() { assert_eq!(moq_origin_close(origin), 0); } -/// A write refuses a timestamp outside `moq_net::Timestamp`'s range, and an untimed write on a -/// timed track. +/// A write refuses a timestamp outside `moq_net::Timestamp`'s range, a NULL one, and an +/// unscaled one on a timed track. #[test] fn timestamp_writes_refuse_out_of_range_and_mismatch() { let origin = id(moq_origin_create()); @@ -2258,14 +2278,20 @@ fn timestamp_writes_refuse_out_of_range_and_mismatch() { id(unsafe { moq_publish_track(broadcast, name.as_ptr() as *const c_char, name.len(), std::ptr::null()) }); for invalid in [ - moq_timestamp { value: 0, scale: 0 }, + moq_timestamp { + value: 0, + scale: 0, + has_scale: true, + }, moq_timestamp { value: 0, scale: 1 << 62, + has_scale: true, }, moq_timestamp { value: 1 << 61, scale: 1_000_000, + has_scale: true, }, ] { assert_eq!( @@ -2276,6 +2302,10 @@ fn timestamp_writes_refuse_out_of_range_and_mismatch() { } assert_eq!( unsafe { moq_publish_track_frame(track, name.as_ptr(), name.len(), std::ptr::null()) }, + MOQ_ERROR_INVALID_POINTER + ); + assert_eq!( + unsafe { moq_publish_track_frame(track, name.as_ptr(), name.len(), &moq_timestamp::unscaled(0)) }, MOQ_ERROR_MOQ ); @@ -2345,7 +2375,7 @@ fn raw_track_publish_consume() { assert_eq!(unsafe { moq_consume_track_frame(frame_id, &mut frame) }, 0); let received = unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }; assert_eq!(received, payload); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(timestamp_us))); + assert_eq!(read_timestamp(frame.timestamp), micros(timestamp_us)); assert!(!frame.keyframe, "raw frames have no keyframe flag"); assert_eq!(moq_consume_track_frame_free(frame_id), 0); @@ -2371,7 +2401,7 @@ fn raw_track_publish_consume() { assert_eq!(unsafe { moq_consume_track_frame(frame_id, &mut frame) }, 0); let received = unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }; assert_eq!(received, expected); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(timestamp_us))); + assert_eq!(read_timestamp(frame.timestamp), micros(timestamp_us)); assert_eq!(moq_consume_track_frame_free(frame_id), 0); } @@ -2528,7 +2558,10 @@ fn a_datagram_only_track_is_read() { .block_on(async { kio::clock::timeout(TIMEOUT, reader).await }) .expect("no datagram reader arrived"); - assert_eq!(datagrams.append(None, &b"only"[..]).unwrap(), 0); + assert_eq!( + datagrams.append(moq_net::Timestamp::unscaled(0), &b"only"[..]).unwrap(), + 0 + ); let dg_id = id(dg_cb.recv()); let mut datagram = moq_datagram { payload: std::ptr::null(), @@ -2601,7 +2634,7 @@ fn raw_track_datagram_publish_consume() { assert_eq!(unsafe { moq_consume_datagram(dg_id, &mut datagram) }, 0); let received = unsafe { std::slice::from_raw_parts(datagram.payload, datagram.payload_size) }; assert_eq!(received, payload); - assert_eq!(read_timestamp(datagram.timestamp), Some(micros(120_000))); + assert_eq!(read_timestamp(datagram.timestamp), micros(120_000)); assert_eq!(datagram.sequence, sequence); assert_eq!(moq_consume_datagram_free(dg_id), 0); @@ -2697,18 +2730,19 @@ fn raw_track_subscription_options_and_update() { let consume = request_broadcast(origin, path); let frame_cb = Callback::new(); - // A read whose ranges all end ends once it read them, so an open range far ahead keeps - // this one open for the update. + // An absolute anchor is a group's start, in the track's microseconds. A read whose ranges + // all end ends once it read them, so an open range far ahead keeps this one open for the + // update. let ranges = [ moq_range { start: moq_anchor { kind: moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32, - group: 1, + group: 20_000, timestamp: moq_timestamp::default(), }, end: moq_anchor { kind: moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32, - group: 2, + group: 20_001, timestamp: moq_timestamp::default(), }, end_present: true, @@ -2716,7 +2750,7 @@ fn raw_track_subscription_options_and_update() { moq_range { start: moq_anchor { kind: moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32, - group: 100, + group: 100_000_000, timestamp: moq_timestamp::default(), }, end: moq_anchor { @@ -2758,14 +2792,14 @@ fn raw_track_subscription_options_and_update() { assert_eq!(unsafe { moq_consume_track_frame(frame_id, &mut frame) }, 0); let received = unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }; assert_eq!(received, b"one"); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(20_000))); + assert_eq!(read_timestamp(frame.timestamp), micros(20_000)); assert_eq!(moq_consume_track_frame_free(frame_id), 0); // Widen the bounded range but keep the open one, so the read stays open for the cancel. let widened = [ moq_range { end: moq_anchor { - group: 3, + group: 40_001, ..ranges[0].end }, ..ranges[0] @@ -2789,7 +2823,7 @@ fn raw_track_subscription_options_and_update() { assert_eq!(unsafe { moq_consume_track_frame(frame_id, &mut frame) }, 0); let received = unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }; assert_eq!(received, b"two"); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(40_000))); + assert_eq!(read_timestamp(frame.timestamp), micros(40_000)); assert_eq!(moq_consume_track_frame_free(frame_id), 0); assert_eq!(moq_consume_track_cancel(consumer), 0); @@ -2800,7 +2834,8 @@ fn raw_track_subscription_options_and_update() { assert_eq!(moq_origin_close(origin), 0); } -/// A time anchor starts at the group containing that presentation time. +/// A time anchor starts at the group containing that presentation time, given unscaled in the +/// track's timescale. #[test] fn raw_track_time_anchor() { let origin = id(moq_origin_create()); @@ -2836,7 +2871,11 @@ fn raw_track_time_anchor() { start: moq_anchor { kind: moq_anchor_kind::MOQ_ANCHOR_TIME as u32, group: 0, - timestamp: micros(30_000), + timestamp: moq_timestamp { + value: 30_000, + scale: 0, + has_scale: false, + }, }, end: moq_anchor { kind: moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32, @@ -2888,10 +2927,11 @@ fn raw_track_time_anchor() { assert_eq!(moq_origin_close(origin), 0); } -/// An untimed track refuses a time anchor update at the call, keeping its ranges. The C -/// API only publishes timed tracks, so the track comes from moq-net. +/// An update with a scaled time anchor is refused at the call, keeping its ranges: a time +/// anchor is an unscaled key in the track's timescale, which the call can't check a scale +/// against. The C API only publishes timed tracks, so the track comes from moq-net. #[test] -fn raw_track_update_refuses_time_on_untimed() { +fn raw_track_update_refuses_a_scaled_time_anchor() { let origin = id(moq_origin_create()); let path = b"raw-track-untimed"; let broadcast = publish_broadcast(origin, path); @@ -2905,13 +2945,11 @@ fn raw_track_update_refuses_time_on_untimed() { Some(moq_net::track::Info::default().with_timescale(None)), ) .unwrap(); + // A scale-less track has no time: each frame opens the group its unscaled key names. + let key = std::cell::Cell::new(0); let write = |payload: &'static [u8]| { - State::lock() - .publish - .track_producer(track) - .unwrap() - .write_frame(None, payload) - .unwrap() + let timestamp = moq_net::Timestamp::unscaled(key.replace(key.get() + 1)); + State::lock().publish.track_frame(track, timestamp, payload).unwrap() }; let consume = request_broadcast(origin, path); @@ -2936,7 +2974,7 @@ fn raw_track_update_refuses_time_on_untimed() { start: moq_anchor { kind: moq_anchor_kind::MOQ_ANCHOR_TIME as u32, group: 0, - timestamp: moq_timestamp::default(), + timestamp: micros(0), }, end: moq_anchor { kind: moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE as u32, @@ -2978,106 +3016,6 @@ fn raw_track_update_refuses_time_on_untimed() { assert_eq!(moq_origin_close(origin), 0); } -#[test] -fn raw_track_relative_update_counts_from_the_track_edge() { - let origin = id(moq_origin_create()); - let path = b"raw-track-relative"; - let broadcast = publish_broadcast(origin, path); - - let track_name = b"data"; - let track = id(unsafe { - moq_publish_track( - broadcast, - track_name.as_ptr() as *const c_char, - track_name.len(), - std::ptr::null(), - ) - }); - let publish = |payload: &[u8]| { - assert_eq!( - unsafe { moq_publish_track_frame(track, payload.as_ptr(), payload.len(), µs(0)) }, - 0 - ); - }; - let anchor = |kind: moq_anchor_kind, group: u64| moq_anchor { - kind: kind as u32, - group, - timestamp: moq_timestamp::default(), - }; - let subscribe = |ranges: &[moq_range]| moq_subscription { - priority: 0, - delay: moq_delay { - max_us: 60_000_000, - ..moq_delay::default() - }, - ranges: ranges.as_ptr(), - ranges_len: ranges.len(), - order: 0, - }; - let recv = |frame_cb: &Callback| { - let frame_id = id(frame_cb.recv()); - let mut frame = moq_frame { - payload: std::ptr::null(), - payload_size: 0, - timestamp: std::ptr::null(), - keyframe: false, - }; - assert_eq!(unsafe { moq_consume_track_frame(frame_id, &mut frame) }, 0); - let received = unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }.to_vec(); - assert_eq!(moq_consume_track_frame_free(frame_id), 0); - received - }; - - publish(b"zero"); - let consume = request_broadcast(origin, path); - - // Group 0 only: the reader's cap sits below where the track will be. A read whose - // ranges all end ends once it read them, so an open range far ahead keeps it open. - let first = [ - moq_range { - start: anchor(moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE, 0), - end: anchor(moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE, 1), - end_present: true, - }, - moq_range { - start: anchor(moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE, 100), - end: anchor(moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE, 0), - end_present: false, - }, - ]; - let frame_cb = Callback::new(); - let consumer = id(unsafe { - moq_consume_track( - consume, - track_name.as_ptr() as *const c_char, - track_name.len(), - &subscribe(&first), - Some(channel_callback), - frame_cb.ptr, - ) - }); - assert_eq!(recv(&frame_cb), b"zero"); - - publish(b"one"); - publish(b"two"); - - // The latest group is the track's, group 2, not the newest one under the old cap. - let latest = [moq_range { - start: anchor(moq_anchor_kind::MOQ_ANCHOR_RELATIVE, 0), - end: anchor(moq_anchor_kind::MOQ_ANCHOR_ABSOLUTE, 0), - end_present: false, - }]; - assert_eq!(unsafe { moq_consume_track_update(consumer, &subscribe(&latest)) }, 0); - assert_eq!(recv(&frame_cb), b"two"); - - assert_eq!(moq_consume_track_cancel(consumer), 0); - assert_eq!(frame_cb.recv_terminal(), 0); - assert_eq!(moq_publish_track_finish(track), 0); - assert_eq!(moq_consume_close(consume), 0); - assert_eq!(moq_publish_close(broadcast), 0); - assert_eq!(moq_origin_close(origin), 0); -} - #[test] fn json_snapshot_publish_consume() { let origin = id(moq_origin_create()); @@ -3131,7 +3069,7 @@ fn json_snapshot_publish_consume() { timestamp: std::ptr::null(), }; assert_eq!(unsafe { moq_consume_json_value(value_id, &mut value) }, 0); - let at = read_timestamp(value.timestamp).expect("a stamped value is timed"); + let at = read_timestamp(value.timestamp); assert!(at.value >= last); last = at.value; let received = unsafe { std::slice::from_raw_parts(value.json.cast::(), value.json_len) }; @@ -3201,7 +3139,7 @@ fn json_stream_publish_consume() { timestamp: std::ptr::null(), }; assert_eq!(unsafe { moq_consume_json_value(value_id, &mut value) }, 0); - let at = read_timestamp(value.timestamp).expect("a stamped value is timed"); + let at = read_timestamp(value.timestamp); assert!(at.value >= last); last = at.value; let received = unsafe { std::slice::from_raw_parts(value.json.cast::(), value.json_len) }; @@ -3217,6 +3155,7 @@ fn json_stream_publish_consume() { let at = moq_timestamp { value: (last + 1_000) * 90 + 45, scale: 90_000, + has_scale: true, }; assert_eq!( unsafe { moq_publish_json_stream_append(producer, expected.as_ptr() as *const c_char, expected.len(), &at) }, @@ -3229,7 +3168,7 @@ fn json_stream_publish_consume() { timestamp: std::ptr::null(), }; assert_eq!(unsafe { moq_consume_json_value(value_id, &mut value) }, 0); - let read = read_timestamp(value.timestamp).expect("a stamped value is timed"); + let read = read_timestamp(value.timestamp); assert_eq!((read.value, read.scale), (last + 1_000, 1_000)); assert_eq!(moq_consume_json_value_free(value_id), 0); @@ -3249,7 +3188,7 @@ type WindowEvent = ( moq_json_window_kind, std::ops::Range, Option, - Option, + moq_net::Timestamp, ); /// Read and free the next event delivered to a JSON window consumer. @@ -3268,7 +3207,7 @@ fn recv_window(cb: &Callback) -> WindowEvent { let json = unsafe { std::slice::from_raw_parts(event.json.cast::(), event.json_len) }; serde_json::from_slice(json).unwrap() }); - let timestamp = read_timestamp(event.timestamp).map(|timestamp| timestamp.try_into().unwrap()); + let timestamp = moq_net::Timestamp::try_from(read_timestamp(event.timestamp)).unwrap(); assert_eq!(moq_consume_json_window_event_free(event_id), 0); assert!( moq_consume_json_window_event_free(event_id) < 0, @@ -3345,7 +3284,7 @@ fn json_window_publish_consume() { MOQ_JSON_WINDOW_KIND_PUSH, n..n + 1, Some(serde_json::json!({ "n": n })), - Some(moq_net::Timestamp::from_millis(n).unwrap()) + moq_net::Timestamp::from_millis(n).unwrap() ) ); } @@ -3356,7 +3295,7 @@ fn json_window_publish_consume() { MOQ_JSON_WINDOW_KIND_POP, 0..2, None, - Some(moq_net::Timestamp::from_millis(3).unwrap()) + moq_net::Timestamp::from_millis(3).unwrap() ) ); @@ -3389,11 +3328,16 @@ fn json_window_caller_timestamps() { let event_cb = Callback::new(); let consumer = window_consume(consume, name, &event_cb); let value = br#"{"n":1}"#; - let invalid = moq_timestamp { value: 1, scale: 0 }; + let invalid = moq_timestamp { + value: 1, + scale: 0, + has_scale: true, + }; // Valid, but too large once converted to the track's milliseconds. let unconvertible = moq_timestamp { value: (1 << 61) - 1, scale: 1, + has_scale: true, }; assert_eq!( @@ -3409,6 +3353,7 @@ fn json_window_caller_timestamps() { moq_timestamp { value: 111_111, scale: 90_000, + has_scale: true, }, ] { assert_eq!( @@ -3438,7 +3383,7 @@ fn json_window_caller_timestamps() { MOQ_JSON_WINDOW_KIND_PUSH, n as u64..n as u64 + 1, Some(serde_json::json!({ "n": 1 })), - Some(moq_net::Timestamp::from_millis(millis).unwrap()), + moq_net::Timestamp::from_millis(millis).unwrap(), ) ); } @@ -3449,7 +3394,7 @@ fn json_window_caller_timestamps() { MOQ_JSON_WINDOW_KIND_POP, 0..1, None, - Some(moq_net::Timestamp::from_millis(2500).unwrap()) + moq_net::Timestamp::from_millis(2500).unwrap() ) ); assert_eq!(event_cb.recv_terminal(), 0); @@ -3982,12 +3927,12 @@ fn media_cut_bounds_audio_groups() { assert_eq!(moq_publish_media_cut(media), 0, "each packet is its own group"); } - // The same boundary, with the next group explicitly numbered. + // The same boundary, with the next group explicitly started past the last. assert_eq!( unsafe { moq_publish_media_frame(media, payload.as_ptr(), payload.len(), micros(60_000)) }, 0 ); - assert_eq!(moq_publish_media_seek(media, 42), 0); + assert_eq!(moq_publish_media_seek(media, 100_000), 0); assert_eq!(moq_publish_media_discontinuity(media), 0); // Both report a missing importer rather than panicking on an unknown id. @@ -4820,7 +4765,7 @@ fn dynamic_serves_track_requests() { unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }, payload ); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(42))); + assert_eq!(read_timestamp(frame.timestamp), micros(42)); assert_eq!(moq_consume_track_frame_free(frame_id), 0); // A second request is aborted, and its subscriber sees the application code. @@ -5044,35 +4989,35 @@ fn a_group_request_nobody_reads_is_withdrawn() { let group_cb = Callback::new(); let dynamic = id(unsafe { moq_publish_track_requests(track, Some(channel_callback), group_cb.ptr) }); - let ascending = |start, end| { + let ascending = |start, end: Option| { moq_net::track::Subscription::default() .with_delay(moq_net::track::Delay::default().with_max(None)) - .with_range(start..end) + .with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(start), + end: end.map(moq_net::track::Anchor::Time), + }]) .with_order(moq_net::track::Order::Ascending) }; let mut reader = { let mut state = State::lock(); let producer = state.publish.track_producer(Id::try_from(track).unwrap()).unwrap(); - let reader = producer.subscribe(ascending(7, u64::MAX)); + let reader = producer.subscribe(ascending(7, None)); producer.start_at(10).unwrap(); - let mut group = producer.create_group(moq_net::group::Info { sequence: 10 }).unwrap(); + let mut group = producer.create_group(moq_net::group::Info { start: 10 }).unwrap(); group .write_frame(moq_net::Timestamp::from_millis(10).unwrap(), &b"live"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); reader }; - let waiter = kio::Waiter::noop(); - while let std::task::Poll::Ready(Ok(Some(_))) = reader.poll_recv_group(&waiter) {} - let first = id(group_cb.recv()); + let first = id(group_cb.recv_reading(&mut reader)); let mut sequence = 0u64; assert_eq!(unsafe { moq_group_request_sequence(first, &mut sequence) }, 0); assert_eq!(sequence, 7); // Reading only 8, so the fill owes nothing once 8 is let go. - reader.update(ascending(8, 9)).unwrap(); - assert!(reader.poll_recv_group(&waiter).is_pending()); - let next = id(group_cb.recv()); + reader.update(ascending(8, Some(8))).unwrap(); + let next = id(group_cb.recv_reading(&mut reader)); assert_eq!(unsafe { moq_group_request_sequence(next, &mut sequence) }, 0); assert_eq!(sequence, 8, "group 7, which nobody reads, held the fill back"); assert!(moq_group_request_accept(first) < 0, "the withdrawn request"); @@ -5103,17 +5048,20 @@ fn a_rejected_group_request_binds_only_its_readers() { let mut state = State::lock(); let producer = state.publish.track_producer(Id::try_from(track).unwrap()).unwrap(); producer.start_at(10).unwrap(); - let mut group = producer.create_group(moq_net::group::Info { sequence: 10 }).unwrap(); + let mut group = producer.create_group(moq_net::group::Info { start: 10 }).unwrap(); group .write_frame(moq_net::Timestamp::from_millis(10).unwrap(), &b"live"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); drop(producer.clone().finish().unwrap()); } let subscribe = || { let subscription = moq_net::track::Subscription::default() .with_delay(moq_net::track::Delay::default().with_max(None)) - .with_range(6..7); + .with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(6), + end: Some(moq_net::track::Anchor::Time(6)), + }]); let mut state = State::lock(); let producer = state.publish.track_producer(Id::try_from(track).unwrap()).unwrap(); producer.subscribe(subscription) @@ -5126,16 +5074,14 @@ fn a_rejected_group_request_binds_only_its_readers() { |request| moq_group_request_free(request), ] { let mut reader = subscribe(); - assert!(reader.poll_recv_group(&waiter).is_pending()); - let request = id(group_cb.recv()); + let request = id(group_cb.recv_reading(&mut reader)); assert_eq!(unsafe { moq_group_request_sequence(request, &mut sequence) }, 0); assert_eq!(sequence, 6); assert_eq!(reject(request), 0); } let mut reader = subscribe(); - assert!(reader.poll_recv_group(&waiter).is_pending()); - let request = id(group_cb.recv()); + let request = id(group_cb.recv_reading(&mut reader)); let group = id(moq_group_request_accept(request)); let payload = b"fetched"; assert_eq!( @@ -5146,7 +5092,7 @@ fn a_rejected_group_request_binds_only_its_readers() { let std::task::Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&waiter) else { panic!("the accepted group reaches the reader"); }; - assert_eq!(group.sequence, 6); + assert_eq!(group.start, 6); drop(reader); assert_eq!(moq_publish_requests_cancel(dynamic), 0); @@ -5290,7 +5236,7 @@ fn local_publish_consume() { }; assert_eq!(unsafe { encoded_frame(frame_id, &mut frame) }, 0); assert_eq!(frame.payload_size, payload.len()); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(timestamp_us))); + assert_eq!(read_timestamp(frame.timestamp), micros(timestamp_us)); let received = unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }; assert_eq!(received, payload, "frame payload should match"); @@ -5473,7 +5419,7 @@ fn consume_audio_follows_a_sibling_broadcast_reference() { assert_eq!(unsafe { encoded_frame(frame_id, &mut frame) }, 0); let received = unsafe { std::slice::from_raw_parts(frame.payload, frame.payload_size) }; assert_eq!(received, payload, "the sibling broadcast's frame should arrive"); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(timestamp_us))); + assert_eq!(read_timestamp(frame.timestamp), micros(timestamp_us)); assert_eq!(moq_consume_read_free(frame_id), 0); assert_eq!(moq_consume_audio_cancel(track), 0); @@ -6087,7 +6033,7 @@ fn video_publish_consume() { keyframe: false, }; assert_eq!(unsafe { encoded_frame(frame_id, &mut frame) }, 0); - assert_eq!(read_timestamp(frame.timestamp), Some(micros(0))); + assert_eq!(read_timestamp(frame.timestamp), micros(0)); assert!(frame.payload_size > 0, "frame should have payload data"); assert_eq!(moq_consume_read_free(frame_id), 0); @@ -7132,7 +7078,7 @@ fn multiple_frames_ordering() { assert_eq!(unsafe { encoded_frame(frame_id, &mut frame) }, 0); assert_eq!( read_timestamp(frame.timestamp), - Some(micros(expected_ts)), + micros(expected_ts), "frame {i} has wrong timestamp" ); @@ -8658,7 +8604,7 @@ fn a_shared_epoch_gives_media_no_delay() { unsafe { moq_publish_json_snapshot_update(json, value.as_ptr() as *const c_char, value.len(), &now) }, 0 ); - let start = moq_net::Timestamp::try_from(now).unwrap().as_micros() as u64; + let start = moq_net::Timestamp::try_from(now).unwrap().as_micros().unwrap() as u64; write_opus(media, start, 50, 100); let delay = audio_rendition(broadcast, &name).delay; diff --git a/rs/moq-c/src/test/parity.rs b/rs/moq-c/src/test/parity.rs index 3bd617cc994..2ade15d3d08 100644 --- a/rs/moq-c/src/test/parity.rs +++ b/rs/moq-c/src/test/parity.rs @@ -33,6 +33,13 @@ fn next_frame(cb: &Callback) -> Vec { payload } +/// Write `payload` into a raw group of a scale-less track, stamped with the group's key. +fn write_unscaled(group: u32, payload: &[u8]) -> i32 { + let mut key = 0; + assert_eq!(unsafe { moq_publish_group_sequence(group, &mut key) }, 0); + unsafe { moq_publish_group_frame(group, payload.as_ptr(), payload.len(), &moq_timestamp::unscaled(key)) } +} + /// Dial `peer` from `endpoint`, reporting to `cb`. fn dial(endpoint: u32, peer: &str, cb: &Callback) -> i32 { unsafe { @@ -115,10 +122,7 @@ fn socket_round_trip() { for part in [b"one".as_slice(), b"two"] { let group = id(moq_publish_track_group(writer)); - assert_eq!( - unsafe { moq_publish_group_frame(group, part.as_ptr(), part.len(), std::ptr::null()) }, - 0 - ); + assert_eq!(write_unscaled(group, part), 0); assert_eq!(moq_publish_group_finish(group), 0); } assert_eq!(moq_publish_track_finish(writer), 0); @@ -145,10 +149,7 @@ fn socket_track_round_trip_and_endpoint_close() { let _reader = id(unsafe { moq_socket_stream_reader(server, Some(channel_callback), frames.ptr) }); for part in [b"first".as_slice(), b"later"] { let group = id(moq_publish_track_group(track)); - assert_eq!( - unsafe { moq_publish_group_frame(group, part.as_ptr(), part.len(), std::ptr::null()) }, - 0 - ); + assert_eq!(write_unscaled(group, part), 0); assert_eq!(moq_publish_group_finish(group), 0); } assert_eq!(next_frame(&frames), b"first"); @@ -191,10 +192,7 @@ fn socket_finished_tracks_are_not_kept_by_the_endpoint() { let frames = Callback::new(); let _reader = id(unsafe { moq_socket_stream_reader(server, Some(channel_callback), frames.ptr) }); let group = id(moq_publish_track_group(track)); - assert_eq!( - unsafe { moq_publish_group_frame(group, &round, 1, std::ptr::null()) }, - 0 - ); + assert_eq!(write_unscaled(group, &[round]), 0); assert_eq!(moq_publish_group_finish(group), 0); assert_eq!(moq_publish_track_finish(track), 0); assert_eq!(next_frame(&frames), [round]); @@ -565,7 +563,7 @@ fn audio_encoder_without_a_broadcast() { // Stereo by default, so a frame is twice the frame size in interleaved samples. let pcm = vec![0.0f32; frame_size as usize * 2]; - let mut packet: moq_frame = unsafe { std::mem::zeroed() }; + let mut packet: moq_audio_packet = unsafe { std::mem::zeroed() }; let mut packets = 0; for _ in 0..3 { assert_eq!(unsafe { moq_audio_encoder_encode(encoder, pcm.as_ptr(), pcm.len()) }, 0); diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs index 2e1ef13647e..246ca566960 100644 --- a/rs/moq-cli/src/archive.rs +++ b/rs/moq-cli/src/archive.rs @@ -408,9 +408,9 @@ mod tests { for path in ["live/a", "live/b", "other"] { let broadcast = origin.route(path, moq_net::origin::RouteOptions::hidden()).unwrap(); let track = broadcast.broadcast().create_track("video", info.clone()).unwrap(); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); group.write_frame(Timestamp::from_millis(0).unwrap(), path).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); broadcast.announce().unwrap(); instances.push(broadcast.instance()); broadcasts.push((broadcast, track)); @@ -496,10 +496,10 @@ mod tests { .with_max_age(None); let video = source.create_track("video", info).unwrap(); for sequence in 0..2 { - let mut group = video.create_group(group::Info { sequence }).unwrap(); + let mut group = video.create_group(group::Info { start: sequence * 1000 }).unwrap(); let pts = Timestamp::from_millis(sequence * 1000).unwrap(); group.write_frame(pts, format!("frame {sequence}")).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let _video = video.finish().unwrap(); let recording = diff --git a/rs/moq-cli/src/fetch.rs b/rs/moq-cli/src/fetch.rs index da702d102af..028c59cb9a9 100644 --- a/rs/moq-cli/src/fetch.rs +++ b/rs/moq-cli/src/fetch.rs @@ -22,7 +22,7 @@ pub struct Args { /// The track name, taken literally: a `/` in it is part of the name. pub track: String, - /// The group sequence to read; the newest group when omitted. + /// The start of the group to read, in the track's timescale; the newest group when omitted. #[usage(long)] pub group: Option, @@ -127,21 +127,21 @@ async fn fetch_from( let mut group = moq_relay::fetch_group(&track, args.group) .await .with_context(|| match args.group { - Some(sequence) => format!("group {sequence} of `{}` not found", args.track), + Some(start) => format!("group {start} of `{}` not found", args.track), None => format!("no group of `{}` found", args.track), })?; - let sequence = group.sequence; + let start = group.start; let mut index = 0; while let Some(frame) = group .read_frame() .await - .with_context(|| format!("failed to read group {sequence} of `{}`", args.track))? + .with_context(|| format!("failed to read group {start} of `{}`", args.track))? { match args.json { true => { let mut line = serde_json::to_vec(&Frame { - group: sequence, + group: start, frame: index, size: frame.payload.len() as u64, payload: base64::engine::general_purpose::STANDARD.encode(&frame.payload), @@ -201,18 +201,19 @@ mod tests { let data = broadcast.broadcast().create_track("data", timed()).expect("data track"); for sequence in 0..3 { - let mut group = data.append_group().expect("group"); + let mut group = data.append_group(sequence).expect("group"); + let at = moq_net::Timestamp::from_millis(sequence).expect("timestamp"); for frame in frames(sequence) { - group.write_frame(moq_net::Timestamp::ZERO, frame).expect("frame"); + group.write_frame(at, frame).expect("frame"); } - group.finish().expect("finish"); + group.close().expect("close"); } let empty = broadcast .broadcast() .create_track("empty", timed()) .expect("empty track"); let live = broadcast.broadcast().create_track("live", timed()).expect("live track"); - let mut open = live.append_group().expect("open group"); + let mut open = live.append_group(0).expect("open group"); open.write_frame(moq_net::Timestamp::ZERO, b"first".as_ref()) .expect("frame"); @@ -265,11 +266,11 @@ mod tests { .route(".hidden", moq_net::origin::RouteOptions::default()) .expect("hidden broadcast"); let secret = hidden.broadcast().create_track("data", timed()).expect("hidden track"); - let mut group = secret.append_group().expect("group"); + let mut group = secret.append_group(0).expect("group"); group .write_frame(moq_net::Timestamp::ZERO, b"secret".as_ref()) .expect("frame"); - group.finish().expect("finish"); + group.close().expect("close"); let ready = fixture.relay.ready(); tokio::spawn(fixture.relay.run()); diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index f2a782fbacf..5210131b3b4 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -1301,11 +1301,11 @@ mod tests { .unwrap(); let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); let track = broadcast.broadcast().create_track("status", info).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group .write_frame(moq_net::Timestamp::ZERO, b"admitted".as_ref()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let (auth, admissions) = moq_relay::auth::Auth::embedded("test-cli"); let (trigger, observer) = moq_relay::shutdown::Observer::new(TIMEOUT); let serving = Serving { @@ -1502,9 +1502,9 @@ mod tests { .unwrap(); let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); let track = broadcast.broadcast().create_track("live", info).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(moq_net::Timestamp::ZERO, b"live".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let reader = relay.consume(); let mut live = reader diff --git a/rs/moq-cli/src/play/buffer.rs b/rs/moq-cli/src/play/buffer.rs index 309e87774e1..218a525f6d4 100644 --- a/rs/moq-cli/src/play/buffer.rs +++ b/rs/moq-cli/src/play/buffer.rs @@ -65,7 +65,8 @@ impl Buffer { pub fn push(&mut self, frame: Frame, max_delay: Duration) { let edge = *self.edge.get_or_insert(frame.timestamp); - self.edge = Some(edge.max(frame.timestamp)); + // Timestamp has no `Ord`; one that doesn't compare keeps the edge. + self.edge = Some(if frame.timestamp > edge { frame.timestamp } else { edge }); if frame.keyframe { if self.waiting_keyframe { self.floor = Some(frame.timestamp); @@ -77,8 +78,11 @@ impl Buffer { } self.index(frame); let edge = self.edge.unwrap(); - let stale = - |oldest: &moq_net::Timestamp| edge.as_micros().saturating_sub(oldest.as_micros()) > max_delay.as_micros(); + // An unscaled timestamp has no age, so it never goes stale. + let stale = |oldest: &moq_net::Timestamp| match (edge.as_micros(), oldest.as_micros()) { + (Ok(edge), Ok(oldest)) => edge.saturating_sub(oldest) > max_delay.as_micros(), + _ => false, + }; if self.oldest.front().is_some_and(stale) { // Dropping a reference picture invalidates the rest of its GOP. Resume // only at a retained keyframe, never feed a broken chain to the codec. @@ -136,7 +140,7 @@ mod tests { buffer.push(frame(50, false), age); buffer.push(frame(100, true), age); buffer.push(frame(150, false), age); - assert_eq!(buffer.frames.front().unwrap().timestamp.as_millis(), 100); + assert_eq!(buffer.frames.front().unwrap().timestamp.as_millis().unwrap(), 100); assert_eq!(buffer.bytes, 14); buffer.pop(); assert_eq!(buffer.bytes, 7); @@ -164,10 +168,10 @@ mod tests { buffer.push(frame(ms, keyframe), age); } buffer.push(frame(95, false), age); - let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis()).collect(); + let kept: Vec<_> = buffer.frames.iter().map(|f| f.timestamp.as_millis().unwrap()).collect(); assert_eq!(kept, [100, 110], "80, 90 and 95 reference the evicted group"); assert_eq!(buffer.bytes, 14); - assert_eq!(buffer.oldest().unwrap().as_millis(), 100); + assert_eq!(buffer.oldest().unwrap().as_millis().unwrap(), 100); buffer.push(frame(200, true), age); buffer.push(frame(180, false), age); assert_eq!(buffer.frames.len(), 4, "a continuous group keeps its leading pictures"); @@ -183,16 +187,16 @@ mod tests { buffer .frames .iter() - .map(|f| f.timestamp.as_millis()) + .map(|f| f.timestamp.as_millis().unwrap()) .collect::>(), [0, 99, 33, 66] ); - assert_eq!(buffer.edge.unwrap().as_millis(), 99); + assert_eq!(buffer.edge.unwrap().as_millis().unwrap(), 99); buffer.clear(); assert_eq!(buffer.bytes, 0); assert!(buffer.frames.is_empty()); buffer.push(frame(1, true), Duration::ZERO); - assert_eq!(buffer.edge.unwrap().as_millis(), 1); + assert_eq!(buffer.edge.unwrap().as_millis().unwrap(), 1); } #[test] fn age_expiry_finds_a_reordered_picture_behind_the_front() { diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index b63eb1dbc35..e5e0f2c4b75 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -274,8 +274,9 @@ impl Media { // Nothing older than the playhead is worth presenting, so the delay // doubles as the staleness budget on the wire. let max_delay = self.args.video_delay(); - // A decoder and its subscription, starting `lookback` groups before the live edge. - let open = async |delay: moq_net::track::Delay, lookback: u64| { + // A decoder and its subscription from `start`, and the live edge when `edge` asks + // to wait for it. + let open = async |delay: moq_net::track::Delay, start: moq_net::track::Anchor, edge: bool| { // The window's renderer refuses a GPU surface it cannot import, so // the decoder downloads those instead, all of them on a device // that imports none. @@ -297,32 +298,37 @@ impl Media { .with_priority(hang::catalog::PRIORITY.video) // The whole delay from the first request, so a cold join never // states a zero minimum before the container's update. - .with_delay(delay), + .with_delay(delay) + .with_ranges([moq_net::track::Range { start, end: None }]), ) .await?; - // Start at the live edge without asking the shared publisher - // subscription to rewind to a cached sequence. The edge is known only - // once the feed is live, so wait for it rather than read the backlog of - // a warm copy still waiting on its source; the wait ends with the track. - if let Some(largest) = subscriber.largest().await { - subscriber = subscriber.with_range(largest.group.saturating_sub(lookback)..); - } + // The live edge is known only once the feed is live: a warm copy still + // waiting on its source reports nothing yet. The wait ends with the track, + // and holds up the rest of playback, so only a warmup lane asks for it. + let edge = match edge { + true => subscriber.largest().await.map(|largest| largest.group), + false => None, + }; let format = catalog::hang::Container::try_from(config)?; // The playout delay is the target latency, and the max how late a group may be. let track = moq_mux::container::Playout::new(delay, Duration::ZERO).consumer(subscriber, format); - Ok::<_, anyhow::Error>((track, decoder)) + Ok::<_, anyhow::Error>((track, decoder, edge)) }; let opened = async { let delay = moq_net::track::Delay { min: Some(self.args.video_delay()), max: Some(max_delay), }; - let (track, decoder) = open(delay, 0).await?; // An intra-refresh rendition also runs a decoder with one more sweep // of budget, joining a group earlier so its pictures stay clean, as // `moq_video::decode::Consumer` does. Both share one upstream request. let warmup = config.warmup.unwrap_or_default().min(moq_mux::container::DELAY_MAX); + // The live edge, from the groups each hop holds from its live feed, never a warm + // copy's backlog, so it names no cached group for the shared publisher + // subscription to rewind to. + let (track, decoder, edge) = + open(delay, moq_net::track::Anchor::Relative(0), !warmup.is_zero()).await?; let refresh = if warmup.is_zero() { None } else { @@ -332,7 +338,13 @@ impl Media { min: delay.min.map(|min| min.saturating_add(warmup)), max: Some(max_delay.saturating_add(warmup)), }; - let (track, decoder) = open(delay, 1).await?; + // The group before the live edge: the one containing the key just below it, as + // starts are sparse. Every wire carries a time start, where a relative one + // past the edge is refused before lite-07; there it reaches the edge alone. + let start = edge.map_or(moq_net::track::Anchor::Relative(0), |edge| { + moq_net::track::Anchor::Time(edge.saturating_sub(1)) + }); + let (track, decoder, _) = open(delay, start, false).await?; let interval = config .framerate .filter(|rate| rate.is_finite() && *rate > 0.0) @@ -792,7 +804,7 @@ async fn play_audio( let delay = consumer.delay(); let crossing = match &event { Ok(Some(moq_audio::decode::Read::End { end })) => { - ends.push_back((Duration::from(*end), role.declare())); + ends.push_back((Duration::try_from(*end)?, role.declare())); None } // Playback restarts at the frame after it, so that is what carries it. @@ -805,9 +817,10 @@ async fn play_audio( // A frame at or past an end comes from the encoder that replaced it; one // before it is the stopped encoder's drain. Only the last end crossed has // replacement media; the ones before it ended empty timelines. + let at = Duration::try_from(frame.timestamp)?; let mut crossed = None; while let Some(&(end, transition)) = ends.front() - && Duration::from(frame.timestamp) >= end + && at >= end { ends.pop_front(); if let Some(skipped) = crossed.replace(transition) { @@ -930,7 +943,7 @@ async fn play_audio( ended = false; // An end this frame did not cross ends the timeline it resumes. if let Some(end) = deferred.take() - && Duration::from(frame.timestamp) < Duration::from(end) + && Duration::try_from(frame.timestamp)? < Duration::try_from(end)? { role.end(end)?; ended = true; @@ -982,7 +995,7 @@ async fn play_audio( let length = frame.data.len() / stride; let mapped = role.map(frame.timestamp)?; - let start = Duration::from(mapped); + let start = Duration::try_from(mapped)?; let timing = timeline.push(start, length, sample_rate, fill_max); // A rewind or a hole too large to fill starts a new playback sink. The old @@ -1080,7 +1093,7 @@ async fn play_audio( // A dropped write leaves the utterance to start on the next frame the sink accepts. if utterance && written.accepted_sample_frames > 0 { utterance = false; - output.send(Event::Utterance(Duration::from(frame.timestamp))); + output.send(Event::Utterance(Duration::try_from(frame.timestamp)?)); } if std::mem::take(&mut declare) { sink.end(); @@ -1090,7 +1103,8 @@ async fn play_audio( if written.accepted_sample_frames > 0 { fresh = false; } - let end = Duration::from(frame.timestamp) + Duration::from_secs_f64(length as f64 / sample_rate as f64); + let end = + Duration::try_from(frame.timestamp)? + Duration::from_secs_f64(length as f64 / sample_rate as f64); wrote = moq_net::Timestamp::try_from(end).ok().map(|end| (end, timing.end)); steer(wrote); playhead.wrote(timing.end, sink.buffered(), Instant::now()); @@ -1239,7 +1253,8 @@ mod tests { producer.write(packet(0, 0.25)).unwrap(); tokio::task::yield_now().await; producer.discontinuity().unwrap(); - producer.write(packet(0, 0.5)).unwrap(); + // The resume starts after the marker: group starts only move forward. + producer.write(packet(1, 0.5)).unwrap(); let _finished = producer.finish().unwrap(); task.await.unwrap(); // The resume keeps the sink, which fills the declared silence with comfort noise. @@ -1520,9 +1535,12 @@ mod tests { kio::clock::sleep(PACKET_DURATION).await; } producer.discontinuity().unwrap(); - // One group per packet and one for the marker, so the next press would be group 11. + // A seek creates the press's first group without appending it, so nothing declares the + // starts between it and the marker skipped: a group there may have been lost. if lost { - producer.seek(12).unwrap(); + producer + .seek(packet(110, 0.5).timestamp.as_micros().unwrap() as u64) + .unwrap(); } // Two seconds muted, with the timeline running on through it. kio::clock::sleep(Duration::from_secs(2)).await; @@ -1710,7 +1728,9 @@ mod tests { assert_eq!(recorder.latencies().len(), 1, "the late burst replaced the sink"); assert!(shown.len() > 50, "video stalled: {} pictures", shown.len()); for (timestamp, at) in shown { - let due = start.saturating_add(delay).saturating_add(Duration::from(timestamp)); + let due = start + .saturating_add(delay) + .saturating_add(Duration::try_from(timestamp).unwrap()); assert!( at.saturating_duration_since(due) <= Duration::from_millis(1) && due.saturating_duration_since(at) <= Duration::from_millis(1), @@ -1727,7 +1747,7 @@ mod tests { moq_video::Surface::rgba(&[128; 16 * 16 * 4], moq_video::Size::new(16, 16))?, frame.timestamp, )); - self.0.sort_by_key(|frame| frame.timestamp); + self.0.sort_by(|a, b| a.timestamp.partial_cmp(&b.timestamp).unwrap()); let ready = self.0.len().saturating_sub(2); self.1.extend(self.0.drain(..ready)); Ok(()) @@ -1867,7 +1887,7 @@ mod tests { assert_eq!( shown[..3] .iter() - .map(|(timestamp, _)| timestamp.as_millis()) + .map(|(timestamp, _)| timestamp.as_millis().unwrap()) .collect::>(), [0, 20, 40] ); @@ -1892,14 +1912,17 @@ mod tests { let shown = shown .iter() .find(|(shown, _)| { - let start = Duration::from(*shown); + let start = Duration::try_from(*shown).unwrap(); start <= timestamp && timestamp < start + PACKET_DURATION }) .expect("continuous video reaches resumed audio"); - (shown.1.saturating_add(timestamp - Duration::from(shown.0)), new[0].from) + ( + shown.1.saturating_add(timestamp - Duration::try_from(shown.0).unwrap()), + new[0].from, + ) } else { - let timestamp = Duration::from(shown[3].0); - let index = (shown[3].0.as_millis() / 20) as usize; + let timestamp = Duration::try_from(shown[3].0).unwrap(); + let index = (shown[3].0.as_millis().unwrap() / 20) as usize; let offset = timestamp - PACKET_DURATION * index as u32; (shown[3].1, new[index - 3].from.saturating_add(offset)) }; @@ -2196,7 +2219,7 @@ mod tests { for _ in 0..2_000 { tokio::task::yield_now().await; if let Some(timestamp) = recorder.present(&media) { - shown.push(timestamp.as_millis()); + shown.push(timestamp.as_millis().unwrap()); } tokio::time::advance(Duration::from_millis(1)).await; } diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index aab4c036253..b94dcfccbf5 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -127,6 +127,35 @@ impl std::ops::DerefMut for Segment { } /// The earlier deadline, where `None` is no deadline. +/// The later of two timestamps, `a` when they don't compare: [`moq_net::Timestamp`] has no `Ord`. +fn later(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { + if b > a { b } else { a } +} + +/// The earlier of two timestamps, `a` when they don't compare; see [`later`]. +fn earlier(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { + if b < a { b } else { a } +} + +/// A timestamp on the broadcast timeline, ordered by time. +/// +/// [`moq_net::Timestamp`] has no `Ord`, since an unscaled one orders against no scaled one. +/// The presentation's `map` only returns scaled timestamps, and scaled ones order totally. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Mapped(moq_net::Timestamp); + +impl Ord for Mapped { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.partial_cmp(&other.0).unwrap_or(std::cmp::Ordering::Equal) + } +} + +impl PartialOrd for Mapped { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + fn soonest(a: Option, b: Option) -> Option { match (a, b) { (Some(a), Some(b)) => Some(a.min(b)), @@ -397,7 +426,11 @@ impl Video { loop { match selector.next(now) { Next::Read(Decoded::Frame(frame)) => { - if floor.is_none_or(|floor| frame.timestamp.as_micros() >= floor.as_micros()) { + // A picture that doesn't compare with the floor isn't known to precede it. + let below = floor.is_some_and( + |floor| matches!((frame.timestamp.as_micros(), floor.as_micros()), (Ok(at), Ok(floor)) if at < floor), + ); + if !below { pending.push_back(frame); } } @@ -440,7 +473,7 @@ impl Video { // decoded before that picture is due, however deep the stream // reorders or the codec holds pictures back. let oldest = role.map(oldest)?; - let owed = held.first().map_or(oldest, |held| oldest.min(*held)); + let owed = held.first().map_or(oldest, |held: &Mapped| earlier(oldest, held.0)); let at = presentation .due(owed) .and_then(|at| at.checked_sub(DECODE_AHEAD)) @@ -497,10 +530,10 @@ impl Video { .expect("no await since inspecting the buffer"); frame.timestamp = role.map(frame.timestamp)?; generation = Some(current); - held.insert(frame.timestamp); + held.insert(Mapped(frame.timestamp)); { let mut edge = edge.lock().unwrap(); - *edge = Some(edge.map_or(frame.timestamp, |edge| edge.max(frame.timestamp))); + *edge = Some(edge.map_or(frame.timestamp, |edge| later(edge, frame.timestamp))); } wake.notify_one(); // Never race a codec call: cancelling Sink::decode poisons it. The @@ -570,7 +603,7 @@ impl Video { while let Some(frame) = decoder.receive()? { // A codec returns display order, so a picture held before one it // returned was dropped rather than delayed. - held.retain(|held| *held > frame.timestamp); + held.retain(|held| held.0 > frame.timestamp); selector.push(Lane::Min, Decoded::Frame(frame), now); } }; @@ -804,7 +837,7 @@ mod tests { let surface = moq_video::Surface::rgba(&vec![128; size * size * 4], moq_video::Size::new(self.size, self.size))?; self.held.push(moq_video::Frame::new(surface, frame.timestamp)); - self.held.sort_by_key(|frame| frame.timestamp); + self.held.sort_by(|a, b| a.timestamp.partial_cmp(&b.timestamp).unwrap()); let bumped = self.held.len().saturating_sub(self.depth); self.ready.extend(self.held.drain(..bumped)); Ok(()) @@ -1030,7 +1063,7 @@ mod tests { if let Some(picture) = media.output.present(&media) && first.is_none() { - first = Some((picture.as_millis(), at)); + first = Some((picture.as_millis().unwrap(), at)); } tokio::time::advance(Duration::from_millis(1)).await; } @@ -1108,7 +1141,11 @@ mod tests { task.await.unwrap().unwrap(); assert_eq!(*flushed.lock().unwrap(), 1, "decoder tail was not flushed exactly once"); let &(timestamp, at) = shown.last().expect("presented video"); - assert_eq!(timestamp.as_micros(), last.as_micros(), "decoder tail was lost"); + assert_eq!( + timestamp.as_micros().unwrap(), + last.as_micros().unwrap(), + "decoder tail was lost" + ); assert!( at.max(last_due).duration_since(at.min(last_due)) <= Duration::from_millis(1), "live edge remained late: {at:?}, expected {last_due:?}" @@ -1168,7 +1205,7 @@ mod tests { for _ in 0..110 { tokio::task::yield_now().await; if let Some(frame) = media.output.present(&media) { - shown.push(frame.as_millis()); + shown.push(frame.as_millis().unwrap()); } tokio::time::advance(Duration::from_millis(1)).await; } @@ -1224,10 +1261,10 @@ mod tests { .lock() .unwrap() .iter() - .map(|frame| frame.timestamp.as_millis()) + .map(|frame| frame.timestamp.as_millis().unwrap()) .collect::>(); assert_eq!(queued, [500], "old decoder output crossed the discontinuity"); - assert_eq!(media.output.present(&media).unwrap().as_millis(), 500); + assert_eq!(media.output.present(&media).unwrap().as_millis().unwrap(), 500); } #[tokio::test] @@ -1248,14 +1285,16 @@ mod tests { .unwrap(); let format = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); let write = |sequence, stamps: Vec| { - let mut group = published.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = published + .create_group(moq_net::group::Info { start: sequence }) + .unwrap(); let frames: Vec<_> = stamps .into_iter() .enumerate() .map(|(index, ms)| frame(ms, index == 0)) .collect(); format.write(&mut group, &frames).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; write(10, vec![100, 80, 90, 110]); write(11, vec![200, 180, 190, 210]); @@ -1284,7 +1323,12 @@ mod tests { tokio::time::advance(Duration::from_millis(1)).await; } task.await.unwrap().unwrap(); - let got: Vec<_> = decoded.lock().unwrap().iter().map(|(at, _)| at.as_millis()).collect(); + let got: Vec<_> = decoded + .lock() + .unwrap() + .iter() + .map(|(at, _)| at.as_millis().unwrap()) + .collect(); assert_eq!(got, [100, 110, 200, 180, 190, 210, 400, 410]); } @@ -1323,7 +1367,7 @@ mod tests { for _ in 0..2_200 { tokio::task::yield_now().await; if let Some(timestamp) = media.output.present(&media) { - shown.push(timestamp.as_millis()); + shown.push(timestamp.as_millis().unwrap()); } tokio::time::advance(Duration::from_millis(1)).await; } @@ -1434,7 +1478,7 @@ mod tests { if media .output .present(&media) - .is_some_and(|picture| picture.as_millis() >= 60) + .is_some_and(|picture| picture.as_millis().unwrap() >= 60) { shown += 1; } @@ -1512,8 +1556,8 @@ mod tests { } task.await.unwrap().unwrap(); assert_eq!( - shown.map(Duration::from), - Some(Duration::from(first) + Duration::from_millis(40)) + shown.map(|shown| Duration::try_from(shown).unwrap()), + Some(Duration::try_from(first).unwrap() + Duration::from_millis(40)) ); } @@ -1630,7 +1674,7 @@ mod tests { for _ in 0..1_000 { tokio::task::yield_now().await; if let Some(timestamp) = media.output.present(&media) { - shown.push(timestamp.as_micros()); + shown.push(timestamp.as_micros().unwrap()); } tokio::time::advance(Duration::from_millis(1)).await; } @@ -1689,10 +1733,11 @@ mod tests { // A publisher finishes a group when it starts the next one. self.seal(sequence); } - let group = self - .groups - .entry(sequence) - .or_insert_with(|| self.track.create_group(moq_net::group::Info { sequence }).unwrap()); + let group = self.groups.entry(sequence).or_insert_with(|| { + self.track + .create_group(moq_net::group::Info { start: sequence }) + .unwrap() + }); let format = moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Video); format .write(group, &[frame(index * 33, index.is_multiple_of(self.sweep))]) @@ -1703,7 +1748,7 @@ mod tests { fn seal(&mut self, sequence: u64) { let open = self.groups.split_off(&sequence); for (_, group) in std::mem::replace(&mut self.groups, open) { - group.finish().unwrap(); + group.close().unwrap(); } } @@ -1755,7 +1800,7 @@ mod tests { } tokio::task::yield_now().await; if let Some(picture) = media.output.present_frame(media) { - shown.push((picture.timestamp.as_millis(), picture.size().width)); + shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); } tokio::time::advance(Duration::from_millis(1)).await; assert!(elapsed < Duration::from_secs(30), "playback never finished"); @@ -1764,7 +1809,7 @@ mod tests { // The window still holds the last pictures the decoder queued. for _ in 0..1_000 { if let Some(picture) = media.output.present_frame(media) { - shown.push((picture.timestamp.as_millis(), picture.size().width)); + shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); } tokio::time::advance(Duration::from_millis(1)).await; } @@ -1891,7 +1936,7 @@ mod tests { } tokio::task::yield_now().await; if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis(), picture.size().width)); + shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); } tokio::time::advance(Duration::from_millis(1)).await; assert!(elapsed < Duration::from_secs(30), "playback never finished: {shown:?}"); @@ -1900,7 +1945,7 @@ mod tests { // The window still holds the last pictures the decoder queued. for _ in 0..1_000 { if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis(), picture.size().width)); + shown.push((picture.timestamp.as_millis().unwrap(), picture.size().width)); } tokio::time::advance(Duration::from_millis(1)).await; } @@ -1971,7 +2016,7 @@ mod tests { } tokio::task::yield_now().await; if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis(), Instant::now() - start)); + shown.push((picture.timestamp.as_millis().unwrap(), Instant::now() - start)); } tokio::time::advance(Duration::from_millis(1)).await; assert!( @@ -1982,7 +2027,7 @@ mod tests { task.await.unwrap().unwrap(); for _ in 0..1_000 { if let Some(picture) = media.output.present_frame(&media) { - shown.push((picture.timestamp.as_millis(), Instant::now() - start)); + shown.push((picture.timestamp.as_millis().unwrap(), Instant::now() - start)); } tokio::time::advance(Duration::from_millis(1)).await; } @@ -2023,7 +2068,7 @@ mod tests { assert_eq!( shown .iter() - .map(|(timestamp, _)| timestamp.as_millis()) + .map(|(timestamp, _)| timestamp.as_millis().unwrap()) .collect::>(), expected, "a picture was dropped" diff --git a/rs/moq-cli/src/publish.rs b/rs/moq-cli/src/publish.rs index c9c6718ae4d..47d1d079fa9 100644 --- a/rs/moq-cli/src/publish.rs +++ b/rs/moq-cli/src/publish.rs @@ -1820,7 +1820,7 @@ pub(crate) mod tests { let first = kio::clock::timeout(WAIT, track.recv_group()).await.unwrap().unwrap(); (track, first) }; - // Each group's sequence and frames, read until the track ends. A read cut off as + // Each group's start and frames, read until the track ends. A read cut off as // stale leaves `None` for that group's frames, and a marker leaves none. let read = |(mut track, first): (moq_net::track::Subscriber, Option)| { tokio::spawn(kio::clock::timeout(WAIT, async move { @@ -1844,10 +1844,10 @@ pub(crate) mod tests { Err(err) => panic!("{err}"), } } - groups.push((group.sequence, frames)); + groups.push((group.start, frames)); next = track.recv_group().await.unwrap(); } - groups.sort_by_key(|(sequence, _)| *sequence); + groups.sort_by_key(|(start, _)| *start); groups })) }; @@ -1865,9 +1865,9 @@ pub(crate) mod tests { let recording: Vec<_> = recording .await .unwrap() - .expect("the recording ends with the track") + .unwrap_or_else(|err| panic!("{version}: the recording ends with the track: {err}")) .into_iter() - .map(|(sequence, frames)| (sequence, frames.expect("the recording reads every group whole"))) + .map(|(start, frames)| (start, frames.expect("the recording reads every group whole"))) .collect(); let output = recording .iter() @@ -1880,24 +1880,30 @@ pub(crate) mod tests { "{version}: byte-identical from the first group" ); let media: Vec<_> = recording.iter().filter(|(_, frames)| !frames.is_empty()).collect(); - let first = |frames: &[hang::container::Frame]| frames[0].timestamp.as_micros(); + let first = |frames: &[hang::container::Frame]| frames[0].timestamp.as_micros().unwrap(); let jump = media .windows(2) .find(|pair| first(&pair[1].1) > first(&pair[0].1) + 10_000_000) .map(|pair| pair[1].0) .unwrap_or_else(|| panic!("{version}: no jump crossed")); - let last = recording.last().unwrap().0; - assert_eq!( - recording.iter().map(|(sequence, _)| *sequence).collect::>(), - (recording[0].0..=last).collect::>(), - "{version}: no group missing" + // Groups are keyed by their starts, so a missing one shows in the bytes above rather than + // in the keys, which only have to rise. + let starts: Vec = recording.iter().map(|(start, _)| *start).collect(); + assert!( + starts.windows(2).all(|pair| pair[0] < pair[1]), + "{version}: groups in start order: {starts:?}" ); + let at = starts.iter().position(|start| *start == jump).unwrap(); let markers: Vec = recording .iter() .filter(|(_, frames)| frames.is_empty()) - .map(|(sequence, _)| *sequence) + .map(|(start, _)| *start) .collect(); - assert_eq!(markers, [jump - 1], "{version}: the break marker precedes the jump"); + assert_eq!( + markers, + [starts[at - 1]], + "{version}: the break marker precedes the jump" + ); // Joining once every group is out, with a budget half the jump, reads every group // after it past the marker, and none of the groups the jump left more than @@ -1917,14 +1923,14 @@ pub(crate) mod tests { let whole: Vec = late .iter() .filter(|(_, frames)| frames.is_some()) - .map(|(sequence, _)| *sequence) + .map(|(start, _)| *start) .collect(); assert!( - whole.ends_with(&(jump..=last).collect::>()), + whole.ends_with(&starts[at..]), "{version}: a late subscriber reads every group from the jump on: {whole:?}" ); assert!( - whole.iter().all(|sequence| *sequence + 2 >= jump), + whole.iter().all(|start| *start >= starts[at - 2]), "{version}: only the group the jump follows is in budget before it: {whole:?}" ); } @@ -1966,7 +1972,7 @@ pub(crate) mod tests { let container = moq_mux::catalog::hang::Container::try_from(config).unwrap(); let first = next_frame(&mut Consumer::new(track, container)).await.timestamp; assert!( - first.as_micros() < Duration::from_secs(5).as_micros(), + first.as_micros().unwrap() < Duration::from_secs(5).as_micros(), "the feed's own PTS: {first:?}" ); // The PES that anchors the mapping need not be this frame: the mux spaces them apart. @@ -2415,8 +2421,11 @@ pub(crate) mod tests { // Its first group is already out with the catalog. let (name, _) = catalog.video.renditions.iter().next().expect("a video rendition"); let mut track = broadcast.track(name).unwrap().subscribe(None).await.unwrap(); - let group = track.recv_group().await.unwrap().expect("the first group"); - assert_eq!(group.sequence, 0, "the new instance's first group"); + track + .recv_group() + .await + .unwrap() + .expect("the new instance's first group"); drop(stdin); import.await.unwrap(); } diff --git a/rs/moq-cli/src/report.rs b/rs/moq-cli/src/report.rs index 6826a21f978..0287efe8ad2 100644 --- a/rs/moq-cli/src/report.rs +++ b/rs/moq-cli/src/report.rs @@ -39,6 +39,10 @@ const PRIORITY: i8 = i8::MIN; /// A compressed snapshot track, `None` once a write failed and closed it. struct ReportTrack { track: Option>, + + /// The track the snapshot producer writes, for its newest group. The track is scale-less, so a + /// write names a key: one past the newest group, where a snapshot opens the next one. + inner: moq_net::track::Producer, } impl ReportTrack { @@ -52,7 +56,8 @@ impl ReportTrack { .with_refresh(8); let track = broadcast.create_track(name, info)?; Ok(Self { - track: Some(moq_json::snapshot::Producer::new(track, Default::default())), + track: Some(moq_json::snapshot::Producer::new(track.clone(), Default::default())), + inner: track, }) } @@ -80,7 +85,12 @@ impl ReportTrack { std::task::Poll::Ready(Err(err)) => Err(err), _ => Ok(()), }; - if let Err(err) = refreshed.and_then(|()| track.update(value)) { + let next = self + .inner + .largest() + .map_or(0, |largest| largest.group.saturating_add(1)); + let at = moq_net::Timestamp::unscaled(next); + if let Err(err) = refreshed.and_then(|()| track.update(moq_json::Timed::new(value, at))) { tracing::warn!(%err, "a report track failed; closing it"); if let Some(track) = self.track.take() { let _ = track.abort(moq_net::Error::Transport(err.to_string())); @@ -601,7 +611,7 @@ async fn read_feedback( lines: tokio::sync::mpsc::Sender, ) -> anyhow::Result<()> { let mut track = reader(broadcast.track(hang::echo::TRACK)?); - while let Some(moq_net::Timed { value: feedback, .. }) = track.latest().await? { + while let Some(moq_json::Timed { value: feedback, .. }) = track.latest().await? { let line = EchoLine { broadcast: path.as_str(), feedback, @@ -632,7 +642,10 @@ mod tests { let mut read = reader(track.consume()); let mut report = moq_json::snapshot::Producer::::new(track, Default::default()); report - .update(&serde_json::json!({ "pad": "x".repeat(MAX_REPORT) })) + .update(moq_json::Timed::new( + &serde_json::json!({ "pad": "x".repeat(MAX_REPORT) }), + moq_net::Timestamp::unscaled(0), + )) .unwrap(); assert!(matches!(read.next().await, Err(moq_json::Error::TooLarge(MAX_REPORT)))); } @@ -670,13 +683,23 @@ mod tests { let mut lines = BufReader::new(read).lines(); tokio::spawn(write_echo(origin.consume(), String::new(), out)); - feedback.update(&serde_json::json!({ "n": 1 })).unwrap(); + feedback + .update(moq_json::Timed::new( + &serde_json::json!({ "n": 1 }), + moq_net::Timestamp::unscaled(1), + )) + .unwrap(); let line = lines.next_line().await.unwrap().unwrap(); assert!(line.contains(r#""n":1"#), "{line}"); broadcast.withdraw(); kio::clock::sleep(DRAIN * 2).await; - feedback.update(&serde_json::json!({ "n": 2 })).unwrap(); + feedback + .update(moq_json::Timed::new( + &serde_json::json!({ "n": 2 }), + moq_net::Timestamp::unscaled(2), + )) + .unwrap(); let late = kio::clock::timeout(Duration::from_secs(1), lines.next_line()).await; assert!(late.is_err(), "a drained reader wrote {late:?}"); } @@ -706,7 +729,12 @@ mod tests { let track = route.broadcast().create_track(hang::echo::TRACK, None).unwrap(); let mut feedback = moq_json::snapshot::Producer::::new(track, Default::default()); route.announce().unwrap(); - feedback.update(&serde_json::json!({ "n": 1 })).unwrap(); + feedback + .update(moq_json::Timed::new( + &serde_json::json!({ "n": 1 }), + moq_net::Timestamp::unscaled(1), + )) + .unwrap(); let line = kio::clock::timeout(Duration::from_secs(5), lines.next_line()).await; let line = line.expect("the takeover was never read").unwrap().unwrap(); assert!(line.contains(r#""n":1"#), "{line}"); @@ -747,7 +775,12 @@ mod tests { let mut lines = BufReader::new(read).lines(); tokio::spawn(write_echo(origin.consume(), ".viewers".to_string(), out)); - feedback.update(&serde_json::json!({ "n": 1 })).unwrap(); + feedback + .update(moq_json::Timed::new( + &serde_json::json!({ "n": 1 }), + moq_net::Timestamp::unscaled(1), + )) + .unwrap(); let line = kio::clock::timeout(Duration::from_secs(5), lines.next_line()).await; let line = line.expect("no feedback read").unwrap().unwrap(); assert!(line.contains(r#""n":1"#), "{line}"); diff --git a/rs/moq-cli/tests/integration/instances.rs b/rs/moq-cli/tests/integration/instances.rs index c3188c9a51d..66164237fcd 100644 --- a/rs/moq-cli/tests/integration/instances.rs +++ b/rs/moq-cli/tests/integration/instances.rs @@ -48,9 +48,12 @@ fn client(version: &str) -> moq_tokio::Client { } fn write(track: &track::Producer, tag: &str) { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, tag.as_bytes().to_vec()).unwrap(); - group.finish().unwrap(); + let start = track.largest().map_or(0, |largest| largest.group + 1); + let mut group = track.append_group(start).unwrap(); + group + .write_frame(Timestamp::from_millis(start).unwrap(), tag.as_bytes().to_vec()) + .unwrap(); + group.close().unwrap(); } async fn read(subscription: &mut track::Subscriber) -> String { diff --git a/rs/moq-crypto/benches/protect.rs b/rs/moq-crypto/benches/protect.rs index 170395d2471..c9611de2a81 100644 --- a/rs/moq-crypto/benches/protect.rs +++ b/rs/moq-crypto/benches/protect.rs @@ -40,11 +40,11 @@ fn bench_protect(c: &mut Criterion) { let mut ms = 0u64; b.iter(|| { ms += 1; - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_group(ms).unwrap(); group .write_frame(Timestamp::from_millis(ms).unwrap(), black_box(&grouped)) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }); }); group.throughput(Throughput::Bytes(opus.len() as u64)); diff --git a/rs/moq-crypto/benches/sign.rs b/rs/moq-crypto/benches/sign.rs index 63c90ad2ecb..de46a1a5514 100644 --- a/rs/moq-crypto/benches/sign.rs +++ b/rs/moq-crypto/benches/sign.rs @@ -53,10 +53,14 @@ fn bench_sign(c: &mut Criterion) { group.throughput(Throughput::Bytes(size as u64)); group.bench_function(format!("write_{}k", size / 1024), |b| { let (producer, _inner) = produce(&key); + let mut start = 0; b.iter(|| { - let mut group = producer.append_group().unwrap(); - group.write_frame(None, black_box(&payload)).unwrap(); - group.finish().unwrap(); + let mut group = producer.append_group(start).unwrap(); + start += 1; + group + .write_frame(moq_net::Timestamp::unscaled(start), black_box(&payload)) + .unwrap(); + group.close().unwrap(); }); }); group.bench_function(format!("roundtrip_{}k", size / 1024), |b| { @@ -66,10 +70,14 @@ fn bench_sign(c: &mut Criterion) { .unwrap(); let (producer, mut consumer, _origin) = runtime.block_on(roundtrip(&key)); let waiter = kio::Waiter::noop(); + let mut start = 0; b.iter(|| { - let mut group = producer.append_group().unwrap(); - group.write_frame(None, black_box(&payload)).unwrap(); - group.finish().unwrap(); + let mut group = producer.append_group(start).unwrap(); + start += 1; + group + .write_frame(moq_net::Timestamp::unscaled(start), black_box(&payload)) + .unwrap(); + group.close().unwrap(); let Poll::Ready(Ok(Some(mut group))) = consumer.poll_recv_group(&waiter) else { panic!("the group is delivered in-process"); }; diff --git a/rs/moq-crypto/src/e2ee/broadcast.rs b/rs/moq-crypto/src/e2ee/broadcast.rs index 574ecb32de1..b0a7e7de51a 100644 --- a/rs/moq-crypto/src/e2ee/broadcast.rs +++ b/rs/moq-crypto/src/e2ee/broadcast.rs @@ -38,7 +38,7 @@ impl Producer { /// Create the track for `semantic` under its physical name. /// /// A name this generation already produced, even on another broadcast, resumes above - /// every sequence it allocated. + /// every group start it allocated. /// /// # Errors /// diff --git a/rs/moq-crypto/src/e2ee/datagram.rs b/rs/moq-crypto/src/e2ee/datagram.rs index 0a9a5157701..8e74516c673 100644 --- a/rs/moq-crypto/src/e2ee/datagram.rs +++ b/rs/moq-crypto/src/e2ee/datagram.rs @@ -21,8 +21,9 @@ use crate::e2ee::window::DatagramWindow; pub struct Datagram { /// The datagram track's own sequence. pub sequence: u64, - /// Presentation timestamp, or `None` when untimed. - pub timestamp: Option, + /// Presentation timestamp, or [`moq_net::Timestamp::unscaled`] at the sequence when the + /// track is scale-less. + pub timestamp: moq_net::Timestamp, /// Decrypted application bytes. pub plaintext: Bytes, } @@ -77,11 +78,16 @@ impl Producer { /// Encrypt `plaintext` at the next sequence and send it, returning that sequence. /// + /// A scale-less track has no time, so there `timestamp` is + /// [`unscaled`](moq_net::Timestamp::unscaled) at the sequence to write, as with + /// [`insert`](Self::insert). + /// /// # Errors /// - /// Same as [`Self::insert`], except [`Error::Reuse`]. - pub fn append(&mut self, timestamp: impl Into>, plaintext: &[u8]) -> Result { - self.write(None, timestamp.into(), plaintext) + /// Same as [`Self::insert`], except [`Error::Reuse`] on a timed track. + pub fn append(&mut self, timestamp: moq_net::Timestamp, plaintext: &[u8]) -> Result { + let sequence = timestamp.scale().is_none().then(|| timestamp.value()); + self.write(sequence, timestamp, plaintext) } /// Encrypt `plaintext` at an explicit sequence, which must be above every sequence this @@ -95,19 +101,17 @@ impl Producer { /// /// [`Error::Reuse`] if `sequence` was already allocated, [`Error::Identity`], /// [`Error::Exhausted`], [`Error::Oversize`], or a net write error. - pub fn insert( - &mut self, - sequence: u64, - timestamp: impl Into>, - plaintext: &[u8], - ) -> Result<()> { - self.write(Some(sequence), timestamp.into(), plaintext)?; + pub fn insert(&mut self, sequence: u64, timestamp: moq_net::Timestamp, plaintext: &[u8]) -> Result<()> { + self.write(Some(sequence), timestamp, plaintext)?; Ok(()) } - fn write(&mut self, sequence: Option, timestamp: Option, plaintext: &[u8]) -> Result { + fn write(&mut self, sequence: Option, timestamp: moq_net::Timestamp, plaintext: &[u8]) -> Result { // Checked before the sequence is spent. let timestamp = self.inner.info().rescale(timestamp)?; + if timestamp.scale().is_none() && sequence != Some(timestamp.value()) { + return Err(moq_net::Error::TimestampMismatch.into()); + } let (sequence, payload) = self.claim.allocate(sequence, |sequence| { self.claim .key diff --git a/rs/moq-crypto/src/e2ee/error.rs b/rs/moq-crypto/src/e2ee/error.rs index 9810d0116f0..354831c03ca 100644 --- a/rs/moq-crypto/src/e2ee/error.rs +++ b/rs/moq-crypto/src/e2ee/error.rs @@ -12,7 +12,8 @@ pub enum Error { #[error("exhausted")] Exhausted, - /// Encrypting at a sequence this generation already allocated for the track's name. + /// Encrypting at a group start or datagram sequence this generation already allocated for the + /// track's name. #[error("reuse")] Reuse, diff --git a/rs/moq-crypto/src/e2ee/generation.rs b/rs/moq-crypto/src/e2ee/generation.rs index 2256a54639d..1ca8107c56e 100644 --- a/rs/moq-crypto/src/e2ee/generation.rs +++ b/rs/moq-crypto/src/e2ee/generation.rs @@ -16,8 +16,8 @@ use rind::{aead, hmac}; /// Grouped-frame versus datagram key domain. /// /// A stream track and a datagram track of one semantic name share a physical name but allocate -/// sequences independently, so the domain alone keeps a group's frame 0 and the datagram at -/// that sequence from sharing a key and nonce. +/// group starts and datagram sequences independently, so the domain alone keeps a group's frame 0 +/// and the datagram at the same number from sharing a key and nonce. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum Domain { Group = 0x00, @@ -177,9 +177,10 @@ fn scope(info: &mut Vec, credential: &Credential, epoch: &Epoch) { /// A publisher-minted generation whose clones share permanent track claims. /// -/// A claim holds a semantic name's key for stream or datagram tracks and that kind's sequence -/// high-water mark, so a track created, reserved, or served again under the name resumes above -/// every sequence its kind allocated before, on any broadcast of the generation. +/// A claim holds a semantic name's key for stream or datagram tracks and that kind's high-water +/// mark (a group start or a datagram sequence), so a track created, reserved, or served again +/// under the name resumes above every one its kind allocated before, on any broadcast of the +/// generation. #[derive(Clone, Debug)] pub struct Producer { // Its keys are the claims: never released, since reopening a track would restart its nonce counters. diff --git a/rs/moq-crypto/src/e2ee/group.rs b/rs/moq-crypto/src/e2ee/group.rs index b3f4680b3d0..628eb7f2de5 100644 --- a/rs/moq-crypto/src/e2ee/group.rs +++ b/rs/moq-crypto/src/e2ee/group.rs @@ -15,20 +15,20 @@ use crate::e2ee::terminal::Terminal; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Frame { /// Presentation timestamp, or `None` when untimed. - pub timestamp: Option, + pub timestamp: moq_net::Timestamp, /// Decrypted application bytes. pub plaintext: Bytes, } /// Exclusive writer for one group; frames are numbered from 0 in write order. /// -/// Dropping without [`Self::finish`] or [`Self::abort`] lets the net group close uncleanly. +/// Dropping without [`Self::close`] or [`Self::abort`] lets the net group close uncleanly. pub struct Producer { // The error that aborted the group once a write failed after encryption. inner: std::result::Result, // The parent track's info, which rescales each timestamp before it is encrypted. track: moq_net::track::Info, - sequence: u64, + start: u64, key: Arc>, next_frame: u32, } @@ -36,7 +36,7 @@ pub struct Producer { impl Producer { pub(crate) fn new(inner: moq_net::group::Producer, track: moq_net::track::Info, key: Arc>) -> Self { Self { - sequence: inner.sequence, + start: inner.start, inner: Ok(inner), track, key, @@ -44,9 +44,9 @@ impl Producer { } } - /// The group's sequence number. - pub fn sequence(&self) -> u64 { - self.sequence + /// Where the group starts, in the track's timescale: its key on the track. + pub fn start(&self) -> u64 { + self.start } /// Next frame index that will be written. @@ -66,16 +66,16 @@ impl Producer { /// /// [`Error::Identity`] if the next frame exceeds 32 bits, [`Error::Exhausted`], /// [`Error::Oversize`], or a net write error. - pub fn write_frame(&mut self, timestamp: impl Into>, plaintext: &[u8]) -> Result<()> { + pub fn write_frame(&mut self, timestamp: moq_net::Timestamp, plaintext: &[u8]) -> Result<()> { let inner = self.inner.as_mut().map_err(|err| err.clone())?; let frame = self.next_frame; if frame == u32::MAX { return Err(Error::Identity); } // Checked before the frame identity is spent. - let timestamp = self.track.rescale(timestamp.into())?; + let timestamp = self.track.rescale_in(timestamp, self.start)?; let payload = self.key.lock().expect("track key").protect( - self.sequence, + self.start, u64::from(frame), plaintext, MAX_GROUPED_PAYLOAD, @@ -91,13 +91,13 @@ impl Producer { Ok(()) } - /// Finish the group. No more frames will be written. + /// Close the group without a known end. No more frames will be written. /// /// # Errors /// /// A net error if the group is already closed, or the error a failed write aborted it with. - pub fn finish(self) -> Result<()> { - self.inner?.finish()?; + pub fn close(self) -> Result<()> { + self.inner?.close()?; Ok(()) } @@ -120,7 +120,7 @@ impl Producer { impl fmt::Debug for Producer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("group::Producer") - .field("sequence", &self.sequence()) + .field("start", &self.start()) .field("next_frame", &self.next_frame) .finish() } @@ -132,17 +132,17 @@ impl fmt::Debug for Producer { /// [`Error::Authentication`] and the underlying subscription is released. pub struct Consumer { inner: Option, - sequence: u64, + start: u64, key: Arc>, terminal: Arc, } impl Consumer { pub(crate) fn new(inner: moq_net::group::Consumer, key: Arc>, terminal: Arc) -> Self { - let sequence = inner.sequence; + let start = inner.start; Self { inner: Some(inner), - sequence, + start, key, terminal, } @@ -153,9 +153,9 @@ impl Consumer { self.inner.take(); } - /// The group's sequence number. - pub fn sequence(&self) -> u64 { - self.sequence + /// Where the group starts, in the track's timescale: its key on the track. + pub fn start(&self) -> u64 { + self.start } /// The next decrypted frame, or `None` once the group finished. @@ -183,7 +183,7 @@ impl Consumer { self.key .lock() .expect("track key") - .open(self.sequence, index, &frame.payload, MAX_GROUPED_PAYLOAD) + .open(self.start, index, &frame.payload, MAX_GROUPED_PAYLOAD) }); (frame.timestamp, result) }; @@ -203,7 +203,7 @@ impl Consumer { impl fmt::Debug for Consumer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("group::Consumer") - .field("sequence", &self.sequence()) + .field("start", &self.start()) .field("index", &self.inner.as_ref().map(|inner| inner.index())) .finish() } diff --git a/rs/moq-crypto/src/e2ee/key.rs b/rs/moq-crypto/src/e2ee/key.rs index d851c37dc28..196596e0462 100644 --- a/rs/moq-crypto/src/e2ee/key.rs +++ b/rs/moq-crypto/src/e2ee/key.rs @@ -8,8 +8,8 @@ use crate::e2ee::error::{Error, Result}; use crate::e2ee::limits::{MAX_INVOCATIONS, MAX_PLAINTEXT_BYTES, TAG_LEN, check_u53}; use crate::e2ee::protect::{nonce, open, protect}; -/// A semantic name's key in one domain and its next unallocated sequence; clones share their -/// accounting. +/// A semantic name's key in one domain and its next unallocated group start or datagram sequence; +/// clones share their accounting. #[derive(Clone, Debug)] pub(crate) struct Claim { pub key: Arc>, diff --git a/rs/moq-crypto/src/e2ee/tests.rs b/rs/moq-crypto/src/e2ee/tests.rs index 7632ebecb65..464ef24f614 100644 --- a/rs/moq-crypto/src/e2ee/tests.rs +++ b/rs/moq-crypto/src/e2ee/tests.rs @@ -414,7 +414,7 @@ impl DatagramPair { self.net .insert(moq_net::datagram::Datagram { sequence, - timestamp: Some(timestamp), + timestamp, payload: payload.into(), }) .unwrap(); @@ -461,15 +461,15 @@ fn path_is_opaque() { #[test] fn grouped_roundtrip() { let mut pair = pair("video"); - let mut group = pair.producer.append_group().unwrap(); - assert_eq!(group.sequence(), 0); + let mut group = pair.producer.append_group(1).unwrap(); + assert_eq!(group.start(), 1); group.write_frame(ms(1), b"frame-zero").unwrap(); group.write_frame(ms(2), b"frame-one").unwrap(); assert_eq!(group.next_frame(), 2); - group.finish().unwrap(); + group.close().unwrap(); let mut group = recv_group(&mut pair.consumer); - assert_eq!(group.sequence(), 0); + assert_eq!(group.start(), 1); assert_eq!(read_frame(&mut group).plaintext, &b"frame-zero"[..]); assert_eq!(read_frame(&mut group).plaintext, &b"frame-one"[..]); assert!(matches!( @@ -486,7 +486,7 @@ fn datagram_roundtrip() { match recv_datagram(&mut pair.consumer) { Event::Datagram(d) => { assert_eq!(d.sequence, 0); - assert_eq!(d.timestamp, Some(ms(5))); + assert_eq!(d.timestamp, ms(5)); assert_eq!(&d.plaintext[..], b"opus"); } other => panic!("{other:?}"), @@ -494,18 +494,14 @@ fn datagram_roundtrip() { } #[test] -fn sequences_allocate_monotonically() { +fn group_starts_only_rise() { let mut pair = pair("video"); - assert_eq!(pair.producer.append_group().unwrap().sequence(), 0); - assert_eq!(pair.producer.create_group(5).unwrap().sequence(), 5); - assert_eq!(pair.producer.append_group().unwrap().sequence(), 6); - for sequence in [6, 3] { + assert_eq!(pair.producer.append_group(0).unwrap().start(), 0); + assert_eq!(pair.producer.create_group(5).unwrap().start(), 5); + assert_eq!(pair.producer.append_group(6).unwrap().start(), 6); + for start in [6, 3] { assert_eq!( - pair.producer - .create_group(sequence) - .map(|_| ()) - .unwrap_err() - .to_string(), + pair.producer.create_group(start).map(|_| ()).unwrap_err().to_string(), "reuse" ); } @@ -548,12 +544,43 @@ fn oversize_datagram_keeps_the_sequence() { assert_eq!(pair.producer.append(ms(1), &max).unwrap(), 0); } +/// On a scale-less datagram track the unscaled timestamp names the sequence, so `append` +/// writes there and `insert` refuses a sequence the timestamp doesn't name. +#[test] +fn scale_less_datagrams_write_at_their_unscaled_sequence() { + let generation = test_generation(); + let net = moq_net::broadcast::Info::new() + .produce() + .create_datagrams( + generation.name("events").unwrap().as_str(), + moq_net::datagram::Info::default(), + ) + .unwrap(); + let mut producer = generation.produce_datagrams(net.clone(), "events").unwrap(); + let mut consumer = consume(&generation).consume_datagrams(net.consume(), "events").unwrap(); + let unscaled = moq_net::Timestamp::unscaled; + assert_eq!(producer.append(unscaled(5), b"five").unwrap(), 5); + let err = producer.insert(6, unscaled(7), b"x").unwrap_err(); + assert!(matches!(err, Error::Net(moq_net::Error::TimestampMismatch)), "{err:?}"); + assert_eq!(producer.invocations(), 1, "a refused timestamp spends no sequence"); + producer.insert(6, unscaled(6), b"six").unwrap(); + for (sequence, payload) in [(5, b"five".as_slice()), (6, b"six".as_slice())] { + match recv_datagram(&mut consumer) { + Event::Datagram(d) => { + assert_eq!((d.sequence, d.timestamp), (sequence, unscaled(sequence))); + assert_eq!(&d.plaintext[..], payload); + } + other => panic!("{other:?}"), + } + } +} + #[test] fn refused_datagram_timestamp_keeps_the_sequence() { let mut pair = datagram_pair("audio"); // (2^61-1) seconds overflows conversion to the milli-scale track. let huge = moq_net::Timestamp::from_secs((1 << 61) - 1).unwrap(); - for timestamp in [None, Some(huge)] { + for timestamp in [moq_net::Timestamp::unscaled(0), huge] { let err = pair.producer.append(timestamp, b"x").unwrap_err(); assert!(matches!(err, Error::Net(moq_net::Error::TimestampMismatch)), "{err:?}"); } @@ -577,9 +604,9 @@ fn one_name_keys_its_stream_and_datagram_tracks_apart() { let mut datagram_reader = consuming.consume_datagrams(net_datagrams.consume(), "audio").unwrap(); // The same plaintext at the same identity in each kind: group 0 frame 0, and datagram 0. - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(ms(1), b"same").unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(datagrams.append(ms(1), b"same").unwrap(), 0); assert_eq!(read_frame(&mut recv_group(&mut track_reader)).plaintext, &b"same"[..]); match recv_datagram(&mut datagram_reader) { @@ -601,9 +628,9 @@ fn one_name_keys_its_stream_and_datagram_tracks_apart() { let forged = net_track(&name); let mut reader = consuming.consume(forged.subscribe(subscribe_all()), "audio").unwrap(); - let mut group = forged.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = forged.create_group(moq_net::group::Info { start: 0 }).unwrap(); group.write_frame(ms(1), sealed).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!( recv_group(&mut reader) .poll_read_frame(&kio::Waiter::noop()) @@ -616,7 +643,7 @@ fn one_name_keys_its_stream_and_datagram_tracks_apart() { forged .insert(moq_net::datagram::Datagram { sequence: 0, - timestamp: Some(ms(1)), + timestamp: ms(1), payload: frame, }) .unwrap(); @@ -635,17 +662,15 @@ fn reproduced_track_resumes_above_its_marks() { // A clone of the generation reopening the name, on any output, resumes above its marks. let mut again = generation.clone().produce(net_track(&name), "video").unwrap(); - for sequence in [0, 4] { - assert_eq!( - again.create_group(sequence).map(|_| ()).unwrap_err().to_string(), - "reuse" - ); + for start in [0, 4] { + assert_eq!(again.create_group(start).map(|_| ()).unwrap_err().to_string(), "reuse"); } - assert_eq!(again.append_group().unwrap().sequence(), 5); - // Two live producers of the name share one sequence namespace. + assert_eq!(again.append_group(5).unwrap().start(), 5); + // Two live producers of the name share one start namespace. let mut other = generation.produce(net_track(&name), "video").unwrap(); - assert_eq!(other.append_group().unwrap().sequence(), 6); - assert_eq!(again.append_group().unwrap().sequence(), 7); + assert_eq!(other.append_group(5).map(|_| ()).unwrap_err().to_string(), "reuse"); + assert_eq!(other.append_group(6).unwrap().start(), 6); + assert_eq!(again.append_group(6).map(|_| ()).unwrap_err().to_string(), "reuse"); // A datagram track of the name has sequences of its own, and resumes above them too. let datagrams = || { @@ -665,7 +690,7 @@ fn reproduced_track_resumes_above_its_marks() { let mut fresh = fresh .produce(net_track(&fresh.name("video").unwrap()), "video") .unwrap(); - assert_eq!(fresh.append_group().unwrap().sequence(), 0); + assert_eq!(fresh.append_group(0).unwrap().start(), 0); // Consuming never claims. consume(&generation) .consume(net.subscribe(subscribe_all()), "video") @@ -724,9 +749,9 @@ fn broadcast_speaks_semantic_names() { assert_eq!(video.name(), physical.as_str()); let mut reader = resolve(&protected.consumer.subscribe("video", subscribe_all()).unwrap()).unwrap(); - let mut group = video.append_group().unwrap(); + let mut group = video.append_group(1).unwrap(); group.write_frame(ms(1), b"frame").unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(read_frame(&mut recv_group(&mut reader)).plaintext, &b"frame"[..]); // The relay's view holds only the physical name. @@ -753,7 +778,7 @@ fn reserved_track_serves_on_demand() { let mut audio = request.accept(timed()); let mut reader = resolve(&subscribing).unwrap(); - let mut group = audio.append_group().unwrap(); + let mut group = audio.append_group(1).unwrap(); group.write_frame(ms(1), b"opus").unwrap(); assert_eq!(read_frame(&mut recv_group(&mut reader)).plaintext, &b"opus"[..]); } @@ -765,17 +790,20 @@ fn broadcast_claims_span_outputs() { let mut video = first.create_track("video", timed()).unwrap(); video.create_group(2).unwrap(); let mut audio = first.reserve_track("audio").unwrap().accept(timed()); - audio.append_group().unwrap(); + audio.append_group(0).unwrap(); let mut voice = first.create_datagrams("voice", timed_datagrams()).unwrap(); voice.append(ms(1), b"opus").unwrap(); // A clone of the generation on another output resumes each name above its marks. let second = crate::e2ee::broadcast::Producer::new(moq_net::broadcast::Info::new().produce(), generation); let mut video = second.create_track("video", timed()).unwrap(); - assert_eq!(video.append_group().unwrap().sequence(), 3); + assert_eq!(video.append_group(2).map(|_| ()).unwrap_err().to_string(), "reuse"); + assert_eq!(video.append_group(3).unwrap().start(), 3); let request = second.reserve_track("audio").unwrap(); assert_eq!(request.name(), "audio"); - assert_eq!(request.accept(timed()).append_group().unwrap().sequence(), 1); + let mut audio = request.accept(timed()); + assert_eq!(audio.append_group(0).map(|_| ()).unwrap_err().to_string(), "reuse"); + assert_eq!(audio.append_group(1).unwrap().start(), 1); let mut voice = second.create_datagrams("voice", timed_datagrams()).unwrap(); assert_eq!(voice.append(ms(1), b"opus").unwrap(), 1); } @@ -823,9 +851,9 @@ fn requests_serve_undeclared_names() { let mut track = request.accept(timed()); assert_eq!(protected.producer.generation().claims(), 1); let mut reader = resolve(&subscribing).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(1).unwrap(); group.write_frame(ms(1), b"42").unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(read_frame(&mut recv_group(&mut reader)).plaintext, &b"42"[..]); // The handler's own consumer reads the broadcast under semantic names too. @@ -908,7 +936,8 @@ fn requests_claim_only_accepted_names() { other.create_track("video", timed()).unwrap().create_group(5).unwrap(); let mut video = request.accept(timed()); assert_eq!(generation.claims(), 1); - assert_eq!(video.append_group().unwrap().sequence(), 6); + assert_eq!(video.append_group(5).map(|_| ()).unwrap_err().to_string(), "reuse"); + assert_eq!(video.append_group(6).unwrap().start(), 6); } #[test] @@ -932,9 +961,9 @@ fn requests_serve_a_dropped_name_above_its_marks() { assert_eq!(request.name(), "presence"); let mut track = request.accept(timed()); assert_eq!(track.create_group(3).map(|_| ()).unwrap_err().to_string(), "reuse"); - assert_eq!(track.append_group().unwrap().sequence(), 4); + assert_eq!(track.append_group(4).unwrap().start(), 4); let mut reader = resolve(&subscribing).unwrap(); - assert_eq!(recv_group(&mut reader).sequence(), 4); + assert_eq!(recv_group(&mut reader).start(), 4); } #[test] @@ -1021,9 +1050,9 @@ fn new_epoch_authenticates_old_does_not() { let net = net_track(&new.name("video").unwrap()); let mut producer = new.produce(net.clone(), "video").unwrap(); - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_group(1).unwrap(); group.write_frame(ms(1), b"fresh").unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut consumer = consume(&new).consume(net.subscribe(subscribe_all()), "video").unwrap(); assert_eq!(read_frame(&mut recv_group(&mut consumer)).plaintext, &b"fresh"[..]); @@ -1099,18 +1128,18 @@ fn failed_opens_spend_the_key() { #[test] fn grouped_authentication_ends_track() { let mut pair = pair("video"); - let mut group = pair.producer.append_group().unwrap(); + let mut group = pair.producer.append_group(0).unwrap(); group.write_frame(ms(1), b"ok").unwrap(); - group.finish().unwrap(); + group.close().unwrap(); // Tamper with a second group by writing ciphertext directly. let semantic = pair.semantic.as_str(); let mut key = pair.generation.key(semantic, Domain::Group).unwrap(); let mut payload = key.protect(1, 0, b"ok", MAX_GROUPED_PAYLOAD).unwrap().to_vec(); payload[0] ^= 1; - let mut bad = pair.net.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut bad = pair.net.create_group(moq_net::group::Info { start: 1 }).unwrap(); bad.write_frame(ms(2), payload).unwrap(); - bad.finish().unwrap(); + bad.close().unwrap(); let waiter = kio::Waiter::noop(); let mut first = recv_group(&mut pair.consumer); @@ -1175,14 +1204,14 @@ fn grouped_failure_wakes_pending_reads_and_releases_demand() { // One authentic frame, then silence. The group and the track stay open. let semantic = pair.semantic.as_str(); let mut key = pair.generation.key(semantic, Domain::Group).unwrap(); - let mut open = pair.net.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut open = pair.net.create_group(moq_net::group::Info { start: 0 }).unwrap(); let open_demand = open.demand(); open.write_frame(ms(1), key.protect(0, 0, b"ok", MAX_GROUPED_PAYLOAD).unwrap()) .unwrap(); let mut payload = key.protect(1, 0, b"nope", MAX_GROUPED_PAYLOAD).unwrap().to_vec(); payload[0] ^= 1; - let mut bad = pair.net.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut bad = pair.net.create_group(moq_net::group::Info { start: 1 }).unwrap(); let bad_demand = bad.demand(); bad.write_frame(ms(2), payload).unwrap(); @@ -1248,7 +1277,7 @@ fn grouped_failure_wakes_pending_reads_and_releases_demand() { fn a_held_group_keeps_demand_after_the_track_drops() { let mut pair = pair("video"); let demand = pair.net.demand(); - let mut open = pair.producer.append_group().unwrap(); + let mut open = pair.producer.append_group(1).unwrap(); open.write_frame(ms(1), b"ok").unwrap(); let mut group = recv_group(&mut pair.consumer); @@ -1376,9 +1405,9 @@ fn duplicate_datagram_is_bounded() { #[test] fn cancellation_ends_consumer() { let mut pair = pair("video"); - let mut group = pair.producer.append_group().unwrap(); + let mut group = pair.producer.append_group(1).unwrap(); group.write_frame(ms(1), b"x").unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pair.producer.finish().unwrap(); let waiter = kio::Waiter::noop(); let mut group = recv_group(&mut pair.consumer); @@ -1401,7 +1430,7 @@ fn secrets_stay_out_of_debug() { #[test] fn failed_group_write_does_not_burn_nonce() { let mut pair = pair("video"); - let mut group = pair.producer.append_group().unwrap(); + let mut group = pair.producer.append_group(1).unwrap(); // (2^61-1) seconds overflows conversion to the milli-scale track. let huge = moq_net::Timestamp::from_secs((1 << 61) - 1).unwrap(); assert!(group.write_frame(huge, b"bad").is_err()); @@ -1412,7 +1441,7 @@ fn failed_group_write_does_not_burn_nonce() { group.write_frame(ms(1), b"ok").unwrap(); assert_eq!(group.invocations(), 1); assert_eq!(group.next_frame(), 1); - group.finish().unwrap(); + group.close().unwrap(); let mut group = recv_group(&mut pair.consumer); assert_eq!(read_frame(&mut group).plaintext, &b"ok"[..]); @@ -1421,14 +1450,14 @@ fn failed_group_write_does_not_burn_nonce() { #[test] fn failed_net_write_after_encryption_aborts_the_group() { let mut pair = pair("video"); - let mut first = pair.producer.append_group().unwrap(); + let mut first = pair.producer.append_group(0).unwrap(); first.write_frame(ms(10), b"start").unwrap(); - first.finish().unwrap(); + first.close().unwrap(); - // The net track refuses a group start that rewinds past the one before it, only after + // The net track refuses a first frame that rewinds past the one before it, only after // the frame was encrypted, so the group can no longer line its transport index up with // the nonce. - let mut group = pair.producer.append_group().unwrap(); + let mut group = pair.producer.append_group(1).unwrap(); let err = group.write_frame(ms(5), b"rewind").unwrap_err(); assert!(matches!(err, Error::Net(moq_net::Error::TimestampMismatch)), "{err:?}"); // The track's grouped key: one frame in each group. @@ -1436,7 +1465,7 @@ fn failed_net_write_after_encryption_aborts_the_group() { let after = group.write_frame(ms(20), b"after").unwrap_err(); assert_eq!(after.to_string(), err.to_string()); assert_eq!(group.invocations(), 2); - assert_eq!(group.finish().unwrap_err().to_string(), err.to_string()); + assert_eq!(group.close().unwrap_err().to_string(), err.to_string()); // Subscribers see the group end with the failure instead of waiting on it. let mut start = recv_group(&mut pair.consumer); @@ -1458,10 +1487,10 @@ fn resumed_group_opens_at_transport_index() { let mut subscriber = net.subscribe(subscribe_all()); let mut producer = generation.produce(net, "video").unwrap(); - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_group(1).unwrap(); group.write_frame(ms(1), b"zero").unwrap(); group.write_frame(ms(2), b"one").unwrap(); - group.finish().unwrap(); + group.close().unwrap(); // Simulate a resume at object 1: skip frame 0 on the net cursor before wrapping. let Poll::Ready(Ok(Some(mut net_group))) = subscriber.poll_recv_group(&kio::Waiter::noop()) else { diff --git a/rs/moq-crypto/src/e2ee/track.rs b/rs/moq-crypto/src/e2ee/track.rs index b4abbd586a5..f34ba1d555f 100644 --- a/rs/moq-crypto/src/e2ee/track.rs +++ b/rs/moq-crypto/src/e2ee/track.rs @@ -12,10 +12,10 @@ use crate::e2ee::terminal::Terminal; /// Exclusive producer for one physical stream track. /// -/// Uses the grouped-frame key and the group sequences of its semantic name, shared with every -/// producer of that name under the generation. Sequences are allocated monotonically, so a -/// re-created or re-served name resumes above every sequence allocated before; reuse and -/// exhaustion are refused before encryption. +/// Uses the grouped-frame key and the group starts of its semantic name, shared with every +/// producer of that name under the generation. Starts only rise, so a re-created or re-served +/// name resumes above every start allocated before; reuse and exhaustion are refused before +/// encryption. pub struct Producer { inner: moq_net::track::Producer, claim: Claim, @@ -31,37 +31,41 @@ impl Producer { self.inner.name() } - /// Allocate the next sequence and start a group there. + /// Start the next group at `start`, in the track's timescale, as + /// [`moq_net::track::Producer::append_group`] does. `start` must be above every start this + /// name allocated under the generation. /// /// # Errors /// - /// [`Error::Identity`](crate::e2ee::Error::Identity) if the next sequence exceeds `2^53-1`, or a net error. - pub fn append_group(&mut self) -> Result { - self.start_group(None) + /// [`Error::Reuse`](crate::e2ee::Error::Reuse) if `start` was already allocated, [`Error::Identity`](crate::e2ee::Error::Identity) if + /// it exceeds `2^53-1`, or a net error. + pub fn append_group(&mut self, start: u64) -> Result { + let (_, inner) = self + .claim + .allocate(Some(start), |start| Ok(self.inner.append_group(start)?))?; + Ok(self.group(inner)) } - /// Start a group at an explicit sequence, which must be above every sequence this name - /// allocated under the generation. + /// Start a group at `start`, in the track's timescale, as + /// [`moq_net::track::Producer::create_group`] does. `start` must be above every start this + /// name allocated under the generation. /// /// # Errors /// - /// [`Error::Reuse`](crate::e2ee::Error::Reuse) if `sequence` was already allocated, [`Error::Identity`](crate::e2ee::Error::Identity) if + /// [`Error::Reuse`](crate::e2ee::Error::Reuse) if `start` was already allocated, [`Error::Identity`](crate::e2ee::Error::Identity) if /// it exceeds `2^53-1`, or a net error. - pub fn create_group(&mut self, sequence: u64) -> Result { - self.start_group(Some(sequence)) + pub fn create_group(&mut self, start: u64) -> Result { + let (_, inner) = self.claim.allocate(Some(start), |start| { + Ok(self.inner.create_group(moq_net::group::Info { start })?) + })?; + Ok(self.group(inner)) } - fn start_group(&mut self, sequence: Option) -> Result { - let (sequence, ()) = self.claim.allocate(sequence, |_| Ok(()))?; - let inner = self.inner.create_group(moq_net::group::Info { sequence })?; - Ok(group::Producer::new( - inner, - self.inner.info().clone(), - Arc::clone(&self.claim.key), - )) + fn group(&self, inner: moq_net::group::Producer) -> group::Producer { + group::Producer::new(inner, self.inner.info().clone(), Arc::clone(&self.claim.key)) } - /// Finish the track after the last allocated sequence. + /// Finish the track after the last allocated start. /// /// # Errors /// diff --git a/rs/moq-crypto/src/sign/broadcast.rs b/rs/moq-crypto/src/sign/broadcast.rs index 40f5711f35e..3c7d36d47dc 100644 --- a/rs/moq-crypto/src/sign/broadcast.rs +++ b/rs/moq-crypto/src/sign/broadcast.rs @@ -14,14 +14,14 @@ use crate::sign::{PublicKey, SecretKey}; /// The publishing side of a signed broadcast. /// /// Signs under the instance of the route that serves it, which a route mints fresh, so a restart -/// never signs a sequence an earlier instance already signed. +/// never signs a group start an earlier instance already signed. #[derive(Clone)] pub struct Producer { inner: moq_net::broadcast::Producer, key: SecretKey, path: moq_net::Path, instance: moq_net::Instance, - // Each track's lowest unsigned sequence, kept across the track's replacements. + // Each track's lowest unsigned group start, kept across the track's replacements. next: Arc>>>>, } @@ -60,7 +60,7 @@ impl Producer { let next = Arc::clone( self.next .lock() - .expect("signed sequences") + .expect("signed starts") .entry(name.to_owned()) .or_default(), ); diff --git a/rs/moq-crypto/src/sign/group.rs b/rs/moq-crypto/src/sign/group.rs index 1efa88bf854..e2b902af71e 100644 --- a/rs/moq-crypto/src/sign/group.rs +++ b/rs/moq-crypto/src/sign/group.rs @@ -14,7 +14,7 @@ use crate::sign::{PublicKey, SecretKey}; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Frame { /// Presentation timestamp at the track's timescale, or `None` when untimed. Signed. - pub timestamp: Option, + pub timestamp: moq_net::Timestamp, /// The application bytes, without the signature. pub payload: Bytes, } @@ -43,9 +43,9 @@ impl Producer { } } - /// The group's sequence number. - pub fn sequence(&self) -> u64 { - self.inner.sequence + /// Where the group starts, in the track's timescale: its key on the track. + pub fn start(&self) -> u64 { + self.inner.start } /// Sign `payload` at the next frame index and write it with the signature appended. @@ -54,24 +54,22 @@ impl Producer { /// /// A net error if the group is closed, the timestamp does not match the track, or the /// signed frame exceeds the frame size limit. - pub fn write_frame(&mut self, timestamp: impl Into>, payload: &[u8]) -> Result<()> { + pub fn write_frame(&mut self, timestamp: moq_net::Timestamp, payload: &[u8]) -> Result<()> { // Signed at the track's timescale, the value the net group stores and every hop carries. - let timestamp = self.track.rescale(timestamp.into())?; + let timestamp = self.track.rescale_in(timestamp, self.inner.start)?; let frame = self.inner.frame_count() as u64; - let body = self - .scope - .sign(&self.key, self.inner.sequence, frame, timestamp, payload); + let body = self.scope.sign(&self.key, self.inner.start, frame, timestamp, payload); self.inner.write_frame(timestamp, body)?; Ok(()) } - /// Finish the group. No more frames will be written. + /// Close the group without a known end. No more frames will be written. /// /// # Errors /// /// A net error if the group is already closed. - pub fn finish(self) -> Result<()> { - self.inner.finish()?; + pub fn close(self) -> Result<()> { + self.inner.close()?; Ok(()) } @@ -89,7 +87,7 @@ impl Producer { impl fmt::Debug for Producer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("group::Producer") - .field("sequence", &self.sequence()) + .field("start", &self.start()) .field("frames", &self.inner.frame_count()) .finish() } @@ -102,7 +100,7 @@ impl fmt::Debug for Producer { pub struct Consumer { // `None` once a frame failed to verify. inner: Option, - sequence: u64, + start: u64, key: PublicKey, scope: Arc, } @@ -110,16 +108,16 @@ pub struct Consumer { impl Consumer { pub(crate) fn new(inner: moq_net::group::Consumer, key: PublicKey, scope: Arc) -> Self { Self { - sequence: inner.sequence, + start: inner.start, inner: Some(inner), key, scope, } } - /// The group's sequence number. - pub fn sequence(&self) -> u64 { - self.sequence + /// Where the group starts, in the track's timescale: its key on the track. + pub fn start(&self) -> u64 { + self.start } /// The next verified frame, or `None` once the group finished. @@ -140,7 +138,7 @@ impl Consumer { let index = inner.index() - 1; match self .scope - .verify(&self.key, self.sequence, index, frame.timestamp, frame.payload) + .verify(&self.key, self.start, index, frame.timestamp, frame.payload) { Ok(payload) => Poll::Ready(Ok(Some(Frame { timestamp: frame.timestamp, @@ -157,7 +155,7 @@ impl Consumer { impl fmt::Debug for Consumer { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("group::Consumer") - .field("sequence", &self.sequence()) + .field("start", &self.start()) .field("index", &self.inner.as_ref().map(|inner| inner.index())) .finish() } diff --git a/rs/moq-crypto/src/sign/scope.rs b/rs/moq-crypto/src/sign/scope.rs index 7fc28dc7db9..f743c5b91a0 100644 --- a/rs/moq-crypto/src/sign/scope.rs +++ b/rs/moq-crypto/src/sign/scope.rs @@ -34,16 +34,12 @@ impl Scope { /// The message signed for `payload` at frame `frame` of group `group`, presented at `timestamp`. /// /// The timestamp is its value at the track's timescale, so a frame keeps its signature only - /// on a hop that carries both unchanged. An untimed frame binds zero for both, which no - /// timescale can be. - pub(crate) fn message( - &self, - group: u64, - frame: u64, - timestamp: Option, - payload: &[u8], - ) -> Vec { - let (timescale, value) = timestamp.map_or((0, 0), |at| (at.scale().as_u64(), at.value())); + /// on a hop that carries both unchanged. A scale-less frame's timestamp is its group key, + /// which `group` already binds, so it binds zero for both, which no timescale can be. + pub(crate) fn message(&self, group: u64, frame: u64, timestamp: moq_net::Timestamp, payload: &[u8]) -> Vec { + let (timescale, value) = timestamp + .scale() + .map_or((0, 0), |scale| (scale.as_u64(), timestamp.value())); let mut message = Vec::with_capacity(self.prefix.len() + 32 + payload.len()); message.extend_from_slice(&self.prefix); for field in [group, frame, timescale, value] { @@ -59,7 +55,7 @@ impl Scope { key: &SecretKey, group: u64, frame: u64, - timestamp: Option, + timestamp: moq_net::Timestamp, payload: &[u8], ) -> Bytes { let signature = key.sign(&self.message(group, frame, timestamp, payload)); @@ -75,7 +71,7 @@ impl Scope { key: &PublicKey, group: u64, frame: u64, - timestamp: Option, + timestamp: moq_net::Timestamp, body: Bytes, ) -> Result { let split = body.len().checked_sub(SIGNATURE_LEN).ok_or(Error::Signature)?; diff --git a/rs/moq-crypto/src/sign/tests.rs b/rs/moq-crypto/src/sign/tests.rs index 8c67bf4e193..9ef958ac4aa 100644 --- a/rs/moq-crypto/src/sign/tests.rs +++ b/rs/moq-crypto/src/sign/tests.rs @@ -26,14 +26,15 @@ fn public(value: &Value) -> PublicKey { value.as_str().unwrap().parse().unwrap() } -/// A vector row's identity: its scope, group, frame, and timestamp (timescale 0 when untimed). -fn identity(row: &Value) -> (Arc, u64, u64, Option) { +/// A vector row's identity: its scope, group, frame, and timestamp (timescale 0 when +/// scale-less, where the timestamp is the group key). +fn identity(row: &Value) -> (Arc, u64, u64, moq_net::Timestamp) { let path = moq_net::Path::new(row["path"].as_str().unwrap()).unwrap(); let instance = row["instance"].as_str().unwrap().parse().unwrap(); let scope = Scope::new(&path, &instance, row["track"].as_str().unwrap()); let timestamp = match row["timescale"].as_u64().unwrap() { - 0 => None, - scale => Some(moq_net::Timestamp::from_scale(row["timestamp"].as_u64().unwrap(), scale).unwrap()), + 0 => moq_net::Timestamp::unscaled(row["group"].as_u64().unwrap()), + scale => moq_net::Timestamp::from_scale(row["timestamp"].as_u64().unwrap(), scale).unwrap(), }; ( scope, @@ -236,7 +237,7 @@ async fn roundtrip_through_an_origin() { assert_eq!(consumer.instance(), published.signed.instance()); let mut track = subscribe(&consumer, "video").await; - let mut group = video.append_group().unwrap(); + let mut group = video.append_group(1).unwrap(); group .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), b"keyframe") .unwrap(); @@ -244,15 +245,15 @@ async fn roundtrip_through_an_origin() { group .write_frame(moq_net::Timestamp::from_micros(2_400).unwrap(), b"") .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut received = recv_group(&mut track).await; let first = received.read_frame().await.unwrap().unwrap(); assert_eq!(first.payload, &b"keyframe"[..]); - assert_eq!(first.timestamp, Some(moq_net::Timestamp::from_millis(1).unwrap())); + assert_eq!(first.timestamp, moq_net::Timestamp::from_millis(1).unwrap()); let second = received.read_frame().await.unwrap().unwrap(); assert_eq!(second.payload, &b""[..]); - assert_eq!(second.timestamp, Some(moq_net::Timestamp::from_millis(2).unwrap())); + assert_eq!(second.timestamp, moq_net::Timestamp::from_millis(2).unwrap()); assert_eq!(read(&mut received).await.unwrap(), None); } @@ -265,10 +266,14 @@ async fn forged_frame_delivers_nothing_and_ends_only_its_group() { let mut track = subscribe(&consumer, "stream").await; // A relay's forgery: an unsigned frame the length of a signed one. - let mut forged = net.append_group().unwrap(); - forged.write_frame(None, vec![0u8; 80]).unwrap(); - forged.write_frame(None, vec![0u8; 80]).unwrap(); - forged.finish().unwrap(); + let mut forged = net.append_group(0).unwrap(); + forged + .write_frame(moq_net::Timestamp::unscaled(0), vec![0u8; 80]) + .unwrap(); + forged + .write_frame(moq_net::Timestamp::unscaled(0), vec![0u8; 80]) + .unwrap(); + forged.close().unwrap(); let mut group = recv_group(&mut track).await; assert!(matches!(read(&mut group).await, Err(Error::Signature))); @@ -276,8 +281,8 @@ async fn forged_frame_delivers_nothing_and_ends_only_its_group() { assert!(matches!(read(&mut group).await, Err(Error::Signature))); let mut next = signed.create_group(1).unwrap(); - next.write_frame(None, b"after").unwrap(); - next.finish().unwrap(); + next.write_frame(moq_net::Timestamp::unscaled(1), b"after").unwrap(); + next.close().unwrap(); let mut group = recv_group(&mut track).await; assert_eq!(read(&mut group).await.unwrap().unwrap(), &b"after"[..]); } @@ -299,9 +304,9 @@ async fn frame_replayed_from_another_group_or_track_is_refused() { let mut a = subscribe(&consumer, "a").await; let mut b = subscribe(&consumer, "b").await; - let mut group = signed.append_group().unwrap(); - group.write_frame(None, b"original").unwrap(); - group.finish().unwrap(); + let mut group = signed.append_group(0).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), b"original").unwrap(); + group.close().unwrap(); let body = raw_frame(&mut raw).await; assert_eq!( read(&mut recv_group(&mut a).await).await.unwrap().unwrap(), @@ -309,28 +314,34 @@ async fn frame_replayed_from_another_group_or_track_is_refused() { ); // The same bytes as group 1 of the same track. - let mut replay = net.append_group().unwrap(); - replay.write_frame(None, body.clone()).unwrap(); - replay.finish().unwrap(); + let mut replay = net.append_group(1).unwrap(); + replay + .write_frame(moq_net::Timestamp::unscaled(1), body.clone()) + .unwrap(); + replay.close().unwrap(); assert!(matches!( read(&mut recv_group(&mut a).await).await, Err(Error::Signature) )); // The same bytes at the same group and index of another track. - let mut moved = other_net.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); - moved.write_frame(None, body.clone()).unwrap(); - moved.finish().unwrap(); + let mut moved = other_net.create_group(moq_net::group::Info { start: 0 }).unwrap(); + moved + .write_frame(moq_net::Timestamp::unscaled(0), body.clone()) + .unwrap(); + moved.close().unwrap(); assert!(matches!( read(&mut recv_group(&mut b).await).await, Err(Error::Signature) )); - // The same bytes at frame 1 of a group at the original sequence's neighbor. - let mut shifted = net.append_group().unwrap(); - shifted.write_frame(None, b"filler".as_slice()).unwrap(); - shifted.write_frame(None, body).unwrap(); - shifted.finish().unwrap(); + // The same bytes at frame 1 of a group at the original start's neighbor. + let mut shifted = net.append_group(2).unwrap(); + shifted + .write_frame(moq_net::Timestamp::unscaled(2), b"filler".as_slice()) + .unwrap(); + shifted.write_frame(moq_net::Timestamp::unscaled(2), body).unwrap(); + shifted.close().unwrap(); assert!(matches!( read(&mut recv_group(&mut a).await).await, Err(Error::Signature) @@ -347,9 +358,9 @@ async fn broadcast_under_another_key_is_refused() { let consumer = published.consume(victim.public()).await; let mut subscribed = subscribe(&consumer, "stream").await; - let mut group = track.append_group().unwrap(); - group.write_frame(None, b"hello").unwrap(); - group.finish().unwrap(); + let mut group = track.append_group(0).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), b"hello").unwrap(); + group.close().unwrap(); assert!(matches!( read(&mut recv_group(&mut subscribed).await).await, Err(Error::Signature) @@ -362,9 +373,9 @@ async fn frame_from_an_earlier_instance_is_refused() { let old = publish(&key, "mesh/node/ssh"); let (old_track, old_net) = old.track(&key, "stream"); let mut raw = old_net.subscribe(subscribe_all()); - let mut group = old_track.append_group().unwrap(); - group.write_frame(None, b"stale").unwrap(); - group.finish().unwrap(); + let mut group = old_track.append_group(0).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), b"stale").unwrap(); + group.close().unwrap(); let body = raw_frame(&mut raw).await; // A restart mints a new instance, so the old instance's group 0 does not verify as the new one's. @@ -373,9 +384,9 @@ async fn frame_from_an_earlier_instance_is_refused() { let (_new_track, new_net) = new.track(&key, "stream"); let consumer = new.consume(key.public()).await; let mut subscribed = subscribe(&consumer, "stream").await; - let mut replay = new_net.append_group().unwrap(); - replay.write_frame(None, body).unwrap(); - replay.finish().unwrap(); + let mut replay = new_net.append_group(0).unwrap(); + replay.write_frame(moq_net::Timestamp::unscaled(0), body).unwrap(); + replay.close().unwrap(); assert!(matches!( read(&mut recv_group(&mut subscribed).await).await, Err(Error::Signature) @@ -383,41 +394,45 @@ async fn frame_from_an_earlier_instance_is_refused() { } #[test] -fn group_sequence_is_never_signed_twice() { +fn group_start_is_never_signed_twice() { let key = SecretKey::generate(); let net = moq_net::broadcast::Info::new().produce(); let path = moq_net::Path::new("mesh/node/ssh").unwrap(); let signed = broadcast::Producer::new(net, key, path, moq_net::Instance::random()); let track = signed.create_track("stream", None).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(None, b"first").unwrap(); + let mut group = track.append_group(0).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), b"first").unwrap(); group.abort(moq_net::Error::Cancel).unwrap(); - // Net would serve the aborted sequence again, but frames of both incarnations would verify, + // Net would serve the aborted start again, but frames of both incarnations would verify, // so a relay holding both could splice them into one group. assert!(matches!( track.create_group(0), Err(Error::Net(moq_net::Error::Duplicate)) )); - assert_eq!(track.append_group().unwrap().sequence(), 1); + assert_eq!(track.append_group(1).unwrap().start(), 1); assert!(matches!( track.create_group(1), Err(Error::Net(moq_net::Error::Duplicate)) )); - assert_eq!(track.create_group(5).unwrap().sequence(), 5); + assert_eq!(track.create_group(5).unwrap().start(), 5); assert!(matches!( track.create_group(3), Err(Error::Net(moq_net::Error::Duplicate)) )); - // A replacement track continues past its predecessor's sequences under the same instance. + // A replacement track continues past its predecessor's starts under the same instance. track.abort(moq_net::Error::Cancel).unwrap(); let replacement = signed.create_track("stream", None).unwrap(); assert!(matches!( replacement.create_group(0), Err(Error::Net(moq_net::Error::Duplicate)) )); - assert_eq!(replacement.append_group().unwrap().sequence(), 6); + assert!(matches!( + replacement.append_group(5), + Err(Error::Net(moq_net::Error::Duplicate)) + )); + assert_eq!(replacement.append_group(6).unwrap().start(), 6); } #[tokio::test(start_paused = true)] @@ -428,12 +443,12 @@ async fn group_resumed_above_frame_zero_verifies_at_its_index() { let scope = Scope::new(&path, &instance, "stream"); let broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast.create_track("stream", None).unwrap(); - let net = track.append_group().unwrap(); + let net = track.append_group(0).unwrap(); let raw = net.consume(); let mut signed = group::Producer::new(net, track.info().clone(), key.clone(), Arc::clone(&scope)); - signed.write_frame(None, b"zero").unwrap(); - signed.write_frame(None, b"one").unwrap(); - signed.finish().unwrap(); + signed.write_frame(moq_net::Timestamp::unscaled(0), b"zero").unwrap(); + signed.write_frame(moq_net::Timestamp::unscaled(0), b"one").unwrap(); + signed.close().unwrap(); let mut resumed = group::Consumer::new(raw.with_range(1..), key.public(), scope); assert_eq!(read(&mut resumed).await.unwrap().unwrap(), &b"one"[..]); @@ -487,8 +502,8 @@ fn sign_interop() { .enumerate() .map(|(frame, payload)| { let (signed, delivered) = match timestamps { - Some((signed, delivered)) => (Some(millis(signed[frame])), Some(delivered[frame])), - None => (None, None), + Some((signed, delivered)) => (millis(signed[frame]), Some(delivered[frame])), + None => (moq_net::Timestamp::unscaled(signed_as), None), }; let body = scope.sign(&key, signed_as, frame as u64, signed, payload); serde_json::json!({ "body": hex::encode(body), "timestamp": delivered }) @@ -535,9 +550,9 @@ fn sign_interop() { for row in groups { let scope = Scope::new(&path, &js_instance, row["track"].as_str().unwrap()); let group = row["group"].as_u64().unwrap(); - let timestamp = |frame: &Value| { - let scale = row["timescale"].as_u64()?; - Some(moq_net::Timestamp::from_scale(frame["timestamp"].as_u64().unwrap(), scale).unwrap()) + let timestamp = |frame: &Value, group: u64| match row["timescale"].as_u64() { + Some(scale) => moq_net::Timestamp::from_scale(frame["timestamp"].as_u64().unwrap(), scale).unwrap(), + None => moq_net::Timestamp::unscaled(group), }; let frames = row["frames"].as_array().unwrap(); let opened: Vec = frames @@ -545,14 +560,14 @@ fn sign_interop() { .enumerate() .map(|(index, frame)| { let body = Bytes::from(hex(&frame["body"])); - let opened = scope.verify(&key.public(), group, index as u64, timestamp(frame), body); + let opened = scope.verify(&key.public(), group, index as u64, timestamp(frame, group), body); hex::encode(opened.unwrap()) }) .collect(); assert_eq!(opened, expected); // The same body does not verify at the neighboring group. let body = Bytes::from(hex(&frames[0]["body"])); - let relocated = scope.verify(&key.public(), group + 1, 0, timestamp(&frames[0]), body); + let relocated = scope.verify(&key.public(), group + 1, 0, timestamp(&frames[0], group + 1), body); assert!(matches!(relocated, Err(Error::Signature))); } let timed: Vec = groups[2]["frames"] @@ -566,7 +581,7 @@ fn sign_interop() { .iter() .map(|µs| { let at = moq_net::Timestamp::from_micros(micros).unwrap(); - track.rescale(Some(at)).unwrap().unwrap().as_millis() as u64 + track.rescale(at).unwrap().as_millis().unwrap() as u64 }) .collect(); assert_eq!(timed, rescaled); diff --git a/rs/moq-crypto/src/sign/track.rs b/rs/moq-crypto/src/sign/track.rs index 5a4a21e85e7..f37c4b1d8e3 100644 --- a/rs/moq-crypto/src/sign/track.rs +++ b/rs/moq-crypto/src/sign/track.rs @@ -11,14 +11,14 @@ use crate::sign::{PublicKey, SecretKey}; /// Producer for one signed track. /// -/// Group sequences only rise under an instance, even across a replacement of the track, so no two +/// Group starts only rise under an instance, even across a replacement of the track, so no two /// groups are signed at the same coordinates. A relay holding an aborted group and a new group at -/// its sequence could otherwise splice their frames into a group that was never published. +/// its start could otherwise splice their frames into a group that was never published. pub struct Producer { inner: moq_net::track::Producer, key: SecretKey, scope: Arc, - // The lowest sequence this instance has not signed on the track, shared with its replacements. + // The lowest start this instance has not signed on the track, shared with its replacements. next: Arc>, } @@ -42,35 +42,43 @@ impl Producer { self.inner.name() } - /// Start a group one past the highest sequence this instance signed on the track. + /// Start the next group at `start`, in the track's timescale, as + /// [`moq_net::track::Producer::append_group`] does, above any start this instance signed on + /// the track. /// /// # Errors /// - /// A net error if the track is closed. - pub fn append_group(&self) -> Result { - self.claim(None) + /// [`moq_net::Error::Duplicate`] if this instance already signed the start or a later one, + /// even in a group since aborted or evicted, or a net error if the track refuses it. + pub fn append_group(&self, start: u64) -> Result { + self.claim(start, |start| self.inner.append_group(start)) } - /// Start a group at an explicit sequence, above any this instance signed on the track. + /// Start a group at `start`, in the track's timescale, as + /// [`moq_net::track::Producer::create_group`] does, above any start this instance signed on + /// the track. /// /// # Errors /// - /// [`moq_net::Error::Duplicate`] if this instance already signed the sequence or a later one, - /// even in a group since aborted or evicted, or a net error if the track is closed. - pub fn create_group(&self, sequence: u64) -> Result { - self.claim(Some(sequence)) - } - - fn claim(&self, sequence: Option) -> Result { - let mut next = self.next.lock().expect("signed sequences"); - let sequence = sequence.unwrap_or(*next); - if sequence < *next { + /// [`moq_net::Error::Duplicate`] if this instance already signed the start or a later one, + /// even in a group since aborted or evicted, or a net error if the track refuses it. + pub fn create_group(&self, start: u64) -> Result { + self.claim(start, |start| self.inner.create_group(moq_net::group::Info { start })) + } + + fn claim( + &self, + start: u64, + create: impl FnOnce(u64) -> moq_net::Result, + ) -> Result { + let mut next = self.next.lock().expect("signed starts"); + if start < *next { return Err(moq_net::Error::Duplicate.into()); } - let after = sequence + let after = start .checked_add(1) .ok_or(moq_net::Error::from(moq_net::BoundsExceeded))?; - let inner = self.inner.create_group(moq_net::group::Info { sequence })?; + let inner = create(start)?; *next = after; Ok(self.group(inner)) } diff --git a/rs/moq-crypto/tests/transport.rs b/rs/moq-crypto/tests/transport.rs index 8683f5645f4..c69356b8765 100644 --- a/rs/moq-crypto/tests/transport.rs +++ b/rs/moq-crypto/tests/transport.rs @@ -101,9 +101,9 @@ async fn grouped_roundtrip(version: &str, payload: &[u8]) { .unwrap() .await .unwrap(); - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_group(1).unwrap(); group.write_frame(Timestamp::from_millis(1).unwrap(), payload).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut group = consumer.recv_group().await.unwrap().unwrap(); let frame = group.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.plaintext[..], payload); diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 84578201892..e13d7bb108f 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -237,7 +237,7 @@ impl From for MoqAudioRead { moq_audio::decode::Read::Frame(frame) => Self::Frame { frame: frame.into() }, moq_audio::decode::Read::Gap { timestamp, duration } => Self::Gap { timestamp: timestamp.into(), - duration: crate::ffi::duration(duration.into()), + duration: crate::ffi::gap(duration), }, moq_audio::decode::Read::End { end } => Self::End { end: end.into() }, } @@ -820,7 +820,10 @@ mod tests { // A partial codec frame makes finish publish the final packet and lookahead. audio .write(MoqAudioFrame { - timestamp: MoqTimestamp { value: 0, scale: 1 }, + timestamp: MoqTimestamp { + value: 0, + scale: Some(1), + }, data: vec![0.1_f32; 860].into_iter().flat_map(f32::to_ne_bytes).collect(), }) .unwrap(); diff --git a/rs/moq-ffi/src/bench.rs b/rs/moq-ffi/src/bench.rs index df8c6bbf973..973d4a85bd1 100644 --- a/rs/moq-ffi/src/bench.rs +++ b/rs/moq-ffi/src/bench.rs @@ -144,9 +144,9 @@ impl Report { } } -fn check(case: &Case, round: u64, frame: usize, want: &[u8], payload: &[u8], timestamp_us: Option) { +fn check(case: &Case, round: u64, frame: usize, want: &[u8], payload: &[u8], timestamp_us: u64) { assert!(payload == want, "corrupt frame {frame} of round {round}"); - assert_eq!(timestamp_us, Some(round * case.frames() as u64 + frame as u64)); + assert_eq!(timestamp_us, round * case.frames() as u64 + frame as u64); } async fn ffi_round( @@ -165,9 +165,10 @@ async fn ffi_round( let mut read = Vec::with_capacity(frames); for _ in 0..frames { let frame = group.read_frame().await.unwrap().expect("group ended"); - let timestamp = frame - .timestamp - .map(|t| moq_net::Timestamp::try_from(t).unwrap().as_micros() as u64); + let timestamp = moq_net::Timestamp::try_from(frame.timestamp) + .unwrap() + .as_micros() + .unwrap() as u64; read.push((frame.payload, timestamp)); } read @@ -179,10 +180,10 @@ async fn ffi_round( for f in 0..case.frames() { let frame = MoqFrame { payload: payload.to_vec(), - timestamp: Some(MoqTimestamp { + timestamp: MoqTimestamp { value: n * case.frames() as u64 + f as u64, - scale: 1_000_000, - }), + scale: Some(1_000_000), + }, }; group.write_frame(frame).unwrap(); } @@ -212,20 +213,21 @@ async fn net_round( let mut read = Vec::with_capacity(frames); for _ in 0..frames { let frame = group.read_frame().await.unwrap().expect("group ended"); - read.push((frame.payload, frame.timestamp.map(|t| t.as_micros() as u64))); + read.push((frame.payload, frame.timestamp.as_micros().unwrap() as u64)); } (consumer, read) }) }) .collect(); for track in &mut tracks { - let mut group = track.append_group().unwrap(); + // The group starts at its first frame's time, in the track's microseconds. + let mut group = track.append_group(n * case.frames() as u64).unwrap(); for f in 0..case.frames() { let timestamp = moq_net::Timestamp::from_micros(n * case.frames() as u64 + f as u64).unwrap(); // Copy, as a binding's write does, so the baseline does not get zero-copy for free. group.write_frame(timestamp, payload.to_vec()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let mut consumers = Vec::with_capacity(readers.len()); for reader in readers { diff --git a/rs/moq-ffi/src/consumer.rs b/rs/moq-ffi/src/consumer.rs index fa5d7b6f2ed..69a27d6da31 100644 --- a/rs/moq-ffi/src/consumer.rs +++ b/rs/moq-ffi/src/consumer.rs @@ -12,7 +12,7 @@ use crate::producer::MoqTrackInfo; fn raw_frame(frame: moq_net::frame::Frame) -> MoqFrame { MoqFrame { payload: frame.payload.to_vec(), - timestamp: frame.timestamp.map(Into::into), + timestamp: frame.timestamp.into(), } } @@ -111,19 +111,21 @@ pub struct MoqRange { /// Where one end of a [`MoqRange`] falls, mirroring [`moq_net::track::Anchor`]. #[derive(Clone, Copy, uniffi::Enum)] pub enum MoqAnchor { - /// This group, delivered from its first frame. + /// The group starting here, delivered from its first frame. Absolute { - /// The absolute group sequence number. + /// The group's start, in its track's timescale, which names it. group: u64, }, - /// The group this many before the latest, clamped to group 0: `0` is the latest group. + /// The group this many before the latest, counting the groups held: `0` is the latest + /// group, and one counting back past every group held reaches back to the oldest. Relative { /// The number of groups back from the live edge. groups: u64, }, - /// The group containing this presentation time. An untimed track refuses it. + /// The group containing this time: the newest one starting at or before it. Time { - /// The presentation time. + /// The time in the track's timescale, given [unscaled](MoqTimestamp::unscaled). A scaled + /// one is refused with a timestamp mismatch until the bindings convert it. timestamp: MoqTimestamp, }, /// The oldest group still around, as far back as the max delay allows. A start only. @@ -140,27 +142,45 @@ pub enum MoqOrder { Descending, } -impl TryFrom for moq_net::track::Anchor { - type Error = MoqError; - - fn try_from(anchor: MoqAnchor) -> Result { - Ok(match anchor { - MoqAnchor::Oldest => Self::Oldest, - MoqAnchor::Absolute { group } => Self::Absolute(group), - MoqAnchor::Relative { groups } => Self::Relative(groups), - MoqAnchor::Time { timestamp } => Self::Time(timestamp.try_into()?), +impl MoqAnchor { + /// This anchor as a start. A group is named by its start, so an absolute group names + /// that time. A time must be unscaled, in the track's timescale: the timescale is unknown + /// here, so a scaled one is refused rather than read in the wrong units. + fn start(self) -> Result { + use moq_net::track::Anchor; + Ok(match self { + Self::Oldest => Anchor::Oldest, + Self::Absolute { group } => Anchor::Time(group), + Self::Relative { groups } => Anchor::Relative(groups), + Self::Time { timestamp } => match timestamp.scale { + None => Anchor::Time(timestamp.value), + Some(_) => return Err(moq_net::Error::TimestampMismatch.into()), + }, }) } -} -impl TryFrom for moq_net::track::Range { - type Error = MoqError; + /// This anchor as an end, or `None` for an absolute end at group 0, which holds nothing. + /// An absolute end is exclusive, so it ends with the group before it. + fn end(self) -> Result, MoqError> { + match self { + Self::Absolute { group } => Ok(group.checked_sub(1).map(moq_net::track::Anchor::Time)), + anchor => anchor.start().map(Some), + } + } +} - fn try_from(range: MoqRange) -> Result { - Ok(Self { - start: range.start.try_into()?, - end: range.end.map(TryInto::try_into).transpose()?, - }) +impl MoqRange { + /// This range as moq-net reads it, or `None` when it holds nothing. + pub(crate) fn resolve(self) -> Result, MoqError> { + let start = self.start.start()?; + let end = match self.end { + Some(end) => match end.end()? { + Some(end) => Some(end), + None => return Ok(None), + }, + None => None, + }; + Ok(Some(moq_net::track::Range { start, end })) } } @@ -212,12 +232,11 @@ impl TryFrom for moq_net::track::Subscription { .with_priority(s.priority) .with_delay(s.delay.into()); if let Some(ranges) = s.ranges { - subscription = subscription.with_ranges( - ranges - .into_iter() - .map(TryInto::try_into) - .collect::, _>>()?, - ); + let ranges = ranges + .into_iter() + .map(MoqRange::resolve) + .collect::, _>>()?; + subscription = subscription.with_ranges(ranges.into_iter().flatten()); } if let Some(order) = s.order { subscription = subscription.with_order(order.into()); @@ -664,7 +683,7 @@ pub struct MoqGroupConsumer { impl MoqGroupConsumer { pub(crate) fn new(group: moq_net::group::Consumer, driver: Background) -> Self { Self { - sequence: group.sequence, + sequence: group.start, reads: Polled::new(group).supervised(driver.clone()), driver, } @@ -688,7 +707,7 @@ impl MoqGroupConsumer { #[uniffi::export] impl MoqGroupConsumer { - /// The sequence number of this group within the track. + /// The start of this group, in its track's timescale, which names it within the track. pub fn sequence(&self) -> u64 { self.sequence } diff --git a/rs/moq-ffi/src/datagram.rs b/rs/moq-ffi/src/datagram.rs index 57e1321d545..c713b70a0a3 100644 --- a/rs/moq-ffi/src/datagram.rs +++ b/rs/moq-ffi/src/datagram.rs @@ -119,10 +119,13 @@ impl MoqDatagramProducer { .ok_or(MoqError::Closed) } - /// Append a datagram, returning its assigned sequence; its timestamp must match the track. - pub fn append(&self, timestamp: Option, payload: Vec) -> Result { + /// Append a datagram, returning its sequence; its timestamp must match the track. + /// + /// A scale-less track has no time, so there the timestamp is the + /// [unscaled](MoqTimestamp::unscaled) sequence to write. + pub fn append(&self, timestamp: MoqTimestamp, payload: Vec) -> Result { let _guard = crate::ffi::enter(); - let timestamp = timestamp.map(TryInto::try_into).transpose()?; + let timestamp = timestamp.try_into()?; Ok(self .inner .peek_mut(|producer| producer.append(timestamp, payload)) @@ -134,7 +137,7 @@ impl MoqDatagramProducer { let _guard = crate::ffi::enter(); let datagram = moq_net::datagram::Datagram { sequence: datagram.sequence, - timestamp: datagram.timestamp.map(TryInto::try_into).transpose()?, + timestamp: datagram.timestamp.try_into()?, payload: datagram.payload.into(), }; Ok(self @@ -230,7 +233,7 @@ impl MoqDatagramConsumer { .wait(|consumer, waiter| { Poll::Ready(Ok(ready!(consumer.poll_recv(waiter))?.map(|datagram| MoqDatagram { sequence: datagram.sequence, - timestamp: datagram.timestamp.map(Into::into), + timestamp: datagram.timestamp.into(), payload: datagram.payload.to_vec(), }))) }) diff --git a/rs/moq-ffi/src/ffi.rs b/rs/moq-ffi/src/ffi.rs index c70807d2d98..5577425f88f 100644 --- a/rs/moq-ffi/src/ffi.rs +++ b/rs/moq-ffi/src/ffi.rs @@ -14,6 +14,12 @@ pub(crate) fn duration(duration: std::time::Duration) -> std::time::Duration { duration.min(std::time::Duration::from_nanos(i64::MAX as u64)) } +/// A gap's length, as [`duration`]. Decoders report gaps only on scaled media, refusing a +/// scale-less track first, so the zero an unscaled length would take never shows. +pub(crate) fn gap(length: moq_net::Timestamp) -> std::time::Duration { + duration(std::time::Duration::try_from(length).unwrap_or_default()) +} + /// A worker runtime driving I/O, timers, and background work beside host-polled calls. static RUNTIME: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/rs/moq-ffi/src/json.rs b/rs/moq-ffi/src/json.rs index 919b8c918d1..436d6fe6c57 100644 --- a/rs/moq-ffi/src/json.rs +++ b/rs/moq-ffi/src/json.rs @@ -206,16 +206,17 @@ impl MoqJsonSnapshotProducer { pub struct MoqJsonValue { /// The JSON document. pub value: String, - /// The frame's timestamp on the broadcast clock, or null when untimed. - pub timestamp: Option, + /// The frame's timestamp on the broadcast clock, or its [unscaled](MoqTimestamp::unscaled) + /// group key on a scale-less track. + pub timestamp: MoqTimestamp, } impl MoqJsonValue { - fn new(timed: Option>) -> Result, MoqError> { + fn new(timed: Option>) -> Result, MoqError> { let Some(timed) = timed else { return Ok(None) }; Ok(Some(Self { value: serde_json::to_string(&timed.value)?, - timestamp: timed.at.map(Into::into), + timestamp: timed.at.into(), })) } } @@ -501,7 +502,7 @@ impl MoqJsonWindowProducer { pub fn push(&self, value: String, timestamp: MoqTimestamp) -> Result<(), MoqError> { let value: Value = serde_json::from_str(&value)?; let at = timestamp.try_into()?; - self.edit(|producer| producer.push(moq_net::Timed::from(&value).at(at))) + self.edit(|producer| producer.push(moq_json::Timed::new(&value, at))) } /// Drop `count` records from the front of the window, clamped to what it holds. @@ -509,7 +510,7 @@ impl MoqJsonWindowProducer { /// [`MoqMediaCatalogProducer::now`](crate::media::MoqMediaCatalogProducer::now) reads. pub fn pop(&self, count: u64, timestamp: MoqTimestamp) -> Result<(), MoqError> { let at = timestamp.try_into()?; - self.edit(|producer| producer.pop(moq_net::Timed::from(count).at(at))) + self.edit(|producer| producer.pop(moq_json::Timed::new(count, at))) } /// Close the open group, so the next push or pop opens a new one whose header restates the @@ -571,14 +572,15 @@ pub enum MoqJsonWindowEvent { pub struct MoqJsonWindowTimed { /// The change to the window. pub event: MoqJsonWindowEvent, - /// The frame's timestamp on the broadcast clock, or null when untimed. + /// The frame's timestamp on the broadcast clock, or its [unscaled](MoqTimestamp::unscaled) + /// group key on a scale-less track. /// A record first learned of from a group's restatement (after a join, or while behind) /// carries that frame's time, not its original push time. - pub timestamp: Option, + pub timestamp: MoqTimestamp, } impl MoqJsonWindowTimed { - fn new(timed: Option>>) -> Result, MoqError> { + fn new(timed: Option>>) -> Result, MoqError> { use moq_json::window::Event; let Some(timed) = timed else { return Ok(None) }; @@ -603,7 +605,7 @@ impl MoqJsonWindowTimed { }; Ok(Some(Self { event, - timestamp: timed.at.map(Into::into), + timestamp: timed.at.into(), })) } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index e04d902831f..14abfe21b5f 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -206,23 +206,36 @@ pub struct MoqAudio { pub warmup: Option, } -/// An exact media timestamp, `value` units at `scale` units per second, mirroring [`moq_net::Timestamp`]. +/// An exact timestamp, `value` units at `scale` units per second, mirroring [`moq_net::Timestamp`]. /// /// Reads carry the scale the publisher wrote, so a tick such as 90 kHz survives a read and /// republish exactly. Writing one past the bounds below is refused with [`MoqError::TimeOverflow`]. +/// +/// A scale-less track has no time, so there `scale` is null and `value` is the frame's group +/// key (a datagram's sequence), as [`moq_net::Timestamp::unscaled`]. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, uniffi::Record)] pub struct MoqTimestamp { - /// The time in `scale` units, at most `2^61 - 1`. + /// The time in `scale` units, at most `2^61 - 1`, or the group key when unscaled. pub value: u64, - /// Units per second, non-zero and at most `2^62 - 1`. - pub scale: u64, + /// Units per second, non-zero and at most `2^62 - 1`, or null on a scale-less track. + pub scale: Option, +} + +impl MoqTimestamp { + /// The scale-less timestamp at group key `key`. + pub fn unscaled(key: u64) -> Self { + Self { + value: key, + scale: None, + } + } } impl From for MoqTimestamp { fn from(timestamp: moq_net::Timestamp) -> Self { Self { value: timestamp.value(), - scale: timestamp.scale().as_u64(), + scale: timestamp.scale().map(moq_net::Timescale::as_u64), } } } @@ -231,7 +244,10 @@ impl TryFrom for moq_net::Timestamp { type Error = moq_net::TimeOverflow; fn try_from(timestamp: MoqTimestamp) -> Result { - Self::from_scale(timestamp.value, timestamp.scale) + match timestamp.scale { + Some(scale) => Self::from_scale(timestamp.value, scale), + None => Ok(Self::unscaled(timestamp.value)), + } } } @@ -244,9 +260,9 @@ impl TryFrom for moq_net::Timestamp { pub struct MoqFrame { /// The frame payload. pub payload: Vec, - /// Presentation timestamp, or null on an untimed track, on both publish and receive. - #[uniffi(default = None)] - pub timestamp: Option, + /// Presentation timestamp, or the [unscaled](MoqTimestamp::unscaled) group key on a + /// scale-less track, on both publish and receive. + pub timestamp: MoqTimestamp, } /// A [`MoqFrame`] plus the codec metadata a media track carries. @@ -260,15 +276,15 @@ pub struct MoqMediaFrame { pub keyframe: bool, } -/// A datagram payload with its own sequence and optional timestamp. +/// A datagram payload with its own sequence and timestamp. #[derive(uniffi::Record)] pub struct MoqDatagram { /// The datagram track's own sequence number. #[uniffi(default = 0)] pub sequence: u64, - /// Presentation timestamp, or null on an untimed track, on both publish and receive. - #[uniffi(default = None)] - pub timestamp: Option, + /// Presentation timestamp, or the [unscaled](MoqTimestamp::unscaled) sequence on a + /// scale-less track, on both publish and receive. + pub timestamp: MoqTimestamp, /// Datagram payload; keep it well below the path MTU so each hop can carry it. pub payload: Vec, } @@ -1142,18 +1158,18 @@ impl MoqMediaTrackProducer { /// Write `frame` to this track. /// /// The importer derives keyframe status from the bitstream, so a [`MoqFrame`] carries only the - /// payload and its caller-supplied timestamp. A missing timestamp is refused. + /// payload and its caller-supplied timestamp, which must be scaled. pub fn write_frame(&self, frame: MoqFrame) -> Result<(), MoqError> { let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); let media = guard.as_mut().ok_or(MoqError::Closed)?; - let timestamp = frame - .timestamp - .ok_or_else(|| MoqError::Codec { - reason: "whole media frames require a timestamp".into(), - })? - .try_into()?; + if frame.timestamp.scale.is_none() { + return Err(MoqError::Codec { + reason: "whole media frames require a scaled timestamp".into(), + }); + } + let timestamp = frame.timestamp.try_into()?; media .import .decode(&frame.payload, Some(timestamp)) @@ -1210,10 +1226,11 @@ impl MoqMediaTrackProducer { Ok(()) } - /// Draw a group boundary and number the next group `sequence`. + /// Draw a group boundary and start the next group at `sequence`, in the track's timescale. /// - /// [`cut`](Self::cut) with an explicit sequence, for a publisher whose group numbers have to - /// be deterministic: two encoders aligning per GOP so a consumer can fail over between them. + /// [`cut`](Self::cut) keyed explicitly rather than at the next keyframe's time, which has to + /// come after every group the track holds. Groups are keyed by their first frame's time, so + /// two encoders of the same content align without it. pub fn seek(&self, sequence: u64) -> Result<(), MoqError> { let _guard = crate::ffi::enter(); let mut guard = self.inner.lock().unwrap(); @@ -1286,7 +1303,8 @@ impl MoqMediaContainerProducer { result } - /// Start a new segment and number its groups `sequence`. + /// Start a new segment whose first group starts at `sequence`, in its track's timescale, after + /// every group the track holds. pub fn seek(&self, sequence: u64) -> Result<(), MoqError> { self.driver.check()?; let result = (|| { @@ -1526,7 +1544,7 @@ impl From for MoqMediaRead { }, moq_mux::container::Read::Gap { timestamp, duration } => Self::Gap { timestamp: timestamp.into(), - duration: crate::ffi::duration(duration.into()), + duration: crate::ffi::gap(duration), }, moq_mux::container::Read::End { end, .. } => Self::End { end: end.into() }, } @@ -1638,7 +1656,7 @@ mod read_tests { }], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; // Newest first, all before the first read. Group 4 covers the min delay alone, so // without the warmup the start would skip group 3; only the warmup's reach keeps it. @@ -1649,7 +1667,7 @@ mod read_tests { let mut micros = Vec::new(); while let Some(read) = media.read().await.unwrap() { if let moq_mux::container::Read::Frame { frame, .. } = read { - micros.push(frame.timestamp.as_micros()); + micros.push(frame.timestamp.as_micros().unwrap()); } } assert_eq!(micros, [4_400_000, 4_700_000, 5_000_000], "video={video}"); @@ -1692,7 +1710,7 @@ mod read_tests { let timestamp = |us| moq_net::Timestamp::from_micros(us).unwrap(); let micros = |value| MoqTimestamp { value, - scale: 1_000_000, + scale: Some(1_000_000), }; let mut first = track.create_group(0u64.into()).unwrap(); format() @@ -1706,7 +1724,7 @@ mod read_tests { }], ) .unwrap(); - first.finish().unwrap(); + first.close().unwrap(); let Some(MoqMediaRead::Frame { frame }) = reader.next().await.unwrap() else { panic!("expected frame") }; @@ -1725,7 +1743,7 @@ mod read_tests { }], ) .unwrap(); - marker.finish().unwrap(); + marker.close().unwrap(); assert!( matches!(reader.next().await.unwrap(), Some(MoqMediaRead::Gap { timestamp, duration }) if timestamp == micros(10_000) && duration == Duration::from_millis(10)) ); @@ -1738,14 +1756,14 @@ mod read_tests { .write( &mut resumed, &[moq_mux::container::Frame { - timestamp: timestamp(20_000), + timestamp: timestamp(20_001), payload: bytes::Bytes::from_static(b"resumed"), keyframe: true, duration: None, }], ) .unwrap(); - resumed.finish().unwrap(); + resumed.close().unwrap(); let Some(MoqMediaRead::Frame { frame }) = reader.next().await.unwrap() else { panic!("end is not EOF") }; @@ -1772,7 +1790,7 @@ impl MoqMediaGroupConsumer { ) -> Self { let inner = moq_mux::container::GroupConsumer::new(group, container); Self { - sequence: inner.sequence(), + sequence: inner.start(), reads: Polled::new(inner).supervised(driver), } } @@ -1780,7 +1798,7 @@ impl MoqMediaGroupConsumer { #[uniffi::export] impl MoqMediaGroupConsumer { - /// The sequence number of this group within the track. + /// The start of this group, in its track's timescale, which names it within the track. pub fn sequence(&self) -> u64 { self.sequence } @@ -2078,7 +2096,7 @@ mod timebase_tests { ) .unwrap(); tokio::time::advance(Duration::from_secs(1)).await; - let first = clock.now().as_micros(); + let first = clock.now().as_micros().unwrap(); // Each write enters the unpaused FFI worker, whose ambient clock is the host's. The // broadcast still stamps on the paused clock it was built under. chunks.write(fragment(&source, 3600, 1).to_vec()).unwrap(); @@ -2115,7 +2133,7 @@ mod timebase_tests { for mut media in readers { while let Some(read) = media.read().await.unwrap() { if let moq_mux::container::Read::Frame { frame, .. } = read { - times.insert(frame.payload[0], frame.timestamp.as_micros()); + times.insert(frame.payload[0], frame.timestamp.as_micros().unwrap()); } } } @@ -2135,8 +2153,14 @@ mod timebase_tests { .unwrap() .ordered(); let mut group = catalogs.next_group().await.unwrap().unwrap(); - let stamp = group.read_frame().await.unwrap().unwrap().timestamp.unwrap(); - assert_eq!(stamp.as_micros(), first + 5_000_000, "shared={shared}"); + let stamp = group.read_frame().await.unwrap().unwrap().timestamp; + // Catalogs published in the same millisecond, such as a rendition and its first estimate, + // start a unit apart, since each starts its own group. + let second = first + 5_000_000; + assert!( + (second..=second + 1_000).contains(&stamp.as_micros().unwrap()), + "shared={shared}" + ); } } diff --git a/rs/moq-ffi/src/producer.rs b/rs/moq-ffi/src/producer.rs index 5f8a0658cfc..0c8eeb4b679 100644 --- a/rs/moq-ffi/src/producer.rs +++ b/rs/moq-ffi/src/producer.rs @@ -100,6 +100,12 @@ impl TryFrom for moq_net::track::Info { } } +/// One past the newest group's start, or 0 on an empty track: where a group the caller gave +/// no start for begins. +fn next_start(track: &moq_net::track::Producer) -> u64 { + track.largest().map_or(0, |largest| largest.group.saturating_add(1)) +} + fn raw_track_info(info: Option) -> Result { info.map(moq_net::track::Info::try_from) .transpose() @@ -474,10 +480,21 @@ impl MoqTrackRequests { /// history a subscription reads that the cache no longer holds. /// /// Cached groups are served without reaching this method. + /// + /// A reader's time start that no cached group places is looked up rather than filled; a + /// fill names groups, so the lookup is dropped unanswered here, and the reader falls back + /// to the groups held. pub async fn requested_fill(&self) -> Result, MoqError> { let fill = self .reads - .wait(|requests, waiter| Poll::Ready(Ok(ready!(requests.poll_requested_fill(waiter))?))) + .wait(|requests, waiter| { + loop { + let fill = ready!(requests.poll_requested_fill(waiter))?; + if fill.time().is_none() { + return Poll::Ready(Ok(fill)); + } + } + }) .await?; Ok(Arc::new(MoqFill::new(fill))) } @@ -825,16 +842,17 @@ impl MoqTrackProducer { Ok(Arc::new(MoqTrackConsumer::new(track, Background::default()))) } - /// Append a new group to the track, returning a producer for writing frames into it. + /// Append a new group to the track, one past the newest group's start, returning a + /// producer for writing frames into it. pub fn append_group(&self) -> Result, MoqError> { let _guard = crate::ffi::enter(); let mut phase = self.phase.lock().unwrap(); let raw = phase.raw()?; - let group = raw.append_group()?; + let group = raw.append_group(next_start(raw))?; Ok(Arc::new(MoqGroupProducer::from_inner(group))) } - /// Create a group with an explicit sequence number. + /// Create a group starting at `sequence`, in the track's timescale, which names it. /// /// Use this for sparse or replayed tracks. [`append_group`](Self::append_group) /// remains the convenient live-stream path. @@ -842,7 +860,7 @@ impl MoqTrackProducer { let _guard = crate::ffi::enter(); let mut phase = self.phase.lock().unwrap(); let raw = phase.raw()?; - let group = raw.create_group(moq_net::group::Info { sequence })?; + let group = raw.create_group(moq_net::group::Info { start: sequence })?; Ok(Arc::new(MoqGroupProducer::from_inner(group))) } @@ -866,10 +884,10 @@ impl MoqTrackProducer { /// [`MoqProtocolKind::TimestampMismatch`](crate::error::MoqProtocolKind::TimestampMismatch). pub fn write_frame(&self, frame: MoqFrame) -> Result<(), MoqError> { let _guard = crate::ffi::enter(); - let timestamp = frame.timestamp.map(TryInto::try_into).transpose()?; + let timestamp: moq_net::Timestamp = frame.timestamp.try_into()?; let mut phase = self.phase.lock().unwrap(); - let raw = phase.raw()?; - raw.write_frame(timestamp, frame.payload)?; + // The group starts at its frame's time, or its unscaled key on a scale-less track. + phase.raw()?.write_frame(timestamp, frame.payload)?; Ok(()) } @@ -972,7 +990,7 @@ impl MoqGroupProducer { /// Wrap `group`, for a track's writer. pub(crate) fn from_inner(group: moq_net::group::Producer) -> Self { Self { - sequence: group.sequence, + sequence: group.start, inner: kio::Bridge::new(GroupPhase::Writing(group)), } } @@ -980,7 +998,7 @@ impl MoqGroupProducer { #[uniffi::export] impl MoqGroupProducer { - /// The sequence number of this group within the track. + /// The start of this group, in its track's timescale, which names it within the track. pub fn sequence(&self) -> u64 { self.sequence } @@ -1003,7 +1021,7 @@ impl MoqGroupProducer { /// [`MoqProtocolKind::TimestampMismatch`](crate::error::MoqProtocolKind::TimestampMismatch). pub fn write_frame(&self, frame: MoqFrame) -> Result<(), MoqError> { let _guard = crate::ffi::enter(); - let timestamp = frame.timestamp.map(TryInto::try_into).transpose()?; + let timestamp = frame.timestamp.try_into()?; let mut guard = self.inner.lock(); let GroupPhase::Writing(group) = &mut *guard else { return Err(MoqError::Closed); @@ -1022,11 +1040,17 @@ impl MoqGroupProducer { /// once while a reader has taken every frame, and once the group is closed, so the next /// write reports why. Fails with `Closed` after [`abort`](Self::abort). pub async fn ready(&self, size: u64) -> Result<(), MoqError> { - let frame = moq_net::frame::Info { size, timestamp: None }; let mut park = kio::Park::default(); crate::ffi::run(kio::wait(|waiter| { self.inner.poll(waiter, |group, cx| match group { - GroupPhase::Writing(group) => group.poll_ready(park.hold(cx), &frame).map(Ok), + GroupPhase::Writing(group) => { + // Only the size paces; the timestamp is checked by the write. + let frame = moq_net::frame::Info { + size, + timestamp: moq_net::Timestamp::unscaled(group.start), + }; + group.poll_ready(park.hold(cx), &frame).map(Ok) + } // The next write reports why. GroupPhase::Finished(_) => Poll::Ready(Ok(())), GroupPhase::Closed => Poll::Ready(Err(MoqError::Closed)), @@ -1042,7 +1066,7 @@ impl MoqGroupProducer { let _guard = crate::ffi::enter(); let mut guard = self.inner.lock(); match std::mem::take(&mut *guard) { - GroupPhase::Writing(group) => *guard = GroupPhase::Finished(group.finish()?), + GroupPhase::Writing(group) => *guard = GroupPhase::Finished(group.close()?), GroupPhase::Finished(group) => *guard = GroupPhase::Finished(group), GroupPhase::Closed => return Err(MoqError::Closed), } @@ -1083,7 +1107,7 @@ mod metadata_tests { fn dropping_a_group_binding_ends_its_unfinished_group() { let broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast.create_track("test", moq_net::track::Info::default()).unwrap(); - let group = track.append_group().unwrap(); + let group = track.append_group(0).unwrap(); let mut consumer = group.consume(); let binding = MoqGroupProducer::from_inner(group); drop(binding); diff --git a/rs/moq-ffi/src/socket.rs b/rs/moq-ffi/src/socket.rs index 74450d0122e..376f0937019 100644 --- a/rs/moq-ffi/src/socket.rs +++ b/rs/moq-ffi/src/socket.rs @@ -176,7 +176,8 @@ impl MoqSocketTrackStream { mod tests { use super::*; use crate::error::MoqProtocolKind; - use crate::media::MoqFrame; + use crate::media::{MoqFrame, MoqTimestamp}; + use crate::producer::MoqGroupProducer; use std::future::Future; use std::pin::Pin; use std::task::{Context, Waker}; @@ -192,10 +193,11 @@ mod tests { (a, b) } - fn frame(payload: &[u8]) -> MoqFrame { + /// `payload` in `group`: socket tracks are scale-less, so a frame carries its group's key. + fn frame(group: &MoqGroupProducer, payload: &[u8]) -> MoqFrame { MoqFrame { payload: payload.into(), - timestamp: None, + timestamp: MoqTimestamp::unscaled(group.sequence()), } } @@ -224,10 +226,10 @@ mod tests { assert!(consumer.info().max_age_unlimited); // Create the later group first: the reliable ascending reader must wait for 0. let later = producer.create_group(1).unwrap(); - later.write_frame(frame(b"later")).unwrap(); + later.write_frame(frame(&later, b"later")).unwrap(); later.finish().unwrap(); let first = producer.create_group(0).unwrap(); - first.write_frame(frame(b"first")).unwrap(); + first.write_frame(frame(&first, b"first")).unwrap(); first.finish().unwrap(); producer.finish().unwrap(); assert_eq!(consumer.read_frame().await.unwrap().unwrap().payload, b"first"); @@ -246,7 +248,13 @@ mod tests { client.producer().abort(42).unwrap(); let error = server.consumer().read_frame().await.err().unwrap(); assert!(matches!(error, MoqError::Protocol { details } if details.kind == MoqProtocolKind::App)); - other.producer().write_frame(frame(b"independent")).unwrap(); + other + .producer() + .write_frame(MoqFrame { + payload: b"independent".as_slice().into(), + timestamp: MoqTimestamp::unscaled(0), + }) + .unwrap(); other.producer().finish().unwrap(); assert_eq!( peer.consumer().read_frame().await.unwrap().unwrap().payload, diff --git a/rs/moq-ffi/src/test.rs b/rs/moq-ffi/src/test.rs index 635a87e3512..3bb602ae61a 100644 --- a/rs/moq-ffi/src/test.rs +++ b/rs/moq-ffi/src/test.rs @@ -31,13 +31,13 @@ mod rung; fn us(value: u64) -> MoqTimestamp { MoqTimestamp { value, - scale: 1_000_000, + scale: Some(1_000_000), } } /// `timestamp` in whole microseconds, whatever scale it was read at. fn micros(timestamp: MoqTimestamp) -> u64 { - moq_net::Timestamp::try_from(timestamp).unwrap().as_micros() as u64 + moq_net::Timestamp::try_from(timestamp).unwrap().as_micros().unwrap() as u64 } /// A millisecond track: the frames written here carry timestamps. @@ -1188,7 +1188,7 @@ async fn raw_track_datagram_roundtrip() { let payload = b"hello datagram".to_vec(); // A local reader receives what is sent after it was created, before its first read. - let sequence = track.append(Some(us(123_456)), payload.clone()).unwrap(); + let sequence = track.append(us(123_456), payload.clone()).unwrap(); let datagram = kio::clock::timeout(TIMEOUT, consumer.recv()) .await .expect("timed out waiting for datagram") @@ -1196,7 +1196,7 @@ async fn raw_track_datagram_roundtrip() { .expect("expected a datagram"); assert_eq!(datagram.sequence, sequence); - assert_eq!(datagram.timestamp.map(micros), Some(123_456)); + assert_eq!(micros(datagram.timestamp), 123_456); assert_eq!(datagram.payload, payload); } @@ -1253,7 +1253,7 @@ async fn raw_track_update_does_not_wait_for_pending_read() { track .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(20_000)), + timestamp: us(20_000), }) .unwrap(); @@ -1264,7 +1264,7 @@ async fn raw_track_update_does_not_wait_for_pending_read() { .unwrap() .expect("expected a frame"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(20_000)); + assert_eq!(micros(frame.timestamp), 20_000); } /// Subscribe to `name` on `broadcast` as a raw track, for a typed consumer to take over. @@ -1282,7 +1282,7 @@ async fn data_writes_preserve_the_callers_timestamp() { // A 90 kHz tick, which no microsecond count holds exactly. let timestamp = MoqTimestamp { value: 123_457, - scale: 90_000, + scale: Some(90_000), }; for kind in ["json snapshot", "json stream"] { let broadcast = MoqBroadcastProducer::new().unwrap(); @@ -1294,7 +1294,7 @@ async fn data_writes_preserve_the_callers_timestamp() { max_age: None, max_age_unlimited: false, refresh: None, - timescale: Some(timestamp.scale), + timescale: timestamp.scale, compression: None, }), ) @@ -1321,8 +1321,8 @@ async fn data_writes_preserve_the_callers_timestamp() { } let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload, payload, "{kind}"); - assert_eq!(frame.timestamp, Some(timestamp), "{kind}"); - assert_eq!(consumer.info().timescale, Some(timestamp.scale)); + assert_eq!(frame.timestamp, timestamp, "{kind}"); + assert_eq!(consumer.info().timescale, timestamp.scale); broadcast.close().unwrap(); } } @@ -1339,14 +1339,14 @@ async fn whole_media_frames_require_the_callers_timestamp() { assert!(matches!( media.write_frame(MoqFrame { payload: b"opus frame".to_vec(), - timestamp: None + timestamp: MoqTimestamp::unscaled(0) }), Err(MoqError::Codec { .. }) )); media .write_frame(MoqFrame { payload: b"opus frame".to_vec(), - timestamp: Some(us(0)), + timestamp: us(0), }) .unwrap(); media.finish().unwrap(); @@ -1373,7 +1373,7 @@ async fn catalog_now_fixes_the_clock_for_later_media() { media .write_frame(MoqFrame { payload: b"opus frame".to_vec(), - timestamp: Some(us(1_000_000)), + timestamp: us(1_000_000), }) .unwrap(); assert_eq!( @@ -1422,7 +1422,7 @@ async fn json_snapshot_roundtrip() { serde_json::from_str::(&value.value).unwrap(), serde_json::json!({ "a": 1 }) ); - assert_eq!(value.timestamp.map(micros), Some(1000)); + assert_eq!(micros(value.timestamp), 1000); let value = kio::clock::timeout(TIMEOUT, consumer.next()) .await @@ -1433,7 +1433,7 @@ async fn json_snapshot_roundtrip() { serde_json::from_str::(&value.value).unwrap(), serde_json::json!({ "a": 2 }) ); - assert_eq!(value.timestamp.map(micros), Some(2000)); + assert_eq!(micros(value.timestamp), 2000); // `latest` skips the buffered third state. let value = kio::clock::timeout(TIMEOUT, consumer.latest()) @@ -1445,7 +1445,7 @@ async fn json_snapshot_roundtrip() { serde_json::from_str::(&value.value).unwrap(), serde_json::json!({ "a": 4 }) ); - assert_eq!(value.timestamp.map(micros), Some(4000)); + assert_eq!(micros(value.timestamp), 4000); producer.finish().unwrap(); assert!(matches!( @@ -1496,7 +1496,7 @@ async fn json_snapshot_refreshes_for_a_joiner() { }) .await .expect("timed out waiting for the refresh"); - assert_eq!(refreshed.sequence(), 1); + assert_eq!(refreshed.sequence(), 3000, "the restatement starts at the value's time"); let consumer = MoqJsonSnapshotConsumer::new(&broadcast.consume().unwrap(), "meta".into(), 0).unwrap(); let value = kio::clock::timeout(TIMEOUT, consumer.next()) .await @@ -1599,7 +1599,7 @@ async fn json_window_roundtrip() { event, MoqJsonWindowTimed { event: window_push(n, n), - timestamp: Some(us(n * 1000)), + timestamp: us(n * 1000), } ); } @@ -1607,7 +1607,7 @@ async fn json_window_roundtrip() { next_window(&consumer).await, Some(MoqJsonWindowTimed { event: MoqJsonWindowEvent::Pop { start: 0, end: 2 }, - timestamp: Some(us(3000)), + timestamp: us(3000), }) ); @@ -1648,7 +1648,7 @@ async fn json_window_cut_restates_the_window() { next_window(&consumer).await, Some(MoqJsonWindowTimed { event, - timestamp: Some(us(at)), + timestamp: us(at), }) ); } @@ -1667,7 +1667,11 @@ async fn json_window_cut_restates_the_window() { }) .await .expect("timed out waiting for the cut's group"); - assert_eq!(group.sequence(), 1, "a repeated cut opens no extra group"); + assert_eq!( + group.sequence(), + 4000, + "a repeated cut opens no extra group: the next starts at the next push" + ); let header = group.read_frame().await.unwrap().expect("expected the header"); assert_eq!( serde_json::from_slice::(&header.payload).unwrap(), @@ -1679,7 +1683,7 @@ async fn json_window_cut_restates_the_window() { next_window(&consumer).await, Some(MoqJsonWindowTimed { event: window_push(3, 3), - timestamp: Some(us(4000)), + timestamp: us(4000), }) ); @@ -1852,7 +1856,7 @@ async fn json_stream_roundtrip() { serde_json::from_str::(&value.value).unwrap(), serde_json::json!({ "n": n }) ); - assert_eq!(value.timestamp.map(micros), Some(n * 1000)); + assert_eq!(micros(value.timestamp), n * 1000); } producer.finish().unwrap(); } @@ -1964,7 +1968,7 @@ async fn requested_datagrams_outlive_the_broadcast_handle() { assert_eq!(request.name().unwrap(), "events"); assert_eq!(request.priority().unwrap(), Some(0)); let producer = request.accept(None).unwrap(); - producer.append(None, b"beep".to_vec()).unwrap(); + producer.append(MoqTimestamp::unscaled(0), b"beep".to_vec()).unwrap(); assert_eq!(consumer.recv().await.unwrap().unwrap().payload, b"beep"); } @@ -2007,7 +2011,7 @@ async fn dynamic_track_request() { track .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(0)), + timestamp: us(0), }) .unwrap(); @@ -2024,7 +2028,7 @@ async fn dynamic_track_request() { .expect("expected a frame"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(0)); + assert_eq!(micros(frame.timestamp), 0); track.finish().unwrap(); } @@ -2034,38 +2038,53 @@ async fn raw_untimed_frames_roundtrip() { let track = broadcast.publish_track("untimed".into(), None).unwrap(); let consumer = track.consume(None).unwrap(); assert_eq!(consumer.info().timescale, None); + // A scale-less frame's timestamp is its group key: a track write opens the group there. track .write_frame(MoqFrame { payload: b"frame".to_vec(), - timestamp: None, + timestamp: MoqTimestamp::unscaled(5), }) .unwrap(); let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload, b"frame"); - assert_eq!(frame.timestamp, None); + assert_eq!(frame.timestamp, MoqTimestamp::unscaled(5)); let group = track.append_group().unwrap(); + assert_eq!(group.sequence(), 6); let reader = group.consume().unwrap(); + let mismatched = group.write_frame(MoqFrame { + payload: b"group".to_vec(), + timestamp: MoqTimestamp::unscaled(5), + }); + assert!(matches!( + mismatched, + Err(MoqError::Protocol { + details: MoqProtocolError { + kind: MoqProtocolKind::TimestampMismatch, + .. + } + }) + )); group .write_frame(MoqFrame { payload: b"group".to_vec(), - timestamp: None, + timestamp: MoqTimestamp::unscaled(6), }) .unwrap(); group.finish().unwrap(); let frame = reader.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload, b"group"); - assert_eq!(frame.timestamp, None); + assert_eq!(frame.timestamp, MoqTimestamp::unscaled(6)); track.finish().unwrap(); } -/// An explicitly timed raw track refuses frames without a timestamp. +/// An explicitly timed raw track refuses unscaled timestamps. #[test] -fn raw_writes_need_a_timestamp() { +fn raw_timed_writes_need_a_scale() { let broadcast = MoqBroadcastProducer::new().unwrap(); let track = broadcast.publish_track("status".into(), Some(timed_info())).unwrap(); let untimed = || MoqFrame { payload: b"ready".to_vec(), - timestamp: None, + timestamp: MoqTimestamp::unscaled(0), }; let refused = |err: MoqError| { matches!( @@ -2093,7 +2112,7 @@ async fn raw_frame_timestamps() { track .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(12_345)), + timestamp: us(12_345), }) .unwrap(); @@ -2103,7 +2122,7 @@ async fn raw_frame_timestamps() { .unwrap() .expect("expected a frame"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(12_345)); + assert_eq!(micros(frame.timestamp), 12_345); let group = track.append_group().unwrap(); let group_consumer = group.consume().unwrap(); @@ -2111,7 +2130,7 @@ async fn raw_frame_timestamps() { group .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(23_456)), + timestamp: us(23_456), }) .unwrap(); group.finish().unwrap(); @@ -2122,7 +2141,7 @@ async fn raw_frame_timestamps() { .unwrap() .expect("expected a frame"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(23_456)); + assert_eq!(micros(frame.timestamp), 23_456); track.finish().unwrap(); } @@ -2140,11 +2159,14 @@ async fn timestamps_are_exact() { // A present zero stays present, and one 90 kHz tick reads back as itself rather than // the 11 µs it floors to. for value in [0, 1] { - let timestamp = MoqTimestamp { value, scale: 90_000 }; + let timestamp = MoqTimestamp { + value, + scale: Some(90_000), + }; track .write_frame(MoqFrame { payload: b"frame".to_vec(), - timestamp: Some(timestamp), + timestamp, }) .unwrap(); let frame = kio::clock::timeout(TIMEOUT, consumer.read_frame()) @@ -2152,36 +2174,31 @@ async fn timestamps_are_exact() { .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.timestamp, Some(timestamp)); + assert_eq!(frame.timestamp, timestamp); } for invalid in [ - MoqTimestamp { value: 0, scale: 0 }, MoqTimestamp { value: 0, - scale: 1 << 62, + scale: Some(0), + }, + MoqTimestamp { + value: 0, + scale: Some(1 << 62), }, MoqTimestamp { value: 1 << 61, - scale: 90_000, + scale: Some(90_000), }, ] { let frame = MoqFrame { payload: b"frame".to_vec(), - timestamp: Some(invalid), + timestamp: invalid, }; assert!( matches!(track.write_frame(frame), Err(MoqError::TimeOverflow)), "{invalid:?}" ); - let range = MoqRange { - start: MoqAnchor::Time { timestamp: invalid }, - end: None, - }; - assert!(matches!( - moq_net::track::Range::try_from(range), - Err(MoqError::TimeOverflow) - )); } } @@ -2224,7 +2241,7 @@ async fn abort_after_finish_keeps_the_track_handle() { track .write_frame(MoqFrame { payload: b"late abort".to_vec(), - timestamp: Some(us(0)), + timestamp: us(0), }) .unwrap(); track.finish().unwrap(); @@ -2242,7 +2259,7 @@ async fn abort_after_finish_reaches_group_consumer() { group .write_frame(MoqFrame { payload: b"late abort".to_vec(), - timestamp: Some(us(0)), + timestamp: us(0), }) .unwrap(); group.finish().unwrap(); @@ -2298,13 +2315,13 @@ async fn fetches_cached_group_without_subscribing() { group .write_frame(MoqFrame { payload: b"first".to_vec(), - timestamp: Some(us(0)), + timestamp: us(0), }) .unwrap(); group .write_frame(MoqFrame { payload: b"second".to_vec(), - timestamp: Some(us(20_000)), + timestamp: us(20_000), }) .unwrap(); group.finish().unwrap(); @@ -2325,10 +2342,10 @@ async fn fetches_cached_group_without_subscribing() { assert_eq!(fetched.sequence(), 0); let frame = fetched.read_frame().await.unwrap().expect("expected first frame"); assert_eq!(frame.payload, b"first".to_vec()); - assert_eq!(frame.timestamp.map(micros), Some(0)); + assert_eq!(micros(frame.timestamp), 0); let frame = fetched.read_frame().await.unwrap().expect("expected second frame"); assert_eq!(frame.payload, b"second".to_vec()); - assert_eq!(frame.timestamp.map(micros), Some(20_000)); + assert_eq!(micros(frame.timestamp), 20_000); assert!(fetched.read_frame().await.unwrap().is_none()); } @@ -2364,7 +2381,8 @@ async fn fetches_cached_media_group_and_decodes_container() { &consumer, MoqMediaGroupConfig { rendition: rendition("media", crate::media::MoqContainer::Legacy), - sequence: 0, + // The group starts at its first frame, a second in, in the track's milliseconds. + sequence: 1000, options: Some(MoqFetchGroupOptions { priority: 7, max_delay: None, @@ -2374,7 +2392,7 @@ async fn fetches_cached_media_group_and_decodes_container() { .await .unwrap(); - assert_eq!(fetched.sequence(), 0); + assert_eq!(fetched.sequence(), 1000); let frame = fetched.next().await.unwrap().expect("expected keyframe"); assert_eq!(frame.payload, b"keyframe"); assert_eq!(micros(frame.timestamp), 1_000_000); @@ -2417,8 +2435,11 @@ async fn fetch_media_group_decodes_multiple_cmaf_samples() { moq_mux::catalog::hang::Container::new(&catalog_container, moq_mux::container::Kind::Video).unwrap(); let broadcast = moq_net::broadcast::Info::new().produce(); - // A CMAF track counts in its init's ticks. - let info = moq_net::track::Info::default().with_timescale(muxer.timescale()); + // A CMAF track counts in its init's ticks. The group outlives its end, which a finished + // track's last group otherwise ages from. + let info = moq_net::track::Info::default() + .with_timescale(muxer.timescale()) + .with_max_age(None); let track = broadcast.create_track("video", info).unwrap(); let consumer = MoqBroadcastConsumer::new(broadcast.consume()); // Buffer both samples into one moof+mdat, which is what this decodes. @@ -2437,14 +2458,18 @@ async fn fetch_media_group_decodes_multiple_cmaf_samples() { .unwrap(); } let _finished = media.finish().unwrap(); - let fetched = kio::clock::timeout( TIMEOUT, MoqMediaGroupConsumer::fetch( &consumer, MoqMediaGroupConfig { rendition: rendition("video", crate::media::MoqContainer::Cmaf { init: init.to_vec() }), - sequence: 0, + // The group starts at its first sample, in the init's ticks. + sequence: moq_net::Timestamp::from_micros(2_000_000) + .unwrap() + .convert(muxer.timescale()) + .unwrap() + .value(), options: None, }, ), @@ -2538,7 +2563,7 @@ async fn next_media(group: &MoqMediaGroupConsumer) -> Result ( fn group_frame(payload: &[u8]) -> MoqFrame { MoqFrame { payload: payload.to_vec(), - timestamp: Some(us(0)), + timestamp: us(0), } } @@ -5354,7 +5383,7 @@ async fn raw_read_frame_skips_empty_group_on_open_track() { track .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(1_000)), + timestamp: us(1_000), }) .unwrap(); @@ -5365,7 +5394,7 @@ async fn raw_read_frame_skips_empty_group_on_open_track() { .unwrap() .expect("expected a frame"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(1_000)); + assert_eq!(micros(frame.timestamp), 1_000); } /// Empty groups already in the cursor are skipped, then the next populated @@ -5383,7 +5412,7 @@ async fn raw_read_frame_skips_empty_then_populated_groups() { track .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(2_000)), + timestamp: us(2_000), }) .unwrap(); @@ -5393,7 +5422,7 @@ async fn raw_read_frame_skips_empty_then_populated_groups() { .unwrap() .expect("expected the populated group's first frame"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(2_000)); + assert_eq!(micros(frame.timestamp), 2_000); } /// A group a hop reset as expired is skipped like an empty one, as a locally expired @@ -5409,13 +5438,13 @@ async fn raw_read_frame_skips_a_group_reset_as_old() { .unwrap(); let consumer = MoqTrackConsumer::new(track.subscribe(None), crate::ffi::Background::default()); - let expired = track.append_group().unwrap(); + let expired = track.append_group(0).unwrap(); // The read holds the group when the reset lands, as one arriving over the wire does. let read = spawn_parked(async move { consumer.read_frame().await }).await; expired .abort(moq_net::Error::Stream(moq_net::StreamError::Old)) .unwrap(); - let mut later = track.append_group().unwrap(); + let mut later = track.append_group(1).unwrap(); later .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), b"later".as_slice()) .unwrap(); @@ -5469,7 +5498,7 @@ async fn raw_read_frame_keeps_group_across_cancelled_call() { group .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(3_000)), + timestamp: us(3_000), }) .unwrap(); group.finish().unwrap(); @@ -5481,7 +5510,7 @@ async fn raw_read_frame_keeps_group_across_cancelled_call() { .unwrap() .expect("cancelled read_frame must not lose the group's first frame"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(3_000)); + assert_eq!(micros(frame.timestamp), 3_000); assert!( kio::clock::timeout(TIMEOUT, consumer.read_frame()) @@ -5527,7 +5556,7 @@ async fn raw_read_frame_drops_aborted_pending_group() { track .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(4_000)), + timestamp: us(4_000), }) .unwrap(); let frame = kio::clock::timeout(TIMEOUT, consumer.read_frame()) @@ -5536,7 +5565,7 @@ async fn raw_read_frame_drops_aborted_pending_group() { .unwrap() .expect("an aborted pending group must not block later groups"); assert_eq!(frame.payload, payload); - assert_eq!(frame.timestamp.map(micros), Some(4_000)); + assert_eq!(micros(frame.timestamp), 4_000); } /// `next_group` and `read_frame` share one ordered cursor. A cancelled @@ -5553,7 +5582,7 @@ async fn raw_read_frame_and_next_group_share_the_cursor() { first .write_frame(MoqFrame { payload: b"first".to_vec(), - timestamp: Some(us(0)), + timestamp: us(0), }) .unwrap(); first.finish().unwrap(); @@ -5573,7 +5602,7 @@ async fn raw_read_frame_and_next_group_share_the_cursor() { track .write_frame(MoqFrame { payload: b"second".to_vec(), - timestamp: Some(us(0)), + timestamp: us(1), }) .unwrap(); let frame = kio::clock::timeout(TIMEOUT, consumer.read_frame()) @@ -5691,7 +5720,7 @@ fn without_runtime() { media .write_frame(MoqFrame { payload: b"hello".to_vec(), - timestamp: Some(us(1000)), + timestamp: us(1000), }) .unwrap(); @@ -5862,7 +5891,7 @@ async fn server_client_run_end(version: &str) { media .write_frame(MoqFrame { payload: payload.clone(), - timestamp: Some(us(1_000_000)), + timestamp: us(1_000_000), }) .unwrap(); @@ -5974,7 +6003,7 @@ async fn roundtrip(media: &MoqMediaTrackProducer, track: &MoqMediaTrackConsumer, media .write_frame(MoqFrame { payload: payload.to_vec(), - timestamp: Some(us(timestamp_us)), + timestamp: us(timestamp_us), }) .unwrap(); loop { @@ -6057,10 +6086,10 @@ async fn datagrams_cross_a_default_connection() { producer.priority_changed(None).await.unwrap(); producer .append( - Some(MoqTimestamp { + MoqTimestamp { value: 1, - scale: 90_000, - }), + scale: Some(90_000), + }, b"tick".to_vec(), ) .unwrap(); @@ -6072,10 +6101,10 @@ async fn datagrams_cross_a_default_connection() { assert_eq!(datagram.payload, b"tick"); assert_eq!( datagram.timestamp, - Some(MoqTimestamp { + MoqTimestamp { value: 1, - scale: 90_000 - }) + scale: Some(90_000) + } ); assert!(matches!(session.state().phase, MoqConnectionPhase::Connected { version } if version == "moq-lite-06")); } @@ -6105,7 +6134,7 @@ async fn requested_datagrams_cross(versions: Vec, negotiated: &str) { .unwrap(); let producer = request.accept(None).unwrap(); consumer.info().await.unwrap(); - producer.append(None, b"only".to_vec()).unwrap(); + producer.append(MoqTimestamp::unscaled(0), b"only".to_vec()).unwrap(); let datagram = kio::clock::timeout(TIMEOUT, consumer.recv()) .await .unwrap() @@ -8411,7 +8440,7 @@ async fn shutdown_without_waiter_survives_session_disposal() { let group = track.append_group().unwrap(); group .write_frame(MoqFrame { - timestamp: Some(us(1)), + timestamp: us(1), payload: vec![42; 64 * 1024], }) .unwrap(); @@ -8520,7 +8549,7 @@ async fn shutdown_ready(track: &MoqTrackProducer, reader: &MoqTrackConsumer) { let group = track.append_group().unwrap(); group .write_frame(MoqFrame { - timestamp: Some(us(0)), + timestamp: us(0), payload: b"ready".to_vec(), }) .unwrap(); @@ -8561,7 +8590,7 @@ async fn shutdown_delivers_the_finished_track() { crate::ffi::run(async move { let group = track.append_group()?; group.write_frame(MoqFrame { - timestamp: Some(us(0)), + timestamp: us(1), payload: b"last".to_vec(), })?; group.finish()?; @@ -8648,7 +8677,7 @@ async fn ascending_subscription_reads_a_cached_backlog_oldest_first() { group .write_frame(MoqFrame { payload: b"x".to_vec(), - timestamp: Some(us(timestamp_us)), + timestamp: us(timestamp_us), }) .unwrap(); group.finish().unwrap(); @@ -9051,7 +9080,9 @@ async fn route_and_claim_credentials_ride_the_moq_transport_announce() { .unwrap(); } -/// A time anchor starts at the group containing it, and an untimed track refuses one. +/// A time anchor starts at the group containing it, and a scale-less track compares it with +/// bare starts. Its time is unscaled, in the track's timescale: a scaled one is refused rather +/// than read in units the subscription can't check. #[tokio::test] async fn time_anchors_start_at_the_containing_group() { let broadcast = MoqBroadcastProducer::new().unwrap(); @@ -9059,11 +9090,12 @@ async fn time_anchors_start_at_the_containing_group() { .publish_track("events".into(), Some(unlimited_info())) .unwrap(); for timestamp_us in [0, 1_000, 2_000] { - let group = track.append_group().unwrap(); + // Each group starts at its frame's time, in the track's microseconds. + let group = track.create_group(timestamp_us).unwrap(); group .write_frame(MoqFrame { payload: b"x".to_vec(), - timestamp: Some(us(timestamp_us)), + timestamp: us(timestamp_us), }) .unwrap(); group.finish().unwrap(); @@ -9077,12 +9109,20 @@ async fn time_anchors_start_at_the_containing_group() { }, ranges: Some(vec![MoqRange { start: MoqAnchor::Time { - timestamp: us(timestamp_us), + timestamp: MoqTimestamp::unscaled(timestamp_us), }, end: None, }]), order: Some(MoqOrder::Ascending), }; + let scaled = MoqRange { + start: MoqAnchor::Time { timestamp: us(1_500) }, + end: None, + }; + assert!(matches!( + scaled.resolve(), + Err(MoqError::Protocol { details }) if details.kind == MoqProtocolKind::TimestampMismatch + )); let consumer = track.consume(Some(since(1_500))).unwrap(); let mut sequences = Vec::new(); for _ in 0..2 { @@ -9093,7 +9133,7 @@ async fn time_anchors_start_at_the_containing_group() { .expect("expected a group"); sequences.push(group.sequence()); } - assert_eq!(sequences, vec![1, 2]); + assert_eq!(sequences, vec![1_000, 2_000]); let untimed = broadcast .publish_track( @@ -9105,15 +9145,7 @@ async fn time_anchors_start_at_the_containing_group() { ) .unwrap(); let consumer = untimed.consume(None).unwrap(); - assert!(matches!( - consumer.update(since(0)), - Err(MoqError::Protocol { - details: MoqProtocolError { - kind: MoqProtocolKind::TimestampMismatch, - .. - } - }) - )); + consumer.update(since(0)).unwrap(); } // Frame-only controls keep checking payloads independently of the breaks between frames. @@ -9447,7 +9479,7 @@ async fn raw_track_concurrent_reads_take_every_group_once() { group .write_frame(MoqFrame { payload: sequence.to_be_bytes().to_vec(), - timestamp: Some(us(sequence)), + timestamp: us(sequence), }) .unwrap(); group.finish().unwrap(); @@ -9717,7 +9749,7 @@ async fn datagram_demand_is_independent_and_aggregates_priority() { let demand = stream.demand().unwrap(); let producer = broadcast.create_datagrams("events".into(), None).unwrap(); assert_eq!(producer.priority().unwrap(), None); - producer.append(None, b"before".to_vec()).unwrap(); + producer.append(MoqTimestamp::unscaled(0), b"before".to_vec()).unwrap(); let first = broadcast.consume().unwrap().datagrams("events".into()).unwrap(); let second = producer.consume().unwrap(); first.set_priority(-5).unwrap(); @@ -9727,15 +9759,18 @@ async fn datagram_demand_is_independent_and_aggregates_priority() { producer .insert(MoqDatagram { sequence: 7, - timestamp: None, + timestamp: MoqTimestamp::unscaled(7), payload: b"seven".to_vec(), }) .unwrap(); let received = first.recv().await.unwrap().unwrap(); assert_eq!(received.sequence, 7); - assert_eq!(received.timestamp, None); + assert_eq!(received.timestamp, MoqTimestamp::unscaled(7)); assert_eq!(received.payload, b"seven"); - assert_eq!(producer.append(None, b"eight".to_vec()).unwrap(), 8); + assert_eq!( + producer.append(MoqTimestamp::unscaled(8), b"eight".to_vec()).unwrap(), + 8 + ); second.cancel(); assert_eq!(producer.priority_changed(Some(10)).await.unwrap(), Some(-5)); stream.finish().unwrap(); @@ -9778,9 +9813,10 @@ async fn write_ahead_of_a_slow_reader( let track = broadcast.publish_track("tcp".into(), None).unwrap(); let group = track.append_group().unwrap(); let reader = group.consume().unwrap(); - let frame = || MoqFrame { + let key = MoqTimestamp::unscaled(group.sequence()); + let frame = move || MoqFrame { payload: vec![7; MIB], - timestamp: None, + timestamp: key, }; // The reader holds the group and takes the first frame, then stalls. @@ -9884,9 +9920,10 @@ async fn concurrent_ready_calls_all_wake() { let track = broadcast.publish_track("tcp".into(), None).unwrap(); let group = track.append_group().unwrap(); let reader = group.consume().unwrap(); - let frame = || MoqFrame { + let key = MoqTimestamp::unscaled(group.sequence()); + let frame = move || MoqFrame { payload: vec![7; MIB], - timestamp: None, + timestamp: key, }; group.write_frame(frame()).unwrap(); assert!( diff --git a/rs/moq-ffi/src/test/rung.rs b/rs/moq-ffi/src/test/rung.rs index e0f06ac9590..7b7168ecad7 100644 --- a/rs/moq-ffi/src/test/rung.rs +++ b/rs/moq-ffi/src/test/rung.rs @@ -166,7 +166,7 @@ async fn shaped_uplink_disables_and_resumes() { let mut payload = if idr { h264_init() } else { Vec::new() }; payload.extend_from_slice(&[0, 0, 0, 1, if idr { 0x65 } else { 0x41 }, 0x88, 0x84, 0x21]); payload.resize((bitrate_bps / 8 / 30) as usize, 0xff); - track.write_frame(MoqFrame { payload, timestamp: Some(us(timestamp)) }).unwrap(); + track.write_frame(MoqFrame { payload, timestamp: us(timestamp) }).unwrap(); keyframe[i] = false; sent[i] += 1; } diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index aad8cbbe057..31e19910f59 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -1234,7 +1234,7 @@ impl From for MoqVideoRead { }, moq_video::decode::Read::Gap { timestamp, duration } => Self::Gap { timestamp: timestamp.into(), - duration: crate::ffi::duration(duration.into()), + duration: crate::ffi::gap(duration), }, moq_video::decode::Read::End { end } => Self::End { end: end.into() }, } @@ -1518,7 +1518,7 @@ mod finish_tests { // NTSC frame times, which no microsecond count holds exactly. let ntsc = |i: u64| MoqTimestamp { value: i * 1001, - scale: 30_000, + scale: Some(30_000), }; let picture = |i: u64| MoqVideoFrame { timestamp: ntsc(i), @@ -1652,7 +1652,7 @@ mod finish_tests { let picture = |width: usize, height: usize, micros: u64| MoqVideoFrame { timestamp: MoqTimestamp { value: micros, - scale: 1_000_000, + scale: Some(1_000_000), }, data: vec![0x80; width * height * 4], }; @@ -1713,7 +1713,7 @@ mod finish_tests { .write(MoqVideoFrame { timestamp: MoqTimestamp { value: 0, - scale: 1_000_000, + scale: Some(1_000_000), }, data: vec![0x80; 320 * 240 * 3 / 2], }) @@ -1896,7 +1896,7 @@ mod rung_tests { MoqVideoFrame { timestamp: MoqTimestamp { value: i * 33_333, - scale: 1_000_000, + scale: Some(1_000_000), }, data: vec![(i * 40) as u8; 320 * 240 * 3 / 2], } diff --git a/rs/moq-gst/src/sink/imp.rs b/rs/moq-gst/src/sink/imp.rs index ed600b1e40e..6a3e50c1ecf 100644 --- a/rs/moq-gst/src/sink/imp.rs +++ b/rs/moq-gst/src/sink/imp.rs @@ -1497,7 +1497,7 @@ mod tests { panic!("first catalog") }; assert_eq!( - first.media.clock.unwrap().wall.as_micros(), + first.media.clock.unwrap().wall.as_micros().unwrap(), (kio::clock::Wall::UNIX_EPOCH + std::time::Duration::from_secs(1_735_689_600)) .saturating_duration_since(kio::clock::Wall::MOQ_EPOCH) .as_micros(), @@ -1540,7 +1540,7 @@ mod tests { ) }; assert_eq!( - clock.wall.as_micros(), + clock.wall.as_micros().unwrap(), (kio::clock::Wall::UNIX_EPOCH + std::time::Duration::from_secs(zero)) .saturating_duration_since(kio::clock::Wall::MOQ_EPOCH) .as_micros() @@ -1558,7 +1558,7 @@ mod tests { let moq_mux::container::Read::Frame { frame, .. } = read else { panic!("media frame") }; - assert_eq!(frame.timestamp.as_micros(), expected * 1_000_000, "{name}"); + assert_eq!(frame.timestamp.as_micros().unwrap(), expected * 1_000_000, "{name}"); } run.0.set_state(gst::State::Null).unwrap(); } @@ -2470,7 +2470,7 @@ mod tests { }; let consumer = broadcast.consume(); let track = broadcast.create_track("video", None).unwrap(); - track.append_group().unwrap().finish().unwrap(); + track.append_group(0).unwrap().close().unwrap(); let track = track.finish().unwrap(); assert!(session.finish()); while bus.pop().is_some() {} diff --git a/rs/moq-gst/src/sink/pad.rs b/rs/moq-gst/src/sink/pad.rs index a26f8875720..2a8e1b03a2d 100644 --- a/rs/moq-gst/src/sink/pad.rs +++ b/rs/moq-gst/src/sink/pad.rs @@ -1520,7 +1520,8 @@ mod tests { // The reader may skip to the latest group, but never sees the track end before the new caps. loop { let group = subscriber.recv_group().await.unwrap().expect("the track continues"); - if group.sequence == 2 { + let start = moq_net::Timestamp::new(group.start, group.timescale().unwrap()).unwrap(); + if start.as_millis().unwrap() == 80 { break; } } @@ -1550,7 +1551,7 @@ mod tests { pad.observe_caps(&broadcast, &catalog, producer_options(&vp8, Some("camera"))), CapsOutcome::Active("camera/1".to_string()) ); - assert_eq!(subscriber.recv_group().await.unwrap().expect("a group").sequence, 0); + assert_eq!(subscriber.recv_group().await.unwrap().expect("a group").start, 0); assert!(subscriber.recv_group().await.unwrap().is_none(), "the old track ended"); } @@ -2075,7 +2076,7 @@ mod tests { "the payload goes out untouched" ); assert_eq!( - std::time::Duration::from(frame.timestamp.expect("timed")).as_micros(), + std::time::Duration::try_from(frame.timestamp).unwrap().as_micros(), 40_000, "the frame carries the PTS mapped through the segment" ); @@ -2088,7 +2089,7 @@ mod tests { let frame = group.read_frame().await.unwrap().expect("a frame in the second group"); assert_eq!(frame.payload.as_ref(), b"second"); assert_eq!( - std::time::Duration::from(frame.timestamp.expect("timed")).as_micros(), + std::time::Duration::try_from(frame.timestamp).unwrap().as_micros(), 80_000 ); } @@ -2194,7 +2195,7 @@ mod tests { let frame = group.read_frame().await.unwrap().expect("a frame"); assert_eq!(frame.payload.as_ref(), b"no pts", "the unstamped buffer was published"); assert_eq!( - std::time::Duration::from(frame.timestamp.expect("timed")).as_micros(), + std::time::Duration::try_from(frame.timestamp).unwrap().as_micros(), 25_000 ); } diff --git a/rs/moq-gst/src/source/imp.rs b/rs/moq-gst/src/source/imp.rs index 531d3c65fd2..7bc52c4ebca 100644 --- a/rs/moq-gst/src/source/imp.rs +++ b/rs/moq-gst/src/source/imp.rs @@ -2010,7 +2010,8 @@ fn build_buffer(frame: moq_mux::container::Frame, kind: TrackKind) -> Result Recording { fn add_segment(recording: &mut Recording, segment: u64) { let pts = segment * 2_000_000; for video in ["360p", "1080p"] { - recording.add(video, &[(segment, &[pts, pts + 1_000_000])]); + recording.add(video, &[&[pts, pts + 1_000_000]]); } - let audio: Vec<(u64, [u64; 1])> = (0..4).map(|i| (segment * 4 + i, [pts + i * 500_000])).collect(); - let audio: Vec<(u64, &[u64])> = audio - .iter() - .map(|(sequence, frames)| (*sequence, &frames[..])) - .collect(); + let audio: Vec<[u64; 1]> = (0..4).map(|i| [pts + i * 500_000]).collect(); + let audio: Vec<&[u64]> = audio.iter().map(|frames| &frames[..]).collect(); recording.add("audio", &audio); } @@ -383,12 +380,12 @@ async fn renditions_share_the_grid_and_segments_get_their_own_objects() { async fn skipped_grid_slots_are_gaps() { let mut recording = Recording::new().await; // 1080p: two 2s groups, a pause, then content resumes at 10s. - recording.add("1080p", &[(0, &[0, 1_000_000]), (1, &[2_000_000, 3_000_000])]); - recording.add("1080p", &[(2, &[10_000_000, 11_000_000])]); + recording.add("1080p", &[&[0, 1_000_000], &[2_000_000, 3_000_000]]); + recording.add("1080p", &[&[10_000_000, 11_000_000]]); // 360p overruns its first group to 3.5s, so it starts segment 2 there, and pauses too. - recording.add("360p", &[(0, &[0, 1_000_000]), (1, &[3_500_000, 4_500_000])]); - recording.add("360p", &[(2, &[10_000_000, 11_000_000])]); - recording.add("audio", &[(0, &[0, 1_000_000])]); + recording.add("360p", &[&[0, 1_000_000], &[3_500_000, 4_500_000]]); + recording.add("360p", &[&[10_000_000, 11_000_000]]); + recording.add("audio", &[&[0, 1_000_000]]); let counting = recording.counting.clone(); let store = recording.finish().await; diff --git a/rs/moq-hls/src/export/grid.rs b/rs/moq-hls/src/export/grid.rs index e8bff016164..d20855368fc 100644 --- a/rs/moq-hls/src/export/grid.rs +++ b/rs/moq-hls/src/export/grid.rs @@ -41,6 +41,68 @@ pub(crate) fn segment_length(interval: Duration) -> Duration { Duration::from_nanos(u64::try_from(length).unwrap_or(u64::MAX)) } +/// A segment's groups, by start, oldest first. +/// +/// Starts are timestamps, so the groups a segment holds are listed rather than spanned: the keys +/// between two groups name none. A segment holding more groups than any request fetches (see +/// [`Rendition::max_groups`](super::rendition::Rendition::max_groups)) is never served, so past +/// that its groups are only counted: groups whose pts stalls in one segment cost no memory. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub(crate) struct Groups { + starts: Vec, + /// The groups past [`MAX_LISTED`] it holds, counted rather than listed. + unlisted: usize, +} + +/// The most starts a segment lists: one more than any request fetches. +const MAX_LISTED: usize = super::rendition::MAX_SEGMENT_GROUPS as usize + 1; + +impl Groups { + /// The one group starting at `start`. + pub fn one(start: u64) -> Self { + Self { + starts: vec![start], + unlisted: 0, + } + } + + /// The first group's start, or `None` for a gap. + pub fn first(&self) -> Option { + self.starts.first().copied() + } + + /// The last group's start listed, or `None` for a gap. + pub fn last(&self) -> Option { + self.starts.last().copied() + } + + /// How many groups the segment holds, listed or not. + pub fn len(&self) -> usize { + self.starts.len() + self.unlisted + } + + /// The groups starting at or below `last`. + pub fn through(&self, last: u64) -> Self { + Self { + starts: self.starts.iter().copied().take_while(|start| *start <= last).collect(), + unlisted: 0, + } + } + + /// Each group's start, oldest first. + pub fn starts(&self) -> std::vec::IntoIter { + self.starts.clone().into_iter() + } + + /// Add a group starting after every one held, counting it once the list is full. + fn push(&mut self, start: u64) { + match self.starts.len() < MAX_LISTED { + true => self.starts.push(start), + false => self.unlisted += 1, + } + } +} + /// One segment the grid completed, oldest first. #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) enum Slot { @@ -48,8 +110,8 @@ pub(crate) enum Slot { Media { /// The segment's number on the grid. segment: u64, - /// Its groups, in sequence order: from its first group up to the next segment's. - groups: std::ops::Range, + /// Its groups, by start. + groups: Groups, /// Its first group's pts. pts: Timestamp, /// Where its `EXTINF` starts: its first group's pts, or its line when that group starts @@ -80,7 +142,7 @@ pub(crate) enum Slot { /// A group seen but not yet placed: placing it needs its end, the next group's pts. #[derive(Clone, Copy, Debug)] struct Group { - sequence: u64, + start: u64, pts: Timestamp, } @@ -88,7 +150,7 @@ struct Group { #[derive(Clone, Debug)] struct Open { segment: u64, - groups: std::ops::Range, + groups: Groups, pts: Timestamp, /// Where a discontinuity marker after its last group stopped the content, if one did. stop: Option, @@ -98,14 +160,14 @@ impl Open { fn new(segment: u64, group: Group) -> Self { Self { segment, - groups: group.sequence..group.sequence + 1, + groups: Groups::one(group.start), pts: group.pts, stop: None, } } } -/// Places one rendition's groups, fed in sequence order, into segments on a grid of one length. +/// Places one rendition's groups, fed in start order, into segments on a grid of one length. pub(crate) struct Grid { /// The segment length in nanoseconds, at least one. length: u128, @@ -143,13 +205,13 @@ impl Grid { /// The segment a group starting at `pts` and ending at `end` belongs to, or `None` while it /// needs an `end` it doesn't have. pub fn segment(&self, pts: Timestamp, end: Option) -> Option { - let pts = pts.as_nanos(); + let pts = super::nanos(pts); let slot = pts / self.length; if pts.is_multiple_of(self.length) { return Some(slot as u64); } let line = (slot + 1) * self.length; - let end = end?.as_nanos(); + let end = super::nanos(end?); Some(if end > line { slot + 1 } else { slot } as u64) } @@ -158,7 +220,7 @@ impl Grid { /// group starts after its line may follow earlier groups of the same segment. pub fn starts(&self, slot: &Slot) -> bool { match slot { - Slot::Media { segment, pts, .. } => (*segment as u128).saturating_mul(self.length) >= pts.as_nanos(), + Slot::Media { segment, pts, .. } => (*segment as u128).saturating_mul(self.length) >= super::nanos(*pts), Slot::Gap { .. } => true, Slot::Reset => false, } @@ -170,15 +232,15 @@ impl Grid { } /// The groups of the open segment so far. - pub fn open_groups(&self) -> Option> { + pub fn open_groups(&self) -> Option { self.open.as_ref().map(|open| open.groups.clone()) } /// The oldest group no completed segment holds: the open segment's first, else the group /// waiting for its end. pub fn unplaced(&self) -> Option { - let open = self.open.as_ref().map(|open| open.groups.start); - open.or(self.pending.map(|group| group.sequence)) + let open = self.open.as_ref().and_then(|open| open.groups.first()); + open.or(self.pending.map(|group| group.start)) } /// Whether a segment waits to be completed: one is open, or a group waits for its end. @@ -196,7 +258,11 @@ impl Grid { /// line can exceed it. fn start(&self, segment: u64, pts: Timestamp) -> Timestamp { let line = self.line(segment); - if pts.as_nanos() > line.as_nanos() { line } else { pts } + if super::nanos(pts) > super::nanos(line) { + line + } else { + pts + } } /// The pts of line `segment`. @@ -205,9 +271,9 @@ impl Grid { Timestamp::from_nanos(u64::try_from(nanos).unwrap_or(u64::MAX)).unwrap_or(Timestamp::ZERO) } - /// A group `sequence` starts at `pts`, after every group already fed. Returns the segments it - /// completes. - pub fn group(&mut self, sequence: u64, pts: Timestamp) -> Vec { + /// The group keyed `start` has its first frame at `pts`, after every group already fed. Returns + /// the segments it completes. + pub fn group(&mut self, start: u64, pts: Timestamp) -> Vec { let mut out = Vec::new(); // A group starting before the one before it, even within one slot, can't share a segment // or a numbering with what came before, since its media would run backwards. @@ -220,7 +286,7 @@ impl Grid { let segment = self.segment(pending.pts, Some(pts)).expect("bounded"); self.place(pending, segment, &mut out); } - let group = Group { sequence, pts }; + let group = Group { start, pts }; // A group starting on a line needs no end to place, so the segment before it completes now. match self.segment(pts, None) { Some(segment) => self.place(group, segment, &mut out), @@ -258,7 +324,7 @@ impl Grid { let end = [open.stop, end, reach] .into_iter() .flatten() - .find(|end| end.as_nanos() > open.pts.as_nanos()) + .find(|end| super::nanos(*end) > super::nanos(open.pts)) .unwrap_or(open.pts); out.push(Slot::Media { segment: open.segment, @@ -280,7 +346,7 @@ impl Grid { return; }; if segment == open.segment { - open.groups.end = group.sequence + 1; + open.groups.push(group.start); // The content resumed within the segment, so it runs to whatever ends this group. open.stop = None; return; @@ -319,6 +385,19 @@ impl Grid { mod test { use super::*; + /// A segment that piles up more groups than any request fetches counts the rest, so its + /// memory stays bounded however long its groups' pts stalls, and it still reads as too long. + #[test] + fn an_overfull_segment_counts_its_groups_past_the_list() { + let mut groups = Groups::one(0); + for start in 1..(MAX_LISTED as u64 + 10) { + groups.push(start); + } + assert_eq!(groups.len(), MAX_LISTED + 10); + assert_eq!(groups.starts.len(), MAX_LISTED); + assert!(groups.len() as u64 > super::super::rendition::MAX_SEGMENT_GROUPS); + } + fn ms(ms: u64) -> Timestamp { Timestamp::from_millis(ms).unwrap() } @@ -336,8 +415,18 @@ mod test { start, end, .. - } => (segment, groups, start.as_millis() as u64, end.as_millis() as u64), - Slot::Gap { segment, pts, end } => (segment, 0..0, pts.as_millis() as u64, end.as_millis() as u64), + } => ( + segment, + groups.first().unwrap()..groups.last().unwrap() + 1, + start.as_millis().unwrap() as u64, + end.as_millis().unwrap() as u64, + ), + Slot::Gap { segment, pts, end } => ( + segment, + 0..0, + pts.as_millis().unwrap() as u64, + end.as_millis().unwrap() as u64, + ), Slot::Reset => panic!("unexpected reset"), }) .collect() @@ -602,7 +691,7 @@ mod test { slots .into_iter() .filter_map(|slot| match slot { - Slot::Media { content, .. } => Some(content.map(|at| at.as_millis() as u64)), + Slot::Media { content, .. } => Some(content.map(|at| at.as_millis().unwrap() as u64)), _ => None, }) .collect() diff --git a/rs/moq-hls/src/export/include.rs b/rs/moq-hls/src/export/include.rs index d961725af61..0d14d6bb674 100644 --- a/rs/moq-hls/src/export/include.rs +++ b/rs/moq-hls/src/export/include.rs @@ -342,7 +342,7 @@ mod tests { .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) .unwrap(); let mut json = moq_json::snapshot::Producer::new(track, Default::default()); - json.update(moq_net::Timed::from(&catalog).at(moq_net::Timestamp::ZERO)) + json.update(moq_json::Timed::new(&catalog, moq_net::Timestamp::ZERO)) .unwrap(); catalog.video.renditions.clear(); catalog.include.insert( diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index ea67cc04e67..94637ca882d 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -53,6 +53,17 @@ use std::time::Duration; use moq_mux::catalog::{self, Stream}; use rand::RngExt; +/// `pts` in nanoseconds. Every timestamp the export places is scaled: a group whose first frame +/// is [unscaled](moq_net::Timestamp::unscaled) is skipped before it reaches the grid. +pub(crate) fn nanos(pts: moq_net::Timestamp) -> u128 { + pts.as_nanos().expect("the export places only scaled timestamps") +} + +/// `pts` as the time since the track's zero; see [`nanos`]. +pub(crate) fn span(pts: moq_net::Timestamp) -> Duration { + Duration::try_from(pts).expect("the export places only scaled timestamps") +} + pub use driver::Driver; pub(crate) use playlist::render_media; #[cfg(feature = "server")] @@ -1479,7 +1490,7 @@ mod tests { for group in 2..5 { let mut writing = media.create_group(group.into()).unwrap(); write_keyframe(&mut writing, group); - writing.finish().unwrap(); + writing.close().unwrap(); } until_listed(&rendition, 4).await; let segment = rendition @@ -1555,7 +1566,7 @@ mod tests { write_keyframe(&mut open, sequence); let mut next = media.create_group((sequence + 1).into()).unwrap(); write_keyframe(&mut next, sequence + 1); - next.finish().unwrap(); + next.close().unwrap(); (media, open) } @@ -1604,7 +1615,7 @@ mod tests { for sequence in 0..3 { let mut group = media.create_group(sequence.into()).unwrap(); write_keyframe(&mut group, sequence); - group.finish().unwrap(); + group.close().unwrap(); } let mut open = media.create_group(3u64.into()).unwrap(); write_keyframe(&mut open, 3); @@ -1639,7 +1650,7 @@ mod tests { let manifest = broadcaster.manifest(None).unwrap().expect("video0 renders"); assert!(!manifest.contains("id=\"video/video1\""), "{manifest}"); - open.finish().unwrap(); + open.close().unwrap(); let manifest = listed_in_manifest(&broadcaster).await; assert!(manifest.contains("id=\"video/video0\""), "{manifest}"); } @@ -1679,7 +1690,7 @@ mod tests { failed(&broken).await; assert!(broadcaster.manifest(None).unwrap().is_none()); - open.finish().unwrap(); + open.close().unwrap(); let manifest = kio::clock::timeout(Duration::from_secs(5), async { loop { if let Some(manifest) = broadcaster.manifest(None).unwrap() { @@ -1787,7 +1798,7 @@ mod tests { assert!(!manifest.contains("id=\"video/video1\""), "{manifest}"); assert!(!manifest.contains("type=\"static\""), "{manifest}"); - open.finish().unwrap(); + open.close().unwrap(); let manifest = listed_in_manifest(&broadcaster).await; assert!(manifest.contains("type=\"static\""), "{manifest}"); } @@ -1815,7 +1826,7 @@ mod tests { .is_err(), "resolved without a listable video" ); - open.finish().unwrap(); + open.close().unwrap(); kio::clock::timeout(Duration::from_secs(1), waiting) .await .expect("video0 is listable once its init is built"); @@ -1843,7 +1854,7 @@ mod tests { for sequence in 0..3 { let mut group = media.create_group(sequence.into()).unwrap(); write_keyframe(&mut group, sequence); - group.finish().unwrap(); + group.close().unwrap(); } kio::clock::timeout(Duration::from_secs(1), waiting) .await @@ -1861,7 +1872,7 @@ mod tests { for sequence in 0..4 { let mut group = media.create_group(sequence.into()).unwrap(); write_keyframe(&mut group, sequence); - group.finish().unwrap(); + group.close().unwrap(); } let broadcaster = Broadcaster::spawned(crate::resolved(&origin, "live").await, Config::default()); @@ -2451,7 +2462,7 @@ mod tests { json, ); catalog - .update(moq_net::Timed::from(&snapshot).at(moq_net::Timestamp::now())) + .update(moq_json::Timed::new(&snapshot, moq_net::Timestamp::ZERO)) .unwrap(); let mut media = Gops { track: broadcast.broadcast().create_track("video0", timed()).unwrap(), @@ -3600,7 +3611,10 @@ mod tests { /// Write group `sequence` with a keyframe at `pts_ms` and, when `frames` is more than one, /// that many frames a tenth of the interval apart. fn group(&mut self, sequence: u64, pts_ms: u64, frames: u64) { - let mut group = self.track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = self + .track + .create_group(moq_net::group::Info { start: sequence }) + .unwrap(); for index in 0..frames.max(1) { let micros = (pts_ms + index * INTERVAL.as_millis() as u64 / 10) * 1_000; let frame = vp8_frame(micros, index == 0); @@ -3611,7 +3625,7 @@ mod tests { .write_to(&mut group) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } /// Write group `sequence` at `pts_ms` with a single keyframe. @@ -3622,7 +3636,10 @@ mod tests { /// Write group `sequence` at `pts_ms` with a single keyframe declaring a `width`x`height` /// picture in band. fn sized(&mut self, sequence: u64, pts_ms: u64, (width, height): (u16, u16)) { - let mut group = self.track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = self + .track + .create_group(moq_net::group::Info { start: sequence }) + .unwrap(); let mut payload = KEYFRAME.to_vec(); payload[6..8].copy_from_slice(&width.to_le_bytes()); payload[8..10].copy_from_slice(&height.to_le_bytes()); @@ -3632,7 +3649,7 @@ mod tests { } .write_to(&mut group) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } /// Write group `sequence` at `2 * sequence` seconds: one GOP per grid line. @@ -3642,7 +3659,10 @@ mod tests { /// Start group `sequence` at `pts_ms` with a keyframe, leaving it open for more frames. fn open(&mut self, sequence: u64, pts_ms: u64) -> moq_net::group::Producer { - let mut group = self.track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = self + .track + .create_group(moq_net::group::Info { start: sequence }) + .unwrap(); let frame = vp8_frame(pts_ms * 1_000, true); hang::container::Frame { timestamp: frame.timestamp, @@ -3984,9 +4004,9 @@ mod tests { let test = Groups::new(); let mut video0 = test.publish("video0"); let mut video1 = test.publish("video1"); - // Group 2 never lands, so segment 1, from groups 1 to 3, can't be served whole. video0.at(0, 0); video0.at(1, 2_000); + let lost = video0.open(2, 2_500); video0.at(3, 3_000); video0.at(4, 4_000); video0.finish(); @@ -3995,6 +4015,8 @@ mod tests { video1.at(1, 7_000); video1.finish(); test.finished("video0").await; + // Group 2 fails once listed, so segment 1, from groups 1 to 3, can't be served whole. + lost.abort(moq_net::Error::Cancel).unwrap(); let rendition = test.rendition("video0"); let mut cursor = rendition.segments(); @@ -4437,7 +4459,7 @@ mod tests { assert_eq!(numbers(&snapshot), [0, 1, 2], "no segment is lost to the hand-over"); let tags: Vec<_> = snapshot.segments.iter().map(|s| s.tag.as_str()).collect(); // The predecessor's part is a cut, named by where it ends. - let cut = segments::cut(old.tag(), 6, Duration::from_secs(3)); + let cut = segments::cut(old.tag(), Some(5), Duration::from_secs(3)); assert_eq!(tags, [old.tag(), &cut, new.tag()]); let durations: Vec<_> = snapshot.segments.iter().map(|s| s.duration.as_millis()).collect(); assert_eq!( @@ -4512,7 +4534,7 @@ mod tests { let snapshot = test.listed("video0", 2).await; assert_eq!(numbers(&snapshot), [0, 1, 2]); let tags: Vec<_> = snapshot.segments.iter().map(|s| s.tag.as_str()).collect(); - let cut = segments::cut(old.tag(), 6, Duration::from_secs(3)); + let cut = segments::cut(old.tag(), Some(5), Duration::from_secs(3)); assert_eq!(tags, [old.tag(), &cut, new.tag()]); let durations: Vec<_> = snapshot.segments.iter().map(|s| s.duration.as_millis()).collect(); assert_eq!( @@ -4527,7 +4549,13 @@ mod tests { #[tokio::test(start_paused = true)] async fn a_segment_too_many_groups_long_is_a_gap() { let test = Groups::with(&["video0"], &[]); - let mut media = test.publish("video0"); + // Groups a microsecond apart: millisecond ones can't outnumber the cap in one segment. + let mut media = Gops { + track: test + .broadcast + .create_track("video0", timed().with_timescale(moq_net::Timescale::MICRO)) + .unwrap(), + }; let cap = test.rendition("video0").max_groups(); assert_eq!( cap, @@ -4535,7 +4563,15 @@ mod tests { "one group per millisecond of the interval" ); for sequence in 0..=cap { - media.at(sequence, 0); + let mut group = media.track.create_group(sequence.into()).unwrap(); + let frame = vp8_frame(sequence, true); + hang::container::Frame { + timestamp: frame.timestamp, + payload: frame.payload, + } + .write_to(&mut group) + .unwrap(); + group.close().unwrap(); } media.at(cap + 1, 2_000); media.at(cap + 2, 4_000); diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index 10b5400047b..e0946ae1b64 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -33,7 +33,7 @@ const MIN_SEGMENT_GROUPS: u64 = 1024; /// The most groups one segment request ever fetches, whatever interval the catalog declares: see /// [`Rendition::max_groups`]. 20 ms audio groups fill it in about 22 minutes. -const MAX_SEGMENT_GROUPS: u64 = 1 << 16; +pub(crate) const MAX_SEGMENT_GROUPS: u64 = 1 << 16; /// DASH timelines are in milliseconds. const TIMESCALE: u32 = 1000; @@ -901,7 +901,12 @@ impl Rendition { let window = self.fanout.window(); while trimmed < inherited { let rest = match rows.get(1) { - Some((_, second)) => rows.back().expect("two rows").1.end.saturating_sub(second.pts.into()), + Some((_, second)) => rows + .back() + .expect("two rows") + .1 + .end + .saturating_sub(super::span(second.pts)), None => Duration::ZERO, }; if !segments::evicts(window, rows.len(), rest) { @@ -1338,8 +1343,8 @@ impl Rendition { .iter() .filter(|row| !row.stop) .map(|row| { - let t = row.pts.as_millis() as u64; - let d = row.end.saturating_sub(row.pts.into()).as_millis() as u64; + let t = super::span(row.pts).as_millis() as u64; + let d = row.end.saturating_sub(super::span(row.pts)).as_millis() as u64; (t, d) }) // A zero-duration segment cannot be requested by DASH timing. @@ -1601,8 +1606,8 @@ enum Fetching { /// every group (its real successor lives in the next group), audible on audio segments /// that span many groups. frames: Vec, - /// The group being read, and the ones after it. - groups: std::ops::Range, + /// The group being read, and the ones after it, by start. + groups: std::vec::IntoIter, group: Option, }, Done, @@ -1700,7 +1705,7 @@ impl Fetch { // publisher grouping far finer than its interval), 404 rather than silently // truncating it to a segment shorter than its advertised duration. The live window // lists such a segment as a gap instead, so only a row from elsewhere reaches this. - if row.groups.end.saturating_sub(row.groups.start) > rendition.max_groups() { + if row.groups.len() as u64 > rendition.max_groups() { tracing::warn!( groups = ?row.groups, "segment spans more groups than one request fetches; refusing to truncate" @@ -1721,7 +1726,7 @@ impl Fetch { track, muxer: rendition.muxer()?, frames: Vec::new(), - groups: row.groups.clone(), + groups: row.groups.starts(), group: None, }; } @@ -2130,7 +2135,7 @@ impl Follow { // A group's place is its first frame's pts: the transport's stamp, or the container's own // where the session carries none. let Some(first) = first.and_then(|frame| first_frame(&self.container, frame, self.probe.as_mut())) else { - tracing::warn!(track = %self.track, group = group.sequence, "skipping a group without a timed first frame"); + tracing::warn!(track = %self.track, group = group.start, "skipping a group without a timed first frame"); return None; }; let (slots, read, size) = match first { @@ -2138,12 +2143,12 @@ impl Follow { // grid numbers what resumes by its own timestamps, as every edge does. First::Marker(at) => { self.latest = None; - (self.grid.stop(at), (group.sequence, None), None) + (self.grid.stop(at), (group.start, None), None) } First::Media(pts, reach, declared) => { - let sequence = group.sequence; + let sequence = group.start; if let Some((_, previous, _)) = &self.latest { - self.spacing = pts.checked_sub(*previous).ok().map(Duration::from).or(self.spacing); + self.spacing = pts.checked_sub(*previous).ok().map(super::span).or(self.spacing); } self.latest = Some((group, pts, reach)); (self.grid.group(sequence, pts), (sequence, Some(pts)), declared) @@ -2180,10 +2185,10 @@ impl Follow { // the segments and gaps before the first holding one, and its own part of that one, then // stops there. let boundary = claim.boundary; - let theirs = |groups: &std::ops::Range| boundary.is_none_or(|boundary| groups.end > boundary + 1); + let theirs = |last: Option| boundary.is_none_or(|boundary| last.is_some_and(|last| last > boundary)); if claim.retired && let Some((sequence, Some(pts))) = read - && theirs(&(sequence..sequence + 1)) + && theirs(Some(sequence)) { claim.theirs_at = claim.theirs_at.or(Some(pts)); } @@ -2206,13 +2211,13 @@ impl Follow { pts, start, .. - } if claim.retired && theirs(&groups) => match (boundary, theirs_at) { + } if claim.retired && theirs(groups.last()) => match (boundary, theirs_at) { // The replacement can't list this segment's number again, so this instance // lists the groups it had read, ending where the replacement's media starts. - (Some(boundary), Some(at)) if groups.start <= boundary => { + (Some(boundary), Some(at)) if groups.first().is_some_and(|first| first <= boundary) => { let slot = grid::Slot::Media { segment, - groups: groups.start..boundary + 1, + groups: groups.through(boundary), pts, start, end: at, @@ -2227,7 +2232,9 @@ impl Follow { if !*whole { // The first segment listed holds only this instance's groups, and starts whole. let own = match &slot { - grid::Slot::Media { groups, .. } => after.is_none_or(|after| groups.start > after), + grid::Slot::Media { groups, .. } => { + after.is_none_or(|after| groups.first().is_some_and(|first| first > after)) + } _ => false, }; if !own || !grid.starts(&slot) { @@ -2242,7 +2249,7 @@ impl Follow { let mut row = row(&rendition, slot, *broken); if last { // Where it stopped reading is this edge's own, so the cut gets a URL of its own. - row.tag = segments::cut(&row.tag, row.groups.end, row.end).into(); + row.tag = segments::cut(&row.tag, row.groups.last(), row.end).into(); } rendition.push(row); *broken = false; @@ -2250,11 +2257,7 @@ impl Follow { return Some(Ok(())); } } - if claim.retired - && grid - .open_groups() - .is_some_and(|groups| theirs(&(groups.start..groups.start + 1))) - { + if claim.retired && grid.open_groups().is_some_and(|groups| theirs(groups.first())) { return Some(Ok(())); } if let Some((sequence, _)) = read { @@ -2471,14 +2474,17 @@ impl Reach { /// Count every sample `frames` decodes to. fn add(&mut self, frames: Vec) { for frame in frames { + // A sample with no time can't reach anywhere on the grid. + let Ok(pts) = frame.timestamp.as_nanos() else { + continue; + }; if frame.payload.is_empty() { self.marker = Some(frame.timestamp); continue; } - let pts = frame.timestamp.as_nanos(); self.stamps.push(pts); - if let Some(duration) = frame.duration { - self.ends = self.ends.max(Some(pts + duration.as_nanos())); + if let Some(Ok(duration)) = frame.duration.map(moq_net::Timestamp::as_nanos) { + self.ends = self.ends.max(Some(pts + duration)); } } } @@ -2537,7 +2543,14 @@ fn first_frame( let stamp = frame.timestamp; let decoded = container.decode(frame).ok()?; let first = decoded.first()?; - let pts = stamp.unwrap_or(first.timestamp); + // A scale-less session carries the group key, not a time, so the container's own is used. + let pts = if stamp.scale().is_some() { + stamp + } else { + first.timestamp + }; + // The grid places only times. + pts.scale()?; if first.payload.is_empty() { return Some(First::Marker(pts)); } @@ -2557,14 +2570,14 @@ fn row(rendition: &Rendition, slot: grid::Slot, broken: bool) -> segments::Row { start, end, .. - } if groups.end.saturating_sub(groups.start) > rendition.max_groups() => { + } if groups.len() as u64 > rendition.max_groups() => { tracing::warn!( rendition = rendition.name, segment, ?groups, "a segment holds more groups than one request fetches; listing it as a gap" ); - (segment, 0..0, start, end, None, true) + (segment, Default::default(), start, end, None, true) } // Timed from where its EXTINF starts, which a group resuming past its line puts at the line. grid::Slot::Media { @@ -2575,11 +2588,11 @@ fn row(rendition: &Rendition, slot: grid::Slot, broken: bool) -> segments::Row { content, .. } => (segment, groups, start, end, content, false), - grid::Slot::Gap { segment, pts, end } => (segment, 0..0, pts, end, None, true), + grid::Slot::Gap { segment, pts, end } => (segment, Default::default(), pts, end, None, true), grid::Slot::Reset => unreachable!("a reset lists no row"), }; - let start = Duration::from(pts); - let end = Duration::from(end).max(start); + let start = super::span(pts); + let end = super::span(end).max(start); let duration = end - start; // RFC 8216 4.3.3.1 requires every EXTINF, rounded to the nearest second, to fit the target // duration. A group overrunning the interval is listed anyway: refusing it loses content, and @@ -2600,7 +2613,7 @@ fn row(rendition: &Rendition, slot: grid::Slot, broken: bool) -> segments::Row { stop, // Listed once a later segment started or the track ended, so its groups are all complete. complete: !stop, - content: content.map(Duration::from), + content: content.map(super::span), } } @@ -2887,7 +2900,7 @@ mod tests { index: 0, segment: 0, tag: rendition.tag.clone(), - groups: 0..1, + groups: grid::Groups::one(0), duration: Duration::from_secs(1), pts: moq_net::Timestamp::ZERO, end: Duration::from_secs(1), @@ -2943,8 +2956,12 @@ mod tests { let Poll::Ready(Ok(request)) = self.poll() else { return; }; - let mut track = request.accept(moq_net::track::Info::default()); - track.write_frame(None, Bytes::from_static(b"retained")).unwrap(); + let track = request.accept(moq_net::track::Info::default()); + let mut group = track.append_group(0).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(0), Bytes::from_static(b"retained")) + .unwrap(); + group.close().unwrap(); self.broadcast.close(); *self.track.lock().unwrap() = Some(track); } @@ -3076,7 +3093,10 @@ mod tests { } fn millis(reach: (Option, Option)) -> (Option, Option) { - (reach.0.map(|t| t.as_millis()), reach.1.map(|t| t.as_millis())) + ( + reach.0.map(|t| t.as_millis().unwrap()), + reach.1.map(|t| t.as_millis().unwrap()), + ) } // A fragment carrying all its samples in one frame reaches as far as its samples say, however diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 03fcdec9c8b..acab431e575 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -128,12 +128,14 @@ pub(crate) fn tag(track: &str, config: &[u8]) -> String { hash.as_ref()[..4].iter().map(|b| format!("{b:02x}")).collect() } -/// The tag of a segment under `tag` cut short before group `groups_end`, ending at `end`. +/// The tag of a segment under `tag` cut short after its group starting at `last`, ending at +/// `end`; `None` for a cut that holds no group. /// /// A retired rendition cuts its last segment where its edge stopped reading, which another edge /// may place elsewhere, so a cut is named by where it ends rather than by its number alone. -pub(crate) fn cut(tag: &str, groups_end: u64, end: Duration) -> String { - self::tag(&format!("{tag}/{groups_end}/{}", end.as_nanos()), b"") +pub(crate) fn cut(tag: &str, last: Option, end: Duration) -> String { + let last = last.map(|last| last.to_string()).unwrap_or_default(); + self::tag(&format!("{tag}/{last}/{}", end.as_nanos()), b"") } /// One playlist segment: its number, timing, and the groups it holds. @@ -146,8 +148,8 @@ pub(crate) struct Row { pub segment: u64, /// The [tag](tag) of the track numbering the segment. pub tag: Arc, - /// The groups the segment holds: from its first up to the next segment's. - pub groups: std::ops::Range, + /// The groups the segment holds, by start. + pub groups: super::grid::Groups, /// Presentation duration. pub duration: Duration, /// The segment's starting presentation timestamp. @@ -269,7 +271,7 @@ impl Producer { if let Some(back) = state.rows.back() { // A backward jump in pts or segment number means the publisher restarted its timeline; // the old window can't be stitched onto the new one, so start over. - if Duration::from(row.pts) < Duration::from(back.pts) || row.segment <= back.segment { + if super::span(row.pts) < super::span(back.pts) || row.segment <= back.segment { tracing::warn!("HLS segments jumped backwards; resetting the playlist window"); state.rows.clear(); state.resets += 1; @@ -282,7 +284,7 @@ impl Producer { loop { let rest = match state.rows.get(1) { - Some(second) => state.rows.back().unwrap().end.saturating_sub(second.pts.into()), + Some(second) => state.rows.back().unwrap().end.saturating_sub(super::span(second.pts)), None => Duration::ZERO, }; if !evicts(window, state.rows.len(), rest) { @@ -394,7 +396,7 @@ impl Producer { state .rows .iter() - .find(|row| !row.stop && row.pts.as_millis() == time as u128 && &*row.tag == tag) + .find(|row| !row.stop && row.pts.as_millis() == Ok(time as u128) && &*row.tag == tag) .map(|row| row.segment) } @@ -404,16 +406,16 @@ impl Producer { /// newest when its group may still be open. pub fn init_groups(&self) -> Vec { let state = self.state.read(); - let complete = |row: &Row| row.complete || row.groups.end > row.groups.start + 1; + let complete = |row: &Row| row.complete || row.groups.len() > 1; let mut media = state.rows.iter().rev().filter(|row| !row.stop); - let Some(newest) = media.next() else { + let Some(newest) = media.next().and_then(|row| Some((row, row.groups.first()?))) else { return Vec::new(); }; - if complete(newest) { - return vec![newest.groups.start]; + if complete(newest.0) { + return vec![newest.1]; } - let older = media.find(|row| complete(row)).map(|row| row.groups.start); - older.into_iter().chain([newest.groups.start]).collect() + let older = media.find(|row| complete(row)).and_then(|row| row.groups.first()); + older.into_iter().chain([newest.1]).collect() } /// Poll until the window lists a row, ends, fails, or closes. @@ -717,10 +719,10 @@ mod tests { index: segment, segment, tag: Arc::from(tag("video.hls", b"")), - groups: group..group + 1, + groups: super::super::grid::Groups::one(group), duration: Duration::from_millis(duration_ms), pts, - end: Duration::from(pts) + Duration::from_millis(duration_ms), + end: Duration::try_from(pts).unwrap() + Duration::from_millis(duration_ms), discontinuity: 0, stop: false, complete: true, @@ -736,7 +738,7 @@ mod tests { let window = Duration::from_secs(30); live.push(row(0, 0, 0, 4_000), window); let mut stop = row(1, 1, 4_000, 0); - stop.groups = 1..1; + stop.groups = Default::default(); stop.stop = true; live.push(stop, window); live.push(row(2, 1, 4_000, 4_000), window); @@ -862,7 +864,7 @@ mod tests { ); let tag = tag("video.hls", b""); - assert_eq!(live.row(&tag, 0).unwrap().groups, 0..1); + assert_eq!(live.row(&tag, 0).unwrap().groups, super::super::grid::Groups::one(0)); assert_eq!(live.row(&tag, 7), None, "unknown segments miss"); assert_eq!(live.row("00000000", 0), None, "another record track's numbering misses"); assert_eq!(live.init_groups(), [0], "a stop holds no media"); diff --git a/rs/moq-hls/src/import.rs b/rs/moq-hls/src/import.rs index 4008e4dc68f..79ddd36b5aa 100644 --- a/rs/moq-hls/src/import.rs +++ b/rs/moq-hls/src/import.rs @@ -574,7 +574,7 @@ impl TrackState { Ok(playlist.segments.len().saturating_sub(ANCHOR_SEGMENTS)) } - /// Forget where we were, so the next pushed segment re-bases the MoQ group sequence. + /// Forget where we were, so the next pushed segment rejoins the importer. /// The byte-range cursor needs no reset: `consume_segments` rebuilds it from the /// playlist prefix on every step. fn reanchor(&mut self) { @@ -706,17 +706,12 @@ impl TrackState { let resource = range.resolve(url, segment.byte_range.as_ref())?; let mut body = fetcher.open(&resource).await?; - // HLS media sequence names the live window, while discontinuity sequence names a - // new media timeline. Whenever we join, skip ahead, or cross a discontinuity, anchor - // the MoQ group sequence to both so consumers do not wait on groups HLS has moved - // past. Contiguous segments let the importer auto-increment instead; we still pack - // (and so validate) the sequence on that path so media sequence can't silently - // auto-increment into the discontinuity bits. - let group_sequence = moq_sequence(discontinuity_sequence, sequence)?; - // `consume_segments` resolves the effective map before every segment, so a missing // importer means the playlist never carried an `EXT-X-MAP`. let generation = self.generation.as_mut().ok_or(Error::MissingMap)?; + // Whenever we join, skip ahead, or cross a discontinuity, close the open groups without an + // end, so a reader sees the media HLS moved past as a gap. The groups after it are keyed + // by their own keyframes' times, as contiguous segments' are. let reanchored = self.next_sequence != Some(sequence) || generation.discontinuity != Some(discontinuity_sequence); let importer = &mut generation.importer; @@ -728,7 +723,7 @@ impl TrackState { importer.discontinuity()?; } if reanchored { - importer.seek(group_sequence)?; + importer.rejoin()?; } // Every playlist entry is a source segment boundary, so declare it even when the media // carries no styp. Every rendition declares the same ones; the HLS segmenter drops a cut @@ -1288,33 +1283,6 @@ fn bump_discontinuity(sequence: u64) -> Result { }) } -/// Pack HLS discontinuity + media sequence into a single MoQ group sequence. -/// -/// HLS media sequence alone can rewind after an upstream reset, and discontinuity -/// sequence alone cannot order segments inside the same epoch. The lower 48 bits hold -/// the media sequence (ample for realistic playlists) while the upper 16 bits hold the -/// discontinuity sequence, so a new epoch always sorts after every segment of the last. -fn moq_sequence(discontinuity_sequence: u64, media_sequence: u64) -> Result { - const MEDIA_BITS: u32 = 48; - const MEDIA_MASK: u64 = (1u64 << MEDIA_BITS) - 1; - const DISCONTINUITY_MASK: u64 = u64::MAX >> MEDIA_BITS; - - if media_sequence > MEDIA_MASK { - return Err(Error::SequenceOverflow { - kind: SequenceKind::Media, - value: media_sequence, - }); - } - if discontinuity_sequence > DISCONTINUITY_MASK { - return Err(Error::SequenceOverflow { - kind: SequenceKind::Discontinuity, - value: discontinuity_sequence, - }); - } - - Ok((discontinuity_sequence << MEDIA_BITS) | media_sequence) -} - #[cfg(test)] mod tests { use super::*; @@ -2431,8 +2399,9 @@ mod tests { ); } - /// The first frame of every buffered group, without waiting for another source event. - fn group_stamps(subscriber: &mut moq_net::track::Ordered) -> (Vec<(u64, u128)>, Option) { + /// The first frame of every buffered group in microseconds, without waiting for another source + /// event. Each group must start at its first frame, in the track's timescale. + fn group_stamps(subscriber: &mut moq_net::track::Ordered) -> (Vec, Option) { let mut out = Vec::new(); loop { let mut group = match subscriber.poll_next_group(&kio::Waiter::noop()) { @@ -2445,7 +2414,12 @@ mod tests { Poll::Ready(Err(error)) => return (out, Some(error)), other => panic!("reading a published group's first frame: {other:?}"), }; - out.push((group.sequence, frame.timestamp.unwrap().as_micros())); + assert_eq!( + group.start, + frame.timestamp.value(), + "a group starts at its first frame" + ); + out.push(frame.timestamp.as_micros().unwrap()); } (out, None) } @@ -2454,9 +2428,9 @@ mod tests { result: Result, before: String, after: String, - old_groups: Vec<(u64, u128)>, + old_groups: Vec, terminal: Option, - new_groups: Vec<(u64, u128)>, + new_groups: Vec, } /// A one-track fMP4 muxer for synthesized VP8 video or Opus audio. @@ -2493,11 +2467,19 @@ mod tests { /// Exercise the playlist/map path with real fMP4 produced by the exporter's muxer. /// - /// The second segment, stamped `after_us`, follows a 5 s one behind the playlist `tags`. + /// The second segment, stamped `after_us`, follows a 5 s one behind the playlist `tags`, and a + /// third follows it 2 s later. async fn discontinuity(video: bool, cmaf: bool, tags: &str, after_us: u64) -> Discontinuity { + let after = format!("#EXTINF:1,\n0.m4s\n{tags}#EXTINF:1,\n1.m4s\n#EXTINF:1,\n2.m4s\n"); + reload(video, cmaf, &after, after_us).await + } + + /// Import a 5 s segment `0.m4s`, then reload the playlist as `after`'s segments, where `1.m4s` + /// is stamped `after_us` and `2.m4s` 2 s later. + async fn reload(video: bool, cmaf: bool, after: &str, after_us: u64) -> Discontinuity { let dir = temp_dir(); let muxer = synth(video); - write_synth(&dir, &muxer, "", &[5_000_000, after_us]); + write_synth(&dir, &muxer, "", &[5_000_000, after_us, after_us + 2_000_000]); std::fs::copy(dir.join("init.mp4"), dir.join("other.mp4")).unwrap(); let mut broadcast = moq_net::broadcast::Info::new().produce(); @@ -2538,9 +2520,7 @@ mod tests { .await .unwrap() .ordered(); - let after = format!( - "#EXTM3U\n#EXT-X-TARGETDURATION:2\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:1,\n0.m4s\n{tags}#EXTINF:1,\n1.m4s\n" - ); + let after = format!("#EXTM3U\n#EXT-X-TARGETDURATION:2\n#EXT-X-MAP:URI=\"init.mp4\"\n{after}"); let after = m3u8_rs::parse_media_playlist_res(after.as_bytes()).unwrap(); let result = track.consume_segments(&local, &after, &base).await; let after_name = name(&catalog.snapshot()); @@ -2578,10 +2558,23 @@ mod tests { case.result.unwrap(); assert!(case.terminal.is_none(), "{terminal:?}", terminal = case.terminal); assert_eq!(case.before, case.after); - assert_eq!( - case.old_groups, - [(0, 5_000_000), (moq_sequence(1, 1).unwrap(), 7_000_000)] - ); + assert_eq!(case.old_groups, [5_000_000, 7_000_000, 9_000_000]); + } + } + } + + /// A playlist that moved past segments rejoins at the next one, and every later keyframe still + /// opens a group of its own. + #[tokio::test] + async fn a_media_sequence_jump_keys_each_later_group_at_its_keyframe() { + for video in [false, true] { + for cmaf in [false, true] { + let after = "#EXT-X-MEDIA-SEQUENCE:5\n#EXTINF:1,\n1.m4s\n#EXTINF:1,\n2.m4s\n"; + let case = reload(video, cmaf, after, 7_000_000).await; + assert_eq!(case.result.unwrap(), 2); + assert!(case.terminal.is_none(), "{terminal:?}", terminal = case.terminal); + assert_eq!(case.before, case.after); + assert_eq!(case.old_groups, [5_000_000, 7_000_000, 9_000_000]); } } } @@ -2615,7 +2608,7 @@ mod tests { ); assert_eq!(case.before, case.after, "{label}"); assert!(case.terminal.is_none(), "{label}: {:?}", case.terminal); - assert_eq!(case.old_groups, [(0, 5_000_000)], "{label}"); + assert_eq!(case.old_groups, [5_000_000], "{label}"); } } } @@ -2677,8 +2670,9 @@ mod tests { case.result.unwrap(); assert!(case.terminal.is_none(), "{terminal:?}", terminal = case.terminal); assert_ne!(case.before, case.after); - assert_eq!(case.old_groups, [(0, 5_000_000)]); - assert_eq!(case.new_groups, [(moq_sequence(1, 1).unwrap(), u128::from(after_us))]); + assert_eq!(case.old_groups, [5_000_000]); + let after_us = u128::from(after_us); + assert_eq!(case.new_groups, [after_us, after_us + 2_000_000]); } } } @@ -2692,7 +2686,7 @@ mod tests { let mut out = Vec::new(); while let Poll::Ready(Ok(Some(mut group))) = track.poll_recv_group(&kio::Waiter::noop()) { while let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&kio::Waiter::noop()) { - out.push(frame.timestamp.unwrap().as_micros()); + out.push(frame.timestamp.as_micros().unwrap()); } } out @@ -2977,41 +2971,4 @@ video.m3u8 let target = request.lines().next()?.split_whitespace().nth(1)?; Some(target.split('?').next().unwrap_or(target)) } - - #[test] - fn moq_sequence_orders_discontinuities_after_media_sequence() { - // A new epoch outranks every segment of the previous one, even a higher media seq. - let last_of_epoch_0 = moq_sequence(0, u64::from(u32::MAX)).unwrap(); - let first_of_epoch_1 = moq_sequence(1, 0).unwrap(); - assert!(first_of_epoch_1 > last_of_epoch_0); - } - - #[test] - fn moq_sequence_preserves_media_order_within_epoch() { - assert!(moq_sequence(3, 10).unwrap() > moq_sequence(3, 9).unwrap()); - } - - #[test] - fn moq_sequence_rejects_unrepresentable_media_sequence() { - let err = moq_sequence(0, 1u64 << 48).unwrap_err(); - assert!(matches!( - err, - Error::SequenceOverflow { - kind: SequenceKind::Media, - .. - } - )); - } - - #[test] - fn moq_sequence_rejects_unrepresentable_discontinuity_sequence() { - let err = moq_sequence(1u64 << 16, 0).unwrap_err(); - assert!(matches!( - err, - Error::SequenceOverflow { - kind: SequenceKind::Discontinuity, - .. - } - )); - } } diff --git a/rs/moq-hls/src/server/include_tests.rs b/rs/moq-hls/src/server/include_tests.rs index 47820582be3..0c913845b89 100644 --- a/rs/moq-hls/src/server/include_tests.rs +++ b/rs/moq-hls/src/server/include_tests.rs @@ -42,7 +42,7 @@ fn publish(origin: &moq_net::origin::Producer, path: &str, instance: &Instance, .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) .unwrap(); let mut catalog = moq_json::snapshot::Producer::new(track, Default::default()); - catalog.update(moq_net::Timed::from(value).at(Timestamp::ZERO)).unwrap(); + catalog.update(moq_json::Timed::new(value, Timestamp::ZERO)).unwrap(); let video = publication .create_track("same", hang::container::track_info(1)) .unwrap(); @@ -61,7 +61,8 @@ fn publish(origin: &moq_net::origin::Producer, path: &str, instance: &Instance, fn frames(track: &track::Producer, first: u64) { for sequence in first..first + 3 { - let mut group = track.create_group(sequence.into()).unwrap(); + // Keyed by its first frame's pts, in the track's microseconds. + let mut group = track.create_group((sequence * 2_000_000).into()).unwrap(); hang::container::Frame { timestamp: Timestamp::from_secs(sequence * 2).unwrap(), payload: bytes::Bytes::from_static(if track.name() == "audio" { @@ -72,7 +73,7 @@ fn frames(track: &track::Producer, first: u64) { } .write_to(&mut group) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } } @@ -166,7 +167,7 @@ async fn an_idle_predecessor_does_not_strand_a_requested_reconfigure() { video.coded_width = Some(640); include .catalog - .update(moq_net::Timed::from(&value).at(Timestamp::ZERO)) + .update(moq_json::Timed::new(&value, Timestamp::ZERO)) .unwrap(); master(&server, "RESOLUTION=640x240", true).await; // The new response lists the predecessor's retained rows while waiting for its @@ -292,7 +293,7 @@ async fn idle_release_keeps_an_ended_playlists_inherited_segments() { video.coded_width = Some(640); include .catalog - .update(moq_net::Timed::from(&value).at(Timestamp::ZERO)) + .update(moq_json::Timed::new(&value, Timestamp::ZERO)) .unwrap(); master(&server, "RESOLUTION=640x240", true).await; let _video = include.video.finish().unwrap(); diff --git a/rs/moq-hls/tests/integration/server.rs b/rs/moq-hls/tests/integration/server.rs index 20410d1d64e..afc15e03be8 100644 --- a/rs/moq-hls/tests/integration/server.rs +++ b/rs/moq-hls/tests/integration/server.rs @@ -37,14 +37,18 @@ async fn respond_serves_a_route_rewritten_into_scope() { /// A recorder's cursor gets the media of a publisher whose moq-lite has no FETCH: the cursor's /// own subscription carries every group to the subscriber's cache. +/// +/// moq-lite-01 to 04 can't say the starts between two groups were skipped, so the reader waits +/// out its feed budget at each gap before it records past one. #[tokio::test(start_paused = true)] async fn a_cursor_records_a_publisher_without_fetch() { - for version in [ - "moq-lite-01", - "moq-lite-02", - "moq-lite-03", - "moq-lite-04", - "moq-lite-05", + const HOLELESS: std::time::Duration = std::time::Duration::from_secs(45); + for (version, wait) in [ + ("moq-lite-01", HOLELESS), + ("moq-lite-02", HOLELESS), + ("moq-lite-03", HOLELESS), + ("moq-lite-04", HOLELESS), + ("moq-lite-05", TIMEOUT), ] { let pair = Pair::new(version.parse().unwrap(), Default::default()).await; let mut publisher = Publisher::new(&pair.publisher, "live", Default::default()); @@ -81,7 +85,7 @@ async fn a_cursor_records_a_publisher_without_fetch() { .unwrap_or_else(|_| panic!("{version}: the cursor never subscribed")); publisher.write(0, 5, 1); - let segment = kio::clock::timeout(TIMEOUT, next) + let segment = kio::clock::timeout(wait, next) .await .unwrap_or_else(|_| panic!("{version}: no segment recorded")) .unwrap_or_else(|err| panic!("{version}: {err}")) diff --git a/rs/moq-json/examples/telemetry.rs b/rs/moq-json/examples/telemetry.rs index 2c72b7853fe..310085307e7 100644 --- a/rs/moq-json/examples/telemetry.rs +++ b/rs/moq-json/examples/telemetry.rs @@ -75,6 +75,11 @@ fn telemetry(tick: u64) -> Value { }) } +/// When `tick` was sampled: the simulated device samples once per second. +fn at(tick: u64) -> moq_net::Timestamp { + moq_net::Timestamp::from_secs(tick).unwrap() +} + /// The telemetry track, optionally compressed. fn track(compress: bool) -> moq_net::track::Producer { let info = moq_net::track::Info::default() @@ -97,7 +102,7 @@ fn frame_bytes(config: snapshot::Config, ticks: u64) -> usize { for tick in 0..ticks { producer - .update(moq_net::Timed::from(&telemetry(tick)).at(moq_net::Timestamp::now())) + .update(moq_json::Timed::new(&telemetry(tick), at(tick))) .unwrap(); } let _finished = producer.finish().unwrap(); @@ -124,9 +129,7 @@ fn verify(producer_config: snapshot::Config, compress: bool, ticks: u64) { for tick in 0..ticks { let expected = telemetry(tick); - producer - .update(moq_net::Timed::from(&expected).at(moq_net::Timestamp::now())) - .unwrap(); + producer.update(moq_json::Timed::new(&expected, at(tick))).unwrap(); // The producer emits exactly one frame per update, so the live consumer yields exactly one // reconstructed value: it must match the input byte-for-byte after patching. match consumer.poll_latest(&waiter) { @@ -155,7 +158,7 @@ fn verify_late_joiner(producer_config: snapshot::Config, compress: bool, ticks: let mut producer = Producer::::new(track, producer_config); for tick in 0..ticks { producer - .update(moq_net::Timed::from(&telemetry(tick)).at(moq_net::Timestamp::now())) + .update(moq_json::Timed::new(&telemetry(tick), at(tick))) .unwrap(); } let _finished = producer.finish().unwrap(); diff --git a/rs/moq-json/examples/teleop.rs b/rs/moq-json/examples/teleop.rs index a04b86145c2..c55e5b891b8 100644 --- a/rs/moq-json/examples/teleop.rs +++ b/rs/moq-json/examples/teleop.rs @@ -4,8 +4,9 @@ //! framing; attach the same broadcast to a moq-tokio origin to carry it over a session. use std::task::Poll; +use moq_json::Timed; use moq_json::{snapshot, stream}; -use moq_net::{Timed, Timescale, Timestamp, broadcast, track}; +use moq_net::{Timescale, Timestamp, broadcast, track}; use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -40,14 +41,14 @@ fn main() -> Result<(), Box> { .iter() .enumerate() { - state.update(Timed::from(control).at(Timestamp::from_millis(tick as u64)?))?; + state.update(Timed::new(control, Timestamp::from_millis(tick as u64)?))?; } let _state = state.finish()?; // Every discrete command is meaningful, including Stop between Arm and Disarm. let expected = [Command::Arm, Command::Stop, Command::Disarm]; for (tick, command) in expected.iter().enumerate() { - commands.append(Timed::from(command).at(Timestamp::from_millis(tick as u64)?))?; + commands.append(Timed::new(command, Timestamp::from_millis(tick as u64)?))?; } let _commands = commands.finish()?; @@ -57,7 +58,7 @@ fn main() -> Result<(), Box> { return Err("finished control track did not yield its latest state".into()); }; assert_eq!(latest.value, Control { speed: 0, turn: 0 }); - assert_eq!(latest.at, Some(Timestamp::from_millis(2)?)); + assert_eq!(latest.at, Timestamp::from_millis(2)?); println!("Current control: {:?} at {:?}", latest.value, latest.at); assert!(matches!(state_reader.poll_latest(&waiter), Poll::Ready(Ok(None)))); for (tick, expected) in expected.into_iter().enumerate() { @@ -65,7 +66,7 @@ fn main() -> Result<(), Box> { return Err("finished command track lost an ordered command".into()); }; assert_eq!(command.value, expected); - assert_eq!(command.at, Some(Timestamp::from_millis(tick as u64)?)); + assert_eq!(command.at, Timestamp::from_millis(tick as u64)?); println!("Command: {:?} at {:?}", command.value, command.at); } assert!(matches!(command_reader.poll_next(&waiter), Poll::Ready(Ok(None)))); diff --git a/rs/moq-json/src/lib.rs b/rs/moq-json/src/lib.rs index ba1641e56cd..ec2ce67cc36 100644 --- a/rs/moq-json/src/lib.rs +++ b/rs/moq-json/src/lib.rs @@ -29,8 +29,11 @@ pub mod snapshot; mod state; pub mod stream; mod strict; +mod timed; pub mod window; +pub use timed::Timed; + /// Test support for the codec bench, enabled by the `bench` feature: the merge-patch diff. #[cfg(feature = "bench")] pub mod bench { @@ -41,6 +44,22 @@ pub mod bench { /// also holds, so a larger frame is refused before it is written. pub(crate) const MAX_GROUP_BYTES: u64 = 32 * 1024 * 1024; +/// Whether a group whose first frame is at `timestamp`, rescaled to `track`, would start past +/// every group on `track`, which a new group must: two groups never share a start. A timed group +/// starts at its first frame's time, and a scale-less one at the key its unscaled timestamp names. +pub(crate) fn starts_past(track: &moq_net::track::Producer, timestamp: moq_net::Timestamp) -> bool { + track.largest().is_none_or(|largest| timestamp.value() > largest.group) +} + +/// `timestamp` for a frame of `group`: a scale-less track's frames carry their group's key, so a +/// frame appended to an open group carries that group's key, whatever key the caller named. +pub(crate) fn frame_time(group: &moq_net::group::Producer, timestamp: moq_net::Timestamp) -> moq_net::Timestamp { + match timestamp.scale() { + Some(_) => timestamp, + None => moq_net::Timestamp::unscaled(group.start), + } +} + /// Errors produced while publishing or consuming JSON. #[derive(thiserror::Error, Debug, Clone)] #[non_exhaustive] diff --git a/rs/moq-json/src/refresh.rs b/rs/moq-json/src/refresh.rs index b945ed91212..05a973eac1e 100644 --- a/rs/moq-json/src/refresh.rs +++ b/rs/moq-json/src/refresh.rs @@ -10,11 +10,28 @@ use crate::Result; /// free sequence, so a producer restating its value in a new group answers it. pub(crate) struct Refresh { demand: moq_net::track::Demand, + + /// Refreshes waiting on a later write, since the one they would restate starts the newest group. + parked: kio::WaiterList, } impl Refresh { pub fn new(track: &moq_net::track::Producer) -> Self { - Self { demand: track.demand() } + Self { + demand: track.demand(), + parked: kio::WaiterList::new(), + } + } + + /// Wait for [`Self::advanced`]: the refresh is wanted, but nothing written yet can start a group. + pub fn park(&mut self, waiter: &kio::Waiter) -> Poll { + waiter.register(&mut self.parked); + Poll::Pending + } + + /// A write moved the newest timestamp on, which a parked refresh may now restate at. + pub fn advanced(&mut self) { + self.parked.wake(); } /// Ready once consumers want a group past `largest`, the newest group written, arming `waiter` @@ -36,3 +53,29 @@ impl Refresh { } } } + +/// A waiter that records whether it was woken, for tests that a parked poll gets a wake. +#[cfg(test)] +pub(crate) fn flagged() -> (kio::Waiter, std::sync::Arc) { + let flag = std::sync::Arc::new(Flag::default()); + (kio::Waiter::new(flag.clone().into()), flag) +} + +#[cfg(test)] +#[derive(Default)] +pub(crate) struct Flag(std::sync::atomic::AtomicBool); + +#[cfg(test)] +impl Flag { + /// Whether a wake arrived since the last call. + pub fn take(&self) -> bool { + self.0.swap(false, std::sync::atomic::Ordering::SeqCst) + } +} + +#[cfg(test)] +impl std::task::Wake for Flag { + fn wake(self: std::sync::Arc) { + self.0.store(true, std::sync::atomic::Ordering::SeqCst); + } +} diff --git a/rs/moq-json/src/snapshot/consumer.rs b/rs/moq-json/src/snapshot/consumer.rs index 1b9618683d7..b4649b0ccd8 100644 --- a/rs/moq-json/src/snapshot/consumer.rs +++ b/rs/moq-json/src/snapshot/consumer.rs @@ -77,7 +77,7 @@ impl Consumer { /// Skips every group a newer one has superseded. Finish the current group's buffered frames /// before taking the next. Lost groups resynchronize from a later snapshot. #[cfg_attr(feature = "async", kio::wrapper)] - pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>> { + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>> { self.poll_read(waiter, false) } @@ -86,11 +86,11 @@ impl Consumer { /// Frames inside the newest group are still decoded in order. What this skips is gone; /// a subsequent `next` resumes after it. An untimed frame keeps `at: None`. #[cfg_attr(feature = "async", kio::wrapper)] - pub fn poll_latest(&mut self, waiter: &kio::Waiter) -> Poll>>> { + pub fn poll_latest(&mut self, waiter: &kio::Waiter) -> Poll>>> { self.poll_read(waiter, true) } - fn poll_read(&mut self, waiter: &kio::Waiter, skip: bool) -> Poll>>> { + fn poll_read(&mut self, waiter: &kio::Waiter, skip: bool) -> Poll>>> { if skip { // Jump to the newest group the reader already has; a newer one supersedes the rest. while let Poll::Ready(Some(group)) = self.track.poll_next_group(waiter)? { @@ -121,7 +121,7 @@ impl Consumer { Poll::Ready(Ok(Some(frame))) => { self.apply(&frame.payload)?; if !skip { - return Poll::Ready(Ok(self.decoder.decode()?.map(|value| moq_net::Timed { + return Poll::Ready(Ok(self.decoder.decode()?.map(|value| crate::Timed { value, at: frame.timestamp, }))); @@ -141,7 +141,7 @@ impl Consumer { ) => { // A later group starts a fresh snapshot. - tracing::warn!(track = self.name, group = group.sequence, error = ?err, + tracing::warn!(track = self.name, group = group.start, error = ?err, "snapshot group lost; waiting for a newer one"); self.group = None; } @@ -150,7 +150,7 @@ impl Consumer { } } if let Some(at) = advanced { - Poll::Ready(Ok(self.decoder.decode()?.map(|value| moq_net::Timed { value, at }))) + Poll::Ready(Ok(self.decoder.decode()?.map(|value| crate::Timed { value, at }))) } else if finished { Poll::Ready(Ok(None)) } else { @@ -242,9 +242,9 @@ mod timestamps { for (n, ticks) in times.into_iter().enumerate() { let at = moq_net::Timestamp::new(ticks, scale).unwrap(); producer - .update(moq_net::Timed { + .update(crate::Timed { value: &serde_json::json!({"n": n, "padding": "large-repeated-value-for-deltas"}), - at: Some(at), + at, }) .unwrap(); if n == 1 { @@ -254,9 +254,9 @@ mod timestamps { panic!("missing state {n}") }; assert_eq!(value.value["n"].as_u64().unwrap(), n as u64); - let at = value.at.unwrap(); + let at = value.at; assert_eq!(at.value(), ticks); - assert_eq!(at.scale(), scale); + assert_eq!(at.scale(), Some(scale)); } let _finished = producer.finish().unwrap(); assert!(matches!( @@ -267,8 +267,8 @@ mod timestamps { panic!("missing latest state") }; assert_eq!(value.value["n"].as_u64().unwrap(), 2); - assert_eq!(value.at.unwrap().value(), 990_007); - assert_eq!(value.at.unwrap().scale(), scale); + assert_eq!(value.at.value(), 990_007); + assert_eq!(value.at.scale(), Some(scale)); } } @@ -286,25 +286,31 @@ mod timestamps { let waiter = kio::Waiter::noop(); for n in 0..3 { producer - .update(&serde_json::json!({"n": n, "padding": "large-repeated-value-for-deltas"})) + .update(crate::Timed::new( + &serde_json::json!({"n": n, "padding": "large-repeated-value-for-deltas"}), + moq_net::Timestamp::unscaled(n), + )) .unwrap(); if n == 0 { let Poll::Ready(Ok(Some(value))) = consumer.poll_next(&waiter) else { panic!("missing first state") }; assert_eq!(value.value["n"].as_u64().unwrap(), 0); - assert_eq!(value.at, None); + assert_eq!(value.at.scale(), None); } } let Poll::Ready(Ok(Some(value))) = consumer.poll_latest(&waiter) else { panic!("missing latest state") }; assert_eq!(value.value["n"].as_u64().unwrap(), 2); - assert_eq!(value.at, None); + assert_eq!(value.at.scale(), None); assert!(consumer.poll_next(&waiter).is_pending()); let n = 3; producer - .update(&serde_json::json!({"n": n, "padding": "large-repeated-value-for-deltas"})) + .update(crate::Timed::new( + &serde_json::json!({"n": n, "padding": "large-repeated-value-for-deltas"}), + moq_net::Timestamp::unscaled(n), + )) .unwrap(); let Poll::Ready(Ok(Some(value))) = consumer.poll_next(&waiter) else { panic!("missing new state") diff --git a/rs/moq-json/src/snapshot/mod.rs b/rs/moq-json/src/snapshot/mod.rs index 35267cee6e1..91d9287a179 100644 --- a/rs/moq-json/src/snapshot/mod.rs +++ b/rs/moq-json/src/snapshot/mod.rs @@ -65,14 +65,31 @@ mod test { use super::encoder::MAX_DELTA_FRAMES; use super::*; + /// `value` at the next key of a scale-less track, whose writes name their group keys: keys + /// only have to increase, so one counter serves every track a test writes. + fn next(value: V) -> crate::Timed { + crate::Timed::new(value, key()) + } + + /// `value` at scale-less `key`. + fn keyed(value: V, key: u64) -> crate::Timed { + crate::Timed::new(value, moq_net::Timestamp::unscaled(key)) + } + + /// The next key of a scale-less track; see [`next`]. + fn key() -> moq_net::Timestamp { + thread_local!(static KEY: std::cell::Cell = const { std::cell::Cell::new(0) }); + moq_net::Timestamp::unscaled(KEY.with(|key| key.replace(key.get() + 1))) + } + #[test] fn editing_keeps_the_callers_timestamp_on_commit_and_drop() { let (mut producer, _) = producer_on(cfg(true), timed()); let mut groups = producer.consume(); let first = moq_net::Timestamp::from_millis(100).unwrap(); let second = moq_net::Timestamp::from_millis(200).unwrap(); - producer.modify(Some(first)).unwrap()["n"] = json!(1); - let mut edit = producer.modify(Some(second)).unwrap(); + producer.modify(first).unwrap()["n"] = json!(1); + let mut edit = producer.modify(second).unwrap(); edit["n"] = json!(2); edit.commit().unwrap(); let waiter = kio::Waiter::noop(); @@ -83,21 +100,21 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected the edited value"); }; - assert_eq!(frame.timestamp, Some(at)); + assert_eq!(frame.timestamp, at); } } #[test] - fn a_missing_edit_timestamp_is_refused_on_a_timed_track() { + fn an_unscaled_edit_timestamp_is_refused_on_a_timed_track() { let (mut producer, _) = producer_on(cfg(false), timed()); - let mut edit = producer.modify(None).unwrap(); + let mut edit = producer.modify(key()).unwrap(); edit["n"] = json!(1); assert!(matches!( edit.commit(), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); let at = moq_net::Timestamp::from_millis(100).unwrap(); - producer.mutate(|value| value["n"] = json!(2), Some(at)).unwrap(); + producer.mutate(|value| value["n"] = json!(2), at).unwrap(); let waiter = kio::Waiter::noop(); let mut groups = producer.consume(); let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { @@ -106,7 +123,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected a timed frame"); }; - assert_eq!(frame.timestamp, Some(at)); + assert_eq!(frame.timestamp, at); } /// Refused before a replacement group is published, so the last value stays readable. @@ -115,11 +132,9 @@ mod test { for deltas in [false, true] { let (mut producer, track) = producer_on(cfg(deltas), timed()); let at = moq_net::Timestamp::from_millis(100).unwrap(); - producer - .update(moq_net::Timed::from(&json!({ "n": 1 })).at(at)) - .unwrap(); + producer.update(crate::Timed::new(&json!({ "n": 1 }), at)).unwrap(); assert!(matches!( - producer.update(&json!({ "n": 2 })), + producer.update(next(&json!({ "n": 2 }))), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); let _finished = producer.finish().unwrap(); @@ -135,11 +150,9 @@ mod test { for deltas in [false, true] { let (mut producer, track) = producer_on(cfg(deltas), timed()); let at = moq_net::Timestamp::from_millis(100).unwrap(); - producer - .update(moq_net::Timed::from(&json!({ "n": 1 })).at(at)) - .unwrap(); + producer.update(crate::Timed::new(&json!({ "n": 1 }), at)).unwrap(); assert!(matches!( - producer.update(moq_net::Timed::from(&json!({ "n": 2 })).at(huge)), + producer.update(crate::Timed::new(&json!({ "n": 2 }), huge)), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); let _finished = producer.finish().unwrap(); @@ -155,7 +168,7 @@ mod test { .unwrap(); let mut groups = track.subscribe(None); let mut producer = Producer::new(track, Config::default()); - producer.update(&json!({ "n": 1 })).unwrap(); + producer.update(next(&json!({ "n": 1 }))).unwrap(); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { panic!("expected an untimed group"); @@ -163,7 +176,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected an untimed frame"); }; - assert_eq!(frame.timestamp, None); + assert_eq!(frame.timestamp.scale(), None); } /// A millisecond track: the frames written here carry timestamps. @@ -225,14 +238,16 @@ mod test { let (mut producer, track) = producer(cfg(true)); for n in [1, 2, 3] { producer - .update(&json!({ "padding": "same-long-value-so-deltas-stay-in-one-group", "n": n })) + .update(next( + &json!({ "padding": "same-long-value-so-deltas-stay-in-one-group", "n": n }), + )) .unwrap(); } let _finished = producer.finish().unwrap(); let mut consumer = Consumer::::new(track, consumer::Config::default()); let mut values = Vec::new(); while let Poll::Ready(Ok(Some(value))) = consumer.poll_next(&kio::Waiter::noop()) { - assert_eq!(value.at, None); + assert_eq!(value.at.scale(), None); values.push(value.value["n"].as_u64().unwrap()); } assert_eq!(values, vec![1, 2, 3]); @@ -255,9 +270,9 @@ mod test { let waiter = kio::Waiter::noop(); // Group 0 delivers a value, then stays open with the reader parked on its next frame. - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); group - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(br#"{"a":1}"#)) + .write_frame(moq_net::Timestamp::ZERO, Bytes::from_static(br#"{"a":1}"#)) .unwrap(); assert!(matches!(consumer.poll_latest(&waiter), Poll::Ready(Ok(Some(v))) if v.value == json!({ "a": 1 }))); assert!(consumer.poll_latest(&waiter).is_pending()); @@ -270,9 +285,12 @@ mod test { ); // The replacement arrives and the reader picks up where the value now lives. - let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); group - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(br#"{"a":2}"#)) + .write_frame( + moq_net::Timestamp::from_millis(1).unwrap(), + Bytes::from_static(br#"{"a":2}"#), + ) .unwrap(); assert!(matches!(consumer.poll_latest(&waiter), Poll::Ready(Ok(Some(v))) if v.value == json!({ "a": 2 }))); } @@ -289,9 +307,9 @@ mod test { let mut consumer = Consumer::::new(track.consume(), consumer::Config::default()); let waiter = kio::Waiter::noop(); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); group - .write_frame(moq_net::Timestamp::now(), Bytes::from_static(br#"{"a":1}"#)) + .write_frame(moq_net::Timestamp::ZERO, Bytes::from_static(br#"{"a":1}"#)) .unwrap(); assert!(matches!(consumer.poll_latest(&waiter), Poll::Ready(Ok(Some(v))) if v.value == json!({ "a": 1 }))); @@ -303,10 +321,10 @@ mod test { #[test] fn a_cut_makes_the_next_update_a_snapshot_group() { let (mut producer, mut track) = producer(cfg(true)); - producer.update(&json!({ "a": 1, "b": 1 })).unwrap(); - producer.update(&json!({ "a": 1, "b": 2 })).unwrap(); + producer.update(keyed(&json!({ "a": 1, "b": 1 }), 0)).unwrap(); + producer.update(keyed(&json!({ "a": 1, "b": 2 }), 1)).unwrap(); producer.cut().unwrap(); - producer.update(&json!({ "a": 1, "b": 3 })).unwrap(); + producer.update(keyed(&json!({ "a": 1, "b": 3 }), 2)).unwrap(); let _finished = producer.finish().unwrap(); // The ratio would have kept every update in one group; the cut rolled it anyway. A consumer @@ -314,7 +332,7 @@ mod test { // that preceded the cut. assert_eq!( track.largest().now_or_never().flatten().map(|position| position.group), - Some(1) + Some(2) ); assert_eq!(drain(track).last().unwrap(), &json!({ "a": 1, "b": 3 })); } @@ -322,12 +340,12 @@ mod test { #[test] fn a_cut_republishes_an_unchanged_value() { let (mut producer, mut track) = producer(cfg(true)); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); producer.cut().unwrap(); // An unchanged value normally writes nothing. After a cut it must still open the replacement // group, or the value would only exist in a group no new consumer reads. - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!( @@ -340,7 +358,7 @@ mod test { #[test] fn a_cut_opens_no_replacement_group() { let (mut producer, mut track) = producer(cfg(true)); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); producer.cut().unwrap(); let _finished = producer.finish().unwrap(); @@ -360,7 +378,7 @@ mod test { // Nothing published yet, so there is no group to cut. producer.cut().unwrap(); producer.cut().unwrap(); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); assert_eq!( track.largest().now_or_never().flatten().map(|position| position.group), Some(0) @@ -369,7 +387,7 @@ mod test { // And a repeated cut rolls once, not once per call. producer.cut().unwrap(); producer.cut().unwrap(); - producer.update(&json!({ "a": 2 })).unwrap(); + producer.update(next(&json!({ "a": 2 }))).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!( @@ -382,11 +400,11 @@ mod test { #[test] fn a_cut_is_inert_without_deltas() { let (mut producer, mut track) = producer(cfg(false)); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); // With deltas off every frame already closes its own group, so there is never one to cut. producer.cut().unwrap(); - producer.update(&json!({ "a": 2 })).unwrap(); + producer.update(next(&json!({ "a": 2 }))).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!( @@ -399,8 +417,8 @@ mod test { #[test] fn deltas_off_snapshot_per_group() { let (mut producer, mut track) = producer(cfg(false)); - producer.update(&json!({ "a": 1 })).unwrap(); - producer.update(&json!({ "a": 2 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); + producer.update(next(&json!({ "a": 2 }))).unwrap(); let _finished = producer.finish().unwrap(); // Two updates => two groups, each a full snapshot. A consumer that joins after both @@ -419,7 +437,7 @@ mod test { let waiter = kio::Waiter::noop(); for n in 1..=3 { - producer.update(&json!({ "a": n })).unwrap(); + producer.update(next(&json!({ "a": n }))).unwrap(); match consumer.poll_latest(&waiter) { Poll::Ready(Ok(Some(value))) => assert_eq!(value.value, json!({ "a": n })), other => panic!("expected value, got {other:?}"), @@ -434,11 +452,11 @@ mod test { #[test] fn a_rejected_update_leaves_the_previous_value_readable() { let (mut producer, track) = producer(cfg(false)); - producer.update(&json!({ "keep": true })).unwrap(); + producer.update(next(&json!({ "keep": true }))).unwrap(); // Serializes past the group cache limit, so the frame cannot be published. let oversized = json!({ "big": "x".repeat(crate::MAX_GROUP_BYTES as usize + 1) }); - assert!(producer.update(&oversized).is_err()); + assert!(producer.update(next(&oversized)).is_err()); let _finished = producer.finish().unwrap(); // A reader arriving now still finds the last good value, not an empty superseding group. @@ -448,9 +466,9 @@ mod test { #[test] fn unchanged_value_writes_nothing() { let (mut producer, mut track) = producer(Config::default()); - assert_eq!(producer.update(&json!({ "a": 1 })).unwrap(), Some(7)); + assert_eq!(producer.update(next(&json!({ "a": 1 }))).unwrap(), Some(7)); assert_eq!( - producer.update(&json!({ "a": 1 })).unwrap(), + producer.update(next(&json!({ "a": 1 }))).unwrap(), None, "an unchanged value reports that nothing was written" ); @@ -472,9 +490,9 @@ mod test { let first = moq_net::Timestamp::from_millis(1_000).unwrap(); let second = moq_net::Timestamp::from_millis(2_000).unwrap(); let value = json!({ "a": 1, "b": "x".repeat(64) }); - let size = producer.update(moq_net::Timed::from(&value).at(first)).unwrap(); + let size = producer.update(crate::Timed::new(&value, first)).unwrap(); let changed = json!({ "a": 2, "b": "x".repeat(64) }); - let delta = producer.update(moq_net::Timed::from(&changed).at(second)).unwrap(); + let delta = producer.update(crate::Timed::new(&changed, second)).unwrap(); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { @@ -484,7 +502,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected a frame"); }; - assert_eq!(frame.timestamp.unwrap().as_micros(), stamp.as_micros()); + assert_eq!(frame.timestamp.as_micros().unwrap(), stamp.as_micros().unwrap()); assert_eq!(size, Some(frame.payload.len())); } } @@ -494,7 +512,7 @@ mod test { fn deltas_never_roll_without_a_joiner() { let (mut producer, mut track) = producer_on(cfg(true), moq_net::track::Info::default().with_refresh(1)); for n in 0..=10 { - producer.update(&json!({ "n": n })).unwrap(); + producer.update(next(&json!({ "n": n }))).unwrap(); } assert!(producer.poll_refresh(&kio::Waiter::noop()).is_pending()); let _finished = producer.finish().unwrap(); @@ -507,7 +525,7 @@ mod test { } /// The frames of `group` received so far: each timestamp and decoded payload. - fn frames(group: &mut moq_net::group::Consumer) -> Vec<(Option, Value)> { + fn frames(group: &mut moq_net::group::Consumer) -> Vec<(moq_net::Timestamp, Value)> { let waiter = kio::Waiter::noop(); let mut out = Vec::new(); while let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) { @@ -534,24 +552,26 @@ mod test { let (mut producer, _track) = producer_on(cfg(true), moq_net::track::Info::default().with_refresh(1)); // A 13 byte snapshot and a 7 byte delta: under the ratio. - producer.update(&json!({ "n": 0, "p": 0 })).unwrap(); - producer.update(&json!({ "n": 1, "p": 0 })).unwrap(); + producer.update(keyed(&json!({ "n": 0, "p": 0 }), 0)).unwrap(); + producer.update(keyed(&json!({ "n": 1, "p": 0 }), 1)).unwrap(); let _early = producer.consume(); assert!(producer.poll_refresh(&waiter).is_pending(), "a join under the ratio"); // A second delta takes the group past it, and the next join asks for a fresh one. - producer.update(&json!({ "n": 2, "p": 0 })).unwrap(); + producer.update(keyed(&json!({ "n": 2, "p": 0 }), 2)).unwrap(); assert!(producer.poll_refresh(&waiter).is_pending(), "no join, no refresh"); let mut late = producer.consume(); assert!(matches!(producer.poll_refresh(&waiter), Poll::Ready(Ok(())))); assert!(producer.poll_refresh(&waiter).is_pending(), "answered once"); let mut fresh = groups(&mut late).pop().expect("the fresh group"); - assert_eq!(fresh.sequence, 1); - assert_eq!(frames(&mut fresh), vec![(None, json!({ "n": 2, "p": 0 }))]); + // Restated at the newest key written, and a later delta rides in it under its key. + let restated = moq_net::Timestamp::unscaled(2); + assert_eq!(fresh.start, 2); + assert_eq!(frames(&mut fresh), vec![(restated, json!({ "n": 2, "p": 0 }))]); - producer.update(&json!({ "n": 3, "p": 0 })).unwrap(); - assert_eq!(frames(&mut fresh), vec![(None, json!({ "n": 3 }))]); + producer.update(keyed(&json!({ "n": 3, "p": 0 }), 3)).unwrap(); + assert_eq!(frames(&mut fresh), vec![(restated, json!({ "n": 3 }))]); } /// A refresh restates the value at the newest timestamp written rather than inventing one. @@ -559,12 +579,10 @@ mod test { fn a_refresh_reuses_the_newest_timestamp() { let ms = |n| moq_net::Timestamp::from_millis(n).unwrap(); let (mut producer, _track) = producer_on(cfg(true), timed().with_refresh(1)); - producer - .update(moq_net::Timed::from(&json!({ "n": 0 })).at(ms(100))) - .unwrap(); + producer.update(crate::Timed::new(&json!({ "n": 0 }), ms(100))).unwrap(); for n in 1..=3 { producer - .update(moq_net::Timed::from(&json!({ "n": n })).at(ms(100 + n))) + .update(crate::Timed::new(&json!({ "n": n }), ms(100 + n))) .unwrap(); } @@ -574,30 +592,35 @@ mod test { Poll::Ready(Ok(())) )); let mut fresh = groups(&mut late).pop().expect("the fresh group"); - assert_eq!(fresh.sequence, 1); - assert_eq!(frames(&mut fresh), vec![(Some(ms(103)), json!({ "n": 3 }))]); + assert_eq!(fresh.start, 103); + assert_eq!(frames(&mut fresh), vec![(ms(103), json!({ "n": 3 }))]); } - /// A delta may carry a timestamp older than its group's first frame, but a new group may not - /// start before that frame, so the restatement takes the newest timestamp written. + /// A delta may carry a timestamp older than its group's first frame, but a new group must start + /// past that frame, so a refresh whose newest timestamp is still the group's own start waits for a + /// later update, then restates the value there. #[test] - fn a_refresh_after_an_older_update_starts_at_the_newest_timestamp() { + fn a_refresh_after_older_updates_waits_for_a_later_one() { let ms = |n| moq_net::Timestamp::from_millis(n).unwrap(); let (mut producer, _track) = producer_on(cfg(true), timed().with_refresh(1)); for (n, at) in [130, 110, 120, 115].into_iter().enumerate() { - producer - .update(moq_net::Timed::from(&json!({ "n": n })).at(ms(at))) - .unwrap(); + producer.update(crate::Timed::new(&json!({ "n": n }), ms(at))).unwrap(); } let mut late = producer.consume(); - assert!(matches!( - producer.poll_refresh(&kio::Waiter::noop()), - Poll::Ready(Ok(())) - )); + let (waiter, woken) = crate::refresh::flagged(); + assert!( + producer.poll_refresh(&waiter).is_pending(), + "no group may share the open group's start" + ); + woken.take(); + + producer.update(crate::Timed::new(&json!({ "n": 4 }), ms(140))).unwrap(); + assert!(woken.take(), "the later update wakes the parked refresh"); + assert!(matches!(producer.poll_refresh(&waiter), Poll::Ready(Ok(())))); let mut fresh = groups(&mut late).pop().expect("the fresh group"); - assert_eq!(fresh.sequence, 1); - assert_eq!(frames(&mut fresh), vec![(Some(ms(130)), json!({ "n": 3 }))]); + assert_eq!(fresh.start, 140); + assert_eq!(frames(&mut fresh), vec![(ms(140), json!({ "n": 4 }))]); } /// A refresh restates the last value written, never one whose update was rejected, and the next @@ -608,13 +631,13 @@ mod test { let (mut producer, track) = producer_on(cfg(true), timed().with_refresh(1)); for n in 0..=3 { producer - .update(moq_net::Timed::from(&json!({ "n": n })).at(ms(100 + n))) + .update(crate::Timed::new(&json!({ "n": n }), ms(100 + n))) .unwrap(); } - // Untimed on a timed track, so the frame is rejected before it reaches a group. + // Unscaled on a timed track, so the frame is rejected before it reaches a group. assert!(matches!( - producer.update(&json!({ "n": 99 })), + producer.update(next(&json!({ "n": 99 }))), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); @@ -624,12 +647,10 @@ mod test { Poll::Ready(Ok(())) )); let mut fresh = groups(&mut late).pop().expect("the fresh group"); - assert_eq!(fresh.sequence, 1); - assert_eq!(frames(&mut fresh), vec![(Some(ms(103)), json!({ "n": 3 }))]); + assert_eq!(fresh.start, 103); + assert_eq!(frames(&mut fresh), vec![(ms(103), json!({ "n": 3 }))]); - producer - .update(moq_net::Timed::from(&json!({ "n": 4 })).at(ms(104))) - .unwrap(); + producer.update(crate::Timed::new(&json!({ "n": 4 }), ms(104))).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!(drain(track).last(), Some(&json!({ "n": 4 }))); } @@ -639,7 +660,7 @@ mod test { fn no_ratio_never_refreshes() { let (mut producer, _track) = producer(cfg(true)); for n in 0..=10 { - producer.update(&json!({ "n": n })).unwrap(); + producer.update(next(&json!({ "n": n }))).unwrap(); } let _late = producer.consume(); assert!(producer.poll_refresh(&kio::Waiter::noop()).is_pending()); @@ -649,9 +670,9 @@ mod test { #[test] fn refresh_resolves_on_a_join_and_errors_on_close() { let (mut producer, _track) = producer_on(cfg(true), moq_net::track::Info::default().with_refresh(1)); - producer.update(&json!({ "n": 0 })).unwrap(); - producer.update(&json!({ "n": 1 })).unwrap(); - producer.update(&json!({ "n": 2 })).unwrap(); + producer.update(next(&json!({ "n": 0 }))).unwrap(); + producer.update(next(&json!({ "n": 1 }))).unwrap(); + producer.update(next(&json!({ "n": 2 }))).unwrap(); let mut refresh = Box::pin(producer.refresh()); assert!(refresh.as_mut().now_or_never().is_none()); @@ -666,8 +687,8 @@ mod test { #[test] fn array_change_is_delta() { let (mut producer, mut track) = producer(cfg(true)); - producer.update(&json!({ "list": [1, 2] })).unwrap(); - producer.update(&json!({ "list": [1, 2, 3] })).unwrap(); + producer.update(next(&json!({ "list": [1, 2] }))).unwrap(); + producer.update(next(&json!({ "list": [1, 2, 3] }))).unwrap(); let _finished = producer.finish().unwrap(); // The array is replaced wholesale in a delta, so it stays in the same group. @@ -683,14 +704,14 @@ mod test { let (mut producer, mut track) = producer(cfg(true)); // First update is the snapshot (frame 0); then MAX_DELTA_FRAMES - 1 deltas fill the group. for i in 0..=MAX_DELTA_FRAMES { - producer.update(&json!({ "n": i })).unwrap(); + producer.update(keyed(&json!({ "n": i }), i as u64)).unwrap(); } let _finished = producer.finish().unwrap(); // The frame cap forced exactly one extra snapshot group despite the huge ratio. assert_eq!( track.largest().now_or_never().flatten().map(|position| position.group), - Some(1) + Some(MAX_DELTA_FRAMES as u64) ); assert_eq!(drain(track).last().unwrap(), &json!({ "n": MAX_DELTA_FRAMES })); } @@ -717,9 +738,9 @@ mod test { .unwrap(); let consumer = Consumer::::new(track.consume(), consumer::Config::default()); let mut producer = Producer::::new(track, Config::default()); - producer.update(&Doc { count: 1 }).unwrap(); + producer.update(next(&Doc { count: 1 })).unwrap(); - assert!(matches!(producer.modify(None), Err(crate::Error::Json(_)))); + assert!(matches!(producer.modify(key()), Err(crate::Error::Json(_)))); // The refused edit published nothing after the original value. let _finished = producer.finish().unwrap(); @@ -735,12 +756,12 @@ mod test { .create_track("test", moq_net::track::Info::default()) .unwrap(); let mut producer = Producer::::new(track.clone(), cfg(false)); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); track.finish_at(moq_net::track::Position::group(2), None).unwrap(); - producer.modify(None).unwrap()["a"] = json!(2); + producer.modify(key()).unwrap()["a"] = json!(2); assert!(matches!( - producer.modify(None), + producer.modify(key()), Err(crate::Error::Net(moq_net::Error::Closed)) )); } @@ -754,16 +775,16 @@ mod test { .create_track("test", moq_net::track::Info::default()) .unwrap(); let mut producer = Producer::::new(track.clone(), cfg(false)); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); track.finish_at(moq_net::track::Position::group(10), None).unwrap(); let _finished = producer.clone().finish().unwrap(); assert!(matches!( - producer.update(&json!({ "a": 2 })), + producer.update(next(&json!({ "a": 2 }))), Err(crate::Error::Net(moq_net::Error::Closed)) )); assert!(matches!( - producer.modify(None), + producer.modify(key()), Err(crate::Error::Net(moq_net::Error::Closed)) )); } @@ -787,13 +808,13 @@ mod test { let mut producer = Producer::::new(track, Config::default()); // First owner sets its field. - producer.modify(None).unwrap().video = Some("v1".to_string()); + producer.modify(key()).unwrap().video = Some("v1".to_string()); // Second owner starts from the latest value and adds its own field without clobbering. - producer.modify(None).unwrap().scte35 = Some(42); + producer.modify(key()).unwrap().scte35 = Some(42); // Locking without mutating publishes nothing (the guard stays clean). - let _ = producer.modify(None).unwrap(); + let _ = producer.modify(key()).unwrap(); let _finished = producer.finish().unwrap(); @@ -830,14 +851,16 @@ mod test { let mut consumer = Consumer::::new(track.consume(), consumer::Config::default()); let mut producer = Producer::::new(track, cfg(false)); - producer.mutate(|doc| doc.video = Some("v1".to_string()), None).unwrap(); + producer + .mutate(|doc| doc.video = Some("v1".to_string()), key()) + .unwrap(); // The second owner starts from the latest value and adds its own field without clobbering. - producer.mutate(|doc| doc.scte35 = Some(42), None).unwrap(); + producer.mutate(|doc| doc.scte35 = Some(42), key()).unwrap(); // A closure that changes nothing publishes nothing: deltas are off, so each publish would // otherwise open a group of its own. - producer.mutate(|_| {}, None).unwrap(); + producer.mutate(|_| {}, key()).unwrap(); assert_eq!( raw.largest().now_or_never().flatten().map(|position| position.group), Some(1) @@ -863,14 +886,14 @@ mod test { fn mutate_returns_the_publish_error() { // Unlike a dropped guard, the closure form hands the failure back to the caller. let (mut producer, _track) = producer(cfg(false)); - producer.update(&json!({ "keep": true })).unwrap(); + producer.update(next(&json!({ "keep": true }))).unwrap(); assert!(matches!( producer.mutate( |value| { *value = json!({ "big": "x".repeat(crate::MAX_GROUP_BYTES as usize + 1) }); }, - None + key() ), Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) )); @@ -880,8 +903,8 @@ mod test { fn a_dropped_edit_publishes_a_snapshot_then_a_delta() { let (mut producer, mut track) = producer(cfg(true)); - *producer.modify(None).unwrap() = json!({ "a": 1, "b": 1 }); - *producer.modify(None).unwrap() = json!({ "a": 1, "b": 2 }); + *producer.modify(key()).unwrap() = json!({ "a": 1, "b": 1 }); + *producer.modify(key()).unwrap() = json!({ "a": 1, "b": 2 }); let _finished = producer.finish().unwrap(); // The ratio keeps both edits in one group: a snapshot plus a merge-patch delta. @@ -895,11 +918,11 @@ mod test { #[test] fn modify_refuses_a_finished_track() { let (mut producer, track) = producer(cfg(false)); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); let _finished = producer.clone().finish().unwrap(); assert!(matches!( - producer.modify(None), + producer.modify(key()), Err(crate::Error::Net(moq_net::Error::Closed)) )); assert_eq!(drain(track), vec![json!({ "a": 1 })]); @@ -908,15 +931,15 @@ mod test { #[test] fn a_dropped_failed_edit_aborts_the_track() { let (mut producer, track) = producer(cfg(false)); - producer.update(&json!({ "keep": true })).unwrap(); + producer.update(next(&json!({ "keep": true }))).unwrap(); let mut subscriber = track.subscribe(None).now_or_never().unwrap().unwrap().ordered(); - *producer.modify(None).unwrap() = json!({ "big": "x".repeat(crate::MAX_GROUP_BYTES as usize + 1) }); + *producer.modify(key()).unwrap() = json!({ "big": "x".repeat(crate::MAX_GROUP_BYTES as usize + 1) }); // The publisher learns the cause at its next edit. The consumer drains the snapshot // that finished, then learns it from the aborted track. assert!(matches!( - producer.modify(None), + producer.modify(key()), Err(crate::Error::Net(moq_net::Error::FrameTooLarge)) )); assert!(matches!( @@ -932,9 +955,9 @@ mod test { #[test] fn a_failed_commit_leaves_the_track_open() { let (mut producer, track) = producer(cfg(false)); - producer.update(&json!({ "keep": true })).unwrap(); + producer.update(next(&json!({ "keep": true }))).unwrap(); - let mut guard = producer.modify(None).unwrap(); + let mut guard = producer.modify(key()).unwrap(); *guard = json!({ "big": "x".repeat(crate::MAX_GROUP_BYTES as usize + 1) }); assert!(matches!( guard.commit(), @@ -942,7 +965,7 @@ mod test { )); // The caller got the error, so the track stays usable for a smaller value. - *producer.modify(None).unwrap() = json!({ "keep": false }); + *producer.modify(key()).unwrap() = json!({ "keep": false }); let _finished = producer.finish().unwrap(); assert_eq!(drain(track).last().unwrap(), &json!({ "keep": false })); } @@ -950,17 +973,18 @@ mod test { #[test] fn a_panicking_edit_does_not_publish() { let (mut producer, mut track) = producer(cfg(false)); - producer.update(&json!({ "a": 1 })).unwrap(); + producer.update(keyed(&json!({ "a": 1 }), 0)).unwrap(); let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let mut guard = producer.modify(None).unwrap(); + let mut guard = producer.modify(moq_net::Timestamp::unscaled(1)).unwrap(); *guard = json!({ "a": 2 }); panic!("torn edit"); })); assert!(panicked.is_err()); - // A discarded edit is not a publish failure, so the track is still open for the next one. - *producer.modify(None).unwrap() = json!({ "a": 3 }); + // A discarded edit is not a publish failure, so the track is still open for the next one, at + // the key the torn edit would have taken had it published. + *producer.modify(moq_net::Timestamp::unscaled(1)).unwrap() = json!({ "a": 3 }); let _finished = producer.finish().unwrap(); assert_eq!( track.largest().now_or_never().flatten().map(|position| position.group), @@ -978,16 +1002,16 @@ mod test { let ms = |n| moq_net::Timestamp::from_millis(n).unwrap(); let (mut producer, _track) = producer_on(cfg(true), timed()); producer - .mutate(|value| *value = json!({ "video": "v1" }), Some(ms(100))) + .mutate(|value| *value = json!({ "video": "v1" }), ms(100)) .unwrap(); - // Untimed on a timed track, so the frame is rejected before it reaches a group. + // Unscaled on a timed track, so the frame is rejected before it reaches a group. assert!(matches!( - producer.mutate(|value| value["scte35"] = json!(42), None), + producer.mutate(|value| value["scte35"] = json!(42), key()), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); - let guard = producer.modify(Some(ms(101))).unwrap(); + let guard = producer.modify(ms(101)).unwrap(); assert_eq!(*guard, json!({ "video": "v1" })); } @@ -997,34 +1021,39 @@ mod test { #[test] fn update_after_finish_errors_instead_of_panicking() { let (mut producer, _track) = producer(cfg(true)); - producer.update(&json!({ "a": 1 })).unwrap(); - producer.update(&json!({ "a": 2 })).unwrap(); // a delta, so a group is open + producer.update(next(&json!({ "a": 1 }))).unwrap(); + producer.update(next(&json!({ "a": 2 }))).unwrap(); // a delta, so a group is open let _finished = producer.clone().finish().unwrap(); - assert!(matches!(producer.update(&json!({ "a": 3 })), Err(crate::Error::Net(_)))); + assert!(matches!( + producer.update(next(&json!({ "a": 3 }))), + Err(crate::Error::Net(_)) + )); } - /// A timed track whose first group already starts at ten seconds, so a group that a frame at - /// [`rewound`] opens is published by `append_group` and then refused by `write_frame` for - /// starting before it. That stands in for any post-`append_group` write failure (the reported - /// one is a frame over moq-net's 32 MB per-group cache) without allocating 32 MB to provoke it. + /// A timed track whose first group starts at 0 but whose first frame is at ten seconds, so a + /// group a frame at [`rewound`] opens is published by `append_group` and then refused by + /// `write_frame` for starting before that frame. That stands in for any post-`append_group` + /// write failure (the reported one is a frame over moq-net's 32 MB per-group cache) without + /// allocating 32 MB to provoke it. fn rejecting_track() -> moq_net::track::Producer { let track = moq_net::broadcast::Info::new() .produce() .create_track("test", timed()) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group .write_frame( moq_net::Timestamp::from_millis(10_000).unwrap(), bytes::Bytes::from_static(b"{}"), ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); track } - /// Before [`rejecting_track`]'s first group starts, so a group opened at it is refused. + /// Past [`rejecting_track`]'s first group start but before its first frame, so a group opened + /// at it is refused. fn rewound() -> moq_net::Timestamp { moq_net::Timestamp::from_millis(1_000).unwrap() } @@ -1039,7 +1068,7 @@ mod test { let mut producer = Producer::::new(track, cfg(false)); assert!(matches!( - producer.update(moq_net::Timed::from(&json!({ "a": 1 })).at(rewound())), + producer.update(crate::Timed::new(&json!({ "a": 1 }), rewound())), Err(crate::Error::Net(_)) )); @@ -1067,7 +1096,7 @@ mod test { let mut consumer = track.subscribe(None); let mut producer = Producer::::new(track, cfg(false)); - let mut guard = producer.modify(None).unwrap(); + let mut guard = producer.modify(key()).unwrap(); guard.a = 1; guard.commit().unwrap(); @@ -1085,22 +1114,22 @@ mod test { #[test] fn newer_group_supersedes_in_progress_reconstruction() { - // A couple of deltas fill group 0, then a cut leaves the last update to open group 1. + // A couple of deltas fill group 0, then a cut leaves the last update to open group 3. let (mut producer, track) = producer(cfg(true)); let mut observer = producer.consume(); let mut consumer = Consumer::::new(track, consumer::Config::default()); let waiter = kio::Waiter::noop(); - producer.update(&json!({ "a": 1 })).unwrap(); // snapshot, group 0 + producer.update(keyed(&json!({ "a": 1 }), 0)).unwrap(); // snapshot, group 0 match consumer.poll_latest(&waiter) { Poll::Ready(Ok(Some(value))) => assert_eq!(value.value, json!({ "a": 1 })), other => panic!("expected first value, got {other:?}"), } - producer.update(&json!({ "a": 2 })).unwrap(); // delta in group 0 - producer.update(&json!({ "a": 3 })).unwrap(); // delta in group 0 + producer.update(keyed(&json!({ "a": 2 }), 1)).unwrap(); // delta in group 0 + producer.update(keyed(&json!({ "a": 3 }), 2)).unwrap(); // delta in group 0 producer.cut().unwrap(); - producer.update(&json!({ "a": 4 })).unwrap(); // snapshot, group 1 + producer.update(keyed(&json!({ "a": 4 }), 3)).unwrap(); // snapshot, group 3 let _finished = producer.finish().unwrap(); assert_eq!( observer @@ -1108,7 +1137,7 @@ mod test { .now_or_never() .flatten() .map(|position| position.group), - Some(1) + Some(3) ); // The consumer jumps to the newest group and never yields a stale value. @@ -1127,7 +1156,7 @@ mod test { .produce() .create_track("test", timed()) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); let mut consumer = Consumer::::new(track.consume(), consumer::Config::default()); let _track = track.finish().unwrap(); let waiter = kio::Waiter::noop(); @@ -1141,7 +1170,7 @@ mod test { Bytes::from(serde_json::to_vec(&json!({ "a": 1 })).unwrap()), ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); match consumer.poll_latest(&waiter) { Poll::Ready(Ok(Some(value))) => assert_eq!(value.value, json!({ "a": 1 })), @@ -1155,7 +1184,7 @@ mod test { // apply them all but yield only the latest value once, not replay every superseded state. let (mut producer, mut track) = producer(cfg(true)); for n in 0..=20 { - producer.update(&json!({ "n": n })).unwrap(); + producer.update(next(&json!({ "n": n }))).unwrap(); } let _finished = producer.finish().unwrap(); @@ -1177,7 +1206,7 @@ mod test { // Same collapse, read through moq-net's inflated copy of the group. let (mut producer, mut track) = producer_on(cfg(true), compressed()); for n in 0..=20 { - producer.update(&json!({ "n": n })).unwrap(); + producer.update(next(&json!({ "n": n }))).unwrap(); } let _finished = producer.finish().unwrap(); @@ -1196,8 +1225,8 @@ mod test { #[test] fn compressed_snapshot_per_group_roundtrips() { let (mut producer, mut track) = producer_on(cfg(false), compressed()); - producer.update(&json!({ "a": 1 })).unwrap(); - producer.update(&json!({ "a": 2 })).unwrap(); + producer.update(next(&json!({ "a": 1 }))).unwrap(); + producer.update(next(&json!({ "a": 2 }))).unwrap(); let _finished = producer.finish().unwrap(); // Deltas disabled: one compressed snapshot per group, latest reconstructs identically. @@ -1215,19 +1244,20 @@ mod test { fn a_compressed_refresh_opens_a_fresh_group() { let (mut producer, mut track) = producer_on(cfg(true), compressed().with_refresh(1)); for n in 0..=40 { - producer.update(&json!({ "n": n })).unwrap(); + producer.update(keyed(&json!({ "n": n }), n)).unwrap(); } let late = Consumer::::new(track.clone(), consumer::Config::default()); assert!(matches!( producer.poll_refresh(&kio::Waiter::noop()), Poll::Ready(Ok(())) )); - producer.update(&json!({ "n": 41 })).unwrap(); + producer.update(keyed(&json!({ "n": 41 }), 41)).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!( track.largest().now_or_never().flatten().map(|position| position.group), - Some(1) + Some(40), + "restated at the newest key" ); assert_eq!(drain_with(late).last().unwrap(), &json!({ "n": 41 })); } @@ -1246,7 +1276,7 @@ mod test { } let (mut producer, track) = producer(cfg(false)); - producer.update(&json!({ "inner": { "count": 300 } })).unwrap(); + producer.update(next(&json!({ "inner": { "count": 300 } }))).unwrap(); let mut consumer = Consumer::::new(track, consumer::Config::default()); let Poll::Ready(Err(err)) = consumer.poll_next(&kio::Waiter::noop()) else { @@ -1261,7 +1291,7 @@ mod test { #[test] fn rejected_root_omits_the_path() { let (mut producer, track) = producer(cfg(false)); - producer.update(&json!("not a map")).unwrap(); + producer.update(next(&json!("not a map"))).unwrap(); let mut consumer = Consumer::>::new(track, consumer::Config::default()); let Poll::Ready(Err(err)) = consumer.poll_next(&kio::Waiter::noop()) else { diff --git a/rs/moq-json/src/snapshot/producer.rs b/rs/moq-json/src/snapshot/producer.rs index 84c9249ba82..0b840024fc4 100644 --- a/rs/moq-json/src/snapshot/producer.rs +++ b/rs/moq-json/src/snapshot/producer.rs @@ -8,8 +8,8 @@ use serde::Serialize; use serde::de::DeserializeOwned; use super::{Encoded, Encoder}; +use crate::Timed; use crate::refresh::Refresh; -use moq_net::Timed; use crate::{Error, Result}; @@ -92,8 +92,10 @@ impl Producer { /// A join wants one once the open group's deltas outweigh the track's /// [`refresh`](moq_net::track::Info::refresh) ratio times its snapshot (see /// [`Demand::wanted`](moq_net::track::Demand::wanted)). It is a soft trigger: that joiner still - /// reads the cached group, then the new one. The snapshot carries the newest timestamp written, so the new group never starts before the last. - /// Nothing happens before the first update, or on a track without a ratio. + /// reads the cached group, then the new one. The snapshot carries the newest timestamp written, + /// where its group starts; while that is the newest group's own start, the refresh waits for a + /// later update, since two groups never share a start. Nothing happens before the first update, + /// or on a track without a ratio. /// /// The producer has no runtime of its own, so the owner drives this: loop on it, or select on it /// beside the updates. Errors once the track is closed. @@ -106,14 +108,14 @@ impl Producer { /// /// Returns the encoded size of the frame written, or `None` if the value is unchanged from the /// previous publish and nothing was written. - pub fn update<'a>(&mut self, value: impl Into>) -> Result> + pub fn update<'a>(&mut self, value: Timed<&'a T>) -> Result> where T: 'a, { - take(&self.inner).update(value.into()) + take(&self.inner).update(value) } - /// Edit the current value in place and publish at `at`, or leave it untimed. + /// Edit the current value in place and publish it at `at` (see [`Timed::at`]). /// /// The closure receives the current value: everything published through this producer so far, /// composed, or `T::default()` if nothing has been. Edit it in place; on return the result is @@ -131,7 +133,7 @@ impl Producer { /// from the latest value, so their sections compose instead of clobbering one another. This is /// [`modify`](Self::modify) opened and committed for you; take the guard to hold the lock across /// several edits, and [`update`](Self::update) to replace the whole value. - pub fn mutate(&mut self, f: impl FnOnce(&mut T), at: Option) -> Result<()> + pub fn mutate(&mut self, f: impl FnOnce(&mut T), at: moq_net::Timestamp) -> Result<()> where T: Default + DeserializeOwned, { @@ -140,7 +142,7 @@ impl Producer { guard.commit() } - /// Edit the current value, publishing on drop at `at`, or untimed when absent. + /// Edit the current value, publishing on drop at `at` (see [`Timed::at`]). /// /// The returned [`Guard`] derefs to the current value: everything published through this producer /// so far, composed, or `T::default()` if nothing has been. Editing it through [`DerefMut`] marks @@ -163,7 +165,7 @@ impl Producer { /// consumers see it instead of a stale value, and the next `modify` returns it here. Call /// [`Guard::commit`] to get the error back immediately instead. A panic while the guard is held /// discards the in-flight edit rather than publishing a torn value. - pub fn modify(&mut self, at: Option) -> Result> + pub fn modify(&mut self, at: moq_net::Timestamp) -> Result> where T: Default + DeserializeOwned, { @@ -186,7 +188,7 @@ impl Producer { }) } - /// Finish the open group, so the deltas already written stop being provisional. + /// Close the open group, so the deltas already written stop being provisional. /// /// No replacement group opens until the next [`update`](Self::update) or [`poll_refresh`](Self::poll_refresh), which emits a full /// snapshot as its first frame even when the value is unchanged. A consumer joining at that group @@ -270,7 +272,7 @@ pub struct Guard<'a, T: Serialize> { inner: MutexGuard<'a, Inner>, value: T, dirty: bool, - at: Option, + at: moq_net::Timestamp, } impl Guard<'_, T> { @@ -341,6 +343,7 @@ struct Inner { /// The newest timestamp written, which a refresh restates the value at. An update may carry an /// older timestamp than its group's first frame, and a new group may not start before that one. + /// `None` before the first write. at: Option, /// Whether this producer still publishes, or how it ended the track. @@ -382,13 +385,13 @@ impl Inner { self.state.abort(err); } - /// Finish the open group, leaving the next update to open a replacement with a full snapshot. + /// Close the open group, leaving the next update to open a replacement with a full snapshot. fn cut(&mut self) -> Result<()> { if self.track.group.is_none() { return Ok(()); } - // The group closes either way below, so reset first: a `finish` error must not leave the + // The group closes either way below, so reset first: a `close` error must not leave the // encoder emitting deltas against a snapshot whose group is gone. self.encoder.reset(); self.track.cut() @@ -413,7 +416,10 @@ impl Inner { track.write(timestamp, &frame)?; let size = frame.payload.len(); frame.commit(); - self.at = self.at.max(timestamp); + if self.at.is_none_or(|newest| timestamp > newest) { + self.at = Some(timestamp); + self.refresh.advanced(); + } Ok(Some(size)) } @@ -423,12 +429,26 @@ impl Inner { self.open()?; std::task::ready!(self.refresh.poll(self.track.largest, waiter))?; - let Inner { track, encoder, at, .. } = self; + let Inner { + track, + encoder, + at, + refresh, + .. + } = self; + // Two groups never share a start, so a refresh at the newest group's own start waits for a + // later write. Checked before encoding, since a snapshot dropped unwritten resets the encoder. + let Some(at) = *at else { + return refresh.park(waiter); + }; + if !crate::starts_past(&track.inner, at) { + return refresh.park(waiter); + } // A group was written, so a value was encoded and there is one to restate. let Some(frame) = encoder.snapshot()? else { return std::task::Poll::Pending; }; - track.write(*at, &frame)?; + track.write(at, &frame)?; frame.commit(); std::task::Poll::Ready(Ok(())) } @@ -460,7 +480,7 @@ struct Track { /// The group a delta would be appended to, open only while deltas are enabled. group: Option, - /// The sequence of the newest group written, which a refresh has to pass. + /// The start of the newest group written, which a refresh has to pass. largest: Option, /// Whether the encoder can emit deltas at all. With them off every frame is a snapshot, so a @@ -469,27 +489,27 @@ struct Track { } impl Track { - /// Finish the open group without opening a replacement. + /// Close the open group without opening a replacement. fn cut(&mut self) -> Result<()> { if let Some(group) = self.group.take() { - group.finish()?; + group.close()?; } Ok(()) } /// `timestamp` at the track's timescale, or [`moq_net::Error::TimestampMismatch`] when it is - /// missing on a timed track, present on an untimed one, or beyond what the scale can hold. + /// unscaled on a timed track, scaled on a scale-less one, or beyond what the scale can hold. /// /// An update checks this before encoding. Found by `write_frame` instead, the mismatch would /// come after a snapshot published its group, leaving an empty newest group in place of the /// last value, and the dropped frame would reset the encoder, rolling a group for nothing. - fn rescale(&self, timestamp: Option) -> Result> { + fn rescale(&self, timestamp: moq_net::Timestamp) -> Result { Ok(self.inner.info().rescale(timestamp)?) } /// Write one encoded frame at a timestamp already [rescaled](Self::rescale) to the track, rolling a /// group when it's a snapshot. - fn write(&mut self, timestamp: Option, encoded: &Encoded) -> Result<()> { + fn write(&mut self, timestamp: moq_net::Timestamp, encoded: &Encoded) -> Result<()> { // Check before touching a group. `write_snapshot` closes the previous group and publishes a // new one before the frame is written, so discovering the limit inside `write_frame` would // leave an empty newest group behind: a snapshot consumer jumps to the newest, so the previous @@ -504,20 +524,26 @@ impl Track { } } - /// Close the open group and write a snapshot as the first frame of a new one. - fn write_snapshot(&mut self, timestamp: Option, payload: bytes::Bytes) -> Result<()> { + /// Write a snapshot as the first frame of a new group starting at `timestamp`, then close the + /// previous group. + /// + /// A start at or before the newest group's is [`moq_net::Error::TimestampMismatch`], refused + /// before the previous group closes. + fn write_snapshot(&mut self, timestamp: moq_net::Timestamp, payload: bytes::Bytes) -> Result<()> { + let start = timestamp.value(); + let mut group = self.inner.append_group(start)?; + self.largest = Some(start); + // The previous group is complete; no more frames will be appended to it. - if let Some(group) = self.group.take() { - group.finish()?; + if let Some(previous) = self.group.take() { + previous.close()?; } - let mut group = self.inner.append_group()?; - self.largest = Some(group.sequence); if let Err(err) = group.write_frame(timestamp, payload) { // `append_group` already published this group, and a rejected frame (too large) doesn't // close the track. Dropping the handle does NOT close the group, so leaving it would strand // any subscriber that advanced into it with nothing to read and no end. - let _ = group.finish(); + let _ = group.close(); return Err(err.into()); } @@ -525,24 +551,25 @@ impl Track { // Keep the group open so future deltas can be appended to it. true => self.group = Some(group), // One frame per group, identical to a plain JSON track. - false => drop(group.finish()?), + false => drop(group.close()?), } Ok(()) } /// Append a delta to the group the last snapshot opened. - fn write_delta(&mut self, timestamp: Option, payload: bytes::Bytes) -> Result<()> { - self.group + fn write_delta(&mut self, timestamp: moq_net::Timestamp, payload: bytes::Bytes) -> Result<()> { + let group = self + .group .as_mut() - .expect("the encoder only emits a delta after a snapshot opened a group") - .write_frame(timestamp, payload)?; + .expect("the encoder only emits a delta after a snapshot opened a group"); + group.write_frame(crate::frame_time(group, timestamp), payload)?; Ok(()) } fn finish(&mut self) -> Result<()> { if let Some(group) = self.group.take() { - group.finish()?; + group.close()?; } // Another handle may have declared the end already, which stands. This shared state // keeps its own clone, whose appends the end refuses. diff --git a/rs/moq-json/src/stream/consumer.rs b/rs/moq-json/src/stream/consumer.rs index 5aa22682b69..306ff923d2c 100644 --- a/rs/moq-json/src/stream/consumer.rs +++ b/rs/moq-json/src/stream/consumer.rs @@ -49,7 +49,7 @@ impl Consumer { /// Get the next record and its exact frame timestamp, or `None` once the track ends. #[cfg_attr(feature = "async", kio::wrapper)] - pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>> { + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>> { loop { if self.rolled { return Poll::Ready(Err(crate::Error::Rolled)); @@ -74,7 +74,7 @@ impl Consumer { match group.poll_read_frame(waiter)? { Poll::Ready(Some(frame)) => { - return Poll::Ready(Ok(Some(moq_net::Timed { + return Poll::Ready(Ok(Some(crate::Timed { value: self.decoder.decode(&frame.payload)?, at: frame.timestamp, }))); @@ -124,9 +124,9 @@ mod timestamps { for (n, ticks) in times.into_iter().enumerate() { let at = moq_net::Timestamp::new(ticks, scale).unwrap(); producer - .append(moq_net::Timed { + .append(crate::Timed { value: &serde_json::json!({"n": n, "padding": "large-repeated-value-for-deltas"}), - at: Some(at), + at, }) .unwrap(); } @@ -136,9 +136,9 @@ mod timestamps { panic!("missing state {n}") }; assert_eq!(value.value["n"].as_u64().unwrap(), n as u64); - let at = value.at.unwrap(); + let at = value.at; assert_eq!(at.value(), ticks); - assert_eq!(at.scale(), scale); + assert_eq!(at.scale(), Some(scale)); } assert!(matches!( consumer.poll_next(&kio::Waiter::noop()), diff --git a/rs/moq-json/src/stream/mod.rs b/rs/moq-json/src/stream/mod.rs index a8b242edb15..38a8bae395a 100644 --- a/rs/moq-json/src/stream/mod.rs +++ b/rs/moq-json/src/stream/mod.rs @@ -50,25 +50,33 @@ mod test { use super::*; + /// `value` at the next key of a scale-less track, whose writes name their group keys: keys + /// only have to increase, so one counter serves every track a test writes. + fn next(value: V) -> crate::Timed { + crate::Timed::new(value, key()) + } + + /// The next key of a scale-less track; see [`next`]. + fn key() -> moq_net::Timestamp { + thread_local!(static KEY: std::cell::Cell = const { std::cell::Cell::new(0) }); + moq_net::Timestamp::unscaled(KEY.with(|key| key.replace(key.get() + 1))) + } + /// Refused before the log's group opens, so the log carries on rather than ending. #[test] fn a_timed_track_refuses_a_bare_publish() { let (mut producer, track) = producer_on(Config::default(), timed()); assert!(matches!( - producer.append(&json!({ "n": 0 })), + producer.append(next(&json!({ "n": 0 }))), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); let at = moq_net::Timestamp::from_millis(100).unwrap(); - producer - .append(moq_net::Timed::from(&json!({ "n": 1 })).at(at)) - .unwrap(); + producer.append(crate::Timed::new(&json!({ "n": 1 }), at)).unwrap(); assert!(matches!( - producer.append(&json!({ "n": 2 })), + producer.append(next(&json!({ "n": 2 }))), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); - producer - .append(moq_net::Timed::from(&json!({ "n": 3 })).at(at)) - .unwrap(); + producer.append(crate::Timed::new(&json!({ "n": 3 }), at)).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!(drain(consume(track)), vec![json!({ "n": 1 }), json!({ "n": 3 })]); } @@ -80,16 +88,12 @@ mod test { let (mut producer, track) = producer_on(Config::default(), timed()); let at = moq_net::Timestamp::from_millis(100).unwrap(); let huge = moq_net::Timestamp::from_scale((1 << 61) - 1, 1).unwrap(); - producer - .append(moq_net::Timed::from(&json!({ "n": 1 })).at(at)) - .unwrap(); + producer.append(crate::Timed::new(&json!({ "n": 1 }), at)).unwrap(); assert!(matches!( - producer.append(moq_net::Timed::from(&json!({ "n": 2 })).at(huge)), + producer.append(crate::Timed::new(&json!({ "n": 2 }), huge)), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); - producer - .append(moq_net::Timed::from(&json!({ "n": 3 })).at(at)) - .unwrap(); + producer.append(crate::Timed::new(&json!({ "n": 3 }), at)).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!(drain(consume(track)), vec![json!({ "n": 1 }), json!({ "n": 3 })]); } @@ -102,7 +106,7 @@ mod test { .unwrap(); let mut groups = track.subscribe(None); let mut producer = Producer::new(track, Config::default()); - producer.append(&json!({ "n": 1 })).unwrap(); + producer.append(next(&json!({ "n": 1 }))).unwrap(); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { panic!("expected an untimed group"); @@ -110,7 +114,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected an untimed frame"); }; - assert_eq!(frame.timestamp, None); + assert_eq!(frame.timestamp.scale(), None); } /// A millisecond track: the frames written here carry timestamps. @@ -168,7 +172,7 @@ mod test { fn plaintext_roundtrip_in_order() { let (mut producer, track) = producer(Config::default()); for n in 0..5 { - producer.append(&json!({ "n": n })).unwrap(); + producer.append(next(&json!({ "n": n }))).unwrap(); } let _finished = producer.finish().unwrap(); @@ -186,7 +190,7 @@ mod test { let mut groups = producer.consume(); let captured = moq_net::Timestamp::from_millis(1_234).unwrap(); let record = json!({ "n": 1 }); - let size = producer.append(moq_net::Timed::from(&record).at(captured)).unwrap(); + let size = producer.append(crate::Timed::new(&record, captured)).unwrap(); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { @@ -195,7 +199,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected a frame"); }; - assert_eq!(frame.timestamp.unwrap().as_micros(), captured.as_micros()); + assert_eq!(frame.timestamp.as_micros().unwrap(), captured.as_micros().unwrap()); assert_eq!(size, frame.payload.len()); } @@ -203,7 +207,7 @@ mod test { fn all_records_ride_one_group() { let (mut producer, mut track) = producer_on(Config::default(), compressed()); for n in 0..50 { - producer.append(&json!({ "n": n })).unwrap(); + producer.append(next(&json!({ "n": n }))).unwrap(); } let _finished = producer.finish().unwrap(); @@ -222,7 +226,7 @@ mod test { let waiter = kio::Waiter::noop(); for n in 0..3 { - producer.append(&json!({ "n": n })).unwrap(); + producer.append(next(&json!({ "n": n }))).unwrap(); match consumer.poll_next(&waiter) { Poll::Ready(Ok(Some(value))) => assert_eq!(value.value, json!({ "n": n })), other => panic!("expected record, got {other:?}"), @@ -249,7 +253,7 @@ mod test { bad.insert((1, 2), 3); assert!( producer - .append(moq_net::Timed::from(&bad).at(moq_net::Timestamp::from_millis(0).unwrap())) + .append(crate::Timed::new(&bad, moq_net::Timestamp::from_millis(0).unwrap())) .is_err() ); @@ -270,34 +274,36 @@ mod test { ); } - /// A timed track whose first group already starts at ten seconds, so a group that a frame at - /// [`rewound`] opens is published by `append_group` and then refused by `write_frame` for - /// starting before it. That stands in for any post-`append_group` write failure (the reported - /// one is a frame over moq-net's 32 MB per-group cache) without allocating 32 MB to provoke it. + /// A timed track whose first group starts at 0 but whose first frame is at ten seconds, so a + /// group a frame at [`rewound`] opens is published by `append_group` and then refused by + /// `write_frame` for starting before that frame. That stands in for any post-`append_group` + /// write failure (the reported one is a frame over moq-net's 32 MB per-group cache) without + /// allocating 32 MB to provoke it. fn rejecting_track() -> moq_net::track::Producer { let track = moq_net::broadcast::Info::new() .produce() .create_track("test", timed()) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group .write_frame( moq_net::Timestamp::from_millis(10_000).unwrap(), bytes::Bytes::from_static(b"{}"), ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); track } - /// Before [`rejecting_track`]'s first group starts, so a group opened at it is refused. + /// Past [`rejecting_track`]'s first group start but before its first frame, so a group opened + /// at it is refused. fn rewound() -> moq_net::Timestamp { moq_net::Timestamp::from_millis(1_000).unwrap() } /// `value` at [`rewound`], so the append gets as far as a published group before it fails. - fn late(value: &Value) -> moq_net::Timed<&Value> { - moq_net::Timed::from(value).at(rewound()) + fn late(value: &Value) -> crate::Timed<&Value> { + crate::Timed::new(value, rewound()) } /// A failed write must reach the consumer, not just the caller. A clean close drains a reader to @@ -358,10 +364,10 @@ mod test { #[test] fn appending_after_finish_fails_without_aborting() { let (mut producer, _track) = producer_on(Config::default(), compressed()); - producer.append(&json!({ "n": 0 })).unwrap(); + producer.append(next(&json!({ "n": 0 }))).unwrap(); let _finished = producer.clone().finish().unwrap(); - assert!(producer.append(&json!({ "n": 1 })).is_err()); + assert!(producer.append(next(&json!({ "n": 1 }))).is_err()); assert_eq!(drain(consume(producer.consume())), vec![json!({ "n": 0 })]); } @@ -384,13 +390,13 @@ mod test { let subscriber = track.subscribe(subscription); // Both groups stay open, the way a publisher writing to two at once leaves them. - let mut first = track.append_group().unwrap(); + let mut first = track.append_group(0).unwrap(); first - .write_frame(moq_net::Timestamp::now(), br#"{"n":0}"#.as_slice()) + .write_frame(moq_net::Timestamp::from_millis(0).unwrap(), br#"{"n":0}"#.as_slice()) .unwrap(); - let mut second = track.append_group().unwrap(); + let mut second = track.append_group(1).unwrap(); second - .write_frame(moq_net::Timestamp::now(), br#"{"n":1}"#.as_slice()) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), br#"{"n":1}"#.as_slice()) .unwrap(); let mut consumer = consume(subscriber); @@ -407,7 +413,7 @@ mod test { // Sticky: a later read must not report the rest of the first group as a whole log. first - .write_frame(moq_net::Timestamp::now(), br#"{"n":2}"#.as_slice()) + .write_frame(moq_net::Timestamp::from_millis(0).unwrap(), br#"{"n":2}"#.as_slice()) .unwrap(); assert!(matches!( consumer.poll_next(&waiter), @@ -433,12 +439,15 @@ mod test { for compression in [moq_net::track::Compression::None, moq_net::track::Compression::Deflate] { let (mut producer, track) = producer_on(Config::default(), info(compression)); producer.set_max_frame_bytes(BUDGET); - producer.append(&json!({ "n": 0 })).unwrap(); + producer.append(next(&json!({ "n": 0 }))).unwrap(); let oversized = Value::String("x".repeat(BUDGET as usize)); - assert!(refused(producer.append(&oversized)), "compression={compression:?}"); + assert!( + refused(producer.append(next(&oversized))), + "compression={compression:?}" + ); - producer.append(&json!({ "n": 1 })).unwrap(); + producer.append(next(&json!({ "n": 1 }))).unwrap(); let _finished = producer.finish().unwrap(); assert_eq!( @@ -455,7 +464,7 @@ mod test { for compression in [moq_net::track::Compression::None, moq_net::track::Compression::Deflate] { let (mut producer, track) = producer_on(Config::default(), info(compression)); for n in 0..10_000 { - producer.append(&json!({ "n": n })).unwrap(); + producer.append(next(&json!({ "n": n }))).unwrap(); } let _finished = producer.finish().unwrap(); @@ -472,7 +481,7 @@ mod test { let (mut producer, track) = producer_on(Config::default(), compressed()); let value = json!({ "s": "line1\nline2\ttab", "u": "a\u{000a}b" }); for _ in 0..4 { - producer.append(&value).unwrap(); + producer.append(next(&value)).unwrap(); } let _finished = producer.finish().unwrap(); diff --git a/rs/moq-json/src/stream/producer.rs b/rs/moq-json/src/stream/producer.rs index d5e351a279b..e6c417b93a1 100644 --- a/rs/moq-json/src/stream/producer.rs +++ b/rs/moq-json/src/stream/producer.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use serde::Serialize; use super::Encoder; -use moq_net::Timed; +use crate::Timed; use crate::Result; @@ -69,8 +69,8 @@ impl Producer { /// /// The log accepts appends for its track's life. A record larger than a frame may be is refused /// with [`moq_net::Error::FrameTooLarge`] before anything is written, leaving the log intact. A - /// timestamp that doesn't match the track's timescale (missing on a timed track, present on an - /// untimed one, or beyond what its scale can hold) is refused with + /// timestamp that doesn't match the track's timescale (unscaled on a timed track, scaled on a + /// scale-less one, or beyond what its scale can hold) is refused with /// [`moq_net::Error::TimestampMismatch`] the same way, so a corrected append carries on. /// /// Any other failure ends the track. A log missing a record is not the lossless log a stream @@ -80,11 +80,11 @@ impl Producer { /// complete. Every later append fails on the ended track. /// /// Returns the encoded size of the frame written. - pub fn append<'a>(&mut self, value: impl Into>) -> Result + pub fn append<'a>(&mut self, value: Timed<&'a T>) -> Result where T: 'a, { - self.inner.lock().unwrap().append(value.into()) + self.inner.lock().unwrap().append(value) } /// Finish the track, closing the group, and return the [`Finished`] log. @@ -185,7 +185,7 @@ impl Inner { } }; - let opened = track.open(); + let opened = track.open(at); let published = opened.is_ok(); let result = match opened { Ok(()) => track.write(at, record.payload()), @@ -230,10 +230,11 @@ struct Track { } impl Track { - /// Open the log's group if it isn't already. - fn open(&mut self) -> std::result::Result<(), moq_net::Error> { + /// Open the log's group if it isn't already, starting at its first record's `timestamp`. + fn open(&mut self, timestamp: moq_net::Timestamp) -> std::result::Result<(), moq_net::Error> { if self.group.is_none() { - self.group = Some(self.inner.append_group()?); + let start = timestamp.value(); + self.group = Some(self.inner.append_group(start)?); } Ok(()) } @@ -241,11 +242,11 @@ impl Track { /// Append one encoded record to the log's group at `timestamp`. fn write( &mut self, - timestamp: Option, + timestamp: moq_net::Timestamp, payload: &bytes::Bytes, ) -> std::result::Result<(), moq_net::Error> { let group = self.group.as_mut().expect("a group is open"); - group.write_frame(timestamp, payload.clone()) + group.write_frame(crate::frame_time(group, timestamp), payload.clone()) } /// End the track with an error, so a consumer sees the failure rather than a clean end. @@ -272,7 +273,7 @@ impl Track { // would leave the track open with `group` already taken, so a later append would open a // second group. That is exactly the split log ending the track exists to prevent. let group = match self.group.take() { - Some(group) => group.finish().map(drop), + Some(group) => group.close().map(drop), None => Ok(()), }; // This shared state keeps its own clone, whose appends the end refuses. diff --git a/rs/moq-json/src/timed.rs b/rs/moq-json/src/timed.rs new file mode 100644 index 00000000000..14f7fadce52 --- /dev/null +++ b/rs/moq-json/src/timed.rs @@ -0,0 +1,23 @@ +use moq_net::Timestamp; + +/// A value to publish or read, and its frame's timestamp. +/// +/// On a timed track `at` is when the value was captured. A scale-less track has no time, so +/// there it is [`Timestamp::unscaled`] at a group key: a write that opens a group starts it +/// at that key, which must pass every group the track holds, as +/// [`append_group`](moq_net::track::Producer::append_group) requires. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Timed

{ + /// The value. + pub value: P, + + /// The frame's timestamp: its capture time, or its group key on a scale-less track. + pub at: Timestamp, +} + +impl

Timed

{ + /// `value` at `at`. + pub fn new(value: P, at: Timestamp) -> Self { + Self { value, at } + } +} diff --git a/rs/moq-json/src/window/consumer.rs b/rs/moq-json/src/window/consumer.rs index 3104a7c65b8..f1355071729 100644 --- a/rs/moq-json/src/window/consumer.rs +++ b/rs/moq-json/src/window/consumer.rs @@ -17,15 +17,16 @@ use crate::Result; /// window yields nothing for records already delivered, so this reads as one continuous stream of /// [`Event`]s regardless of how the publisher framed them. /// -/// Each event carries the timestamp of the frame that produced it, `None` on an untimed track. The -/// wire has no per-record time, so a record a reader first learns of from a group header (one it -/// joined after, or missed while behind) gets the header frame's time, not its original push time. +/// Each event carries the timestamp of the frame that produced it, its group key on a scale-less +/// track. The wire has no per-record time, so a record a reader first learns of from a group header +/// (one it joined after, or missed while behind) gets the header frame's time, not its original +/// push time. pub struct Consumer { track: moq_net::track::Ordered, group: Option, codec: Option, decoder: Decoder, - /// Timestamp of the frame whose events the decoder is draining. + /// Timestamp of the frame whose events the decoder is draining, `None` before the first. at: Option, } @@ -48,12 +49,13 @@ impl Consumer { /// Get the next event and its frame's timestamp, or `None` once the track ends. #[cfg_attr(feature = "async", kio::wrapper)] - pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>>> { + pub fn poll_next(&mut self, waiter: &kio::Waiter) -> Poll>>>> { loop { // Drain what the last frame produced before reading more, so every queued event is that // frame's and shares its timestamp. if let Some(value) = self.decoder.next_event() { - return Poll::Ready(Ok(Some(moq_net::Timed { value, at: self.at }))); + let at = self.at.expect("an event follows the frame that produced it"); + return Poll::Ready(Ok(Some(crate::Timed { value, at }))); } let Some(group) = &mut self.group else { @@ -88,8 +90,8 @@ impl Consumer { Poll::Ready(Err(err)) => return Poll::Ready(Err(err.into())), Poll::Ready(Ok(Some(frame))) => { let codec = self.codec.as_mut().expect("an open MoQ group has a window codec"); + self.at = Some(frame.timestamp); self.decoder.decode(codec, &frame.payload)?; - self.at = frame.timestamp; } Poll::Ready(Ok(None)) => { // This group is exhausted. Clear it and poll for a later one, which restates the diff --git a/rs/moq-json/src/window/mod.rs b/rs/moq-json/src/window/mod.rs index 8433ffc68d5..391330354b3 100644 --- a/rs/moq-json/src/window/mod.rs +++ b/rs/moq-json/src/window/mod.rs @@ -74,15 +74,30 @@ mod test { use super::*; + /// `value` at the next key of a scale-less track, whose writes name their group keys: keys + /// only have to increase, so one counter serves every track a test writes. + fn next(value: V) -> crate::Timed { + crate::Timed::new(value, key()) + } + + /// `value` at scale-less `key`. + fn keyed(value: V, key: u64) -> crate::Timed { + crate::Timed::new(value, moq_net::Timestamp::unscaled(key)) + } + + /// The next key of a scale-less track; see [`next`]. + fn key() -> moq_net::Timestamp { + thread_local!(static KEY: std::cell::Cell = const { std::cell::Cell::new(0) }); + moq_net::Timestamp::unscaled(KEY.with(|key| key.replace(key.get() + 1))) + } + #[test] fn pushes_and_pops_keep_the_callers_timestamps() { let (mut producer, mut groups) = producer_on(ProducerConfig::default(), timed()); let first = moq_net::Timestamp::from_millis(100).unwrap(); let second = moq_net::Timestamp::from_millis(200).unwrap(); - producer - .push(moq_net::Timed::from(&json!({ "n": 1 })).at(first)) - .unwrap(); - producer.pop(moq_net::Timed::from(1u64).at(second)).unwrap(); + producer.push(crate::Timed::new(&json!({ "n": 1 }), first)).unwrap(); + producer.pop(crate::Timed::new(1u64, second)).unwrap(); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { panic!("expected the window group"); @@ -91,7 +106,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected the timestamped window edit"); }; - assert_eq!(frame.timestamp, Some(at)); + assert_eq!(frame.timestamp, at); } } @@ -100,7 +115,7 @@ mod test { fn a_timed_track_refuses_a_bare_publish() { let (mut producer, mut track) = producer_on(ProducerConfig::default(), timed()); assert!(matches!( - producer.push(&json!({ "n": 0 })), + producer.push(next(&json!({ "n": 0 }))), Err(crate::Error::Net(moq_net::Error::TimestampMismatch)) )); assert_eq!( @@ -109,8 +124,8 @@ mod test { "a refused header published a group" ); let at = moq_net::Timestamp::from_millis(100).unwrap(); - producer.push(moq_net::Timed::from(&rec(1)).at(at)).unwrap(); - producer.push(moq_net::Timed::from(&rec(2)).at(at)).unwrap(); + producer.push(crate::Timed::new(&rec(1), at)).unwrap(); + producer.push(crate::Timed::new(&rec(2), at)).unwrap(); let _finished = producer.finish().unwrap(); let pushed: Vec<_> = drain(&mut consumer(track)) .into_iter() @@ -139,14 +154,14 @@ mod test { )); }; - refused(producer.push(moq_net::Timed::from(&rec(1)).at(huge))); + refused(producer.push(crate::Timed::new(&rec(1), huge))); assert_eq!(largest(&mut track), None, "a refused header published a group"); - producer.push(moq_net::Timed::from(&rec(1)).at(at)).unwrap(); - refused(producer.push(moq_net::Timed::from(&rec(2)).at(huge))); - refused(producer.pop(moq_net::Timed::from(1u64).at(huge))); - producer.push(moq_net::Timed::from(&rec(2)).at(at)).unwrap(); - assert_eq!(largest(&mut track), Some(0), "a refused op rolled the group"); + producer.push(crate::Timed::new(&rec(1), at)).unwrap(); + refused(producer.push(crate::Timed::new(&rec(2), huge))); + refused(producer.pop(crate::Timed::new(1u64, huge))); + producer.push(crate::Timed::new(&rec(2), at)).unwrap(); + assert_eq!(largest(&mut track), Some(100), "a refused op rolled the group"); let _finished = producer.finish().unwrap(); let pushed: Vec<_> = drain(&mut consumer(track)) @@ -170,9 +185,9 @@ mod test { let mut producer = Producer::new(track, ProducerConfig::default()); let at = |ms| moq_net::Timestamp::from_millis(ms).unwrap(); - producer.push(moq_net::Timed::from(&rec(1)).at(at(100))).unwrap(); - producer.push(moq_net::Timed::from(&rec(2)).at(at(200))).unwrap(); - producer.pop(moq_net::Timed::from(1u64).at(at(300))).unwrap(); + producer.push(crate::Timed::new(&rec(1), at(100))).unwrap(); + producer.push(crate::Timed::new(&rec(2), at(200))).unwrap(); + producer.pop(crate::Timed::new(1u64, at(300))).unwrap(); let events: Vec<_> = drain_timed(&mut live).into_iter().map(|e| (e.value, e.at)).collect(); assert_eq!( events, @@ -182,23 +197,23 @@ mod test { index: 0, value: rec(1) }, - Some(at(100)) + at(100) ), ( Event::Push { index: 1, value: rec(2) }, - Some(at(200)) + at(200) ), - (Event::Pop(0..1), Some(at(300))), + (Event::Pop(0..1), at(300)), ] ); // A reader joining at a later group learns of the retained record from its header, so it // gets the header frame's time rather than the record's original push time. producer.cut().unwrap(); - producer.push(moq_net::Timed::from(&rec(3)).at(at(400))).unwrap(); + producer.push(crate::Timed::new(&rec(3), at(400))).unwrap(); let late = late.with_range(1..); let mut late = consumer(late); let events: Vec<_> = drain_timed(&mut late).into_iter().map(|e| (e.value, e.at)).collect(); @@ -210,27 +225,27 @@ mod test { index: 1, value: rec(2) }, - Some(at(400)) + at(400) ), ( Event::Push { index: 2, value: rec(3) }, - Some(at(400)) + at(400) ), ] ); } #[test] - fn untimed_events_have_no_timestamp() { + fn scale_less_events_carry_their_keys() { let (mut producer, track) = producer(ProducerConfig::default()); - producer.push(&rec(1)).unwrap(); - producer.pop(1).unwrap(); + producer.push(next(&rec(1))).unwrap(); + producer.pop(next(1)).unwrap(); let events = drain_timed(&mut consumer(track)); assert_eq!(events.len(), 2); - assert!(events.iter().all(|event| event.at.is_none())); + assert!(events.iter().all(|event| event.at.scale().is_none())); } #[test] @@ -241,7 +256,7 @@ mod test { .unwrap(); let mut groups = track.subscribe(None); let mut producer = Producer::new(track, ProducerConfig::default()); - producer.push(&json!({ "n": 1 })).unwrap(); + producer.push(next(&json!({ "n": 1 }))).unwrap(); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { panic!("expected an untimed group"); @@ -249,7 +264,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("expected an untimed frame"); }; - assert_eq!(frame.timestamp, None); + assert_eq!(frame.timestamp.scale(), None); } /// A millisecond track: the frames written here carry timestamps. @@ -287,27 +302,29 @@ mod test { Consumer::new(track, ConsumerConfig::default()) } - /// A timed track whose first group already starts at ten seconds, so a group that a frame at - /// [`rewound`] opens is published by `append_group` and then refused by `write_frame` for - /// starting before it. That stands in for any post-`append_group` write failure (the reported - /// one is a frame over moq-net's 32 MB per-group cache) without allocating 32 MB to provoke it. + /// A timed track whose first group starts at 0 but whose first frame is at ten seconds, so a + /// group a frame at [`rewound`] opens is published by `append_group` and then refused by + /// `write_frame` for starting before that frame. That stands in for any post-`append_group` + /// write failure (the reported one is a frame over moq-net's 32 MB per-group cache) without + /// allocating 32 MB to provoke it. fn rejecting_track() -> moq_net::track::Producer { let track = moq_net::broadcast::Info::new() .produce() .create_track("test", timed()) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group .write_frame( moq_net::Timestamp::from_millis(10_000).unwrap(), bytes::Bytes::from_static(b"{}"), ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); track } - /// Before [`rejecting_track`]'s first group starts, so a group opened at it is refused. + /// Past [`rejecting_track`]'s first group start but before its first frame, so a group opened + /// at it is refused. fn rewound() -> moq_net::Timestamp { moq_net::Timestamp::from_millis(1_000).unwrap() } @@ -318,7 +335,7 @@ mod test { } /// Drain every event currently available, with its frame timestamp. - fn drain_timed(consumer: &mut Consumer) -> Vec>> { + fn drain_timed(consumer: &mut Consumer) -> Vec>> { let waiter = kio::Waiter::noop(); let mut out = Vec::new(); while let Poll::Ready(Ok(Some(event))) = consumer.poll_next(&waiter) { @@ -357,12 +374,12 @@ mod test { } fn push(&mut self, n: u64) { - self.producer.push(&rec(n)).unwrap(); + self.producer.push(next(&rec(n))).unwrap(); self.read(); } fn pop(&mut self, count: u64) { - self.producer.pop(count).unwrap(); + self.producer.pop(next(count)).unwrap(); self.read(); } @@ -425,9 +442,9 @@ mod test { fn the_window_slides() { let (mut producer, _track) = producer(ProducerConfig::default()); for n in 0..5 { - producer.push(&rec(n)).unwrap(); + producer.push(next(&rec(n))).unwrap(); if n >= 2 { - producer.pop(1).unwrap(); + producer.pop(next(1)).unwrap(); } } @@ -539,9 +556,9 @@ mod test { let mut producer = Producer::::new(track, ProducerConfig::default().with_ops(false)); for n in 0..5 { - producer.push(&rec(n)).unwrap(); + producer.push(next(&rec(n))).unwrap(); } - producer.pop(3).unwrap(); + producer.pop(next(3)).unwrap(); let mut subscriber = producer.consume(); let largest = subscriber .largest() @@ -660,7 +677,7 @@ mod test { let first = encoder.push(&rec(0)).unwrap(); let payload = first.payload.clone(); first.commit(); - let mut lost = track.append_group().unwrap(); + let mut lost = track.append_group(0).unwrap(); lost.write_frame(moq_net::Timestamp::ZERO, payload).unwrap(); assert_eq!( drain(&mut consumer), @@ -674,9 +691,11 @@ mod test { let second = encoder.push(&rec(1)).unwrap(); let payload = second.payload.clone(); second.commit(); - let mut checkpoint = track.append_group().unwrap(); - checkpoint.write_frame(moq_net::Timestamp::ZERO, payload).unwrap(); - checkpoint.finish().unwrap(); + let mut checkpoint = track.append_group(1).unwrap(); + checkpoint + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), payload) + .unwrap(); + checkpoint.close().unwrap(); let _track = track.finish().unwrap(); assert_eq!( @@ -700,13 +719,13 @@ mod test { ); // A 32 byte header and a 16 byte push: under the ratio. - producer.push(&rec(0)).unwrap(); - producer.push(&rec(1)).unwrap(); + producer.push(keyed(&rec(0), 0)).unwrap(); + producer.push(keyed(&rec(1), 1)).unwrap(); let _early = producer.consume(); assert!(producer.poll_refresh(&waiter).is_pending(), "a join under the ratio"); - producer.push(&rec(2)).unwrap(); - producer.pop(1).unwrap(); + producer.push(keyed(&rec(2), 2)).unwrap(); + producer.pop(keyed(1, 3)).unwrap(); assert!(producer.poll_refresh(&waiter).is_pending(), "no join, no refresh"); let mut late = producer.consume(); assert!(matches!(producer.poll_refresh(&waiter), Poll::Ready(Ok(())))); @@ -718,7 +737,8 @@ mod test { }) .last() .expect("the fresh group"); - assert_eq!(fresh.sequence, 1); + // Restated at the newest key written. + assert_eq!(fresh.start, 3); let Poll::Ready(Ok(Some(header))) = fresh.poll_read_frame(&waiter) else { panic!("expected the restating header"); }; @@ -728,7 +748,7 @@ mod test { ); // The next edit rides as an op in the fresh group. - producer.push(&rec(3)).unwrap(); + producer.push(next(&rec(3))).unwrap(); let Poll::Ready(Ok(Some(op))) = fresh.poll_read_frame(&waiter) else { panic!("expected an op in the fresh group"); }; @@ -745,9 +765,9 @@ mod test { let ms = |n| moq_net::Timestamp::from_millis(n).unwrap(); let (mut producer, _track) = producer_on(ProducerConfig::default(), timed().with_refresh(1)); for n in 0..4 { - producer.push(moq_net::Timed::from(&rec(n)).at(ms(100 + n))).unwrap(); + producer.push(crate::Timed::new(&rec(n), ms(100 + n))).unwrap(); } - producer.pop(moq_net::Timed::from(1).at(ms(200))).unwrap(); + producer.pop(crate::Timed::new(1, ms(200))).unwrap(); let mut late = producer.consume(); assert!(matches!(producer.poll_refresh(&waiter), Poll::Ready(Ok(())))); @@ -760,21 +780,29 @@ mod test { let Poll::Ready(Ok(Some(header))) = fresh.poll_read_frame(&waiter) else { panic!("expected the restating header"); }; - assert_eq!(header.timestamp, Some(ms(200))); + assert_eq!(header.timestamp, ms(200)); } - /// An op may carry a timestamp older than its group's first frame, but a new group may not start - /// before that frame, so the restating header takes the newest timestamp written. + /// An op may carry a timestamp older than its group's first frame, but a new group must start + /// past that frame, so a refresh whose newest timestamp is still the group's own start waits for + /// a later edit, then restates the window there. #[test] - fn a_refresh_after_an_older_edit_starts_at_the_newest_timestamp() { - let waiter = kio::Waiter::noop(); + fn a_refresh_after_older_edits_waits_for_a_later_one() { + let (waiter, woken) = crate::refresh::flagged(); let ms = |n| moq_net::Timestamp::from_millis(n).unwrap(); let (mut producer, _track) = producer_on(ProducerConfig::default(), timed().with_refresh(1)); for (n, at) in [130, 110, 120, 115].into_iter().enumerate() { - producer.push(moq_net::Timed::from(&rec(n as u64)).at(ms(at))).unwrap(); + producer.push(crate::Timed::new(&rec(n as u64), ms(at))).unwrap(); } let mut late = producer.consume(); + assert!( + producer.poll_refresh(&waiter).is_pending(), + "no group may share the open group's start" + ); + woken.take(); + producer.push(crate::Timed::new(&rec(4), ms(140))).unwrap(); + assert!(woken.take(), "the later edit wakes the parked refresh"); assert!(matches!(producer.poll_refresh(&waiter), Poll::Ready(Ok(())))); let mut fresh = std::iter::from_fn(|| match late.poll_recv_group(&waiter) { Poll::Ready(Ok(Some(group))) => Some(group), @@ -785,7 +813,8 @@ mod test { let Poll::Ready(Ok(Some(header))) = fresh.poll_read_frame(&waiter) else { panic!("expected the restating header"); }; - assert_eq!(header.timestamp, Some(ms(130))); + assert_eq!(fresh.start, 140); + assert_eq!(header.timestamp, ms(140)); } /// Without a ratio no join asks for a group, and a finished producer stops refreshing. @@ -793,7 +822,7 @@ mod test { fn refresh_needs_a_ratio_and_an_open_track() { let (mut producer, _track) = producer(ProducerConfig::default()); for n in 0..8 { - producer.push(&rec(n)).unwrap(); + producer.push(next(&rec(n))).unwrap(); } let _late = producer.consume(); assert!(producer.poll_refresh(&kio::Waiter::noop()).is_pending()); @@ -826,7 +855,7 @@ mod test { #[test] fn an_empty_pop_writes_nothing() { let (mut producer, mut track) = producer(ProducerConfig::default()); - producer.pop(5).unwrap(); + producer.pop(next(5)).unwrap(); let _finished = producer.finish().unwrap(); // Nothing was ever pushed, so there is nothing to drop and no group to publish. @@ -843,7 +872,7 @@ mod test { let mut producer = Producer::::new(track, ProducerConfig::default()); // Rewound past the first group, so the header group is published before the write fails. - assert!(producer.push(moq_net::Timed::from(&rec(1)).at(rewound())).is_err()); + assert!(producer.push(crate::Timed::new(&rec(1), rewound())).is_err()); assert_eq!(producer.range(), 0..0); assert!(producer.window().is_empty()); @@ -858,7 +887,7 @@ mod test { fn the_window_stays_readable_after_finish() { let (producer, mut track) = producer(ProducerConfig::default()); let mut clone = producer.clone(); - clone.push(&rec(1)).unwrap(); + clone.push(next(&rec(1))).unwrap(); let finished = producer.finish().unwrap(); assert_eq!(finished.window(), vec![rec(1)]); @@ -870,7 +899,7 @@ mod test { ); // A clone sharing the finished state refuses edits, and finishing it again does nothing. assert!(matches!( - clone.push(&rec(2)), + clone.push(next(&rec(2))), Err(crate::Error::Net(moq_net::Error::Closed)) )); let _finished = clone.finish().unwrap(); @@ -885,13 +914,13 @@ mod test { .create_track("test", moq_net::track::Info::default()) .unwrap(); let mut producer = Producer::::new(track.clone(), ProducerConfig::default()); - producer.push(&rec(1)).unwrap(); + producer.push(next(&rec(1))).unwrap(); // Another handle ends the track, so finishing it fails. track.abort(moq_net::Error::Cancel).unwrap(); let err = producer.clone().finish().err().unwrap().to_string(); assert_eq!(producer.clone().finish().err().unwrap().to_string(), err); - assert_eq!(producer.push(&rec(2)).unwrap_err().to_string(), err); + assert_eq!(producer.push(next(&rec(2))).unwrap_err().to_string(), err); } /// A track another handle already ended still has its open group closed by this producer's @@ -904,7 +933,7 @@ mod test { .unwrap(); let mut subscriber = track.subscribe(None).ordered(); let mut producer = Producer::::new(track.clone(), ProducerConfig::default()); - producer.push(&rec(1)).unwrap(); + producer.push(next(&rec(1))).unwrap(); let _track = track.finish().unwrap(); let _finished = producer.clone().finish().unwrap(); @@ -918,7 +947,7 @@ mod test { "the group ends rather than waiting for a producer that finished" ); assert!(matches!( - producer.push(&rec(2)), + producer.push(next(&rec(2))), Err(crate::Error::Net(moq_net::Error::Closed)) )); } @@ -927,11 +956,11 @@ mod test { #[test] fn edits_after_an_abort_report_its_cause() { let (mut producer, _track) = producer(ProducerConfig::default()); - producer.push(&rec(1)).unwrap(); + producer.push(next(&rec(1))).unwrap(); producer.clone().abort(moq_net::Error::Cancel).unwrap(); assert!(matches!( - producer.push(&rec(2)), + producer.push(next(&rec(2))), Err(crate::Error::Net(moq_net::Error::Cancel)) )); assert!(matches!( @@ -943,15 +972,15 @@ mod test { #[test] fn writes_after_another_clone_finishes_are_rejected() { let (mut producer, _track) = producer(ProducerConfig::default()); - producer.push(&rec(1)).unwrap(); + producer.push(next(&rec(1))).unwrap(); let _finished = producer.clone().finish().unwrap(); assert!(matches!( - producer.push(&rec(2)), + producer.push(next(&rec(2))), Err(crate::Error::Net(moq_net::Error::Closed)) )); assert!(matches!( - producer.pop(1), + producer.pop(next(1)), Err(crate::Error::Net(moq_net::Error::Closed)) )); assert_eq!(producer.window(), vec![rec(1)]); @@ -1043,11 +1072,11 @@ mod test { let mut producer = Producer::::new(track, ProducerConfig::default()); producer.cut().unwrap(); - producer.push(&rec(0)).unwrap(); - producer.push(&rec(1)).unwrap(); + producer.push(next(&rec(0))).unwrap(); + producer.push(next(&rec(1))).unwrap(); producer.cut().unwrap(); producer.cut().unwrap(); - producer.push(&rec(2)).unwrap(); + producer.push(next(&rec(2))).unwrap(); let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut first))) = groups.poll_recv_group(&waiter) else { diff --git a/rs/moq-json/src/window/producer.rs b/rs/moq-json/src/window/producer.rs index 2f3827541f6..bd7a0047869 100644 --- a/rs/moq-json/src/window/producer.rs +++ b/rs/moq-json/src/window/producer.rs @@ -3,7 +3,7 @@ use std::marker::PhantomData; use std::sync::{Arc, Mutex}; -use moq_net::Timed; +use crate::Timed; use serde::Serialize; use serde_json::Value; @@ -71,15 +71,15 @@ impl Producer { self.inner.lock().unwrap().encoder.range() } - /// Drop records from the front at `count`'s timestamp, or untimed when absent. + /// Drop records from the front at `count`'s timestamp (see [`Timed::at`]). /// /// A no-op when the window is already empty, and clamped to what it holds, so a caller can trim /// unconditionally. - pub fn pop(&mut self, count: impl Into>) -> Result<()> { - self.inner.lock().unwrap().pop(count.into()) + pub fn pop(&mut self, count: Timed) -> Result<()> { + self.inner.lock().unwrap().pop(count) } - /// Finish the open group, leaving the next edit to open a replacement with a header. + /// Close the open group, leaving the next edit to open a replacement with a header. /// /// Idempotent: cutting when no group is open does nothing. A caller that stores complete groups /// cuts after its edits, so every edit so far sits in a group no later frame can extend. @@ -92,8 +92,10 @@ impl Producer { /// A join wants one once the open group's ops outweigh the track's /// [`refresh`](moq_net::track::Info::refresh) ratio times its header (see /// [`Demand::wanted`](moq_net::track::Demand::wanted)). It is a soft trigger: that joiner still - /// reads the cached group, then the new one. The header carries the newest timestamp written. Nothing happens before - /// the first edit, or on a track without a ratio. + /// reads the cached group, then the new one. The header carries the newest timestamp written, where + /// its group starts; while that is the newest group's own start, the refresh waits for a later + /// edit, since two groups never share a start. Nothing happens before the first edit, or on a + /// track without a ratio. /// /// The producer has no runtime of its own, so the owner drives this: loop on it, or select on it /// beside the edits. Errors once the track is finished or closed. @@ -125,12 +127,12 @@ impl Producer { } impl Producer { - /// Append a record at its timestamp, or untimed when absent. - pub fn push<'a>(&mut self, value: impl Into>) -> Result<()> + /// Append a record at its timestamp (see [`Timed::at`]). + pub fn push<'a>(&mut self, value: Timed<&'a T>) -> Result<()> where T: 'a, { - self.inner.lock().unwrap().push(value.into()) + self.inner.lock().unwrap().push(value) } } @@ -187,6 +189,7 @@ struct Inner { /// The newest timestamp written, which a refresh restates the window at. An edit may carry an /// older timestamp than its group's first frame, and a new group may not start before that one. + /// `None` before the first write. at: Option, /// Whether this producer still publishes, or how it ended the track. @@ -207,7 +210,10 @@ impl Inner { // new group, since a consumer may hold the group the frame was meant for. track.write(at, &frame)?; frame.commit(); - self.at = self.at.max(at); + if self.at.is_none_or(|newest| at > newest) { + self.at = Some(at); + self.refresh.advanced(); + } Ok(()) } @@ -217,9 +223,23 @@ impl Inner { self.ensure_open()?; std::task::ready!(self.refresh.poll(self.track.largest, waiter))?; - let Inner { track, encoder, at, .. } = self; + let Inner { + track, + encoder, + at, + refresh, + .. + } = self; + // Two groups never share a start, so a refresh at the newest group's own start waits for a + // later write. Checked before encoding, since a header dropped unwritten resets the encoder. + let Some(at) = *at else { + return refresh.park(waiter); + }; + if !crate::starts_past(&track.inner, at) { + return refresh.park(waiter); + } let frame = encoder.snapshot()?; - track.write(*at, &frame)?; + track.write(at, &frame)?; frame.commit(); std::task::Poll::Ready(Ok(())) } @@ -228,10 +248,10 @@ impl Inner { let Some(group) = self.track.group.take() else { return Ok(()); }; - // The group closes either way, so reset first: a `finish` error must not leave the encoder + // The group closes either way, so reset first: a `close` error must not leave the encoder // appending ops to a group that is gone. self.encoder.reset(); - group.finish()?; + group.close()?; Ok(()) } @@ -298,7 +318,10 @@ impl Inner { let frame = encoder.push(value.value)?; track.write(at, &frame)?; frame.commit(); - self.at = self.at.max(at); + if self.at.is_none_or(|newest| at > newest) { + self.at = Some(at); + self.refresh.advanced(); + } Ok(()) } @@ -312,44 +335,50 @@ struct Track { /// encoder too, or the encoder's next op would have no group to land in. group: Option, - /// The sequence of the newest group written, which a refresh has to pass. + /// The start of the newest group written, which a refresh has to pass. largest: Option, } impl Track { /// `timestamp` at the track's timescale, or [`moq_net::Error::TimestampMismatch`] when it is - /// missing on a timed track, present on an untimed one, or beyond what the scale can hold. + /// unscaled on a timed track, scaled on a scale-less one, or beyond what the scale can hold. /// /// An edit checks this before encoding. Found by `write_frame` instead, the mismatch would /// come after a header published its group, leaving an empty newest group behind, and the /// dropped frame would reset the encoder, rolling a group for an edit that changed nothing. - fn rescale(&self, timestamp: Option) -> Result> { + fn rescale(&self, timestamp: moq_net::Timestamp) -> Result { Ok(self.inner.info().rescale(timestamp)?) } /// Write one encoded frame at a timestamp already [rescaled](Self::rescale) to the track, rolling a /// group when it is a header. - fn write(&mut self, timestamp: Option, encoded: &Encoded) -> Result<()> { + fn write(&mut self, timestamp: moq_net::Timestamp, encoded: &Encoded) -> Result<()> { match encoded.keyframe { true => self.write_header(timestamp, encoded.payload.clone()), false => self.write_op(timestamp, encoded.payload.clone()), } } - /// Close the open group and write the header as the first frame of a new one. - fn write_header(&mut self, timestamp: Option, payload: bytes::Bytes) -> Result<()> { + /// Write the header as the first frame of a new group starting at `timestamp`, then close the + /// previous group. + /// + /// A start at or before the newest group's is [`moq_net::Error::TimestampMismatch`], refused + /// before the previous group closes. + fn write_header(&mut self, timestamp: moq_net::Timestamp, payload: bytes::Bytes) -> Result<()> { + let start = timestamp.value(); + let mut group = self.inner.append_group(start)?; + self.largest = Some(start); + // The previous group is complete; no more frames will be appended to it. - if let Some(group) = self.group.take() { - group.finish()?; + if let Some(previous) = self.group.take() { + previous.close()?; } - let mut group = self.inner.append_group()?; - self.largest = Some(group.sequence); if let Err(err) = group.write_frame(timestamp, payload) { // `append_group` already published this group, and a rejected frame (too large) doesn't // close the track. Dropping the handle does NOT close the group, so leaving it would strand // any subscriber that advanced into it with nothing to read and no end. - let _ = group.finish(); + let _ = group.close(); return Err(err.into()); } @@ -358,17 +387,18 @@ impl Track { } /// Append an op to the group the last header opened. - fn write_op(&mut self, timestamp: Option, payload: bytes::Bytes) -> Result<()> { - self.group + fn write_op(&mut self, timestamp: moq_net::Timestamp, payload: bytes::Bytes) -> Result<()> { + let group = self + .group .as_mut() - .expect("the encoder only emits an op after a header opened a group") - .write_frame(timestamp, payload)?; + .expect("the encoder only emits an op after a header opened a group"); + group.write_frame(crate::frame_time(group, timestamp), payload)?; Ok(()) } fn finish(&mut self) -> Result<()> { if let Some(group) = self.group.take() { - group.finish()?; + group.close()?; } // Another handle may have declared the end already, which stands. This shared state // keeps its own clone, whose appends the end refuses. diff --git a/rs/moq-mux/benches/containers.rs b/rs/moq-mux/benches/containers.rs index 5360ad0e34f..f8e72f2a4fc 100644 --- a/rs/moq-mux/benches/containers.rs +++ b/rs/moq-mux/benches/containers.rs @@ -366,7 +366,7 @@ fn cmaf_receive(c: &mut Criterion) { } .encode(&mut data) .unwrap(); - let frame = moq_net::frame::Frame::new(None, Bytes::from(data)); + let frame = moq_net::frame::Frame::new(moq_net::Timestamp::unscaled(0), Bytes::from(data)); assert_eq!(wire.decode(frame.clone()).unwrap().len(), samples); group.throughput(Throughput::Elements(samples as u64)); group.bench_function(BenchmarkId::new(samples.to_string(), size), |b| { diff --git a/rs/moq-mux/benches/jitter.rs b/rs/moq-mux/benches/jitter.rs index fef7f0e50ab..5e73414d5e8 100644 --- a/rs/moq-mux/benches/jitter.rs +++ b/rs/moq-mux/benches/jitter.rs @@ -26,7 +26,7 @@ async fn steer(tracks: u16, floors: u64) -> u64 { let arrival = jitter::Arrival { arrived: at, read: at, - decode: Timestamp::from_millis(second * 1_000).unwrap().into(), + decode: Timestamp::from_millis(second * 1_000).unwrap().try_into().unwrap(), restart: 0, sync: true, leading: false, @@ -87,7 +87,7 @@ fn acquisition(c: &mut Criterion) { jitter::Arrival { arrived: start, read: start, - decode: Timestamp::from_millis(frame * 20).unwrap().into(), + decode: Timestamp::from_millis(frame * 20).unwrap().try_into().unwrap(), restart: 0, sync: frame % 15 == 0, leading: false, @@ -144,7 +144,8 @@ fn handoff(c: &mut Criterion) { u64::from(instance) * 10_000 + frame * 20, ) .unwrap() - .into(), + .try_into() + .unwrap(), restart: 0, sync: frame % 15 == 0, leading: false, @@ -193,7 +194,7 @@ fn admission(c: &mut Criterion) { let frame = |decode| jitter::Arrival { arrived: start, read: kio::clock::now(), - decode: Timestamp::from_millis(decode).unwrap().into(), + decode: Timestamp::from_millis(decode).unwrap().try_into().unwrap(), restart: 0, sync: true, leading: false, diff --git a/rs/moq-mux/src/catalog/data.rs b/rs/moq-mux/src/catalog/data.rs index a0783866366..36e02aea1ac 100644 --- a/rs/moq-mux/src/catalog/data.rs +++ b/rs/moq-mux/src/catalog/data.rs @@ -110,7 +110,8 @@ impl Listing { if let Some(captured) = captured { // A source clock running fast stamps ahead of now. Ordered as given, it would put every // media track behind it by however fast it runs. It was written no later than now. - self.estimator.order(captured.min(now)); + // An unscaled capture time doesn't compare, and the estimator skips it. + self.estimator.order(if captured > now { now } else { captured }); } self.rendition.estimate(self.estimator.reading()) } diff --git a/rs/moq-mux/src/catalog/estimate.rs b/rs/moq-mux/src/catalog/estimate.rs index c81f0a902df..0658766cc41 100644 --- a/rs/moq-mux/src/catalog/estimate.rs +++ b/rs/moq-mux/src/catalog/estimate.rs @@ -128,7 +128,8 @@ impl Estimator { } /// Observe a frame of `bytes` encoded bytes at presentation time `timestamp`, as written by - /// [`container::Producer::write`](crate::container::Producer::write). + /// [`container::Producer::write`](crate::container::Producer::write). An + /// [unscaled](Timestamp::unscaled) timestamp has no time, so it measures nothing. pub fn write(&mut self, timestamp: Timestamp, bytes: usize) { self.rate(timestamp, bytes); self.order(timestamp); @@ -136,7 +137,7 @@ impl Estimator { /// Count a write toward the bitrate and framerate only. pub(crate) fn rate(&mut self, timestamp: Timestamp, bytes: usize) { - let timestamp = nanos(timestamp); + let Some(timestamp) = nanos(timestamp) else { return }; self.bitrate.write(timestamp, bytes); self.window.write(timestamp); } @@ -152,7 +153,8 @@ impl Estimator { /// A frame with no other track to trail trails nothing, so a window holding frames from before /// another track joined never reports the trail of the few after it. pub(crate) fn order(&mut self, timestamp: Timestamp) { - let behind = self.seat.write(nanos(timestamp)).unwrap_or_default(); + let Some(timestamp) = nanos(timestamp) else { return }; + let behind = self.seat.write(timestamp).unwrap_or_default(); self.window.trail(behind); } @@ -167,7 +169,7 @@ impl Estimator { /// at the first cut once the highest timestamp has advanced a second past where it opened. /// Nothing is reported for either before then. pub fn cut(&mut self, end: Option) { - self.bitrate.cut(end.map(nanos)); + self.bitrate.cut(end.and_then(nanos)); self.window.cut(); } @@ -185,9 +187,11 @@ impl Estimator { /// Observe a frame's reorder delay (`PTS - DTS`), which raises the jitter to the decode buffer a /// B-frame stream needs. Only a container knows this; the elementary stream carries no decode - /// time. + /// time. An [unscaled](Timestamp::unscaled) delay has no time, so it raises nothing. pub fn reorder(&mut self, delay: Timestamp) { - self.burst(Duration::from(delay)); + if let Ok(delay) = Duration::try_from(delay) { + self.burst(delay); + } } /// Record the media duration emitted together by a container importer. @@ -218,7 +222,7 @@ impl Estimator { /// Note that written content reaches `end`, as a fragment or a frame with a duration says. pub(crate) fn reach(&mut self, end: Timestamp) { - let end = nanos(end); + let Some(end) = nanos(end) else { return }; self.window.reach = Some(self.window.reach.map_or(end, |reach| reach.max(end))); } @@ -273,8 +277,11 @@ impl From for Reading { /// a span anchored on one scale could never be timed against a frame on another. Since a span now /// survives a boundary it can't time, that would block the bitrate for the life of the track rather /// than for one group. Normalizing on the way in is what makes the carry-forward safe. -fn nanos(timestamp: Timestamp) -> u128 { - timestamp.as_nanos() +/// +/// `None` for a scale-less track's [unscaled](Timestamp::unscaled) key, which has no time to +/// measure, so every observation of one is skipped. +fn nanos(timestamp: Timestamp) -> Option { + timestamp.as_nanos().ok() } /// The time from `start` to `end`, or `None` if it didn't advance (a B-frame presenting earlier, a diff --git a/rs/moq-mux/src/catalog/hang/consumer.rs b/rs/moq-mux/src/catalog/hang/consumer.rs index ff443bd9646..99fcab42ce3 100644 --- a/rs/moq-mux/src/catalog/hang/consumer.rs +++ b/rs/moq-mux/src/catalog/hang/consumer.rs @@ -113,13 +113,13 @@ mod test { .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) .expect("catalog track should create"); let mut consumer = Consumer::new(track.consume()); - let mut group = track.append_group().expect("catalog group should append"); + let mut group = track.append_group(0).expect("catalog group should append"); let payload = serde_json::to_string(&published).expect("catalog should serialize"); group .write_frame(moq_net::Timestamp::ZERO, payload) .expect("catalog frame should write"); - group.finish().expect("catalog group should finish"); + group.close().expect("catalog group should finish"); match consumer.poll_next(&kio::Waiter::noop()) { Poll::Ready(result) => result, @@ -168,12 +168,12 @@ mod test { .expect("catalog track should resolve"); let mut consumer = Consumer::new(catalog_track); - let mut group = track.append_group().expect("catalog group should append"); + let mut group = track.append_group(0).expect("catalog group should append"); let payload = serde_json::to_string(&published).expect("catalog should serialize"); group .write_frame(moq_net::Timestamp::ZERO, payload) .expect("catalog frame should write"); - group.finish().expect("catalog group should finish"); + group.close().expect("catalog group should finish"); consumer.next().await } @@ -347,7 +347,7 @@ mod test { fn waits_for_pending_catalog_group_payload() { let track = track_producer(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()); let mut consumer = Consumer::new(track.consume()); - let mut group = track.append_group().expect("catalog group should append"); + let mut group = track.append_group(0).expect("catalog group should append"); let waiter = kio::Waiter::noop(); assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); @@ -356,7 +356,7 @@ mod test { group .write_frame(moq_net::Timestamp::ZERO, payload) .expect("catalog frame should write"); - group.finish().expect("catalog group should finish"); + group.close().expect("catalog group should finish"); assert_eq!(expect_catalog(consumer.poll_next(&waiter)), catalog); } @@ -365,11 +365,11 @@ mod test { fn a_newer_version_is_a_typed_refusal() { let track = track_producer(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()); let mut consumer: Consumer = Consumer::new(track.consume()); - let mut group = track.append_group().expect("catalog group should append"); + let mut group = track.append_group(0).expect("catalog group should append"); group .write_frame(moq_net::Timestamp::ZERO, r#"{"version":5}"#) .expect("catalog frame should write"); - group.finish().expect("catalog group should finish"); + group.close().expect("catalog group should finish"); assert!(matches!( consumer.poll_next(&kio::Waiter::noop()), @@ -381,7 +381,7 @@ mod test { fn waits_for_pending_catalog_group_payload_after_track_finish() { let track = track_producer(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()); let mut consumer = Consumer::new(track.consume()); - let mut group = track.append_group().expect("catalog group should append"); + let mut group = track.append_group(0).expect("catalog group should append"); let _track = track.finish().expect("catalog track should finish"); @@ -392,7 +392,7 @@ mod test { group .write_frame(moq_net::Timestamp::ZERO, payload) .expect("catalog frame should write"); - group.finish().expect("catalog group should finish"); + group.close().expect("catalog group should finish"); assert_eq!(expect_catalog(consumer.poll_next(&waiter)), catalog); } @@ -406,17 +406,17 @@ mod test { let (_old, old_payload) = catalog_payload("old"); let (latest, latest_payload) = catalog_payload("latest"); - let mut old_group = track.append_group().expect("old catalog group should append"); + let mut old_group = track.append_group(0).expect("old catalog group should append"); old_group .write_frame(moq_net::Timestamp::ZERO, old_payload) .expect("old catalog frame should write"); - old_group.finish().expect("old catalog group should finish"); + old_group.close().expect("old catalog group should finish"); - let mut latest_group = track.append_group().expect("latest catalog group should append"); + let mut latest_group = track.append_group(1).expect("latest catalog group should append"); latest_group - .write_frame(moq_net::Timestamp::ZERO, latest_payload) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), latest_payload) .expect("latest catalog frame should write"); - latest_group.finish().expect("latest catalog group should finish"); + latest_group.close().expect("latest catalog group should finish"); let _track = track.finish().expect("catalog track should finish"); assert_eq!(expect_catalog(consumer.poll_next(&waiter)), latest); @@ -432,20 +432,20 @@ mod test { let (_old, old_payload) = catalog_payload("old"); let (latest, latest_payload) = catalog_payload("latest"); - let mut old_group = track.append_group().expect("old catalog group should append"); + let mut old_group = track.append_group(0).expect("old catalog group should append"); old_group .write_frame(moq_net::Timestamp::ZERO, old_payload) .expect("old catalog frame should write"); - old_group.finish().expect("old catalog group should finish"); + old_group.close().expect("old catalog group should finish"); - let mut latest_group = track.append_group().expect("latest catalog group should append"); + let mut latest_group = track.append_group(1).expect("latest catalog group should append"); assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); latest_group - .write_frame(moq_net::Timestamp::ZERO, latest_payload) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), latest_payload) .expect("latest catalog frame should write"); - latest_group.finish().expect("latest catalog group should finish"); + latest_group.close().expect("latest catalog group should finish"); assert_eq!(expect_catalog(consumer.poll_next(&waiter)), latest); } @@ -459,15 +459,15 @@ mod test { let (_old, old_payload) = catalog_payload("old"); let (latest, latest_payload) = catalog_payload("latest"); - let mut old_group = track.append_group().expect("old catalog group should append"); + let mut old_group = track.append_group(0).expect("old catalog group should append"); assert!(matches!(consumer.poll_next(&waiter), Poll::Pending)); - let mut latest_group = track.append_group().expect("latest catalog group should append"); + let mut latest_group = track.append_group(1).expect("latest catalog group should append"); latest_group - .write_frame(moq_net::Timestamp::ZERO, latest_payload) + .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), latest_payload) .expect("latest catalog frame should write"); - latest_group.finish().expect("latest catalog group should finish"); + latest_group.close().expect("latest catalog group should finish"); let _track = track.finish().expect("catalog track should finish"); assert_eq!(expect_catalog(consumer.poll_next(&waiter)), latest); @@ -475,7 +475,7 @@ mod test { old_group .write_frame(moq_net::Timestamp::ZERO, old_payload) .expect("old catalog frame should write"); - old_group.finish().expect("old catalog group should finish"); + old_group.close().expect("old catalog group should finish"); assert!(matches!(consumer.poll_next(&waiter), Poll::Ready(Ok(None)))); } diff --git a/rs/moq-mux/src/catalog/hang/container.rs b/rs/moq-mux/src/catalog/hang/container.rs index 0186bd8e140..5946633eaa7 100644 --- a/rs/moq-mux/src/catalog/hang/container.rs +++ b/rs/moq-mux/src/catalog/hang/container.rs @@ -4,8 +4,12 @@ use crate::container::{Container as ContainerTrait, Frame, Kind, fmp4, legacy, l /// /// Built from a track's audio or video configuration, including its container. pub enum Container { - /// varint timestamp + raw codec bitstream. The original hang wire format. + /// varint timestamp + raw codec bitstream. The original hang wire format. Its frames carry no + /// duration, so audio built from [`Kind`] alone reads none. Legacy(Kind), + /// [`Legacy`](Self::Legacy) audio whose codec states each packet's duration, which its reads + /// carry. Built from an [`AudioConfig`](hang::catalog::AudioConfig) whose codec does. + LegacyAudio(legacy::Audio), /// ISO-BMFF moof+mdat fragments. The wrapped [`fmp4::Wire`] holds /// the track's `trak` box so per-frame writes and reads have the /// timescale and track id available. @@ -36,7 +40,10 @@ impl TryFrom<&hang::catalog::VideoConfig> for Container { impl TryFrom<&hang::catalog::AudioConfig> for Container { type Error = crate::Error; fn try_from(config: &hang::catalog::AudioConfig) -> crate::Result { - Self::new(&config.container, Kind::Audio) + match (&config.container, legacy::Audio::new(config)) { + (hang::catalog::Container::Legacy, Some(audio)) => Ok(Self::LegacyAudio(audio)), + (container, _) => Self::new(container, Kind::Audio), + } } } @@ -53,6 +60,7 @@ impl Container { pub(crate) fn prefix(&self, timestamp: moq_net::Timestamp) -> crate::Result> { match self { Self::Legacy(kind) => legacy::Wire(*kind).prefix(timestamp).map(Some), + Self::LegacyAudio(_) => legacy::Wire(Kind::Audio).prefix(timestamp).map(Some), Self::Loc(kind) => loc::Wire(*kind).prefix(timestamp).map(Some), Self::Cmaf(_) => Ok(None), } @@ -79,6 +87,7 @@ impl ContainerTrait for Container { fn end(&self, frame: &Frame) -> Option { match self { Self::Legacy(kind) => legacy::Wire(*kind).end(frame), + Self::LegacyAudio(_) => legacy::Wire(Kind::Audio).end(frame), Self::Cmaf(cmaf) => cmaf.end(frame), Self::Loc(kind) => loc::Wire(*kind).end(frame), } @@ -87,6 +96,7 @@ impl ContainerTrait for Container { fn kind(&self) -> Kind { match self { Self::Legacy(kind) | Self::Loc(kind) => *kind, + Self::LegacyAudio(_) => Kind::Audio, Self::Cmaf(wire) => ::kind(wire), } } @@ -98,6 +108,7 @@ impl ContainerTrait for Container { ) -> Result<(), Self::Error> { match self { Self::Legacy(kind) => legacy::Wire(*kind).finish_group(group, end), + Self::LegacyAudio(_) => legacy::Wire(Kind::Audio).finish_group(group, end), Self::Cmaf(wire) => wire.finish_group(group, end).map_err(Into::into), Self::Loc(kind) => loc::Wire(*kind).finish_group(group, end), } @@ -106,6 +117,7 @@ impl ContainerTrait for Container { fn write(&self, group: &mut moq_net::group::Producer, frames: &[Frame]) -> Result<(), Self::Error> { match self { Self::Legacy(kind) => legacy::Wire(*kind).write(group, frames), + Self::LegacyAudio(_) => legacy::Wire(Kind::Audio).write(group, frames), Self::Cmaf(cmaf) => cmaf.write(group, frames).map_err(Into::into), Self::Loc(kind) => loc::Wire(*kind).write(group, frames), } @@ -114,6 +126,7 @@ impl ContainerTrait for Container { fn decode(&self, frame: moq_net::frame::Frame) -> Result, Self::Error> { match self { Self::Legacy(kind) => legacy::Wire(*kind).decode(frame), + Self::LegacyAudio(audio) => audio.decode(frame), Self::Cmaf(cmaf) => Ok(cmaf.decode(frame)?), Self::Loc(kind) => loc::Wire(*kind).decode(frame), } diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index c0f92185124..75795805588 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -290,6 +290,10 @@ struct Outputs { /// Keeps the broadcast open while the catalog is, so a publisher may hand its own handle to a /// media producer and drop it. broadcast: moq_net::broadcast::Producer, + /// How many groups the hang and MSF tracks each took, for tests to count publishes by: group + /// starts are timestamps. + #[cfg(test)] + published: Arc<[std::sync::atomic::AtomicU64; 2]>, } impl Clone for Outputs { @@ -299,6 +303,8 @@ impl Clone for Outputs { hang_track: self.hang_track.clone(), msf_track: self.msf_track.clone(), broadcast: self.broadcast.clone(), + #[cfg(test)] + published: self.published.clone(), } } } @@ -310,19 +316,30 @@ impl Outputs { // Every consumer refuses a catalog that fails this, so refuse it before it reaches one. catalog.check_renditions()?; // The media timeline, so the catalog in effect at a media timestamp is the one stamped at or - // before it, whichever thread or runtime publishes. - let at = clock.now(); + // before it, whichever thread or runtime publishes. Each catalog starts a group on both + // tracks, so one landing in the same millisecond as the last is stamped a unit later + // instead of refused. + let (_, at) = crate::start::next(&self.hang_track, clock.now())?; + let (start, at) = crate::start::next(&self.msf_track, at)?; // One snapshot per group while deltas are disabled. - self.hang.update(moq_net::Timed::from(catalog).at(at))?; + #[cfg(test)] + let before = self.hang_track.largest(); + self.hang.update(moq_json::Timed::new(catalog, at))?; + #[cfg(test)] + if self.hang_track.largest() != before { + self.published[0].fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } // `abort` below is what keeps the tracks consistent if this fails after the hang track was // updated. let Some(msf) = msf.render(catalog, &self.msf_track.broadcast().path)? else { return Ok(()); }; - let mut group = self.msf_track.append_group()?; + let mut group = self.msf_track.append_group(start)?; + #[cfg(test)] + self.published[1].fetch_add(1, std::sync::atomic::Ordering::Relaxed); group.write_frame(at, msf)?; - group.finish()?; + group.close()?; Ok(()) } @@ -514,6 +531,8 @@ impl Producer { hang_track, msf_track, broadcast: broadcast.clone(), + #[cfg(test)] + published: Arc::default(), }, current: Arc::new(Mutex::new(State { catalog, @@ -660,11 +679,11 @@ impl Producer { let clock = &guard.state.clock; let at = match wall { Some(wall) => clock.reading(wall)?, - None => clock.now().as_micros() as i128, + None => clock.now().as_micros().expect("the broadcast clock is scaled") as i128, }; - super::Offset::from_micros(at - pts.as_micros() as i128)? + super::Offset::from_micros(at - pts.as_micros()? as i128)? } else { - let since = std::time::Duration::from_micros(u64::try_from(pts.as_micros()).unwrap_or(u64::MAX)); + let since = std::time::Duration::from_micros(u64::try_from(pts.as_micros()?).unwrap_or(u64::MAX)); // Moved, not replaced: it keeps reading the clock the broadcast was created under. let clock = match wall { Some(wall) => guard.state.clock.placed(since, wall)?, @@ -1977,10 +1996,7 @@ mod test { // The caller got the error, so the tracks stay usable once the catalog fits again. catalog.modify().unwrap().remove_section("big").unwrap(); catalog.finish().unwrap(); - assert_eq!( - catalog.outputs.hang_track.largest(), - Some(moq_net::track::Position::group(0)) - ); + assert_eq!(sent(&catalog), Some(0)); } /// Every consumer refuses an ID naming both a video and an audio rendition, so the producer @@ -2024,10 +2040,7 @@ mod test { // The drop that follows `commit` must not publish a second snapshot. catalog.finish().unwrap(); - assert_eq!( - catalog.outputs.hang_track.largest(), - Some(moq_net::track::Position::group(0)) - ); + assert_eq!(sent(&catalog), Some(0)); } /// A video rendition `video0` on `catalog`, with a keyframe group at each of `starts` (ms). @@ -2237,7 +2250,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("captured frame is published"); }; - let timestamp = frame.timestamp.expect("captured frame is timed"); + let timestamp = frame.timestamp; assert_eq!(catalog.clock().wall_clock(timestamp).unwrap(), wall); assert_eq!( catalog.snapshot().media.clock.unwrap().wall_clock(timestamp).unwrap(), @@ -2323,7 +2336,7 @@ mod test { while let Some(mut group) = track.next_group().await.unwrap() { let first = stamps.len(); while let Some(frame) = group.read_frame().await.unwrap() { - stamps.push(frame.timestamp.unwrap().as_micros()); + stamps.push(frame.timestamp.as_micros().unwrap()); } starts.extend(stamps.get(first).copied()); } @@ -2343,8 +2356,8 @@ mod test { /// How far `pts` lands from `clock`'s reading now, which is zero for a frame live on arrival. fn lateness(clock: &crate::Clock, pts: moq_net::Timestamp) -> std::time::Duration { - let now = clock.now().as_micros(); - std::time::Duration::from_micros(now.abs_diff(pts.as_micros()) as u64) + let now = clock.now().as_micros().unwrap(); + std::time::Duration::from_micros(now.abs_diff(pts.as_micros().unwrap()) as u64) } /// A `with_clock` catalog is fixed from the start: even the first importer shifts onto it, @@ -2452,7 +2465,7 @@ mod test { .unwrap(); assert_eq!(catalog.snapshot().media.clock, published); assert!( - shifted.as_micros() >= 10_000_000, + shifted.as_micros().unwrap() >= 10_000_000, "{shifted:?} lands at the clock's reading" ); } @@ -2477,7 +2490,7 @@ mod test { "it placed the clock, so it stays verbatim" ); let placed = catalog.clock(); - assert_eq!(placed.reading(wall).unwrap(), pts.as_micros() as i128); + assert_eq!(placed.reading(wall).unwrap(), pts.as_micros().unwrap() as i128); // Placed again on the now fixed clock, the same mapping shifts by nothing. let again = catalog.timebase(); @@ -2576,7 +2589,7 @@ mod test { // The same mapping from `with_clock` agrees in any units, without reading the host's wall // clock, which may be unset. - let zero = kio::clock::Wall::MOQ_EPOCH + Duration::from_micros(input.wall.as_micros() as u64); + let zero = kio::clock::Wall::MOQ_EPOCH + Duration::from_micros(input.wall.as_micros().unwrap() as u64); let fixed = crate::Clock::at(kio::clock::now(), zero).unwrap(); seed.media.clock = Some(an_hour_ago(moq_net::Timescale::MILLI)); crate::clock::fake_host_wall(kio::clock::Wall::UNIX_EPOCH + Duration::from_secs(86_400)); @@ -2701,7 +2714,7 @@ mod test { let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("the frame is published"); }; - assert_eq!(frame.timestamp, Some(timestamp)); + assert_eq!(frame.timestamp, timestamp); // The catalog clock is fixed now, and has no wall to offset a placement onto. assert!(matches!( @@ -3329,8 +3342,11 @@ mod test { } /// How many catalogs have gone out on the wire. - fn sent(catalog: &Producer) -> Option { - catalog.outputs.hang_track.largest().map(|position| position.group) + /// How many hang catalogs went out before the newest, or `None` before the first. + fn sent(catalog: &Producer) -> Option { + catalog.outputs.published[0] + .load(std::sync::atomic::Ordering::Relaxed) + .checked_sub(1) } /// The catalog a viewer joining now receives. @@ -3589,8 +3605,11 @@ mod test { } /// How many MSF catalogs have gone out on the wire. - fn msf_sent(catalog: &Producer) -> Option { - catalog.outputs.msf_track.largest().map(|position| position.group) + /// How many MSF catalogs went out before the newest, or `None` before the first. + fn msf_sent(catalog: &Producer) -> Option { + catalog.outputs.published[1] + .load(std::sync::atomic::Ordering::Relaxed) + .checked_sub(1) } /// The last MSF catalog on the wire. diff --git a/rs/moq-mux/src/catalog/timebase.rs b/rs/moq-mux/src/catalog/timebase.rs index 61ed913d4d3..366458ef834 100644 --- a/rs/moq-mux/src/catalog/timebase.rs +++ b/rs/moq-mux/src/catalog/timebase.rs @@ -96,17 +96,17 @@ impl Offset { (self.0 as i128 * units_per_second as i128).div_euclid(1_000_000) } - /// Shift `pts`, keeping its scale. Refuses a result below zero rather than clamping it. + /// Shift `pts`, keeping its scale. Refuses a result below zero rather than clamping it, and an + /// [unscaled](moq_net::Timestamp::unscaled) `pts`, which has no time to shift. pub(crate) fn apply(self, pts: moq_net::Timestamp) -> crate::Result { if self.0 == 0 { return Ok(pts); } - let scale = pts.scale(); + let scale = pts.scale().ok_or(moq_net::TimeOverflow)?; let value = pts.value() as i128 + self.ticks(scale.as_u64()); let value = u64::try_from(value).map_err(|_| { crate::Error::UnmappableTimestamp(format!( - "{} µs shifted by {} µs lands before the broadcast clock's zero", - pts.as_micros(), + "{pts:?} shifted by {} µs lands before the broadcast clock's zero", self.0 )) })?; diff --git a/rs/moq-mux/src/clock.rs b/rs/moq-mux/src/clock.rs index 550cbeb4813..1e983f16e1b 100644 --- a/rs/moq-mux/src/clock.rs +++ b/rs/moq-mux/src/clock.rs @@ -176,7 +176,11 @@ impl Clock { /// Refuses a wall that is no whole number of those units (some 90 kHz values). pub(crate) fn exact(wall: hang::catalog::Clock) -> crate::Result { let unmappable = |why: &str| crate::Error::UnmappableTimestamp(format!("root clock {wall:?} {why}")); - let scale = wall.wall.scale().as_u64() as u128; + let scale = wall + .wall + .scale() + .ok_or_else(|| unmappable("has no timescale"))? + .as_u64() as u128; let scaled = wall.wall.value() as u128 * Self::TIMESCALE.as_u64() as u128; if !scaled.is_multiple_of(scale) { return Err(unmappable("is no whole microsecond")); @@ -203,8 +207,9 @@ impl Clock { let instant = self.source.now(); let now = set(self.source.wall()).ok_or_else(|| unmappable("needs the host wall clock, which is unset"))?; // Saturating would stamp everything early by the skew without saying so. - let reading = u64::try_from(moq_micros(now) - zero.wall.as_micros() as i128) - .map_err(|_| unmappable("puts PTS zero after now"))?; + let zero_micros = zero.wall.as_micros().map_err(|_| unmappable("has no timescale"))?; + let reading = + u64::try_from(moq_micros(now) - zero_micros as i128).map_err(|_| unmappable("puts PTS zero after now"))?; Ok(Self { source: self.source.clone(), instant, @@ -217,7 +222,7 @@ impl Clock { /// /// Pure in the stored mapping, like [`wall_clock`](Self::wall_clock), and refused without one. pub(crate) fn reading(&self, wall: Wall) -> crate::Result { - Ok(moq_micros(wall) - self.mapping()?.wall.as_micros() as i128) + Ok(moq_micros(wall) - self.mapping()?.wall.as_micros()? as i128) } /// The wall mapping, refused when the host's wall clock was unset. @@ -328,7 +333,10 @@ mod tests { kio::clock::sleep(Duration::from_secs(3)).await; assert_eq!(clock.now(), us(3_000_000)); - assert_eq!(fresh.now().as_micros() - before.as_micros(), 3_000_000); + assert_eq!( + fresh.now().as_micros().unwrap() - before.as_micros().unwrap(), + 3_000_000 + ); assert_eq!(clock.wall(), wall); assert_eq!( clock.wall_clock(clock.now()).unwrap(), @@ -361,7 +369,7 @@ mod tests { // PTS zero at exactly the moq epoch advertises 0. let clock = Clock::at(epoch(), moq_epoch()).unwrap(); assert_eq!(clock.wall().unwrap().wall.value(), 0); - assert_eq!(clock.wall().unwrap().wall.scale(), Clock::TIMESCALE); + assert_eq!(clock.wall().unwrap().wall.scale(), Some(Clock::TIMESCALE)); // A second later is a second's worth of clock units. let clock = Clock::at(epoch(), moq_epoch() + Duration::from_secs(1)).unwrap(); @@ -385,7 +393,7 @@ mod tests { let clock = Clock::new(); // PTS zero sits before construction, so a frame stamped a little before now still maps, // and the mapping still names the current wall time. - assert!(clock.now().as_micros() >= Clock::LEAD.as_micros()); + assert!(clock.now().as_micros().unwrap() >= Clock::LEAD.as_micros()); let now = clock.wall_clock(clock.now()).unwrap(); let host = kio::clock::wall(); let drift = now.max(host).saturating_duration_since(now.min(host)); @@ -399,7 +407,7 @@ mod tests { let since = Duration::from_secs(10 * 3600); let clock = Clock::new().arrival(since).unwrap(); let now = clock.now(); - assert!(now.as_micros() >= since.as_micros()); + assert!(now.as_micros().unwrap() >= since.as_micros()); let actual = clock.wall_clock(now).unwrap(); let host = kio::clock::wall(); let drift = actual.max(host).saturating_duration_since(actual.min(host)); @@ -409,7 +417,7 @@ mod tests { let earlier = clock .capture(kio::clock::now().saturating_sub(Duration::from_secs(1))) .unwrap(); - assert!(earlier.as_micros() < now.as_micros()); + assert!(earlier.as_micros().unwrap() < now.as_micros().unwrap()); // PTS zero before 2020 cannot be named on the wire. assert!(matches!( @@ -426,7 +434,7 @@ mod tests { let clock = Clock::new(); assert_eq!(clock.wall(), None); - assert!(clock.now().as_micros() >= Clock::LEAD.as_micros()); + assert!(clock.now().as_micros().unwrap() >= Clock::LEAD.as_micros()); assert!(matches!( clock.wall_clock(clock.now()), Err(crate::Error::UnmappableTimestamp(_)) diff --git a/rs/moq-mux/src/codec/aac/import.rs b/rs/moq-mux/src/codec/aac/import.rs index a0500fa4e98..487e49dffaf 100644 --- a/rs/moq-mux/src/codec/aac/import.rs +++ b/rs/moq-mux/src/codec/aac/import.rs @@ -85,9 +85,9 @@ impl Import { Ok(()) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/annexb/export.rs b/rs/moq-mux/src/codec/annexb/export.rs index 3237d5b2574..64c8764d5ef 100644 --- a/rs/moq-mux/src/codec/annexb/export.rs +++ b/rs/moq-mux/src/codec/annexb/export.rs @@ -88,7 +88,7 @@ impl Export { match ready!(track.source.poll_read(waiter))? { Some(frame) => { - let timestamp = std::time::Duration::from(frame.timestamp); + let timestamp = std::time::Duration::try_from(frame.timestamp)?; if track.joining { if !frame.keyframe || self.playhead.is_some_and(|playhead| timestamp <= playhead) { continue; diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index 6e887823619..9930a2efcc9 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -227,9 +227,9 @@ impl Import { self.catalog.set_enabled(&mut self.track, enabled) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/flac/import.rs b/rs/moq-mux/src/codec/flac/import.rs index d5b540c35b3..5a3cace0319 100644 --- a/rs/moq-mux/src/codec/flac/import.rs +++ b/rs/moq-mux/src/codec/flac/import.rs @@ -68,9 +68,9 @@ impl Import { self.track.discontinuity() } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/h264/export.rs b/rs/moq-mux/src/codec/h264/export.rs index 1f854c6508f..b8e77bd31cf 100644 --- a/rs/moq-mux/src/codec/h264/export.rs +++ b/rs/moq-mux/src/codec/h264/export.rs @@ -218,16 +218,16 @@ mod tests { .unwrap(); // Group 0 (keyframe-starting group): one IDR frame. - let mut g0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut g0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_length_prefixed(&mut g0, 0, &[idr]); - g0.finish().unwrap(); + g0.close().unwrap(); // Group 1 (next group): one P-slice. Consumer marks the first frame // of every group as keyframe by protocol invariant, so the exporter // MUST treat both group-starts as keyframes and inject SPS+PPS twice. - let mut g1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut g1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_length_prefixed(&mut g1, 33_000, &[p_slice]); - g1.finish().unwrap(); + g1.close().unwrap(); let _track = track.finish().unwrap(); // Consumer side: run the exporter. @@ -332,10 +332,10 @@ mod tests { let track = broadcast .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); let nals: &[&[u8]] = &[&[0x67, 0x42, 0, 0x1f], &[0x68, 0xce], &[0x65, 0x88]]; write_length_prefixed(&mut group, 0, nals); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let consumer = broadcast.consume(); let mut exporter = Export::new(crate::source::announced(&consumer).await, Once(Some(catalog))); diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index 331988bea31..f44411da656 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -177,9 +177,9 @@ impl Import { self.track.report_bitrate(bitrate); } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/h265/export.rs b/rs/moq-mux/src/codec/h265/export.rs index 84ba32f4920..46d376b7055 100644 --- a/rs/moq-mux/src/codec/h265/export.rs +++ b/rs/moq-mux/src/codec/h265/export.rs @@ -208,13 +208,13 @@ mod tests { .create_track("video.hvc1", hang::container::track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let mut g0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut g0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_length_prefixed(&mut g0, 0, &[idr]); - g0.finish().unwrap(); + g0.close().unwrap(); - let mut g1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut g1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_length_prefixed(&mut g1, 33_000, &[trail]); - g1.finish().unwrap(); + g1.close().unwrap(); let _track = track.finish().unwrap(); let consumer = broadcast.consume(); @@ -328,10 +328,10 @@ mod tests { let track = broadcast .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) .unwrap(); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); let nals: &[&[u8]] = &[&[0x40, 1, 0x0c], &[0x42, 1, 1], &[0x44, 1, 0xc0], &[0x26, 1, 0x88]]; write_length_prefixed(&mut group, 0, nals); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let consumer = broadcast.consume(); let mut exporter = Export::new(crate::source::announced(&consumer).await, Once(Some(catalog))); diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index cc491e88c8a..cdde9f7018c 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -180,9 +180,9 @@ impl Import { self.track.report_bitrate(bitrate); } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/legacy.rs b/rs/moq-mux/src/codec/legacy.rs index 538e9ddc414..21f868bbafd 100644 --- a/rs/moq-mux/src/codec/legacy.rs +++ b/rs/moq-mux/src/codec/legacy.rs @@ -183,9 +183,9 @@ impl Import { Ok(()) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/mp3.rs b/rs/moq-mux/src/codec/mp3.rs index 40f22612ea0..6e26dba466c 100644 --- a/rs/moq-mux/src/codec/mp3.rs +++ b/rs/moq-mux/src/codec/mp3.rs @@ -157,9 +157,9 @@ impl Import { self.track.discontinuity() } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/opus/import.rs b/rs/moq-mux/src/codec/opus/import.rs index 6e58476edf0..34621d04aa0 100644 --- a/rs/moq-mux/src/codec/opus/import.rs +++ b/rs/moq-mux/src/codec/opus/import.rs @@ -73,9 +73,9 @@ impl Import { Ok(()) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { + self.track.seek(start)?; Ok(()) } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index bfc2222ef63..7acda2efb84 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -125,9 +125,9 @@ impl Import { self.catalog.set_enabled(&mut self.track, enabled) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { + self.track.seek(start)?; Ok(()) } } diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index cb17472208b..a6c5e331e6c 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -126,9 +126,9 @@ impl Import { self.catalog.set_enabled(&mut self.track, enabled) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { - self.track.seek(sequence)?; + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { + self.track.seek(start)?; Ok(()) } } diff --git a/rs/moq-mux/src/container/consumer.rs b/rs/moq-mux/src/container/consumer.rs index 0cc1c909f1d..81e0f9b2dc1 100644 --- a/rs/moq-mux/src/container/consumer.rs +++ b/rs/moq-mux/src/container/consumer.rs @@ -7,7 +7,7 @@ use kio::clock::Instant; use moq_net::Timestamp; use moq_net::track::Delay; -use super::{Container, Frame, Read, TimestampRewind}; +use super::{Container, Frame, Read, TimestampRewind, time}; /// Media and group boundaries in delivery order. pub(crate) enum Event { @@ -60,18 +60,21 @@ pub(crate) enum Item { /// ## Ordering & age skipping /// /// Groups can arrive on the wire out of order. The consumer always reads frames *within* -/// a group in arrival order, but across groups it advances by sequence number, skipping +/// a group in arrival order, but across groups it advances by start, skipping /// a stalled or missing group once everything it could still present (bounded by where /// the next group begins) falls a full budget behind the newest content, the same reach /// [`Delay::max`] measures. With the default max delay of zero, /// the consumer skips aggressively: any group that has a newer alternative is dropped. /// With a non-zero max delay, slow groups are tolerated up to that budget before being -/// skipped. A missing sequence gets the same tolerance: there is no way to tell a stream that -/// lost the delivery race from one the cache evicted, and a reader starved of CPU may take the -/// missing group in the same burst as the ones past it. Its wait, [`set_wait`](Self::set_wait) -/// or the max delay, also expires that much monotonic time after the first group past it -/// arrived, so a marker behind a hole is delivered even when the track is paused, and the holes -/// a stalled source left behind expire together rather than one wait each. +/// skipped. Starts are timestamps, so keys lie between any two groups: a run the publisher +/// declared skipped ([`skip_groups`](moq_net::track::Producer::skip_groups), which appending +/// does) holds no group, and the consumer steps over it at once. Any other run may hold a +/// missing group, which gets the same tolerance: there is no way to tell a stream that lost the +/// delivery race from one the cache evicted, and a reader starved of CPU may take the missing +/// group in the same burst as the ones past it. Its wait, [`set_wait`](Self::set_wait) or the +/// max delay, also expires that much monotonic time after the first group past it arrived, so a +/// marker behind a hole is delivered even when the track is paused, and the holes a stalled +/// source left behind expire together rather than one wait each. /// /// Delivery starts at the floor the /// [`Subscription::ranges`](moq_net::track::Subscription::ranges) name, raised to where the @@ -105,8 +108,8 @@ pub(crate) enum Item { /// ## Gaps and ends /// /// [`poll_read`](Self::poll_read) yields [`Read::Gap`] wherever media after an item already read -/// will never arrive: a latency skip, an evicted current group, or a sequence hole whose -/// boundary is not contiguous within 1 ms. Whatever is lost before the first item is no +/// will never arrive: a latency skip, an evicted current group, or a hole the publisher did not +/// declare skipped whose boundary is not contiguous within 1 ms. Whatever is lost before the first item is no /// gap, since nothing was read before it. A marker group (no decodable frames: one empty /// payload, or on a data track one object with no bytes) is the publisher declaring a stop, /// read as [`Read::End`]; nothing is missing. A data track's marker carries no timestamp, so @@ -295,9 +298,10 @@ fn pts_contiguous(end: Option, next_start: std::time::Durat end.is_some_and(|end| next_start <= end.saturating_add(CONTIGUITY_TOLERANCE)) } -/// A gap from the delivered `edge` to `resume`, in the resume's scale. -fn gap(edge: std::time::Duration, resume: Timestamp) -> Item { - let scale = resume.scale(); +/// A gap from the delivered `edge` to `resume`, in the resume's scale. Refused for an unscaled +/// `resume`, which has no time to resume at. +fn gap(edge: std::time::Duration, resume: Timestamp) -> Result { + let scale = resume.scale().ok_or(moq_net::Error::TimestampMismatch)?; // A next group may start before the edge (an overlapping endpoint), so the span bottoms out at zero. let start = edge .as_nanos() @@ -308,7 +312,7 @@ fn gap(edge: std::time::Duration, resume: Timestamp) -> Item { let duration = resume .checked_sub(timestamp) .expect("at or below the resume, in its scale"); - Item::Gap { timestamp, duration } + Ok(Item::Gap { timestamp, duration }) } /// A [`Consumer`]'s [`Delay`] and the catalog rendition's warmup, applied together. @@ -435,6 +439,18 @@ impl Consumer { } } + /// Continue groups an earlier reader of this track read up to `reached`: the first group + /// starting there reads as continuing them rather than past a hole. + pub(crate) fn continuing(mut self, reached: Option) -> Self { + self.presented_end = reached; + self + } + + /// Where the groups read through ended, for [`Self::continuing`]. + pub(crate) fn reached(&self) -> Option { + self.presented_end + } + /// Start at the newest group, the live edge, rather than the floor or the oldest group the /// publisher serves: the publisher's largest when it reports one past what has arrived. The /// subscription keeps its budget, so every group after the start arrives as it would for any @@ -536,7 +552,7 @@ impl Consumer { && let Some(edge) = edge { self.next = Some(item); - return Poll::Ready(Ok(Some(gap(edge, resume)))); + return Poll::Ready(Ok(Some(gap(edge, resume)?))); } Poll::Ready(Ok(Some(item))) } @@ -603,7 +619,7 @@ impl Consumer { self.join(waiter); let (joined, backlog) = (self.joined, &mut self.backlog); self.pending.retain(|group| { - let keep = group.sequence >= start; + let keep = group.start >= start; if !keep { Self::skip(joined, backlog, group); } @@ -631,7 +647,7 @@ impl Consumer { let start = match matches!(self.startup, Startup::Newest) { true => { self.poll_largest(waiter); - let back = self.pending.back().expect("a group has a frame").sequence; + let back = self.pending.back().expect("a group has a frame").start; back.max(self.largest.unwrap_or(back)) } // The start may still wait for a group earlier than any that arrived. @@ -642,14 +658,14 @@ impl Consumer { }; self.join(waiter); // Nothing has been read, so discarding what lies below the start is no gap. - self.discard(|group| group.sequence < start)?; + self.discard(|group| group.start < start)?; self.current = start; self.startup = Startup::Started; } } loop { - if self.pending.front().is_none_or(|group| group.sequence <= self.current) { + if self.pending.front().is_none_or(|group| group.start <= self.current) { self.hole_timer = None; } @@ -665,14 +681,15 @@ impl Consumer { // Return the next frame from the current group if possible. // If the current group is finished or errored, advance to the next group. if let Some(group) = self.pending.front_mut() - && group.sequence <= self.current + && group.start <= self.current { match group.poll_read(waiter, &self.format) { Poll::Ready(Ok(Some(Event::Frame { frame, arrived }))) => { - let seq = group.group.sequence; + let seq = group.group.start; let ts = frame.timestamp; + let first = time(ts).ok_or(moq_net::Error::TimestampMismatch)?; if let Some(floor) = self.floor - && ts.as_micros() < floor.as_micros() + && time(floor).is_some_and(|floor| first.as_micros() < floor.as_micros()) { return Poll::Ready(Err(TimestampRewind { timestamp: ts, floor }.into())); } @@ -680,18 +697,16 @@ impl Consumer { self.start = Some((seq, ts)); } // A group covering the target began before it, so the reader warms until the target too. - let first = std::time::Duration::from(ts); self.warmed .get_or_insert_with(|| first.max(self.target.unwrap_or(first)).saturating_add(self.warmup)); - let end = std::time::Duration::from(ts) - + frame.duration.map(std::time::Duration::from).unwrap_or_default(); + let end = first + frame.duration.and_then(time).unwrap_or_default(); self.advance(end); self.group = seq; return Poll::Ready(Ok(Some(Event::Frame { frame, arrived }))); } Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))) => { - self.group = group.group.sequence; - self.advance(end.into()); + self.group = group.group.start; + self.advance(time(end).ok_or(moq_net::Error::TimestampMismatch)?); return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens }))); } Poll::Ready(Ok(Some(event))) => return Poll::Ready(Ok(Some(event))), @@ -702,7 +717,7 @@ impl Consumer { Poll::Ready(Err(Failure::Decode(e))) => { tracing::warn!( track = &*self.name, - group = group.group.sequence, + group = group.group.start, error = ?e, "group payload failed to decode; ending the reader" ); @@ -716,22 +731,22 @@ impl Consumer { // that will never arrive. tracing::warn!( track = &*self.name, - group = group.group.sequence, + group = group.group.start, error = ?e, "current group evicted; skipping to next buffered group" ); // Whatever the group still held is lost, so this is a skip like any other. self.lose(); let current = self.current; - self.discard(|group| group.sequence <= current)?; - self.current = self.pending.front().map_or(self.current + 1, |g| g.sequence); + self.discard(|group| group.start <= current)?; + self.current = self.pending.front().map_or(self.current + 1, |g| g.start); // Said now rather than with the gap, which waits for the next group: what // the caller holds of this one need not wait for a successor. return Poll::Ready(Ok(Some(Event::Cut { observed: kio::clock::now(), }))); } - // Cleanly finished group: advance to the next sequence. + // Cleanly finished group: advance past its key. Poll::Ready(Ok(None)) => { // A data track's marker carries no timestamp, so it ends where this reader // got to. Before the first item there is nothing to end. @@ -743,10 +758,18 @@ impl Consumer { .ok() .and_then(|micros| Timestamp::from_micros(micros).ok()) .ok_or(moq_net::Error::Decode(moq_net::DecodeError::BoundsExceeded))?; - self.group = group.group.sequence; + self.group = group.group.start; return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: true }))); } - if let Some(end) = group.max_end { + // A group its producer finished whole presents up to its end, which can pass + // what its frames reach when they carry no duration. + let whole = group + .group + .end() + .zip(group.group.timescale()) + .and_then(|(end, scale)| Timestamp::new(end, scale).ok()) + .and_then(time); + if let Some(end) = group.max_end.max(whole) { self.presented_end = Some(end); } self.pending.pop_front(); @@ -763,7 +786,7 @@ impl Consumer { // Live groups register the waiter here so a later abort wakes the reader. let current = self.current; self.discard(|group| { - group.sequence > current + group.start > current && group.buffered.is_empty() && group.fault.is_none() && group.poll_aborted(waiter) @@ -771,9 +794,15 @@ impl Consumer { // The current group's furthest presentation point (timestamp + duration). let current_end = if let Some(current) = self.pending.front_mut() - && current.sequence <= self.current + && current.start <= self.current { - match current.poll_min_timestamp(waiter, &self.format, true) { + let polled = current.poll_min_timestamp(waiter, &self.format, true); + // A frame that landed since the read above is delivered, not judged: its end can + // reach the next group, and skipping there would throw it away as lost. + if current.ready() { + continue; + } + match polled { Poll::Ready(Some(_)) => current.max_end, _ => None, } @@ -786,24 +815,24 @@ impl Consumer { let next_group = self.poll_next_group(waiter); let max_timestamp = self.poll_newest(waiter); - // Walk the cursor over missing sequences below the first arrived group. + // Walk the cursor over missing keys below the first arrived group. // Groups race on independent QUIC streams (newer ones at higher priority), - // so a buffered higher sequence proves nothing: the missing one may be + // so a buffered higher key proves nothing: the missing one may be // merely late, and there is no way to tell that from an eviction. The wait, // in wall time since a later group arrived, is the gate. Media buffered past // the hole proves it stale only past the max delay, not the shorter wait: a // reader starved of CPU takes in a burst of groups at once, and the missing // one may land in the same burst. A finished track closes // the gap outright, since no new group can arrive. That proof covers only - // sequences that never arrived: finishing the track ends new groups, not + // keys that never arrived: finishing the track ends new groups, not // the frames still flowing on ones already open, so arrived groups are - // settled below by their own FIN, abort, or the budget. Sequences the + // settled below by their own FIN, abort, or the budget. Keys the // subscriber's own ranges exclude never arrive either, so a gap between two // ranges closes at once. if let Some(front) = self.pending.front() - && front.sequence > self.current + && front.start > self.current { - let sequence = front.sequence; + let sequence = front.start; // Late since a group past it arrived, not since the cursor got here: once the edge // stops moving, a backlog with every other group missing would otherwise drain one // budget per hole, which reads downstream as a source far slower than real time. @@ -820,14 +849,29 @@ impl Consumer { // The groups past the hole already fill the max buffer, so it can hold none of what // would arrive into it. let full = self.memory.full(); - if finished || elapsed || aged || excluded || full { + // Group starts are timestamps, so the key past the cursor is rarely the next group's. + // The publisher declares the starts it never produced, so a run it skipped holds no + // group to wait for or lose. Nor does one ending at a group whose first frame, or a + // marker's bound, continues where the content read so far ended. Either way the + // cursor steps to it without waiting. + let skipped = self.is_skipped(self.current..sequence); + let front = self.pending.front_mut().unwrap(); + let first = front.poll_min_timestamp(waiter, &self.format, false); + let start = front.min_timestamp.and_then(time).or(front.max_end); + let continues = skipped || start.is_some_and(|start| pts_contiguous(self.presented_end, start)); + // A run nothing will fill, such as one dropped over a wire that can't say it was + // skipped, still waits for the group past it to show where it starts: stepping + // before that frame lands would call a contiguous timeline a loss. + let excluded = excluded && first.is_ready(); + if finished || elapsed || aged || excluded || full || continues { let _ = self.pending.front_mut().unwrap().look_all(waiter, &self.format); let front = self.pending.front_mut().unwrap(); let empty = front.is_empty() && matches!(front.group.poll_finished(waiter), Poll::Ready(Ok(_))); let front = self.pending.front().unwrap(); - let next_start = front.min_timestamp.map(std::time::Duration::from).or(front.max_end); + let next_start = front.min_timestamp.and_then(time).or(front.max_end); // A data marker has no timestamp to prove continuity across the hole. - if !empty && next_start.is_none_or(|start| !pts_contiguous(self.presented_end, start)) { + if !empty && !continues && next_start.is_none_or(|start| !pts_contiguous(self.presented_end, start)) + { self.lose(); } tracing::debug!( @@ -838,6 +882,7 @@ impl Consumer { aged, excluded, full, + continues, "skipping missing groups" ); self.current = sequence; @@ -877,8 +922,8 @@ impl Consumer { false if self.memory.full() => self .pending .iter() - .position(|group| group.sequence > self.current) - .map(|i| (i, self.pending[i].min_timestamp.map(std::time::Duration::from))), + .position(|group| group.start > self.current) + .map(|i| (i, self.pending[i].min_timestamp.and_then(time))), false => None, }; @@ -912,27 +957,28 @@ impl Consumer { let mut newest = None; let mut moved = false; while let Some(group) = self.pending.front_mut() { - if group.sequence > self.current { + if group.start > self.current { break; } let _ = group.poll_min_timestamp(waiter, &self.format, true); - let Some((frame, ..)) = group.buffered.front() else { + let Some(first) = group.buffered.front().and_then(|(frame, ..)| time(frame.timestamp)) else { break; }; - let first = std::time::Duration::from(frame.timestamp); let current_end = group.max_end; // The next group that arrived, even one whose first frame is still in flight: that frame // may land within the budget, so skipping on to a later group could drop the one the start // belongs on. Until it lands, nothing is skipped, as `@moq/hang`'s consumer waits on its // successor's start. - let Some(idx) = self.pending.iter().position(|group| group.sequence > self.current) else { + let Some(idx) = self.pending.iter().position(|group| group.start > self.current) else { break; }; let Poll::Ready(Some(next_start)) = self.pending[idx].poll_min_timestamp(waiter, &self.format, false) else { break; }; - let next_start = std::time::Duration::from(next_start); + let Some(next_start) = time(next_start) else { + break; + }; let newest = *newest.get_or_insert_with(|| self.poll_newest(waiter)); // Nothing has been read, so the reader still warms. if newest.saturating_sub(first) <= self.max_delay.saturating_add(self.warmup) { @@ -952,12 +998,14 @@ impl Consumer { // The first group past the cursor with a timestamp, the skip target, and where it starts. fn poll_next_group(&mut self, waiter: &kio::Waiter) -> Option<(usize, std::time::Duration)> { for (i, group) in self.pending.iter_mut().enumerate() { - if group.sequence <= self.current { + if group.start <= self.current { continue; } - if let Poll::Ready(Some(ts)) = group.poll_min_timestamp(waiter, &self.format, false) { - return Some((i, ts.into())); + if let Poll::Ready(Some(ts)) = group.poll_min_timestamp(waiter, &self.format, false) + && let Some(ts) = time(ts) + { + return Some((i, ts)); } } None @@ -966,13 +1014,15 @@ impl Consumer { // The max timestamp across the groups past the cursor, or zero without one. fn poll_newest(&mut self, waiter: &kio::Waiter) -> std::time::Duration { for group in self.pending.iter_mut().rev() { - if group.sequence <= self.current { + if group.start <= self.current { break; } // Older groups won't be newer than this. - if let Poll::Ready(Some(ts)) = group.poll_max_timestamp(waiter, &self.format) { - return ts.into(); + if let Poll::Ready(Some(ts)) = group.poll_max_timestamp(waiter, &self.format) + && let Some(ts) = time(ts) + { + return ts; } } std::time::Duration::ZERO @@ -1000,7 +1050,7 @@ impl Consumer { if hole || had_marker || discarded { self.lose(); } - let new_current = self.pending.front().map(|g| g.sequence).unwrap(); + let new_current = self.pending.front().map(|g| g.start).unwrap(); tracing::debug!(old = self.current, new = new_current, "skipping slow groups"); @@ -1012,7 +1062,7 @@ impl Consumer { // Where an unfloored start begins, the lowest group that arrived, or `None` while it waits for // one covering the reach (see `Playout`), registering the waiter on whatever would end the wait. fn reach_start(&mut self, waiter: &kio::Waiter, finished: bool) -> Option { - let lowest = self.pending.front().expect("a group has a frame").sequence; + let lowest = self.pending.front().expect("a group has a frame").start; // The publisher serves nothing older than the max delay, so a min delay past it reaches no // further. let reach = self.min_delay.min(self.max_delay).saturating_add(self.warmup); @@ -1025,7 +1075,7 @@ impl Consumer { let mut edge = None; for group in self.pending.iter_mut().rev() { if let Poll::Ready(Some(ts)) = group.poll_max_timestamp(waiter, &self.format) { - edge = Some(std::time::Duration::from(ts)); + edge = time(ts); break; } } @@ -1035,11 +1085,10 @@ impl Consumer { }; let target = edge.saturating_sub(reach); self.target = Some(target); - let covered = self.pending.iter().any(|group| { - group - .min_timestamp - .is_some_and(|start| std::time::Duration::from(start) <= target) - }); + let covered = self + .pending + .iter() + .any(|group| group.min_timestamp.and_then(time).is_some_and(|start| start <= target)); if covered { return Some(lowest); } @@ -1113,12 +1162,12 @@ impl Consumer { } let first = self .pending - .partition_point(|group| self.received.is_some_and(|last| group.sequence <= last)); + .partition_point(|group| self.received.is_some_and(|last| group.start <= last)); for group in self.pending.range_mut(first..) { if !matches!(group.group.poll_finished(waiter), Poll::Ready(Ok(_))) { return Poll::Pending; } - self.received = Some(group.sequence); + self.received = Some(group.start); } Poll::Ready(()) } @@ -1155,7 +1204,7 @@ impl Consumer { }; let mut reader = GroupBuffer::new(group, self.memory.clone()); - let sequence = reader.group.sequence; + let sequence = reader.group.start; if sequence < self.current { tracing::debug!(old = ?sequence, current = ?self.current, "skipping old group"); @@ -1163,9 +1212,7 @@ impl Consumer { continue; } - let idx = self - .pending - .partition_point(|g| g.group.sequence < reader.group.sequence); + let idx = self.pending.partition_point(|g| g.group.start < reader.group.start); // Each group is late from the earliest arrival among it and the groups past it. A drained // backlog comes in sequence order, not arrival order, so an arrival can lower those below it. if let Some(next) = self.pending.get(idx) { @@ -1190,7 +1237,7 @@ impl Consumer { self.pending.retain_mut(|group| { // Keep the cursor's gap and anything already decoded, including markers and faults. // A never-read evicted group holds no event that its missing sequence cannot say. - group.sequence == current || !group.is_empty() || !group.poll_aborted(waiter) + group.start == current || !group.is_empty() || !group.poll_aborted(waiter) }); self.collect_at = self.pending.len().saturating_mul(2).max(1); } @@ -1204,7 +1251,7 @@ impl Consumer { // Note `group` dropped without delivering all of it: backlog when the publisher's largest at the // subscription's start reached it. One above was produced since, such as an outage's. fn skip(joined: Option, backlog: &mut Option, group: &GroupBuffer) { - if joined.is_some_and(|joined| group.sequence <= joined) { + if joined.is_some_and(|joined| group.start <= joined) { *backlog = (*backlog).max(Some(group.group.arrived())); } } @@ -1264,7 +1311,12 @@ impl Consumer { // The fault a discarded group would hide: its own, or frames below the latest delivered start. fn discard_fault(start: Option<(u64, Timestamp)>, group: &mut GroupBuffer) -> Option { match (start, group.min_timestamp) { - (Some((prev, floor)), Some(min)) if group.sequence > prev && min.as_micros() < floor.as_micros() => { + (Some((prev, floor)), Some(min)) + if group.start > prev + && time(min) + .zip(time(floor)) + .is_some_and(|(min, floor)| min.as_micros() < floor.as_micros()) => + { Some(TimestampRewind { timestamp: min, floor }.into()) } _ => group.fault.take(), @@ -1281,6 +1333,16 @@ impl Consumer { } } + /// Whether the publisher declared every group start in `groups` skipped, so no group lies + /// there; see [`moq_net::track::Subscriber::is_skipped`]. A group with nothing possible + /// below it is its track's first. False once an error ended the reader. + pub fn is_skipped(&self, groups: std::ops::Range) -> bool { + match &self.reader { + Reader::Reading(track) | Reader::Failed(track, _) => track.is_skipped(groups), + Reader::Ended => false, + } + } + /// Set the [`Delay`], mid-stream or before the first read. /// /// [`Delay::max`] is clamped to the track's max age like [`new`](Self::new). The @@ -1420,6 +1482,12 @@ impl GroupBuffer { } /// Poll for the next frame from this group. + /// Whether [`Self::poll_read`] would hand out an item without reading the wire: a frame + /// looked ahead, or a marker it reached. + fn ready(&self) -> bool { + !self.buffered.is_empty() || self.markers.front().is_some_and(|(index, _)| *index <= self.delivered) + } + fn poll_read(&mut self, waiter: &kio::Waiter, format: &F) -> Poll, Failure>> { loop { if self.markers.front().is_some_and(|(index, _)| *index <= self.delivered) { @@ -1475,31 +1543,39 @@ impl GroupBuffer { // held, so the last one read also holds whatever of it the views leave out, such as a header. let mut views = 0; let first = self.buffered.len(); + // Media is timed: a scale-less track's unscaled keys carry no time to present at. + let unscaled = || Failure::Decode(moq_net::Error::TimestampMismatch.into()); for mut frame in frames { if let Some(bound) = format.end(&frame) { - self.note_end(bound); + let end = time(bound).ok_or_else(unscaled)?; + self.note_end(end); self.markers.push_back((self.index, bound)); self.charge(MARKER_COST); continue; } + let at = time(frame.timestamp).ok_or_else(unscaled)?; + let duration = frame + .duration + .map(|duration| time(duration).ok_or_else(unscaled)) + .transpose()?; + // Every buffered timestamp has a time, so any two of them compare. self.min_timestamp = Some(match self.min_timestamp { - Some(existing) => existing.min(frame.timestamp), - None => frame.timestamp, + Some(existing) if existing <= frame.timestamp => existing, + _ => frame.timestamp, }); self.max_timestamp = Some(match self.max_timestamp { - Some(existing) => existing.max(frame.timestamp), - None => frame.timestamp, + Some(existing) if existing > frame.timestamp => existing, + _ => frame.timestamp, }); // Furthest presentation point, in wall-clock terms so timestamp and // duration can be at different scales without extra conversions. A frame // with no duration contributes only its timestamp. - self.note_end(frame.timestamp); - if let Some(duration) = frame.duration { - let end = std::time::Duration::from(frame.timestamp) + std::time::Duration::from(duration); - self.max_end = Some(self.max_end.map_or(end, |existing| existing.max(end))); + self.note_end(at); + if let Some(duration) = duration { + self.note_end(at + duration); } // First frame of a group is always a keyframe by protocol invariant; trust @@ -1611,8 +1687,7 @@ impl GroupBuffer { matches!(self.group.poll_finished(waiter), Poll::Ready(Err(_))) } - fn note_end(&mut self, timestamp: Timestamp) { - let end = std::time::Duration::from(timestamp); + fn note_end(&mut self, end: std::time::Duration) { self.max_end = Some(self.max_end.map_or(end, |existing| existing.max(end))); } @@ -1707,8 +1782,8 @@ mod tests { /// Encode a `[timestamp][duration][payload]` DurationWire frame. fn encode_duration_frame(timestamp: Timestamp, duration: Timestamp) -> Vec { let mut buf = Vec::with_capacity(18); - buf.extend_from_slice(&(timestamp.as_micros() as u64).to_le_bytes()); - buf.extend_from_slice(&(duration.as_micros() as u64).to_le_bytes()); + buf.extend_from_slice(&(timestamp.as_micros().unwrap() as u64).to_le_bytes()); + buf.extend_from_slice(&(duration.as_micros().unwrap() as u64).to_le_bytes()); buf.extend_from_slice(&[0xDE, 0xAD]); buf } @@ -1751,7 +1826,7 @@ mod tests { /// Write a finished group with explicit sequence and timestamps (Container::Legacy(crate::container::Kind::Data) format). fn write_group(track: &mut moq_net::track::Producer, sequence: u64, timestamps: &[Timestamp]) { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); for ×tamp in timestamps { let frame = Frame { timestamp, @@ -1763,7 +1838,7 @@ mod tests { .write(&mut group, &[frame]) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } /// Drain all available frames with a per-read timeout. @@ -1846,7 +1921,7 @@ mod tests { } fn write_marker_group(track: &mut moq_net::track::Producer, sequence: u64, timestamp: Timestamp) { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); Container::Legacy(crate::container::Kind::Audio) .write( &mut group, @@ -1858,7 +1933,7 @@ mod tests { }], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } /// Every item until the track ends: a frame as its timestamp in µs, a gap as its start and @@ -1867,9 +1942,13 @@ mod tests { let mut items = Vec::new(); while let Some(read) = consumer.read().await.unwrap() { items.push(match read { - Read::Frame { frame, .. } => frame.timestamp.as_micros().to_string(), - Read::Gap { timestamp, duration } => format!("gap {}+{}", timestamp.as_micros(), duration.as_micros()), - Read::End { end, .. } => format!("end {}", end.as_micros()), + Read::Frame { frame, .. } => frame.timestamp.as_micros().unwrap().to_string(), + Read::Gap { timestamp, duration } => format!( + "gap {}+{}", + timestamp.as_micros().unwrap(), + duration.as_micros().unwrap() + ), + Read::End { end, .. } => format!("end {}", end.as_micros().unwrap()), }); } items @@ -1904,7 +1983,10 @@ mod tests { let track = track_producer("data", info); let subscription = moq_net::track::Subscription::default() .with_delay(Delay::default().with_max(hour)) - .with_range(0..); + .with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(0), + end: None, + }]); let mut reader = Consumer::new( track.subscribe(subscription), Container::Legacy(crate::container::Kind::Data), @@ -1927,22 +2009,25 @@ mod tests { assert_eq!(items(&mut reader).await, ["0", "10000", "end 10000", "2405070000"]); } - /// A data marker has no timestamp, so the sequences missing before it are lost, not covered - /// by the break it declares. + /// A data marker has no timestamp, so a group missing between the content read and the marker's + /// start is lost, not covered by the break it declares. #[tokio::test(start_paused = true)] async fn a_hole_before_a_terminal_data_marker_is_a_gap() { let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = moq_net::track::Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(10))) - .with_range(0..); + .with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(0), + end: None, + }]); let mut reader = Consumer::new( track.subscribe(subscription), Container::Legacy(crate::container::Kind::Data), ); write_group(&mut track, 0, &[ts(0), ts(10_000)]); - let mut group = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); - group.write_frame(ts(0), Bytes::new()).unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 20_000 }).unwrap(); + group.write_frame(ts(20_000), Bytes::new()).unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); assert_eq!(items(&mut reader).await, ["0", "10000", "gap 10000+0", "end 10000"]); @@ -1957,9 +2042,9 @@ mod tests { let mut reader = Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(10_000)]); assert!(matches!(reader.read().await.unwrap(), Some(Read::Frame { .. }))); - let mut marker = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); - marker.write_frame(ts(10_000), Bytes::new()).unwrap(); - marker.finish().unwrap(); + let mut marker = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); + marker.write_frame(ts(20_000), Bytes::new()).unwrap(); + marker.close().unwrap(); if !budget.is_zero() { assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); @@ -2013,9 +2098,9 @@ mod tests { reader.read().await.unwrap(), Some(Read::Frame { group: 2, .. }) )); - let mut marker = track.create_group(moq_net::group::Info { sequence: 4 }).unwrap(); - marker.write_frame(ts(20_000), Bytes::new()).unwrap(); - marker.finish().unwrap(); + let mut marker = track.create_group(moq_net::group::Info { start: 4 }).unwrap(); + marker.write_frame(ts(30_000), Bytes::new()).unwrap(); + marker.close().unwrap(); assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); tokio::time::advance(Duration::from_millis(99)).await; assert!(reader.poll_read(&kio::Waiter::noop()).is_pending()); @@ -2056,9 +2141,9 @@ mod tests { )); assert_eq!(reader.arrival(), Some(second)); - let mut marker = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + let mut marker = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); marker.write_frame(ts(66_000), Bytes::new()).unwrap(); - marker.finish().unwrap(); + marker.close().unwrap(); assert!(matches!(reader.read().await.unwrap(), Some(Read::End { group: 2, .. }))); assert_eq!(reader.arrival(), Some(second), "an end carries no arrival of its own"); } @@ -2148,7 +2233,7 @@ mod tests { async fn the_join_is_the_largest_at_the_answer_not_at_the_start() { let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); let landed = kio::clock::now(); - let mut open = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + let mut open = track.create_group(moq_net::group::Info { start: 3 }).unwrap(); let subscriber = track.subscribe( moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))), ); @@ -2173,7 +2258,7 @@ mod tests { Container::Legacy(crate::container::Kind::Audio) .write(&mut open, &[frame]) .unwrap(); - open.finish().unwrap(); + open.close().unwrap(); assert!(matches!( reader.read().await.unwrap(), Some(Read::Frame { group: 5, .. }) @@ -2295,7 +2380,7 @@ mod tests { }; // A stalled group 500 ms behind the newest content is still within the max delay. - let mut stalled = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut stalled = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); let frame = Frame { timestamp: ts(0), payload: Bytes::from_static(&[0xDE, 0xAD]), @@ -2308,7 +2393,7 @@ mod tests { write_group(&mut track, 1, &[ts(500_000)]); tokio::time::advance(Duration::from_millis(500)).await; assert_eq!(drain(&mut reader), ["frame 0"]); - stalled.finish().unwrap(); + stalled.close().unwrap(); assert_eq!(drain(&mut reader), ["frame 1"]); // A missing group expires the wait after the first group past it arrived. @@ -2338,21 +2423,24 @@ mod tests { let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = moq_net::track::Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(10))) - .with_range(0..); + .with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(0), + end: None, + }]); let mut reader = Consumer::new( track.subscribe(subscription), Container::Legacy(crate::container::Kind::Data), ); reader.max_delay = Duration::ZERO; // The current group stays open, so only a skip moves past it. - let mut slow = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut slow = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); hang::container::Frame { timestamp: ts(0), payload: Bytes::from_static(b"a"), } .write_to(&mut slow) .unwrap(); - let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); group.write_frame(ts(10_000), Bytes::new()).unwrap(); hang::container::Frame { timestamp: ts(10_000), @@ -2360,7 +2448,7 @@ mod tests { } .write_to(&mut group) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); write_group(&mut track, 2, &[ts(1_000_000)]); let read = loop { @@ -2398,13 +2486,16 @@ mod tests { let mut track = track_producer("data", hang::container::track_info(hang::catalog::PRIORITY.audio)); let subscription = moq_net::track::Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(10))) - .with_range(0..); + .with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(0), + end: None, + }]); let mut reader = Consumer::new( track.subscribe(subscription), Container::Legacy(crate::container::Kind::Data), ); write_group(&mut track, 0, &[ts(0)]); - let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); let mut objects = [Bytes::new(), second.clone()]; if !marker_first { objects.reverse(); @@ -2420,7 +2511,7 @@ mod tests { .unwrap(), } } - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let error = loop { @@ -2444,14 +2535,14 @@ mod tests { fn a_gap_spans_the_edge_to_the_resume_in_its_scale() { let scale = moq_net::Timescale::new(90_000).unwrap(); let resume = Timestamp::new(90_000, scale).unwrap(); - let Item::Gap { timestamp, duration } = gap(Duration::from_millis(500), resume) else { + let Item::Gap { timestamp, duration } = gap(Duration::from_millis(500), resume).unwrap() else { unreachable!() }; assert_eq!(timestamp, Timestamp::new(45_000, scale).unwrap()); assert_eq!(duration, Timestamp::new(45_000, scale).unwrap()); // A next group overlapping the edge leaves an empty span at the resume point. - let Item::Gap { timestamp, duration } = gap(Duration::from_secs(2), resume) else { + let Item::Gap { timestamp, duration } = gap(Duration::from_secs(2), resume).unwrap() else { unreachable!() }; assert_eq!(timestamp, resume); @@ -2487,6 +2578,65 @@ mod tests { ); } + /// Starts are timestamps, so keys lie between any two groups. A run the publisher declared + /// skipped holds nothing, so the reader steps over it without a gap even where the content + /// jumps; one it never declared may have held a group that was lost. + #[tokio::test(start_paused = true)] + async fn only_an_undeclared_run_between_groups_is_a_gap() { + let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = track.subscribe( + moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))), + ); + let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); + write_group(&mut track, 0, &[ts(0), ts(10_000)]); + track.skip_groups(1..1_000_000); + write_group(&mut track, 1_000_000, &[ts(1_000_000), ts(1_010_000)]); + write_group(&mut track, 2_000_000, &[ts(2_000_000), ts(2_010_000)]); + let _track = track.finish().unwrap(); + assert_eq!( + items(&mut reader).await, + [ + "0", + "10000", + "1000000", + "1010000", + "gap 1010000+990000", + "2000000", + "2010000" + ] + ); + } + + /// Legacy frames carry no duration, so an audio reader reads each packet's from its codec. + /// One-packet groups then run back to back even where nothing declared the starts between + /// them skipped, as a moq-lite-06 hop can't, while a lost group is still a gap. + #[tokio::test(start_paused = true)] + async fn codec_durations_bridge_undeclared_runs_between_audio_groups() { + let track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let subscription = track.subscribe( + moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))), + ); + let config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + let mut reader = Consumer::new(subscription, Container::try_from(&config).unwrap()); + // One 20 ms CELT packet per group. The group starting at 40 ms never arrives. + for start in [0, 20_000, 60_000] { + let mut group = track.create_group(moq_net::group::Info { start }).unwrap(); + let packet = Frame { + timestamp: ts(start), + payload: Bytes::from_static(&[31 << 3, 0]), + keyframe: false, + duration: None, + }; + Container::try_from(&config) + .unwrap() + .write(&mut group, &[packet]) + .unwrap(); + group.close().unwrap(); + } + let _track = track.finish().unwrap(); + assert_eq!(items(&mut reader).await, ["0", "20000", "gap 40000+20000", "60000"]); + } + #[tokio::test] async fn a_latency_skip_is_a_gap() { let mut track = track_producer("gaps", hang::container::track_info(hang::catalog::PRIORITY.audio)); @@ -2496,7 +2646,7 @@ mod tests { let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); reader.max_delay = Duration::ZERO; // Group 0 stays open, and is read before the newer group lands: the skip comes after the start. - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Audio) .write( &mut group, @@ -2524,7 +2674,7 @@ mod tests { let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Video)); let wire = Container::Legacy(crate::container::Kind::Video); for (sequence, start) in [(0, 0), (1, 33_000)] { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); let frame = |timestamp: u64, payload| Frame { timestamp: ts(timestamp), payload, @@ -2539,7 +2689,7 @@ mod tests { ], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let _track = track.finish().unwrap(); assert_eq!(items(&mut reader).await, ["0", "33000"]); @@ -2554,7 +2704,7 @@ mod tests { ); let mut reader = Consumer::new(subscription, Container::Legacy(crate::container::Kind::Audio)); write_group(&mut track, 0, &[ts(0), ts(20_000)]); - let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); let frame = |timestamp: u64, payload| Frame { timestamp: ts(timestamp), payload, @@ -2567,7 +2717,7 @@ mod tests { &[frame(50_000, Bytes::new()), frame(40_000, Bytes::from_static(&[1]))], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); assert_eq!(items(&mut reader).await, ["0", "20000", "end 50000", "40000"]); } @@ -2581,11 +2731,11 @@ mod tests { let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Audio)); write_group(&mut track, 0, &[ts(0)]); - write_marker_group(&mut track, 1, ts(0)); + write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(1_000_000)]); let _track = track.finish().unwrap(); - assert_eq!(items(&mut consumer).await, ["0", "end 0", "1000000"]); + assert_eq!(items(&mut consumer).await, ["0", "end 10000", "1000000"]); } /// Missing media reads as a gap; only the publisher can declare an end. @@ -2600,13 +2750,13 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); // Group 1 never arrives. write_group(&mut track, 2, &[ts(1_000_000)]); - write_marker_group(&mut track, 3, ts(1_000_000)); + write_marker_group(&mut track, 3, ts(1_010_000)); write_group(&mut track, 4, &[ts(2_000_000)]); let _track = track.finish().unwrap(); assert_eq!( items(&mut consumer).await, - ["0", "gap 0+1000000", "1000000", "end 1000000", "2000000"] + ["0", "gap 0+1000000", "1000000", "end 1010000", "2000000"] ); } @@ -2620,9 +2770,9 @@ mod tests { write_group(&mut track, 0, &[ts(0)]); track - .create_group(moq_net::group::Info { sequence: 1 }) + .create_group(moq_net::group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); write_group(&mut track, 2, &[ts(1_000)]); let _track = track.finish().unwrap(); @@ -2636,7 +2786,7 @@ mod tests { async fn a_live_edge_start_on_an_empty_group_moves_on() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); write_group(&mut track, 0, &[ts(0)]); - let _empty = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let _empty = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); let mut subscriber = track.subscribe( moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500))), @@ -2666,7 +2816,7 @@ mod tests { consumer.max_delay = Duration::ZERO; // Group 0 stays open, and is read before the newer group lands: the skip comes after the start. - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Audio) .write( &mut group, @@ -2735,7 +2885,7 @@ mod tests { let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); // Group 0: 5 frames, NOT finished (blocks consumer) - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); for f in 0..5u64 { Container::Legacy(crate::container::Kind::Data) .write( @@ -2760,7 +2910,7 @@ mod tests { // Finish group 0 after consumer has had time to accumulate pending groups let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = read_all(&mut consumer).await.unwrap(); @@ -2778,7 +2928,7 @@ mod tests { // Group 0 at ts 0 keeps timestamps monotonic with sequence (groups 1-9 follow at // g*50 ms), so the test exercises age skipping and not rewind detection. - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -2804,7 +2954,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = read_all(&mut consumer).await.unwrap(); @@ -2820,7 +2970,7 @@ mod tests { let consumer_track = track.subscribe(None); let mut consumer = container_max_delay_only(consumer_track, Duration::from_millis(100)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -2845,7 +2995,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = read_all(&mut consumer).await.unwrap(); @@ -2888,11 +3038,11 @@ mod tests { // A 2.5 s group, half arrived, and the next one already 0.5 s in. The reader started // before the next one landed, so this is a stall, not a join's backlog, which starts // within the budget instead (see `a_buffered_backlog_starts_within_the_budget`). - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write(&mut group0, ts(0)); write(&mut group0, ts(500_000)); assert_eq!(read(&mut consumer), Some(ts(0))); - let mut group1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write(&mut group1, ts(2_500_000)); write(&mut group1, ts(3_000_000)); @@ -2901,17 +3051,17 @@ mod tests { assert_eq!(read(&mut consumer), None, "cut group 0 short"); write(&mut group0, ts(1_000_000)); assert_eq!(read(&mut consumer), Some(ts(1_000_000))); - group0.finish().unwrap(); + group0.close().unwrap(); assert_eq!(read(&mut consumer), Some(ts(2_500_000))); // Once the reach falls a full budget behind, the stalled group is skipped. - let mut group2 = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + let mut group2 = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); write(&mut group2, ts(5_000_000)); write(&mut group2, ts(7_000_000)); assert_eq!(read(&mut consumer), Some(ts(3_000_000))); assert_eq!(read(&mut consumer), Some(ts(5_000_000))); - group1.finish().unwrap(); - group2.finish().unwrap(); + group1.close().unwrap(); + group2.close().unwrap(); } /// A backlog that lands whole before the first read starts at the first group whose first @@ -2929,7 +3079,7 @@ mod tests { let timestamps: Vec<_> = (0..10).map(|f| ts(g * 1_000_000 + f * 100_000)).collect(); write_group(&mut track, g, ×tamps); } - let mut last = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); + let mut last = track.create_group(moq_net::group::Info { start: 5 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut last, @@ -2942,7 +3092,7 @@ mod tests { ) .unwrap(); if !open { - last.finish().unwrap(); + last.close().unwrap(); } let mut read = Vec::new(); @@ -2973,7 +3123,7 @@ mod tests { for g in 0..4u64 { write_group(&mut track, g, &frames(g)); } - let mut fifth = track.create_group(moq_net::group::Info { sequence: 4 }).unwrap(); + let mut fifth = track.create_group(moq_net::group::Info { start: 4 }).unwrap(); write_group(&mut track, 5, &[ts(5_000_000)]); let mut read = Vec::new(); @@ -2999,7 +3149,7 @@ mod tests { ) .unwrap(); } - fifth.finish().unwrap(); + fifth.close().unwrap(); drain(&mut consumer); let expected: Vec<_> = frames(3).into_iter().chain(frames(4)).chain([ts(5_000_000)]).collect(); @@ -3016,7 +3166,7 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); write_group(&mut track, 0, &[ts(100_000)]); assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); - write_group(&mut track, 1, &[ts(100_000), ts(0)]); + write_group(&mut track, 1, &[ts(133_000), ts(0)]); let _track = track.finish().unwrap(); let err = loop { match consumer.read().await { @@ -3049,17 +3199,6 @@ mod tests { ); } - #[tokio::test] - async fn a_group_starting_at_the_previous_start_is_accepted() { - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); - let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Data)); - write_group(&mut track, 0, &[ts(100_000)]); - assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); - write_group(&mut track, 1, &[ts(100_000)]); - let _track = track.finish().unwrap(); - assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(100_000)); - } - /// A keyframe one frame below the previous group's last frame, but above its start, is an /// overlap rather than a rewind. #[tokio::test] @@ -3076,7 +3215,10 @@ mod tests { let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); assert_eq!( - frames.iter().map(|f| f.timestamp.as_micros()).collect::>(), + frames + .iter() + .map(|f| f.timestamp.as_micros().unwrap()) + .collect::>(), vec![50_000, 83_000] ); } @@ -3095,7 +3237,10 @@ mod tests { let _track = track.finish().unwrap(); let frames = continuous_frames(&mut consumer).await; assert_eq!( - frames.iter().map(|f| f.timestamp.as_micros()).collect::>(), + frames + .iter() + .map(|f| f.timestamp.as_micros().unwrap()) + .collect::>(), vec![66_000, 50_000] ); } @@ -3110,10 +3255,10 @@ mod tests { Container::Legacy(crate::container::Kind::Audio), ); write_group(&mut track, 0, &[ts(0)]); - write_marker_group(&mut track, 1, ts(0)); + write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(1_000_000)]); let _track = track.finish().unwrap(); - assert_eq!(items(&mut consumer).await, ["0", "end 0", "1000000"]); + assert_eq!(items(&mut consumer).await, ["0", "end 10000", "1000000"]); } /// A pause's timestamp jump is not lateness: the group that resumes it can arrive behind the @@ -3130,7 +3275,7 @@ mod tests { consumer.max_delay = Duration::from_millis(300); write_group(&mut track, 0, &[ts(0)]); // The end, then the stopped encoder's final packet, as `moq_audio`'s producer writes them. - let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); let frame = |timestamp, payload| Frame { timestamp, payload, @@ -3146,7 +3291,7 @@ mod tests { ], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); assert!(matches!( consumer.read().await.unwrap(), @@ -3168,7 +3313,7 @@ mod tests { consumer.max_delay = Duration::ZERO; write_group(&mut track, 0, &[ts(0)]); assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); - write_marker_group(&mut track, 1, ts(0)); + write_marker_group(&mut track, 1, ts(10_000)); write_group(&mut track, 2, &[ts(1_000_000)]); let _track = track.finish().unwrap(); assert!(matches!(consumer.read().await.unwrap(), Some(Read::Gap { .. }))); @@ -3181,7 +3326,7 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Audio)); consumer.max_delay = Duration::ZERO; - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Audio) .write( &mut group0, @@ -3195,12 +3340,12 @@ mod tests { .unwrap(); assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(0)); - write_marker_group(&mut track, 1, ts(10_000)); + write_marker_group(&mut track, 1, ts(9_999)); write_group(&mut track, 2, &[ts(10_000)]); assert!(matches!(consumer.read().await.unwrap(), Some(Read::Gap { .. }))); assert_eq!(consumer.read_frame().await.unwrap().unwrap().timestamp, ts(10_000)); - group0.finish().unwrap(); + group0.close().unwrap(); } #[tokio::test] @@ -3251,7 +3396,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Video)); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); let media = |timestamp| Frame { timestamp, payload: Bytes::from_static(&[0xDE, 0xAD]), @@ -3266,7 +3411,7 @@ mod tests { .write(&mut group, &[media(ts(33_000))]) .unwrap(); write_marker(&mut group, ts(50_000)); // the group's end - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); @@ -3283,7 +3428,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Video)); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_marker(&mut group, ts(20_000)); Container::Legacy(crate::container::Kind::Video) .write( @@ -3296,7 +3441,7 @@ mod tests { }], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let frame = consumer.read_frame().await.unwrap().unwrap(); @@ -3315,7 +3460,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Video)); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Video) .write( &mut group, @@ -3330,12 +3475,12 @@ mod tests { for i in 1..5u64 { write_marker(&mut group, ts(i * 1_000)); } - group.finish().unwrap(); + group.close().unwrap(); write_group(&mut track, 1, &[ts(100_000)]); let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, vec![0, 100_000], "markers skipped, next group reached"); } @@ -3348,7 +3493,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Loc(crate::container::Kind::Video)); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); let media = Frame { timestamp: ts(0), payload: Bytes::from_static(&[0xDE, 0xAD]), @@ -3369,7 +3514,7 @@ mod tests { }], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); @@ -3388,7 +3533,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -3407,7 +3552,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(10)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = read_all(&mut consumer).await.unwrap(); @@ -3628,7 +3773,7 @@ mod tests { let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); // Group 0: a frame the consumer reads, positioning it there. - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -3740,42 +3885,10 @@ mod tests { let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, vec![100_000, 200_000], "the late group is delivered in order"); } - /// An explicit start floor pins where delivery begins: when a later group's stream - /// wins the arrival race, the consumer waits for the requested head under the age - /// budget instead of dropping it on arrival (#3258). - #[tokio::test] - async fn startup_keeps_late_arriving_head_group_within_max_delay() { - tokio::time::pause(); - let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); - let consumer_track = track.subscribe( - moq_net::track::Subscription::default() - .with_range(0..) - .with_delay(Delay::default().with_max(Duration::from_millis(500))), - ); - let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - - // Group 1's stream wins the race, and the consumer polls before group 0 lands. - write_group(&mut track, 1, &[ts(100_000)]); - assert!( - kio::clock::timeout(Duration::from_millis(50), consumer.read()) - .await - .is_err(), - "the requested head is within budget and must be waited for" - ); - - // Group 0 arrives moments later, well within the 500ms budget. - write_group(&mut track, 0, &[ts(0)]); - let _track = track.finish().unwrap(); - - let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); - assert_eq!(micros, vec![0, 100_000], "the head group is delivered first"); - } - /// A group whose frames lost the race to a newer group's stream is still read once /// they land: the cursor starts at group 0 and waits under the budget (#3258). #[tokio::test] @@ -3788,7 +3901,7 @@ mod tests { let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); // Group 0's stream opens first but carries no frames yet; group 1's frame wins. - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_group(&mut track, 1, &[ts(100_000)]); // Startup latches onto the lowest arrived sequence and waits under the budget. @@ -3810,11 +3923,11 @@ mod tests { }], ) .unwrap(); - group0.finish().unwrap(); + group0.close().unwrap(); let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, vec![0, 100_000], "the slow stream's frames are delivered"); } @@ -3862,7 +3975,7 @@ mod tests { let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!( micros, vec![4_800_000, 5_000_000], @@ -3920,7 +4033,7 @@ mod tests { let frames = first.ok().map(|frame| frame.unwrap().unwrap()); let frames = frames.into_iter().chain(read_all(&mut consumer).await.unwrap()); - let micros: Vec = frames.map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, expected, "warmup {warmup:?}"); } } @@ -3962,7 +4075,7 @@ mod tests { write_group(&mut track, 5, &[ts(5_000_000)]); // Group 4 covers the 4.92s target, and its first frame is read before the rest of it lands. - let mut covering = track.create_group(moq_net::group::Info { sequence: 4 }).unwrap(); + let mut covering = track.create_group(moq_net::group::Info { start: 4 }).unwrap(); let write = |group: &mut moq_net::group::Producer, micros| { let frame = Frame { timestamp: ts(micros), @@ -3989,13 +4102,13 @@ mod tests { ); write(&mut covering, 4_950_000); - covering.finish().unwrap(); + covering.close().unwrap(); let _track = track.finish().unwrap(); let micros: Vec = read_all(&mut consumer) .await .unwrap() .iter() - .map(|f| f.timestamp.as_micros()) + .map(|f| f.timestamp.as_micros().unwrap()) .collect(); assert_eq!(micros, vec![4_950_000, 5_000_000], "the warmup is not skipped as stale"); } @@ -4269,7 +4382,7 @@ mod tests { write_group(&mut track, 4, &[ts(4_500_000)]); let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, vec![4_500_000, 5_000_000]); } @@ -4334,14 +4447,14 @@ mod tests { let mut consumer = Consumer::new(consumer_track, FailingDecode); // A decodable frame first (so the consumer reads the group), then a malformed one. - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); group .write_frame(moq_net::Timestamp::ZERO, Bytes::from(0u64.to_le_bytes().to_vec())) .unwrap(); group .write_frame(moq_net::Timestamp::ZERO, Bytes::from_static(b"FAIL")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); // The first frame decodes; the malformed second frame must surface as an error. @@ -4357,13 +4470,16 @@ mod tests { ); } - /// Write one FailingDecode frame: a microsecond timestamp, or `None` for a malformed payload. + /// Write one FailingDecode frame: a microsecond timestamp, or `None` for a malformed payload, + /// stamped a microsecond past a second per group start so a group it opens starts after the + /// groups before it. fn write_failing(group: &mut moq_net::group::Producer, micros: Option) { let payload = match micros { Some(micros) => Bytes::from(micros.to_le_bytes().to_vec()), None => Bytes::from_static(b"FAIL"), }; - group.write_frame(ts(micros.unwrap_or(0)), payload).unwrap(); + let stamp = micros.unwrap_or(group.start * 1_000_000 + 1); + group.write_frame(ts(stamp), payload).unwrap(); } /// Read until something other than a frame, failing if the reader stalls. @@ -4389,10 +4505,10 @@ mod tests { moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut group, None); write_failing(&mut group, Some(0)); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let read = read_past_frames(&mut consumer).await; @@ -4409,10 +4525,10 @@ mod tests { let track = track_producer("test", timed()); let mut consumer = Consumer::new(track.subscribe(None), FailingDecode); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut group, None); write_failing(&mut group, Some(0)); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let event = kio::wait(|waiter| consumer.poll_event(waiter)).await; @@ -4431,11 +4547,11 @@ mod tests { moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode); - let mut malformed = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut malformed = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut malformed, None); let malformed_demand = malformed.demand(); - let _malformed = malformed.finish().unwrap(); - let mut later = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let _malformed = malformed.close().unwrap(); + let mut later = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_failing(&mut later, Some(10_000)); let read = read_past_frames(&mut consumer).await; @@ -4456,13 +4572,13 @@ mod tests { let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode); // The current group stays open, so the reader looks ahead into the next one. - let mut slow = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut slow = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut slow, Some(0)); - let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_failing(&mut group, Some(10_000)); write_failing(&mut group, None); write_failing(&mut group, Some(20_000)); - group.finish().unwrap(); + group.close().unwrap(); let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); @@ -4473,7 +4589,7 @@ mod tests { .is_err() ); - slow.finish().unwrap(); + slow.close().unwrap(); let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp, ts(10_000)); let read = kio::clock::timeout(Duration::from_millis(50), consumer.read()).await; @@ -4493,22 +4609,22 @@ mod tests { moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode); - let mut current = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut current = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut current, Some(0)); write_failing(&mut current, Some(10_000)); write_failing(&mut current, Some(20_000)); let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); - let mut later = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut later = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_failing(&mut later, None); - later.finish().unwrap(); + later.close().unwrap(); for expected in [10_000, 20_000] { let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp, ts(expected)); } - current.finish().unwrap(); + current.close().unwrap(); let read = read_past_frames(&mut consumer).await; assert!(matches!(read, Err(crate::Error::UnknownFormat(_))), "{read:?}"); } @@ -4523,20 +4639,20 @@ mod tests { moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode); - let mut current = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut current = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut current, Some(100_000)); let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(100_000)); - let mut malformed = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut malformed = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_failing(&mut malformed, Some(110_000)); malformed.write_frame(ts(110_000), Bytes::from_static(b"FAIL")).unwrap(); - malformed.finish().unwrap(); + malformed.close().unwrap(); // The transport only checks a group's first frame, so the rewind is its second. - let mut rewound = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + let mut rewound = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); write_failing(&mut rewound, Some(150_000)); write_failing(&mut rewound, Some(50_000)); - rewound.finish().unwrap(); + rewound.close().unwrap(); // The reader looks ahead into both while it waits on the open group. assert!( kio::clock::timeout(Duration::from_millis(50), consumer.read()) @@ -4544,7 +4660,7 @@ mod tests { .is_err() ); - current.finish().unwrap(); + current.close().unwrap(); let _track = track.finish().unwrap(); let read = read_past_frames(&mut consumer).await; assert!(matches!(read, Err(crate::Error::UnknownFormat(_))), "{read:?}"); @@ -4560,12 +4676,12 @@ mod tests { moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode).live(); - let mut old = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut old = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut old, None); - old.finish().unwrap(); - let mut newest = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + old.close().unwrap(); + let mut newest = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_failing(&mut newest, Some(10_000)); - newest.finish().unwrap(); + newest.close().unwrap(); let _track = track.finish().unwrap(); let read = read_past_frames(&mut consumer).await; @@ -4582,12 +4698,12 @@ mod tests { moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode); - let mut current = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut current = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut current, Some(0)); let first = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(first.timestamp, ts(0)); - let mut later = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut later = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_failing(&mut later, None); // The reader looks ahead into the later group while it waits on the open one. assert!( @@ -4601,7 +4717,7 @@ mod tests { let frame = consumer.read_frame().await.unwrap().unwrap(); assert_eq!(frame.timestamp, ts(10_000)); - current.finish().unwrap(); + current.close().unwrap(); let _track = track.finish().unwrap(); let read = read_past_frames(&mut consumer).await; assert!(matches!(read, Err(crate::Error::UnknownFormat(_))), "{read:?}"); @@ -4617,7 +4733,7 @@ mod tests { moq_net::track::Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut consumer = Consumer::new(track.subscribe(subscription), FailingDecode); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut group, Some(0)); write_failing(&mut group, None); let first = consumer.read_frame().await.unwrap().unwrap(); @@ -4635,11 +4751,11 @@ mod tests { fn the_lookahead_keeps_a_decode_error_before_an_evicted_tail() { let waiter = kio::Waiter::noop(); let track = track_producer("test", timed()); - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_failing(&mut group, Some(0)); write_failing(&mut group, None); write_failing(&mut group, Some(10_000)); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); // Finding the start buffers one frame; the read behind it already holds the rest. let mut buffer = GroupBuffer::::new(group.consume(), Memory::new(DEFAULT_MAX_BUFFER)); @@ -4668,7 +4784,7 @@ mod tests { let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); let payload_bytes = vec![0x01, 0x02, 0x03, 0x04, 0x05]; - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group, @@ -4681,7 +4797,7 @@ mod tests { }], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); @@ -4707,7 +4823,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -4727,7 +4843,7 @@ mod tests { // Write group 2: recv_group fires, drops current buffer_until for group 1 write_group(&mut track, 2, &[ts(200_000)]); kio::clock::sleep(Duration::from_millis(20)).await; - group0.finish().unwrap(); + group0.close().unwrap(); let _track = track.finish().unwrap(); }); @@ -4762,7 +4878,7 @@ mod tests { let frames = read_all(&mut consumer).await.unwrap(); assert_eq!(frames.len(), 1); assert_eq!(frames[0].timestamp, ts(one_hour)); - assert_eq!(frames[0].timestamp.as_micros(), one_hour as u128); + assert_eq!(frames[0].timestamp.as_micros().unwrap(), one_hour as u128); } #[tokio::test] @@ -4798,7 +4914,7 @@ mod tests { // to avoid the age skip and test B-frame timestamp tracking. let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); for ×tamp in &[ts(0), ts(66_000), ts(33_000)] { Container::Legacy(crate::container::Kind::Data) .write( @@ -4818,7 +4934,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = kio::clock::timeout(Duration::from_secs(2), async { @@ -4853,7 +4969,7 @@ mod tests { write_group(&mut track, 3, &[ts(0)]); write_group(&mut track, 5, &[ts(150_000)]); - let mut group7 = track.create_group(moq_net::group::Info { sequence: 7 }).unwrap(); + let mut group7 = track.create_group(moq_net::group::Info { start: 7 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group7, @@ -4879,7 +4995,7 @@ mod tests { }], ) .unwrap(); - group7.finish().unwrap(); + group7.close().unwrap(); let _track = track.finish().unwrap(); }); @@ -4908,7 +5024,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let _group5 = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); + let _group5 = track.create_group(moq_net::group::Info { start: 5 }).unwrap(); write_group(&mut track, 7, &[ts(210_000)]); // Group 5 is open and within budget: its frames may still arrive, so wait. @@ -4926,7 +5042,7 @@ mod tests { let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, vec![210_000, 710_000], "the expired frameless group is skipped"); } @@ -4943,7 +5059,7 @@ mod tests { let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); // Sequence 0 never arrives; sequence 1's stream opens but never carries a frame. - let group1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let group1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); let _track = track.finish().unwrap(); // While group 1 is open its frames may still come, so the consumer waits. @@ -4976,7 +5092,7 @@ mod tests { // Group 0's stream is open but its frames lose the race; the track boundary // arrives before them. - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_group(&mut track, 1, &[ts(100_000)]); let _track = track.finish().unwrap(); @@ -4999,10 +5115,10 @@ mod tests { }], ) .unwrap(); - group0.finish().unwrap(); + group0.close().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, vec![0, 100_000], "the open group's late frames are delivered"); } @@ -5015,8 +5131,14 @@ mod tests { let subscription = moq_net::track::Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(10))) .with_ranges([ - moq_net::track::Range::groups(0..1), - moq_net::track::Range::groups(10..11), + moq_net::track::Range { + start: moq_net::track::Anchor::Time(0), + end: Some(moq_net::track::Anchor::Time(0)), + }, + moq_net::track::Range { + start: moq_net::track::Anchor::Time(10), + end: Some(moq_net::track::Anchor::Time(10)), + }, ]); let mut consumer = Consumer::new( track.subscribe(subscription), @@ -5033,7 +5155,7 @@ mod tests { .expect("the open track still delivers the later range") .unwrap() .expect("a frame"); - micros.push(frame.timestamp.as_micros()); + micros.push(frame.timestamp.as_micros().unwrap()); } assert_eq!(micros, vec![0, 1_000_000]); } @@ -5059,7 +5181,7 @@ mod tests { let _track = track.finish().unwrap(); let frames = read_all(&mut consumer).await.unwrap(); - let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let micros: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert_eq!(micros, vec![700_000, 800_000, 900_000]); } @@ -5203,7 +5325,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -5224,7 +5346,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(20)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = read_all(&mut consumer).await.unwrap(); @@ -5241,7 +5363,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -5260,7 +5382,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(20)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = read_all(&mut consumer).await.unwrap(); @@ -5280,7 +5402,7 @@ mod tests { let mut consumer = container_max_delay_only(consumer_track, Duration::from_millis(100)); // Group 0: stalled at ts=0, NOT finished - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group0, @@ -5302,7 +5424,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(50)).await; - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = read_all(&mut consumer).await.unwrap(); @@ -5339,7 +5461,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); group0.abort(moq_net::Error::Cancel).unwrap(); write_group(&mut track, 1, &[ts(30_000)]); @@ -5361,7 +5483,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); for timestamp in [ts(0), ts(10_000)] { let frame = Frame { timestamp, @@ -5373,7 +5495,7 @@ mod tests { .write(&mut group0, &[frame]) .unwrap(); } - let group0 = group0.finish().unwrap(); + let group0 = group0.close().unwrap(); write_group(&mut track, 1, &[ts(30_000)]); let _track = track.finish().unwrap(); @@ -5400,7 +5522,7 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); let first = Frame { timestamp: ts(0), payload: Bytes::from(vec![0xDEu8; 1024]), @@ -5460,8 +5582,8 @@ mod tests { ); let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); - let group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); - group0.finish().unwrap(); + let group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); + group0.close().unwrap(); write_group(&mut track, 1, &[ts(30_000)]); let _track = track.finish().unwrap(); @@ -5483,7 +5605,7 @@ mod tests { let mut consumer = Consumer::new(consumer_track, Container::Legacy(crate::container::Kind::Data)); // Write frames using Container::Legacy(crate::container::Kind::Data) encoding - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); for i in 0..3u64 { let frame = Frame { timestamp: ts(i * 33_333), @@ -5495,7 +5617,7 @@ mod tests { .write(&mut group, &[frame]) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let mut frames = Vec::new(); @@ -5530,13 +5652,13 @@ mod tests { let mut consumer = Consumer::new(consumer_track, DurationWire); // Group 0: one frame at ts=0 lasting 33ms, never finished. - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_duration_frame(&mut group0, ts(0), ts(33_000)); // Group 1: finished, starts exactly where group 0's frame ends. - let mut group1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_duration_frame(&mut group1, ts(33_000), ts(33_000)); - group1.finish().unwrap(); + group1.close().unwrap(); let _track = track.finish().unwrap(); @@ -5561,16 +5683,12 @@ mod tests { ), DurationWire, ); - let mut group0 = track - .create_group(moq_net::group::Info { sequence: 1_000_000 }) - .unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 1_000_000 }).unwrap(); write_duration_frame(&mut group0, ts(0), ts(33_000)); - group0.finish().unwrap(); - let mut group1 = track - .create_group(moq_net::group::Info { sequence: 1_090_000 }) - .unwrap(); + group0.close().unwrap(); + let mut group1 = track.create_group(moq_net::group::Info { start: 1_090_000 }).unwrap(); write_duration_frame(&mut group1, ts(33_000), ts(33_000)); - group1.finish().unwrap(); + group1.close().unwrap(); let _track = track.finish().unwrap(); let frames = continuous_frames(&mut consumer).await; assert_eq!(frames.len(), 2); @@ -5590,13 +5708,13 @@ mod tests { let mut consumer = Consumer::new(consumer_track, DurationWire); // Group 0: frame at ts=0 lasting only 10ms, far short of group 1 at 33ms. - let mut group0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut group0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); write_duration_frame(&mut group0, ts(0), ts(10_000)); // Group 1: finished at 33ms. - let mut group1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_duration_frame(&mut group1, ts(33_000), ts(33_000)); - group1.finish().unwrap(); + group1.close().unwrap(); let _track = track.finish().unwrap(); // A second frame lands on group 0 and finishes it after the consumer has @@ -5604,7 +5722,7 @@ mod tests { let finisher = tokio::spawn(async move { kio::clock::sleep(Duration::from_millis(20)).await; write_duration_frame(&mut group0, ts(20_000), ts(10_000)); - group0.finish().unwrap(); + group0.close().unwrap(); }); let frames = kio::clock::timeout(Duration::from_secs(2), async { @@ -5628,7 +5746,7 @@ mod tests { async fn live_duration_marker_follows_an_immediately_delivered_frame() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let mut consumer = Consumer::new(track.subscribe(None), Container::Legacy(crate::container::Kind::Video)); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); Container::Legacy(crate::container::Kind::Video) .write( &mut group, @@ -5659,7 +5777,7 @@ mod tests { sequence: u64, timestamp: Timestamp, ) -> moq_net::group::Producer { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group, @@ -5770,7 +5888,7 @@ mod tests { consumer.set_max_buffer(MAX_BUFFER); let _stalled = open_group(&mut track, 0, ts(0)); - let mut markers = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut markers = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); for _ in 0..2_000 { write_marker(&mut markers, ts(10_000)); } @@ -5785,7 +5903,7 @@ mod tests { }], ) .unwrap(); - markers.finish().unwrap(); + markers.close().unwrap(); // The markers fill the max buffer behind the stalled group, which moves the cursor on to them, // and the media past them still plays. @@ -5832,7 +5950,7 @@ mod tests { ); let mut consumer = Consumer::new(subscriber, Container::Loc(crate::container::Kind::Data)); - let mut padded = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); + let mut padded = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); for i in 0..8 { let timestamp = i * 1_000; // An unknown video-config property (0x0d) padding the frame, then the timestamp (0x10). @@ -5848,7 +5966,7 @@ mod tests { wire.push(0xAD); padded.write_frame(ts(timestamp as u64), wire).unwrap(); } - padded.finish().unwrap(); + padded.close().unwrap(); let mut reads = Vec::new(); while let Poll::Ready(read) = consumer.poll_read(&waiter) { @@ -5876,7 +5994,7 @@ mod tests { consumer.set_max_buffer(2 * std::mem::size_of::>()); let _stalled = open_group(&mut track, 0, ts(0)); - let _empty = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let _empty = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); write_group(&mut track, 2, &[ts(20_000)]); let mut reads = Vec::new(); @@ -5902,8 +6020,8 @@ mod tests { consumer.poll_read(waiter), Poll::Ready(Ok(Some(Read::Frame { group: 0, .. }))) )); - let between = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); - let mut later = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + let between = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); + let mut later = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut later, @@ -5915,7 +6033,7 @@ mod tests { }], ) .unwrap(); - later.finish().unwrap(); + later.close().unwrap(); (stalled, between) } @@ -5955,11 +6073,11 @@ mod tests { let (_stalled, mut between) = fill_behind_a_stalled_group(&mut track, &mut consumer, &waiter); between.write_frame(ts(10_000), Bytes::new()).unwrap(); - between.finish().unwrap(); + between.close().unwrap(); let mut reads = Vec::new(); while let Poll::Ready(read) = consumer.poll_read(&waiter) { reads.push(match read.unwrap() { - Some(Read::Frame { group, frame }) => format!("frame {group} {}", frame.timestamp.as_micros()), + Some(Read::Frame { group, frame }) => format!("frame {group} {}", frame.timestamp.as_micros().unwrap()), Some(Read::Gap { .. }) => "gap".to_string(), Some(Read::End { group, .. }) => format!("end {group}"), None => break, @@ -5977,7 +6095,7 @@ mod tests { let mut consumer = container_max_delay_only(track.subscribe(None), max_delay); consumer.set_max_buffer(4096); for sequence in 0..2u64 { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group, @@ -5989,11 +6107,11 @@ mod tests { }], ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - let mut marker = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + let mut marker = track.create_group(moq_net::group::Info { start: 3 }).unwrap(); marker.write_frame(ts(20_000), Bytes::new()).unwrap(); - marker.finish().unwrap(); + marker.close().unwrap(); // The reader gets to the track only after the hole before group 3 has expired. tokio::time::advance(2 * max_delay).await; @@ -6001,7 +6119,7 @@ mod tests { let mut reads = Vec::new(); while let Poll::Ready(read) = consumer.poll_read(&waiter) { reads.push(match read.unwrap() { - Some(Read::Frame { group, frame }) => format!("frame {group} {}", frame.timestamp.as_micros()), + Some(Read::Frame { group, frame }) => format!("frame {group} {}", frame.timestamp.as_micros().unwrap()), Some(Read::Gap { .. }) => "gap".to_string(), Some(Read::End { group, .. }) => format!("end {group}"), None => break, @@ -6016,9 +6134,9 @@ mod tests { async fn a_hole_in_a_drained_backlog_is_late_from_the_earliest_arrival_past_it() { let mut track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let marker = |track: &mut moq_net::track::Producer, sequence: u64| { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); group.write_frame(ts(sequence * 10_000), Bytes::new()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; write_group(&mut track, 0, &[ts(0)]); marker(&mut track, 5); @@ -6073,7 +6191,7 @@ mod tests { Container::Legacy(crate::container::Kind::Data), ); for sequence in 0..256 { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group, @@ -6108,7 +6226,7 @@ mod tests { Container::Legacy(crate::container::Kind::Data), ); write_group(&mut track, 0, &[ts(0)]); - let mut group = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); Container::Legacy(crate::container::Kind::Data) .write( &mut group, @@ -6126,7 +6244,7 @@ mod tests { consumer.pending.iter().all(|group| group.is_empty()), "receipt must not decode payload" ); - group.finish().unwrap(); + group.close().unwrap(); assert!(consumer.poll_received(&kio::Waiter::noop()).is_ready()); let mut frames = Vec::new(); while let Some(frame) = consumer.read_frame().await.unwrap() { diff --git a/rs/moq-mux/src/container/declared.rs b/rs/moq-mux/src/container/declared.rs index d95a81be746..4d614c535ee 100644 --- a/rs/moq-mux/src/container/declared.rs +++ b/rs/moq-mux/src/container/declared.rs @@ -91,7 +91,7 @@ impl DeclaredClock { if self.held.len() > self.depth { return Err(ClockError::Undrained.into()); } - let source = source.as_nanos(); + let source = source.as_nanos()?; // Only `highest` matters: a frame below all of it reorders past the depth. let mut pts = pts; for &(tick, prior) in &self.highest { @@ -236,7 +236,7 @@ mod tests { let mut out = Vec::new(); for &source in read { let source = micros(source); - clock.push((), source.as_millis() as u64, source)?; + clock.push((), source.as_millis().unwrap() as u64, source)?; while let Some(((), dts, pts)) = clock.pop(false)? { out.push((dts, pts)); } diff --git a/rs/moq-mux/src/container/flv/export.rs b/rs/moq-mux/src/container/flv/export.rs index 5ef73085bac..9c3d03e77f6 100644 --- a/rs/moq-mux/src/container/flv/export.rs +++ b/rs/moq-mux/src/container/flv/export.rs @@ -263,7 +263,7 @@ impl FlvTrack { let released = (|| { while let Some(((mut arrival, record, bill), dts, pts)) = clock.pop(flush)? { // A tie the clock moved presents on the tick it moved to. - if arrival.item.timestamp.as_millis() != u128::from(pts) { + if arrival.item.timestamp.as_millis()? != u128::from(pts) { arrival.item.timestamp = moq_net::Timestamp::from_millis(pts)?; } if let Some(settled) = settled { @@ -307,7 +307,7 @@ impl FlvTrack { crate::jitter::Arrival { arrived, read, - decode: moq_net::Timestamp::from_millis(u64::from(timestamp))?.into(), + decode: moq_net::Timestamp::from_millis(u64::from(timestamp))?.try_into()?, restart, sync, leading, @@ -522,7 +522,7 @@ impl Export { let arrival = crate::jitter::Arrival { arrived: track.empty, read: kio::clock::now(), - decode: frame.timestamp.into(), + decode: frame.timestamp.try_into()?, restart: track.source.markers(), sync: kind == 1 || frame.keyframe, leading: false, @@ -545,7 +545,7 @@ impl Export { // Clocked in FLV's milliseconds, so the leading frames step a tag tick apart. let source = arrival.item.timestamp; let arrived = arrival.arrived; - clock.push((arrival, record, bill), source.as_millis() as u64, source)?; + clock.push((arrival, record, bill), source.as_millis()? as u64, source)?; track.release(key, Some(arrived), self.shift, false, &mut self.jitter)?; } None => { @@ -1021,7 +1021,7 @@ fn audio_flavor(config: &hang::catalog::AudioConfig) -> crate::Result { /// The frame's presentation time on the tag clock, `shift` milliseconds past its own. fn frame_timestamp_ms(frame: &Frame, shift: u32) -> crate::Result { - u32::try_from(frame.timestamp.as_millis()) + u32::try_from(frame.timestamp.as_millis()?) .ok() .and_then(|pts| pts.checked_add(shift)) .ok_or_else(|| Error::TimestampRange.into()) diff --git a/rs/moq-mux/src/container/flv/import.rs b/rs/moq-mux/src/container/flv/import.rs index fb6efe4d927..6f90e553453 100644 --- a/rs/moq-mux/src/container/flv/import.rs +++ b/rs/moq-mux/src/container/flv/import.rs @@ -627,14 +627,14 @@ impl Import { Ok(track.end()?) } - /// Close the current group on every track and reopen at `sequence`. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { + /// Close the current group on every track and reopen at `start`. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { self.input.check()?; for stream in self.video.values_mut() { - stream.track.seek(sequence)?; + stream.track.seek(start)?; } for stream in self.audio.values_mut() { - stream.track.seek(sequence)?; + stream.track.seek(start)?; } Ok(()) } diff --git a/rs/moq-mux/src/container/flv/import_test.rs b/rs/moq-mux/src/container/flv/import_test.rs index c6d8aca7c7e..a735ec972dc 100644 --- a/rs/moq-mux/src/container/flv/import_test.rs +++ b/rs/moq-mux/src/container/flv/import_test.rs @@ -724,7 +724,7 @@ async fn import_reports_negative_pts_and_can_resume() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read_frame().await.unwrap().expect("the good frame"); - assert_eq!(frame.timestamp.as_millis(), 10); + assert_eq!(frame.timestamp.as_millis().unwrap(), 10); } #[tokio::test(start_paused = true)] @@ -767,7 +767,7 @@ async fn import_enhanced_hvc1_applies_composition_time() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = decoder.read_frame().await.unwrap().expect("a video frame"); - assert_eq!(frame.timestamp.as_millis(), 17); + assert_eq!(frame.timestamp.as_millis().unwrap(), 17); } #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 6917ba7c73a..a79de8b8db9 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -449,7 +449,7 @@ impl Fmp4Track { self.buffer .first() .or(self.pending.as_ref()) - .map(|frame| Duration::from(frame.timestamp)) + .and_then(|frame| crate::container::time(frame.timestamp)) } /// The subscription, while the rendition is in the catalog. @@ -884,9 +884,8 @@ impl Export { .tracks .iter() .filter_map(|(name, t)| { - t.pending - .as_ref() - .map(|f| (Duration::from(f.timestamp), !t.is_video, name.clone())) + let f = t.pending.as_ref()?; + Some((crate::container::time(f.timestamp)?, !t.is_video, name.clone())) }) .min() .map(|(_, _, name)| name); @@ -897,8 +896,9 @@ impl Export { let frame = track.pending.take().unwrap(); // Everything a track reads before the init is queued or buffered behind it. if !self.init.written() { - let start = *track.backlog.get_or_insert(frame.timestamp.into()); - if Duration::from(frame.timestamp).saturating_sub(start) > INIT_QUEUE { + let at = Duration::try_from(frame.timestamp)?; + let start = *track.backlog.get_or_insert(at); + if at.saturating_sub(start) > INIT_QUEUE { return Poll::Ready(Err(Error::TrackUndescribed(self.undescribed()).into())); } } @@ -946,9 +946,8 @@ impl Export { .tracks .iter() .filter_map(|(name, t)| { - let first = t.buffer.first()?; - (t.finished() || t.buffer.is_closed()) - .then(|| (Duration::from(first.timestamp), !t.is_video, name.clone())) + let first = crate::container::time(t.buffer.first()?.timestamp)?; + (t.finished() || t.buffer.is_closed()).then(|| (first, !t.is_video, name.clone())) }) .min() .map(|(_, _, name)| name); @@ -1005,13 +1004,13 @@ impl Export { self.tracks .iter() .filter_map(|(name, track)| { - let first = track.buffer.first()?; + let first = crate::container::time(track.buffer.first()?.timestamp)?; // A stopped track's pending frame joins its buffer first. let complete = (track.finished() && track.pending.is_none()) || track.buffer.is_closed() || (self.fragment_duration == Some(Duration::ZERO) && track.buffer.last().is_some_and(|frame| frame.duration.is_some())); - complete.then(|| (Duration::from(first.timestamp), !track.is_video, name)) + complete.then_some((first, !track.is_video, name)) }) .filter(|(start, _, _)| *start <= earliest) .min() @@ -1226,13 +1225,13 @@ fn should_flush(buffer: &Buffer, frame: &Frame, fragment_duration: Option= cap } @@ -1296,7 +1295,7 @@ fn emit_fragment( // independent; video is only when its buffer opened on a keyframe (a GOP boundary). let independent = !track.is_video || independent; if let (Some(first), Some(written)) = (frames.first(), track.written) { - let start = Duration::from(first.timestamp); + let start = Duration::try_from(first.timestamp)?; if independent && start <= written { return Err(Error::TrackRewound { track: name.to_string(), @@ -1306,8 +1305,9 @@ fn emit_fragment( .into()); } } - let latest = frames.iter().map(|frame| Duration::from(frame.timestamp)).max(); - track.written = track.written.max(latest); + for frame in &frames { + track.written = track.written.max(Some(Duration::try_from(frame.timestamp)?)); + } let timescale = moq_net::Timescale::new(track.timescale)?; let run = if partial && track.config.may_reorder() { Run::PartialGop @@ -1344,12 +1344,12 @@ fn fragment_duration(frames: &[Frame], default_frame: Duration, timescale: moq_n sum.checked_add(u64::from(ticks)) .ok_or_else(|| moq_net::TimeOverflow.into()) })?; - return Ok(Timestamp::new(ticks, timescale)?.into()); + return Ok(Timestamp::new(ticks, timescale)?.try_into()?); } let mut min = Duration::MAX; let mut max = Duration::ZERO; for f in frames { - let pts = Duration::from(f.timestamp); + let pts = Duration::try_from(f.timestamp)?; min = min.min(pts); max = max.max(pts); } @@ -1422,7 +1422,8 @@ pub(crate) fn infer_missing_durations( if run == Run::PartialGop && !stated && let (Some(inferred), Some(cadence)) = (frames[i].duration, cadence) - && Duration::from(inferred) > Duration::from(cadence) + && let (Some(longer), Some(shorter)) = (crate::container::time(inferred), crate::container::time(cadence)) + && longer > shorter { frames[i].duration = Some(cadence); } @@ -1556,7 +1557,7 @@ mod tests { } fn duration_micros(frame: &Frame) -> u128 { - frame.duration.unwrap().as_micros() + frame.duration.unwrap().as_micros().unwrap() } fn frame(timestamp_us: u64, duration_us: Option) -> Frame { @@ -1604,7 +1605,12 @@ mod tests { assert_eq!(frames[1].duration, Some(ts(41_667))); assert_eq!(frames[2].duration, Some(ts(33_000))); assert_eq!( - fragment_duration(&frames, Duration::from_millis(33), frames[0].duration.unwrap().scale()).unwrap(), + fragment_duration( + &frames, + Duration::from_millis(33), + frames[0].duration.unwrap().scale().unwrap() + ) + .unwrap(), Duration::from_micros(116_334) ); } @@ -1623,7 +1629,12 @@ mod tests { assert_eq!(frames[0].duration, Some(ts(40_000))); assert_eq!( - fragment_duration(&frames, Duration::from_millis(40), frames[0].duration.unwrap().scale()).unwrap(), + fragment_duration( + &frames, + Duration::from_millis(40), + frames[0].duration.unwrap().scale().unwrap() + ) + .unwrap(), Duration::from_millis(40) ); } @@ -1646,7 +1657,12 @@ mod tests { assert_eq!(frames[0].duration, Some(ts(41_667))); assert_eq!( - fragment_duration(&frames, Duration::from_millis(33), frames[0].duration.unwrap().scale()).unwrap(), + fragment_duration( + &frames, + Duration::from_millis(33), + frames[0].duration.unwrap().scale().unwrap() + ) + .unwrap(), Duration::from_micros(41_667) ); } @@ -1699,7 +1715,7 @@ mod tests { (duration.value(), duration.scale()) }) .collect(); - assert_eq!(durations, vec![(1_000, timescale); 4]); + assert_eq!(durations, vec![(1_000, Some(timescale)); 4]); } /// The regression for the stalled HLS player: a subscriber that got a stale cached group @@ -1721,7 +1737,12 @@ mod tests { assert_eq!(frames[0].duration, Some(ts(33_000))); assert_eq!( - fragment_duration(&frames, Duration::from_millis(33), frames[0].duration.unwrap().scale()).unwrap(), + fragment_duration( + &frames, + Duration::from_millis(33), + frames[0].duration.unwrap().scale().unwrap() + ) + .unwrap(), Duration::from_millis(33) ); } @@ -1773,7 +1794,7 @@ mod tests { for f in &frames { assert_eq!( - f.duration.unwrap().as_micros(), + f.duration.unwrap().as_micros().unwrap(), 20_000, "TOC duration, not the fallback" ); @@ -1820,7 +1841,12 @@ mod tests { assert_eq!(frames[1].duration, Some(ts(33_000))); assert_eq!(frames[2].duration, Some(ts(33_000))); assert_eq!( - fragment_duration(&frames, Duration::from_millis(33), frames[0].duration.unwrap().scale()).unwrap(), + fragment_duration( + &frames, + Duration::from_millis(33), + frames[0].duration.unwrap().scale().unwrap() + ) + .unwrap(), Duration::from_micros(99_000) ); } diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index 067feea4912..2ece4b923a0 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -645,7 +645,10 @@ async fn zero_fragment_duration_uses_the_successor_then_the_cut() { /// `reorder: 0` says it presents in decode order, so the 50 ms gap is a real duration. #[tokio::test(start_paused = true)] async fn zero_fragment_duration_trusts_a_long_gap_only_without_reordering() { - let cadence = moq_net::Timestamp::from_scale(1_000, 30_000).unwrap().into(); + let cadence = moq_net::Timestamp::from_scale(1_000, 30_000) + .unwrap() + .try_into() + .unwrap(); for (reorder, expected) in [(None, cadence), (Some(0), std::time::Duration::from_millis(50))] { let mut live = Live::avc3(); for config in live.catalog.modify().unwrap().video.renditions.values_mut() { @@ -720,7 +723,7 @@ async fn unusable_framerate_uses_the_standard_fallback_rate() { super::Track::new(timescale, crate::container::fmp4::Kind::Video), ) .unwrap(); - assert_eq!(decoded[0].duration.unwrap().as_scale(timescale), 3_000); + assert_eq!(decoded[0].duration.unwrap().as_scale(timescale).unwrap(), 3_000); } /// A one-packet audio group is a one-sample fragment, timed by the catalog cadence @@ -2208,11 +2211,11 @@ async fn cmaf_source_exports_at_the_frame_timestamp() { }; let fragment = super::encode_fragment(info, &samples).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(10_000).unwrap(); group .write_frame(moq_net::Timestamp::from_secs(10).unwrap(), fragment) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let catalog_stream = crate::catalog::Consumer::<()>::new(&consumer, crate::catalog::CatalogFormat::Hang) @@ -2227,7 +2230,7 @@ async fn cmaf_source_exports_at_the_frame_timestamp() { let presented: Vec = fragments .into_iter() .flat_map(|fragment| super::decode(fragment.data, None, super::Track::new(scale, super::Kind::Video)).unwrap()) - .map(|sample| sample.timestamp.as_micros()) + .map(|sample| sample.timestamp.as_micros().unwrap()) .collect(); assert_eq!(presented, [10_000_000, 10_066_000, 10_033_000]); } @@ -2672,7 +2675,7 @@ fn presentation_times( times .entry(id) .or_default() - .extend(samples.iter().map(|sample| sample.timestamp.as_micros())); + .extend(samples.iter().map(|sample| sample.timestamp.as_micros().unwrap())); } times } diff --git a/rs/moq-mux/src/container/fmp4/import.rs b/rs/moq-mux/src/container/fmp4/import.rs index cc4bf56ad93..e1d3202aeeb 100644 --- a/rs/moq-mux/src/container/fmp4/import.rs +++ b/rs/moq-mux/src/container/fmp4/import.rs @@ -166,8 +166,8 @@ struct Passthrough { // `container::Producer`), so it reports what its estimator measured itself. member: Option, - // Sequence to use for the next group, set by `Import::seek`. - pending_sequence: Option, + // Start to key the next group with instead of its fragment's timestamp, set by `Import::seek`. + pending_start: Option, // Measures the track's catalog jitter and bitrate from the fragments; the descriptor's own // bitrate, when it declares one, still wins over what this measures. @@ -192,12 +192,12 @@ impl Passthrough { self.rendition.estimate(reading) } - /// Close the open group, so the next fragment opens one at `pending_sequence` or the next. + /// Close the open group, so the next fragment opens one at `pending_start` or its own timestamp. fn close(&mut self) -> crate::Result<()> { self.estimator.cut(None); self.publish_estimate()?; if let Some(group) = self.group.take() { - group.finish()?; + group.close()?; } Ok(()) } @@ -639,7 +639,7 @@ impl Import { group: None, start: None, member, - pending_sequence: None, + pending_start: None, estimator: self.catalog.estimator(), claim: crate::catalog::Claim::new(self.catalog.bandwidth()), }) @@ -1262,7 +1262,8 @@ impl Import { }; producer.commit(sample)?; if last { - producer.burst(pending.run.end.checked_sub(pending.run.earliest)?.into())?; + let span = pending.run.end.checked_sub(pending.run.earliest)?; + producer.burst(span.try_into()?)?; } }, _ => unreachable!()), } @@ -1414,7 +1415,10 @@ impl Fmp4Track { .map(|duration| Timestamp::new(duration as u64, timescale)) .transpose()?; if let Some(duration) = duration_ts { - self.sample_duration = Some(self.sample_duration.map_or(duration, |min| min.min(duration))); + self.sample_duration = Some( + self.sample_duration + .map_or(duration, |min| crate::container::earlier(min, duration)), + ); } let presented = self.edit.present(pts)?; @@ -1427,8 +1431,8 @@ impl Fmp4Track { Some(duration) => timestamp.checked_add(duration)?, None => timestamp, }; - end = Some(end.map_or(sample_end, |max: Timestamp| max.max(sample_end))); - earliest = Some(earliest.map_or(timestamp, |min: Timestamp| min.min(timestamp))); + end = Some(end.map_or(sample_end, |max| crate::container::later(max, sample_end))); + earliest = Some(earliest.map_or(timestamp, |min| crate::container::earlier(min, timestamp))); samples.push(Sample { dts, @@ -1593,7 +1597,7 @@ impl Passthrough { // Audio follows segment boundaries; video also waits for a sync first sample so a new // group's first fragment can be decoded independently. let keyframe = placement.keyframe(kind, run.samples.first().ok_or(Error::MissingTrun)?); - let start_group = self.group.is_none() || (placement.new_segment && keyframe); + let mut start_group = self.group.is_none() || (placement.new_segment && keyframe); // A group may not start below the one before it. Refused before the open group closes or // a new one opens, so a rewound fragment publishes nothing, as an unwrapped track does. if start_group @@ -1602,13 +1606,28 @@ impl Passthrough { { return Err(crate::container::TimestampRewind { timestamp, floor }.into()); } + // A group is keyed by its first fragment's timestamp, which has to start after every group + // the track holds. A segment that doesn't continues the open group instead; with none open + // it is refused. + let start = match start_group && self.pending_start.is_none() { + true => match (crate::start::opens(&self.track, timestamp)?, &self.group) { + (Some(start), _) => Some(start), + (None, Some(_)) => { + start_group = false; + None + } + (None, None) => Some(crate::start::after(&self.track, timestamp)?), + }, + false => None, + }; let group = if start_group { if let Some(prev) = self.group.take() { - prev.finish()?; + prev.close()?; } - match self.pending_sequence.take() { - Some(sequence) => self.track.create_group(moq_net::group::Info { sequence })?, - None => self.track.append_group()?, + match (self.pending_start.take(), start) { + (Some(start), _) => self.track.create_group(moq_net::group::Info { start })?, + (None, Some(start)) => self.track.append_group(start)?, + (None, None) => unreachable!("a new group without a pending start has one"), } } else { self.group.take().ok_or(Error::NoKeyframe)? @@ -1624,7 +1643,7 @@ impl Passthrough { .checked_add(run.samples.iter().map(|sample| sample.range.len()).sum()) .ok_or(Error::InvalidDataOffset)?; let mut writer = group.create_frame(moq_net::frame::Info { - timestamp: Some(timestamp), + timestamp, size: size as u64, })?; writer.write(prefix)?; @@ -1650,7 +1669,7 @@ impl Passthrough { self.estimator.write(timestamp, pending.size); let bytes = run.samples.iter().map(|sample| sample.range.len()).sum(); self.sent.add(run.samples.len() as u64, bytes, keyframes); - self.estimator.burst(end.checked_sub(timestamp)?.into()); + self.estimator.burst(end.checked_sub(timestamp)?.try_into()?); self.estimator.reach(end); self.publish_estimate()?; Ok(()) @@ -1764,39 +1783,63 @@ impl Import { } } - /// Close the current group on every track and open the next one at `sequence`. + /// Close the current group on every track and open the next one at `start`, in each track's timescale. /// /// Broadcast-wide: every track inside this fMP4 import advances together; per-track - /// control is intentionally not exposed. Skipping sequences is not a new timeline, so the + /// control is intentionally not exposed. Skipping ahead is not a new timeline, so the /// next fragment must still advance past the last decode time; see /// [`discontinuity`](Self::discontinuity). Text spends a group per cue, so a text track that - /// is already past `sequence` continues from its own next group instead. Video resumes at its + /// is already past `start` continues from its own next start instead. Video resumes at its /// next sync sample, dropping the samples before it. /// /// A fragment whose samples are still arriving owns its groups until its last sample, so the /// seek waits for it to finish publishing. - pub fn seek(&mut self, sequence: u64) -> Result<()> { + pub fn seek(&mut self, start: u64) -> Result<()> { + self.reposition(Seek::Start(start)) + } + + /// Close the current group on every track, as [`seek`](Self::seek) does, but key the groups + /// after it at their own times: for a source that lost what came between and can't name where + /// it resumes until it reads it, such as an HLS reader that fell behind its playlist. + /// + /// Each group closes without an end, so a reader sees the gap rather than a group that seems to + /// run up to the media after it. + pub fn rejoin(&mut self) -> Result<()> { + self.reposition(Seek::Own) + } + + fn reposition(&mut self, seek: Seek) -> Result<()> { self.input.check()?; if self.publishing() { - self.deferred.seek = Some(sequence); + self.deferred.seek = Some(seek); return Ok(()); } - self.apply_seek(sequence) + self.apply_seek(seek) } - fn apply_seek(&mut self, sequence: u64) -> Result<()> { + fn apply_seek(&mut self, seek: Seek) -> Result<()> { for track in self.text.values_mut() { - let next = track.producer.next_sequence(); - track.producer.seek(sequence.max(next))?; + match seek { + Seek::Start(start) => { + let next = track.producer.next_start(); + track.producer.seek(start.max(next))?; + } + Seek::Own => track.producer.cut(None)?, + } } for track in self.tracks.values_mut() { // The next group opens on a sync sample, so video drops the samples before one. if track.kind == Kind::Video { track.orphaned = true; } - media!(&mut track.output, producer => producer.seek(sequence)?, passthrough => { + media!(&mut track.output, producer => match seek { + Seek::Start(start) => producer.seek(start)?, + Seek::Own => producer.cut(None)?, + }, passthrough => { passthrough.close()?; - passthrough.pending_sequence = Some(sequence); + if let Seek::Start(start) = seek { + passthrough.pending_start = Some(start); + } }); } Ok(()) @@ -1839,8 +1882,8 @@ impl Import { if discontinuity { self.apply_discontinuity(); } - if let Some(sequence) = seek { - self.apply_seek(sequence)?; + if let Some(seek) = seek { + self.apply_seek(seek)?; } Ok(()) } @@ -1849,10 +1892,19 @@ impl Import { /// Controls called while a fragment was still publishing, applied once it has. #[derive(Default)] struct Deferred { - seek: Option, + seek: Option, discontinuity: bool, } +/// Where the groups after a [`seek`](Import::seek) or [`rejoin`](Import::rejoin) start. +#[derive(Clone, Copy)] +enum Seek { + /// The next group starts here, in each track's timescale. + Start(u64), + /// Each group starts at its own first sample. + Own, +} + /// An fMP4 import that [finished](Import::finish): every track has ended, and they can still be /// read or aborted, but it takes no more input. /// @@ -2361,7 +2413,17 @@ mod tests { } } - /// Each closed group's sequence and frame count, or `None` for a group that failed. + /// Where a group at `ticks` of the source's 44.1 kHz audio starts on `consumer`'s track. + fn start(consumer: &moq_net::track::Subscriber, ticks: u64) -> u64 { + let timescale = consumer.info().timescale.expect("a timed track"); + Timestamp::from_scale(ticks, 44_100) + .unwrap() + .convert(timescale) + .unwrap() + .value() + } + + /// Each closed group's start and frame count, or `None` for a group that failed. fn groups(consumer: &mut moq_net::track::Subscriber) -> Vec<(u64, Option)> { use futures::FutureExt; let mut groups = Vec::new(); @@ -2375,10 +2437,10 @@ mod tests { frames = None; break; } - None => panic!("group {} left open", group.sequence), + None => panic!("group {} left open", group.start), } } - groups.push((group.sequence, frames)); + groups.push((group.start, frames)); } groups } @@ -2399,9 +2461,10 @@ mod tests { import.seek(100).unwrap(); import.decode(&sample[SIZE / 2..]).unwrap(); assert_eq!(groups(&mut consumer), [(0, Some(1))], "passthrough: {passthrough}"); - // The discontinuity lets the next fragment restate the decode time. + // The discontinuity lets the next fragment restate a decode time inside the last + // fragment, past its start: group starts strictly increase. import - .decode(&media_header(id, 0, &[0], SIZE as u32, Some(SIZE))) + .decode(&media_header(id, 512, &[0], SIZE as u32, Some(SIZE))) .unwrap(); import.decode(&sample).unwrap(); let _finished = import.finish().unwrap(); @@ -2426,9 +2489,10 @@ mod tests { // In place, so the tracks can be inspected after the failure. let err = import.finish_in_place().unwrap_err(); assert!(matches!(err, crate::Error::Mp4(_)), "{err:?}"); + let second = start(&consumer, 1024); assert_eq!( groups(&mut consumer), - [(0, Some(1)), (1, None)], + [(0, Some(1)), (second, None)], "passthrough: {passthrough}" ); let finished = match &import.tracks[&id].output { @@ -2461,7 +2525,8 @@ mod tests { err.to_string() ); let _finished = import.finish().unwrap(); - assert_eq!(groups(&mut consumer), [(0, Some(1))], "passthrough: {passthrough}"); + let first = start(&consumer, 48_000); + assert_eq!(groups(&mut consumer), [(first, Some(1))], "passthrough: {passthrough}"); } } } diff --git a/rs/moq-mux/src/container/fmp4/import_test.rs b/rs/moq-mux/src/container/fmp4/import_test.rs index ecab2bc6a44..04d73bf81a8 100644 --- a/rs/moq-mux/src/container/fmp4/import_test.rs +++ b/rs/moq-mux/src/container/fmp4/import_test.rs @@ -18,12 +18,18 @@ fn replay() -> moq_net::track::Subscription { /// Drain every group currently buffered on the consumer without waiting for new ones. /// Used in tests where the producer is still alive after writing. #[cfg(test)] -fn drain_group_sequences(consumer: &mut moq_net::track::Subscriber) -> Vec { - let mut sequences = Vec::new(); +fn drain_group_starts(consumer: &mut moq_net::track::Subscriber) -> Vec { + let mut starts = Vec::new(); while let Some(group) = consumer.recv_group().now_or_never().and_then(|r| r.ok().flatten()) { - sequences.push(group.sequence); + starts.push(group.start); } - sequences + starts +} + +/// A group `start` on a track counting in `timescale`, in microseconds. +fn micros(start: u64, timescale: Option) -> u64 { + let timescale = timescale.expect("a timed track"); + moq_net::Timestamp::new(start, timescale).unwrap().as_micros().unwrap() as u64 } fn run_fmp4(data: &[u8]) -> crate::catalog::hang::Catalog { @@ -717,7 +723,7 @@ async fn published( let track = consumer.track(name).unwrap().subscribe(replay()).await.unwrap(); let mut reader = crate::container::Consumer::new(track, container); while let Some(frame) = reader.read_frame().await.unwrap() { - samples.push((frame.timestamp.as_micros(), frame.keyframe, frame.payload)); + samples.push((frame.timestamp.as_micros().unwrap(), frame.keyframe, frame.payload)); } } out @@ -1088,7 +1094,7 @@ async fn unwrapped_samples_present_at_their_composition_time() { let mut frames = Vec::new(); while let Some(frame) = group.read_frame().await.unwrap() { let frame = hang::container::Frame::decode(frame.payload).unwrap(); - frames.push((frame.timestamp.as_micros(), frame.payload.to_vec())); + frames.push((frame.timestamp.as_micros().unwrap(), frame.payload.to_vec())); } // 25, 27, and 26 frames at 24 per second, in whole microseconds. assert_eq!( @@ -1386,7 +1392,8 @@ async fn video_joined_mid_gop_starts_at_a_sync_sample() { for fragment in &fragments { fmp4.decode(fragment).unwrap(); } - fmp4.seek(10).unwrap(); + // Past every start the tracks hold, in either's timescale: group starts strictly increase. + fmp4.seek(1_000_000).unwrap(); for fragment in &after_seek { fmp4.decode(fragment).unwrap(); } @@ -1594,11 +1601,11 @@ async fn test_seek_sets_initial_sequence_in(passthrough: bool) { let _ = fmp4.decode(&frag_buf); let _finished = fmp4.finish().unwrap(); - let sequences = drain_group_sequences(&mut video_track); - assert!(!sequences.is_empty(), "expected at least one group"); - assert_eq!(sequences[0], 100, "first group should land at the seeked sequence"); - for win in sequences.windows(2) { - assert_eq!(win[1], win[0] + 1, "subsequent groups should auto-increment"); + let starts = drain_group_starts(&mut video_track); + assert!(!starts.is_empty(), "expected at least one group"); + assert_eq!(starts[0], 100, "first group should land at the seeked start"); + for win in starts.windows(2) { + assert!(win[1] > win[0], "subsequent groups start later, at their keyframes"); } } @@ -1655,8 +1662,8 @@ async fn test_msf_catalog_roundtrip() { assert!(matches!(audio.container, Container::Cmaf { .. })); } -/// The sequence of every group a finished track published. -async fn sequences(consumer: &moq_net::broadcast::Consumer, track: &str) -> Vec { +/// The start of every group a finished track published, in microseconds. +async fn starts(consumer: &moq_net::broadcast::Consumer, track: &str) -> Vec { let replay = moq_net::track::Subscription::default() .with_delay(moq_net::track::Delay::default().with_max(std::time::Duration::from_secs(3600))); let mut track = consumer @@ -1666,11 +1673,12 @@ async fn sequences(consumer: &moq_net::broadcast::Consumer, track: &str) -> Vec< .await .unwrap() .ordered(); - let mut sequences = Vec::new(); + let timescale = track.info().timescale; + let mut starts = Vec::new(); while let Some(group) = track.next_group().await.unwrap() { - sequences.push(group.sequence); + starts.push(micros(group.start, timescale)); } - sequences + starts } // ---- Sample-duration handling in decode() ---- @@ -2113,13 +2121,13 @@ async fn segmented_source_writes_one_group_per_segment_in(passthrough: bool) { catalog.finish().unwrap(); assert_eq!( - sequences(&consumer, &video_name).await, - [0, 1, 2], + starts(&consumer, &video_name).await, + [0, 1_000_000, 2_000_000], "one video group per segment" ); assert_eq!( - sequences(&consumer, &audio_name).await, - [0, 1, 2], + starts(&consumer, &audio_name).await, + [0, 1_000_000, 2_000_000], "one audio group per segment, not one per fragment" ); } @@ -2170,9 +2178,12 @@ async fn groups_with_skew(video_offset_us: i64, passthrough: bool) -> (Vec, let _finished = fmp4.finish().unwrap(); catalog.finish().unwrap(); + // The second each group starts in, which a skew of a few frames either way leaves alone. + let segments = + |starts: Vec| -> Vec { starts.into_iter().map(|start| (start + 500_000) / 1_000_000).collect() }; ( - sequences(&consumer, &video_name).await, - sequences(&consumer, &audio_name).await, + segments(starts(&consumer, &video_name).await), + segments(starts(&consumer, &audio_name).await), ) } @@ -2308,9 +2319,18 @@ async fn a_mid_gop_join_keeps_tracks_on_the_same_groups_in(passthrough: bool) { } let _finished = fmp4.finish().unwrap(); - let video = drain_group_sequences(&mut video_track); - let audio = drain_group_sequences(&mut audio_track); - assert_eq!(audio, video, "audio and video groups line up"); + let video: Vec<_> = drain_group_starts(&mut video_track) + .into_iter() + .map(|start| micros(start, video_track.info().timescale)) + .collect(); + let audio: Vec<_> = drain_group_starts(&mut audio_track) + .into_iter() + .map(|start| micros(start, audio_track.info().timescale)) + .collect(); + // Audio's first group also holds the fragment before video's first keyframe, so the groups + // line up from there: as many of them, cut where video cuts. + assert_eq!(audio.len(), video.len(), "audio and video groups line up"); + assert_eq!(audio[1..], video[1..], "audio and video groups line up"); } /// One audio fragment for bbb's track 2, carrying a single sample at the track's 44100 timescale. @@ -2683,9 +2703,9 @@ async fn join_after(earlier: Earlier, start_us: u64) { let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); fmp4.decode(&init).unwrap(); - let before = clock.now().as_micros(); + let before = clock.now().as_micros().unwrap(); fmp4.decode(&live_session(start_us, 20)).unwrap(); - let after = clock.now().as_micros(); + let after = clock.now().as_micros().unwrap(); if let Some(stream) = data.as_mut() { stream .append(&serde_json::json!("after"), catalog.clock().now()) @@ -2716,7 +2736,7 @@ async fn join_after(earlier: Earlier, start_us: u64) { let mut times = Vec::new(); while let std::task::Poll::Ready(Ok(Some(mut group))) = stamps.poll_recv_group(&waiter) { while let std::task::Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) { - times.push(frame.timestamp.unwrap().as_micros()); + times.push(frame.timestamp.as_micros().unwrap()); } } assert_eq!(times.len(), 2); @@ -2752,9 +2772,9 @@ async fn an_import_shifts_onto_an_explicit_clock() { let mut fmp4 = crate::container::fmp4::Import::new(broadcast, catalog.reserve()); fmp4.decode(&init).unwrap(); - let before = clock.now().as_micros(); + let before = clock.now().as_micros().unwrap(); fmp4.decode(&live_session(3_600_000_000, 5)).unwrap(); - let after = clock.now().as_micros(); + let after = clock.now().as_micros().unwrap(); let _finished = fmp4.finish().unwrap(); let snapshot = catalog.snapshot(); @@ -2964,7 +2984,11 @@ async fn imports_the_sample_defaults_fixtures_in(passthrough: bool) { let frame = hang::container::Frame::decode(frame.payload).unwrap(); // The duration marker closing a video group is no sample. if !frame.payload.is_empty() { - samples.push((frame.timestamp.as_micros() as u64, index == 0, frame.payload.to_vec())); + samples.push(( + frame.timestamp.as_micros().unwrap() as u64, + index == 0, + frame.payload.to_vec(), + )); } index += 1; } @@ -2985,7 +3009,7 @@ async fn imports_the_sample_defaults_fixtures_in(passthrough: bool) { let mut first = track.recv_group().await.unwrap().unwrap(); let frame = first.read_frame().await.unwrap().unwrap(); - let frames = super::decode(frame.payload, frame.timestamp, wire.track().unwrap()).unwrap(); + let frames = super::decode(frame.payload, Some(frame.timestamp), wire.track().unwrap()).unwrap(); let samples: Vec = frames.iter().map(super::FixtureSample::from).collect(); assert_eq!(samples, fixture.samples, "{}", fixture.name); @@ -3125,7 +3149,7 @@ async fn webvtt_samples_become_timed_text_with_source_metadata() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = reader.read_frame().await.unwrap().unwrap(); - assert_eq!(frame.timestamp.as_micros(), 250_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 250_000); assert_eq!( &frame.payload[..], b"WEBVTT\n\ncue-1\n00:00:00.250 --> 00:00:01.500 align:start\nHello\n\n" @@ -3496,7 +3520,7 @@ async fn ttml_samples_become_timed_text_with_source_metadata() { assert_eq!(config.format, hang::catalog::TextFormat::Ttml); assert_eq!(config.lang.as_deref(), Some("eng")); assert_eq!(config.label.as_deref(), Some("English captions")); - assert_eq!(frame.timestamp.as_micros(), 100_000_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 100_000_000); assert_eq!( &frame.payload[..], br#"

Hello

"# @@ -3519,7 +3543,7 @@ async fn ttml_follows_a_shifted_source_clock() { .place(moq_net::Timestamp::from_secs(100).unwrap(), wall) .unwrap(); let (_, frame) = imported_ttml(timebase, broadcast, &catalog, &consumer).await; - assert_eq!(frame.timestamp.as_micros(), 10_000_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 10_000_000); assert_eq!( &frame.payload[..], br#"

Hello

"# @@ -3578,7 +3602,7 @@ async fn ttml_body_begins_at_the_serialized_frame_timestamp() { crate::catalog::hang::Container::Legacy(crate::container::Kind::Data), ); let frame = reader.read_frame().await.unwrap().unwrap(); - assert_eq!(frame.timestamp.as_micros(), 100_000_011); + assert_eq!(frame.timestamp.as_micros().unwrap(), 100_000_011); let payload = std::str::from_utf8(&frame.payload).unwrap(); assert!( payload.contains(r#""#), diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 96637d6b197..6553822e643 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -546,7 +546,9 @@ impl Container for Wire { fn decode(&self, frame: moq_net::frame::Frame) -> std::result::Result, Self::Error> { // CMAF carries no data track, so an object with no bytes is no marker and fails to decode. - decode(frame.payload, frame.timestamp, self.track()?) + // A scale-less track's unscaled key is no broadcast time, so its samples keep their `tfdt`. + let timestamp = frame.timestamp.scale().map(|_| frame.timestamp); + decode(frame.payload, timestamp, self.track()?) } } @@ -585,7 +587,7 @@ impl Track { /// `timestamp` is the moq-lite frame timestamp, which is the broadcast timeline: the fragment's /// earliest sample presents at it. `tfdt` and the `trun` offsets only place the samples relative /// to each other, since a publisher may move a passthrough track to another timeline without -/// rewriting the payload. An untimed frame (`None`) has no broadcast time, so its samples +/// rewriting the payload. A scale-less track's frame (`None`) has no broadcast time, so its samples /// present at the time `tfdt` gives them. pub(crate) fn decode(data: Bytes, timestamp: Option, track: Track) -> Result> { metadata::check(&data)?; @@ -670,7 +672,11 @@ fn encode(group: &mut moq_net::group::Producer, frames: &[Frame], info: Fragment // The fragment may carry several samples; the net frame's timestamp is the // fragment's earliest presentation time, which `decode` anchors its samples to. // It is rounded to the track's ticks the same way the samples are, so the two agree. - let Some(earliest) = frames.iter().map(|f| f.timestamp).min() else { + let Some(earliest) = frames + .iter() + .map(|f| f.timestamp) + .reduce(|earliest, timestamp| if timestamp < earliest { timestamp } else { earliest }) + else { return Ok(()); }; let timestamp = Timestamp::new(timestamp_ticks(earliest, info.timescale)?, info.timescale)?; @@ -721,9 +727,10 @@ fn base_ticks(frame: &Frame, timescale: moq_net::Timescale) -> Result { timestamp_ticks(frame.timestamp, timescale) } -/// Round an absolute timestamp to its nearest tick at the output scale. +/// Round an absolute timestamp to its nearest tick at the output scale. Refuses an +/// [unscaled](Timestamp::unscaled) one, which has no time. fn timestamp_ticks(timestamp: Timestamp, timescale: moq_net::Timescale) -> Result { - let source_scale = u128::from(timestamp.scale().as_u64()); + let source_scale = u128::from(timestamp.scale().ok_or(moq_net::TimeOverflow)?.as_u64()); let scaled = u128::from(timestamp.value()) * u128::from(timescale.as_u64()); let ticks = (scaled + source_scale / 2) / source_scale; u64::try_from(ticks).map_err(|_| Error::PtsOverflow) @@ -833,7 +840,7 @@ pub(super) fn encode_at(info: FragmentInfo, base_dts: u64, frames: &[Frame]) -> /// Convert a duration to the exact value stored in a 32-bit `trun` sample-duration field. fn trun_duration(duration: Timestamp, timescale: moq_net::Timescale) -> Result { - let source_timescale = duration.scale().as_u64(); + let source_timescale = duration.scale().ok_or(moq_net::TimeOverflow)?.as_u64(); let output_timescale = timescale.as_u64(); let scaled = u128::from(duration.value()) * u128::from(output_timescale); let ticks = scaled / u128::from(source_timescale); @@ -1558,8 +1565,8 @@ pub(crate) struct FixtureSample { impl From<&Frame> for FixtureSample { fn from(frame: &Frame) -> Self { Self { - timestamp: frame.timestamp.as_micros() as u64, - duration: frame.duration.map_or(0, |d| d.as_micros() as u64), + timestamp: frame.timestamp.as_micros().unwrap() as u64, + duration: frame.duration.map_or(0, |d| d.as_micros().unwrap() as u64), keyframe: frame.keyframe, data: frame.payload.to_vec(), } @@ -1601,7 +1608,8 @@ mod tests { let wire = Wire::new(synthesize_audio_trak(1, 48_000, &config).unwrap()).unwrap(); for (name, data) in super::metadata::tests::attacks(b"moof") { bounded &= super::metadata::tests::measure(name, || { - wire.decode(moq_net::frame::Frame::new(None, data.clone())).is_err() + wire.decode(moq_net::frame::Frame::new(Timestamp::unscaled(0), data.clone())) + .is_err() }); } assert!(bounded); @@ -2118,7 +2126,7 @@ mod tests { } fn micros(frames: &[Frame]) -> Vec { - frames.iter().map(|f| f.timestamp.as_micros()).collect() + frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect() } /// The frame timestamp is the broadcast timeline: a passthrough fragment whose `tfdt` still @@ -2147,16 +2155,16 @@ mod tests { let broadcast = moq_net::broadcast::Info::new().produce(); let info = moq_net::track::Info::default().with_timescale(timescale); let track = broadcast.create_track("video", info).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); let input = [sample(100_000), sample(33_000), sample(66_000)]; wire.write(&mut group, &input).unwrap(); let frame = group.consume().read_frame().await.unwrap().unwrap(); - assert_eq!(frame.timestamp.unwrap().as_micros(), 33_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 33_000); let frames = decode( frame.payload.clone(), - frame.timestamp, + Some(frame.timestamp), Track::new(timescale, Kind::Video), ) .unwrap(); @@ -2166,7 +2174,7 @@ mod tests { assert_eq!(micros(&shifted), [5_067_000, 5_000_000, 5_033_000]); } - /// An untimed track's frames carry no broadcast time, so their samples present at `tfdt`. + /// A scale-less track's frames carry no broadcast time, so their samples present at `tfdt`. #[test] fn an_untimed_fragment_decodes_at_its_tfdt() { let mut config = VideoConfig::new(hang::catalog::VideoCodec::VP8); @@ -2179,7 +2187,9 @@ mod tests { let input = [sample(source), sample(source + 66_000), sample(source + 33_000)]; let fragment = encode_fragment(info(1, timescale, 0), &input).unwrap(); - let frames = wire.decode(moq_net::frame::Frame::new(None, fragment)).unwrap(); + let frames = wire + .decode(moq_net::frame::Frame::new(Timestamp::unscaled(0), fragment)) + .unwrap(); let source = u128::from(source); assert_eq!(micros(&frames), [source, source + 66_000, source + 33_000]); } @@ -2196,7 +2206,7 @@ mod tests { let broadcast = moq_net::broadcast::Info::new().produce(); let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); let track = broadcast.create_track("video", info).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); let err = wire.write(&mut group, &[sample(0)]).unwrap_err(); assert!(matches!(err, Error::TimescaleMismatch { .. }), "got {err:?}"); @@ -2293,7 +2303,7 @@ mod tests { let frames = decode(Bytes::from(buf), None, Track::new(timescale, Kind::Video)).unwrap(); assert_eq!(frames.len(), 1); - assert_eq!(frames[0].timestamp.as_micros(), 83_333); + assert_eq!(frames[0].timestamp.as_micros().unwrap(), 83_333); assert_eq!(frames[0].duration, None); } } diff --git a/rs/moq-mux/src/container/fmp4/muxer.rs b/rs/moq-mux/src/container/fmp4/muxer.rs index 96a66dd1f8d..433557ad3b6 100644 --- a/rs/moq-mux/src/container/fmp4/muxer.rs +++ b/rs/moq-mux/src/container/fmp4/muxer.rs @@ -353,7 +353,7 @@ impl Muxer { let mut frames = frames.to_vec(); apply_codec_durations(&mut frames, self.opus); if let Some(end) = end - && let Some(last) = frames.iter_mut().max_by_key(|frame| frame.timestamp.as_micros()) + && let Some(last) = frames.iter_mut().max_by_key(|frame| frame.timestamp.as_micros().ok()) && last.duration.is_none() // Quantized at the output scale like the gaps between samples, so any end gap is // whole ticks; one within a tick falls back like a sample with no successor. @@ -384,9 +384,12 @@ mod tests { use super::*; use hang::catalog::{AudioCodec, VideoCodec}; - /// A millisecond track: the frames written here carry timestamps. + /// A millisecond track: the frames written here carry timestamps. It sets no max age, so a + /// group finished whole stays readable after its span has passed. fn timed() -> moq_net::track::Info { - moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) + moq_net::track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None) } fn frame(micros: u64, keyframe: bool) -> Frame { @@ -435,9 +438,9 @@ mod tests { ) .unwrap(); assert_eq!(decoded.len(), 2); - assert_eq!(decoded[0].timestamp.as_micros(), 10_000_000); + assert_eq!(decoded[0].timestamp.as_micros().unwrap(), 10_000_000); assert!(decoded[0].keyframe); - assert_eq!(decoded[1].timestamp.as_micros(), 10_033_000); + assert_eq!(decoded[1].timestamp.as_micros().unwrap(), 10_033_000); } /// A duration marker times the group's last sample through fmp4 (and HLS, which @@ -459,7 +462,7 @@ mod tests { let mut muxer = video_muxer(); let frames = muxer.read(&mut group).await.unwrap(); assert_eq!(frames.len(), 2, "the marker is not a sample"); - assert_eq!(frames[1].duration.unwrap().as_micros(), 33_000); + assert_eq!(frames[1].duration.unwrap().as_micros().unwrap(), 33_000); let fragment = muxer.fragment(0, &frames, None).unwrap(); assert_eq!(super::super::sample_durations(&fragment), vec![Some(990), Some(990)]); @@ -761,7 +764,7 @@ mod tests { super::super::Track::new(timescale, crate::container::fmp4::Kind::Video), ) .unwrap(); - assert_eq!(decoded[0].duration.unwrap().as_scale(timescale), 3_000); + assert_eq!(decoded[0].duration.unwrap().as_scale(timescale).unwrap(), 3_000); } // A downstream timeline fixed at 90 kHz overrides the framerate-derived default, and the @@ -791,7 +794,7 @@ mod tests { super::super::Track::new(timescale, crate::container::fmp4::Kind::Video), ) .unwrap(); - assert_eq!(decoded[0].timestamp.as_micros(), 33_333); + assert_eq!(decoded[0].timestamp.as_micros().unwrap(), 33_333); } #[test] @@ -811,7 +814,7 @@ mod tests { super::super::Track::new(timescale, crate::container::fmp4::Kind::Video), ) .unwrap(); - assert_eq!(decoded[0].duration.unwrap().as_scale(timescale), 3_000); + assert_eq!(decoded[0].duration.unwrap().as_scale(timescale).unwrap(), 3_000); } // mdhd.timescale is 32 bits, but moq_net::Timescale spans the whole QUIC varint range. A @@ -902,7 +905,7 @@ mod tests { assert_eq!(decoded.len(), 4); for f in &decoded { assert_eq!( - f.duration.unwrap().as_micros(), + f.duration.unwrap().as_micros().unwrap(), 20_000, "TOC duration, not the fallback" ); diff --git a/rs/moq-mux/src/container/group.rs b/rs/moq-mux/src/container/group.rs index 96c4801079d..5757b366a76 100644 --- a/rs/moq-mux/src/container/group.rs +++ b/rs/moq-mux/src/container/group.rs @@ -8,7 +8,7 @@ use super::{Container, Frame}; /// This is the group-scoped counterpart to [`Consumer`](super::Consumer). Where that one /// subscribes to a track and juggles group ordering, age skipping, and rewinds, this one /// reads exactly the group it was handed, in arrival order, and ends. That is what a caller -/// wants after a FETCH, a group already chosen by sequence, or for a live group taken from +/// wants after a FETCH, a group already chosen by start, or for a live group taken from /// [`recv_group`](moq_net::track::Subscriber::recv_group), whose frames return as they land. /// /// A batch of frames decoded from one wire frame (a CMAF fragment carrying several samples) is @@ -35,9 +35,9 @@ impl GroupConsumer { } } - /// The sequence number of this group within its track. - pub fn sequence(&self) -> u64 { - self.group.sequence + /// Where this group starts within its track, in the track's timescale. + pub fn start(&self) -> u64 { + self.group.start } /// Read the next frame, or `None` once the group ends. @@ -88,9 +88,12 @@ mod tests { use super::*; use crate::catalog::hang::Container as Hang; - /// A millisecond track: the frames written here carry timestamps. + /// A millisecond track: the frames written here carry timestamps. It sets no max age, so a + /// group finished whole stays readable after its span has passed. fn timed() -> moq_net::track::Info { - moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) + moq_net::track::Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None) } fn frame(timestamp_us: u64, payload: &'static [u8], keyframe: bool) -> Frame { @@ -114,9 +117,10 @@ mod tests { media.write(frame(1_020_000, b"delta", false)).unwrap(); let _finished = media.finish().unwrap(); - let group = consumer.track("media").unwrap().fetch_group(0, None).await.unwrap(); + // The group starts at its keyframe, a second in, in the track's milliseconds. + let group = consumer.track("media").unwrap().fetch_group(1_000, None).await.unwrap(); let mut group = GroupConsumer::new(group, Hang::Legacy(crate::container::Kind::Data)); - assert_eq!(group.sequence(), 0); + assert_eq!(group.start(), 1_000); let first = group.read().await.unwrap().unwrap(); assert_eq!(first.payload, b"keyframe".as_slice()); @@ -173,7 +177,7 @@ mod tests { .unwrap(); let _finished = media.finish().unwrap(); - let group = consumer.track("media").unwrap().fetch_group(0, None).await.unwrap(); + let group = consumer.track("media").unwrap().fetch_group(1_000, None).await.unwrap(); let mut group = GroupConsumer::new(group, Hang::Legacy(crate::container::Kind::Video)); let first = group.read().await.unwrap().unwrap(); @@ -196,7 +200,7 @@ mod tests { let broadcast = moq_net::broadcast::Info::new().produce(); let track = broadcast.create_track("media", timed()).unwrap(); let format = Hang::Legacy(crate::container::Kind::Video); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); let mut live = GroupConsumer::new(group.consume(), Hang::Legacy(crate::container::Kind::Video)); format @@ -211,7 +215,7 @@ mod tests { assert_eq!(second.payload, b"delta".as_slice()); assert!(live.read().now_or_never().is_none(), "the open group has nothing more"); - group.finish().unwrap(); + group.close().unwrap(); assert!(live.read().await.unwrap().is_none()); } @@ -230,6 +234,12 @@ mod tests { let broadcast = moq_net::broadcast::Info::new().produce(); // A CMAF track counts in its init's ticks. let info = moq_net::track::Info::default().with_timescale(muxer.timescale()); + // The group starts at its keyframe, two seconds in. + let start = moq_net::Timestamp::from_secs(2) + .unwrap() + .convert(muxer.timescale()) + .unwrap() + .value(); let track = broadcast.create_track("video", info).unwrap(); let consumer = broadcast.consume(); @@ -250,7 +260,7 @@ mod tests { } let _finished = media.finish().unwrap(); - let group = consumer.track("video").unwrap().fetch_group(0, None).await.unwrap(); + let group = consumer.track("video").unwrap().fetch_group(start, None).await.unwrap(); let mut group = GroupConsumer::new(group, Hang::new(&cmaf, crate::container::Kind::Video).unwrap()); let first = group.read().await.unwrap().unwrap(); diff --git a/rs/moq-mux/src/container/legacy/mod.rs b/rs/moq-mux/src/container/legacy/mod.rs index d7fcd521a2b..efa4f8366d1 100644 --- a/rs/moq-mux/src/container/legacy/mod.rs +++ b/rs/moq-mux/src/container/legacy/mod.rs @@ -83,3 +83,116 @@ impl Container for Wire { }]) } } + +/// How long each packet of a Legacy audio track plays, as its codec states. +/// +/// Legacy frames carry no duration; the hang draft leaves it to the codec. Knowing it, a reader +/// tells a group that ends where the next one starts from one followed by lost audio. Group keys +/// can't say: they are starts, and a wire without skips (moq-lite-05 and 06) reports the starts +/// between two groups as it does a loss. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Audio(Packets); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Packets { + /// Read from each packet's TOC byte, in 48 kHz samples. + Opus, + /// The same number of samples in every packet, at the track's rate. + Fixed { samples: u32, rate: u32 }, + /// Interleaved 32-bit samples, so the payload's length says. + Pcm { frame_bytes: u32, rate: u32 }, +} + +impl Audio { + /// The packet clock of `config`'s codec, or `None` when a packet doesn't state its duration + /// without decoding it. + pub fn new(config: &hang::catalog::AudioConfig) -> Option { + let rate = config.sample_rate; + if rate == 0 { + return None; + } + let packets = match &config.codec { + hang::catalog::AudioCodec::Opus => Packets::Opus, + // AAC-LC codes 1024 samples per packet. SBR and PS packets decode to twice that, at a + // rate the catalog may or may not double, so they state nothing here. + hang::catalog::AudioCodec::AAC(aac) if aac.profile == 2 => Packets::Fixed { samples: 1024, rate }, + hang::catalog::AudioCodec::Pcm => Packets::Pcm { + frame_bytes: config.channel_count.checked_mul(4).filter(|bytes| *bytes > 0)?, + rate, + }, + _ => return None, + }; + Some(Self(packets)) + } + + /// How long `payload` plays, or `None` when it doesn't parse as one packet. + pub fn duration(&self, payload: &[u8]) -> Option { + if payload.is_empty() { + return None; + } + let (samples, rate) = match self.0 { + Packets::Opus => (crate::codec::opus::packet_samples(payload)?, 48_000), + Packets::Fixed { samples, rate } => (samples, rate), + Packets::Pcm { frame_bytes, rate } => { + let frame_bytes = frame_bytes as usize; + if !payload.len().is_multiple_of(frame_bytes) { + return None; + } + (u32::try_from(payload.len() / frame_bytes).ok()?, rate) + } + }; + moq_net::Timestamp::from_scale(samples.into(), rate.into()).ok() + } + + /// Decode a frame as [`Wire`] does for audio, stating each packet's duration. + pub(crate) fn decode(&self, frame: moq_net::frame::Frame) -> crate::Result> { + let mut frames = Wire(Kind::Audio).decode(frame)?; + for frame in &mut frames { + frame.duration = self.duration(&frame.payload); + } + Ok(frames) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(codec: hang::catalog::AudioCodec, sample_rate: u32, channel_count: u32) -> hang::catalog::AudioConfig { + hang::catalog::AudioConfig::new(codec, sample_rate, channel_count) + } + + fn micros(duration: Option) -> Option { + duration.map(|duration| std::time::Duration::try_from(duration).unwrap().as_micros()) + } + + #[test] + fn opus_states_each_packet_in_its_toc() { + let audio = Audio::new(&config(hang::catalog::AudioCodec::Opus, 48_000, 2)).unwrap(); + // CELT FB at 2.5 ms and 20 ms, one frame each. + assert_eq!(micros(audio.duration(&[16 << 3, 0])), Some(2_500)); + assert_eq!(micros(audio.duration(&[31 << 3, 0])), Some(20_000)); + // Two 20 ms frames. + assert_eq!(micros(audio.duration(&[(31 << 3) | 1, 0])), Some(40_000)); + // An empty payload is a marker, not a packet. + assert_eq!(audio.duration(&[]), None); + } + + #[test] + fn aac_lc_and_pcm_state_a_fixed_or_measured_count() { + let aac = hang::catalog::AudioCodec::AAC(hang::catalog::AAC { profile: 2 }); + let audio = Audio::new(&config(aac, 48_000, 2)).unwrap(); + assert_eq!(micros(audio.duration(&[0xff])), Some(21_333)); + + let audio = Audio::new(&config(hang::catalog::AudioCodec::Pcm, 1_000, 2)).unwrap(); + // Ten stereo frames of 32-bit samples are 10 ms at 1 kHz. + assert_eq!(micros(audio.duration(&[0; 80])), Some(10_000)); + assert_eq!(audio.duration(&[0; 81]), None, "a partial sample frame"); + } + + #[test] + fn a_codec_whose_packets_vary_states_nothing() { + let he = hang::catalog::AudioCodec::AAC(hang::catalog::AAC { profile: 5 }); + assert_eq!(Audio::new(&config(he, 48_000, 2)), None); + } +} diff --git a/rs/moq-mux/src/container/mkv/export.rs b/rs/moq-mux/src/container/mkv/export.rs index 2b40636146c..7d5b3ac6181 100644 --- a/rs/moq-mux/src/container/mkv/export.rs +++ b/rs/moq-mux/src/container/mkv/export.rs @@ -181,7 +181,7 @@ impl MkvTrack { arrival: crate::jitter::Arrival, jitter: &mut crate::jitter::Buffer, ) -> Result<()> { - let pts = (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64; + let pts = (arrival.item.timestamp.as_nanos()? * 90_000 / 1_000_000_000) as u64; match &mut self.declared { Some(clock) => { let source = arrival.item.timestamp; @@ -199,7 +199,7 @@ impl MkvTrack { if let Some(clock) = &mut self.declared { while let Some((mut arrival, dts, pts)) = clock.pop(flush)? { // A tie the clock moved presents on the tick it moved to. - if (arrival.item.timestamp.as_nanos() * 90_000 / 1_000_000_000) as u64 != pts { + if (arrival.item.timestamp.as_nanos()? * 90_000 / 1_000_000_000) as u64 != pts { arrival.item.timestamp = moq_net::Timestamp::from_scale(pts, 90_000)?; } if !self.finished && !flush { @@ -244,7 +244,7 @@ impl MkvTrack { if !self.finished && !flush { arrival.arrived = arrival.arrived.max(self.latest); } - arrival.decode = moq_net::Timestamp::from_scale(dts, 90_000)?.into(); + arrival.decode = moq_net::Timestamp::from_scale(dts, 90_000)?.try_into()?; jitter.push(name.to_owned(), arrival)?; } Ok(()) @@ -541,7 +541,7 @@ impl Export { crate::jitter::Arrival { arrived, read: kio::clock::now(), - decode: frame.timestamp.into(), + decode: frame.timestamp.try_into()?, restart: track.source.markers(), sync: track.kind == TrackKind::Audio || frame.keyframe, leading: false, @@ -768,7 +768,7 @@ impl Export { // frame's timestamp directly at MILLI rather than going through micros. let frame_ticks: u64 = frame .timestamp - .as_millis() + .as_millis()? .try_into() .map_err(|_| Error::TimestampU64)?; diff --git a/rs/moq-mux/src/container/mkv/import.rs b/rs/moq-mux/src/container/mkv/import.rs index 3e7e5246336..89dd5d29fc1 100644 --- a/rs/moq-mux/src/container/mkv/import.rs +++ b/rs/moq-mux/src/container/mkv/import.rs @@ -889,19 +889,19 @@ impl Import { Ok(()) } - /// Close the current group on every track and open the next one at `sequence`. + /// Close the current group on every track and open the next one at `start`. /// /// Broadcast-wide: every track inside this MKV import advances together; per-track /// control is intentionally not exposed. While a block streams in, the seek waits for it to end, /// since its later frames continue its group. - pub fn seek(&mut self, sequence: u64) -> Result<()> { + pub fn seek(&mut self, start: u64) -> Result<()> { self.input.check()?; if self.block.is_some() { - self.seek = Some(sequence); + self.seek = Some(start); return Ok(()); } for track in self.tracks.values_mut() { - track.track.seek(sequence)?; + track.track.seek(start)?; } Ok(()) } @@ -1700,8 +1700,15 @@ mod tests { /// A SimpleBlock header for a keyframe on `track` declaring `size` payload bytes. fn simple_block(track: u8, size: u64) -> Vec { + simple_block_at(track, size, 0) + } + + /// A keyframe block `timecode` ticks past its cluster's. + fn simple_block_at(track: u8, size: u64, timecode: i16) -> Vec { let mut block = header(&[0xa3], size + 4); - block.extend_from_slice(&[0x80 | track, 0, 0, 0x80]); + block.push(0x80 | track); + block.extend_from_slice(&timecode.to_be_bytes()); + block.push(0x80); block } @@ -2128,9 +2135,11 @@ mod tests { let (mut import, _catalog, _consumer) = import(); import.decode(&live(vec![entry(1, 1, "V_VP9")])).unwrap(); let payload = vec![3; 1024 * 1024]; - // Twice the most a track holds. - for _ in 0..64 { - import.decode(&simple_block(1, payload.len() as u64)).unwrap(); + // Twice the most a track holds, one group a millisecond. + for timecode in 0..64 { + import + .decode(&simple_block_at(1, payload.len() as u64, timecode)) + .unwrap(); import.decode(&payload).unwrap(); } let _import = import.finish().unwrap(); diff --git a/rs/moq-mux/src/container/mkv/import_test.rs b/rs/moq-mux/src/container/mkv/import_test.rs index 53d99923e44..2740a0e000e 100644 --- a/rs/moq-mux/src/container/mkv/import_test.rs +++ b/rs/moq-mux/src/container/mkv/import_test.rs @@ -1185,7 +1185,7 @@ async fn laced_opus_frames() { for (index, packet) in packets.iter().enumerate() { let frame = reader.read_frame().await.unwrap().expect("one frame per packet"); assert_eq!(frame.payload.as_ref(), *packet, "{lacing:?}, grouped={grouped}"); - assert_eq!(frame.timestamp.as_micros(), 107_000 + index as u128 * 20_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 107_000 + index as u128 * 20_000); } assert!(reader.read_frame().await.unwrap().is_none()); } @@ -1298,7 +1298,7 @@ async fn unlaced_block_duration_does_not_use_track_default() { while let Some(mut group) = track.recv_group().await.unwrap() { while let Some(frame) = group.read_frame().await.unwrap() { let frame = hang::container::Frame::decode(frame.payload).unwrap(); - last = Some((frame.timestamp.as_micros(), frame.payload.len())); + last = Some((frame.timestamp.as_micros().unwrap(), frame.payload.len())); } } assert_eq!(last, Some((80_000, 0))); @@ -1887,7 +1887,7 @@ async fn a_seek_mid_block_waits_for_the_block() { while group.read_frame().await.unwrap().is_some() { frames += 1; } - groups.push((group.sequence, frames)); + groups.push((group.start, frames)); } assert_eq!(groups, [(0, 2), (100, 1)]); } diff --git a/rs/moq-mux/src/container/mod.rs b/rs/moq-mux/src/container/mod.rs index cfebb38e2c4..7b50b537f77 100644 --- a/rs/moq-mux/src/container/mod.rs +++ b/rs/moq-mux/src/container/mod.rs @@ -127,12 +127,42 @@ pub struct Frame { pub keyframe: bool, } +/// A media timestamp's time since its track's zero, or `None` when [unscaled](moq_net::Timestamp::unscaled). +/// +/// A [`Consumer`] refuses an unscaled frame as it decodes, so every frame an export reads has a +/// time; where nothing can return the error, skipping one only has to never panic. +pub(crate) fn time(timestamp: moq_net::Timestamp) -> Option { + std::time::Duration::try_from(timestamp).ok() +} + +/// `timestamp` in microseconds for a message, or its debug form when unscaled. +fn micros(timestamp: moq_net::Timestamp) -> String { + match timestamp.as_micros() { + Ok(micros) => format!("{micros} µs"), + Err(_) => format!("{timestamp:?}"), + } +} + +/// The later of two timestamps, `a` when they don't compare: [`moq_net::Timestamp`] has no +/// [`Ord`], since an unscaled one orders against no scaled one. +pub(crate) fn later(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { + if b > a { b } else { a } +} + +/// The earlier of two timestamps, `a` when they don't compare; see [`later`]. +pub(crate) fn earlier(a: moq_net::Timestamp, b: moq_net::Timestamp) -> moq_net::Timestamp { + if b < a { b } else { a } +} + /// Stamp `frame` with the duration that ends at `bound`, unless it already has one. pub(crate) fn close_duration(frame: &mut Frame, bound: moq_net::Timestamp) { if frame.duration.is_some() { return; } - let Some(delta) = bound.as_micros().checked_sub(frame.timestamp.as_micros()) else { + let (Ok(bound), Ok(timestamp)) = (bound.as_micros(), frame.timestamp.as_micros()) else { + return; + }; + let Some(delta) = bound.checked_sub(timestamp) else { return; }; if delta == 0 { @@ -164,9 +194,9 @@ pub struct InvalidEnd; /// A group starts before the previous group did: the source restarted, which is a new broadcast. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] #[error( - "frame timestamp {} µs is below the previous group's start {} µs", - .timestamp.as_micros(), - .floor.as_micros() + "frame timestamp {} is below the previous group's start {}", + micros(*.timestamp), + micros(*.floor) )] pub struct TimestampRewind { /// The refused frame's timestamp. diff --git a/rs/moq-mux/src/container/producer.rs b/rs/moq-mux/src/container/producer.rs index def867c2120..4936a67ddb2 100644 --- a/rs/moq-mux/src/container/producer.rs +++ b/rs/moq-mux/src/container/producer.rs @@ -3,13 +3,23 @@ use bytes::Bytes; use super::{Container, Frame, Kind}; fn add_micros(timestamp: moq_net::Timestamp, extra: moq_net::Timestamp) -> Option { - let micros = timestamp.as_micros().saturating_add(extra.as_micros()); + let micros = timestamp.as_micros().ok()?.saturating_add(extra.as_micros().ok()?); let micros = u64::try_from(micros).ok()?; moq_net::Timestamp::from_micros(micros).ok() } +/// Whether `left` is before `right` to the microsecond; false when either is unscaled. fn timestamp_lt(left: moq_net::Timestamp, right: moq_net::Timestamp) -> bool { - left.as_micros() < right.as_micros() + matches!((left.as_micros(), right.as_micros()), (Ok(left), Ok(right)) if left < right) +} + +/// Refuse a frame without a time with [`TimestampMismatch`](moq_net::Error::TimestampMismatch): +/// media is timed, and a scale-less track's [unscaled](moq_net::Timestamp::unscaled) key is not. +fn timed(timestamp: moq_net::Timestamp, duration: Option) -> Result<(), moq_net::Error> { + match timestamp.scale().is_some() && duration.is_none_or(|duration| duration.scale().is_some()) { + true => Ok(()), + false => Err(moq_net::Error::TimestampMismatch), + } } /// A producer for media tracks that manages group boundaries. @@ -54,9 +64,9 @@ pub struct Producer { buffer_duration: std::time::Duration, - /// Sequence to use for the next group opened by [`Self::write`]. - /// Set by [`Self::seek`] and consumed on the next group creation. - pending_sequence: Option, + /// Start to key the next group opened by [`Self::write`] with, instead of its keyframe's + /// timestamp. Set by [`Self::seek`] and consumed on the next group creation. + pending_start: Option, /// This track's place in the broadcast's join interval, for a video or audio track a catalog /// minted: it reports the longest group measured. @@ -69,6 +79,9 @@ pub struct Producer { /// Exclusive presentation end of finished groups, where a payload with no timestamp of its /// own lands. Not a floor: a group may start below it. live_edge: Option, + /// The latest frame timestamp written, which a discontinuity marker never falls below: a frame + /// without a duration reaches only its own timestamp, so the live edge can sit right on it. + newest: Option, /// The keyframe timestamp that opened the latest group. The next group may not start below it. start: Option, @@ -151,11 +164,12 @@ impl Producer { Some(prefix) if self.buffer_duration.is_zero() => prefix, _ => return Err(crate::Error::UnsupportedContainer("a streamed frame".into())), }; + timed(timestamp, duration)?; self.prepare(timestamp, keyframe)?; let group = self.group.take().expect("prepare opens a group"); let total = prefix.len().checked_add(size).ok_or(moq_net::Error::FrameTooLarge)?; let mut writer = group.create_frame(moq_net::frame::Info { - timestamp: Some(timestamp), + timestamp, size: total as u64, })?; writer.write(prefix)?; @@ -250,10 +264,11 @@ impl Producer { group: None, buffer: Vec::new(), buffer_duration: std::time::Duration::ZERO, - pending_sequence: None, + pending_start: None, member: None, end: None, live_edge: None, + newest: None, start: None, floor: None, last_duration: None, @@ -288,10 +303,11 @@ where group: None, buffer: Vec::new(), buffer_duration: std::time::Duration::ZERO, - pending_sequence: None, + pending_start: None, member: None, end: None, live_edge: None, + newest: None, start: None, floor: None, last_duration: None, @@ -336,12 +352,17 @@ where }) } - /// Resolve a timestamp on the broadcast's shared clock. + /// Resolve a timestamp on the broadcast's shared clock: `hint`, or the clock's reading now. + /// + /// A reading comes from the clock rather than media, so one landing on the newest group's start, + /// as a burst of untimed frames can, is moved one unit of the track's timescale past it: group + /// starts strictly increase. pub fn timestamp(&self, hint: Option) -> crate::Result { - self.rendition - .as_ref() - .ok_or(crate::Error::NotPublished)? - .timestamp(hint) + let rendition = self.rendition.as_ref().ok_or(crate::Error::NotPublished)?; + match hint { + Some(pts) => rendition.timestamp(Some(pts)), + None => Ok(crate::start::next(&self.inner, rendition.timestamp(None)?)?.1), + } } fn publish_estimate(&mut self) -> crate::Result<()> { @@ -480,9 +501,9 @@ where self.group.is_none() } - /// One past the track's largest group, even an empty one: where a group continues after - /// what the track holds. - pub(crate) fn next_sequence(&self) -> u64 { + /// One past the track's largest group start, even an empty group's: the lowest start a group + /// after what the track holds can take. + pub(crate) fn next_start(&self) -> u64 { self.inner.largest().map_or(0, |largest| largest.group + 1) } @@ -578,6 +599,9 @@ where /// independently decodable (audio) marks only the first frame of each group a keyframe (see /// [`needs_keyframe`](Self::needs_keyframe)) so the group spans more than one frame. /// + /// A group starts at its keyframe's timestamp, in the track's timescale, and starts strictly + /// increase: a keyframe that doesn't start after the open group continues it. + /// /// Group starts never go backwards: a keyframe below the start of the previous group, or any /// frame below the start of the group before its own, returns /// [`TimestampRewind`](super::TimestampRewind) with the refused timestamp and that start, @@ -585,6 +609,7 @@ where /// broadcast. Frames may still dip below the previous group's content: B-frames, open-GOP /// leading pictures, and a keyframe that overlaps the previous group's last frame. pub fn write(&mut self, frame: Frame) -> crate::Result<()> { + timed(frame.timestamp, frame.duration)?; self.prepare(frame.timestamp, frame.keyframe)?; // Buffer or write the frame. @@ -626,6 +651,19 @@ where return Err(super::TimestampRewind { timestamp, floor }.into()); } + // A keyframe opens a group keyed by its timestamp, which has to start after every group the + // track holds. One that doesn't, such as parameter sets and the picture they precede written + // as two keyframes at one timestamp, continues the open group instead. With none open it is + // refused, before anything changes. + let start = match keyframe && self.pending_start.is_none() { + true => match (crate::start::opens(&self.inner, timestamp)?, &self.group) { + (Some(start), _) => Some(start), + (None, Some(_)) => return Ok(()), + (None, None) => Some(crate::start::after(&self.inner, timestamp)?), + }, + false => None, + }; + // A keyframe cuts the previous group, using its timestamp as the boundary where the // previous group's content ends, unless it overlaps that content. if keyframe { @@ -645,9 +683,17 @@ where // mid-stream join) decides whether to skip until the first keyframe. return Err(super::MissingKeyframe.into()); } - let group = match self.pending_sequence.take() { - Some(sequence) => self.inner.create_group(moq_net::group::Info { sequence })?, - None => self.inner.append_group()?, + let group = match (self.pending_start.take(), start) { + (Some(start), _) => self.inner.create_group(moq_net::group::Info { start })?, + (None, Some(start)) => { + // A track's first group says nothing came before it, so a reader can tell it + // is the first: a decoder starting there needs no warmup. + if self.inner.largest().is_none() { + self.inner.skip_groups(0..start); + } + self.inner.append_group(start)? + } + (None, None) => unreachable!("a keyframe without a pending start has one"), }; self.floor = self.start; @@ -674,10 +720,7 @@ where /// Place a written frame in the write order, unless it is history; see [`Self::history`]. fn order(&mut self, timestamp: moq_net::Timestamp) { - if self - .history - .is_none_or(|until| timestamp.as_micros() > until.as_micros()) - { + if self.history.is_none_or(|until| timestamp_lt(until, timestamp)) { self.estimator.order(timestamp); } } @@ -693,11 +736,20 @@ where /// returns [`InvalidEnd`](super::InvalidEnd) without flushing or closing the group. pub fn cut(&mut self, end: Option) -> crate::Result<()> { let marker_at = end.or_else(|| self.estimated_end()); - self.close(end, marker_at) + self.close(end, marker_at, None) } /// Close the current group, ending a video track's group with a duration marker at `marker_at`. - fn close(&mut self, end: Option, marker_at: Option) -> crate::Result<()> { + /// + /// The group ends whole at `whole`, else at a bound `end` the caller knows, such as the next + /// keyframe, else where a media track's duration marker goes, so a reader can tell the group + /// starting there continues it. + fn close( + &mut self, + end: Option, + marker_at: Option, + whole: Option, + ) -> crate::Result<()> { if self.container.kind() == Kind::Video && !self.reordered && let Some((end, previous)) = end.zip(self.previous_timestamp) @@ -715,12 +767,18 @@ where let reordered = std::mem::take(&mut self.reordered); // Where this group's content stops: the duration marker when we write one, else the // caller's bound, else the furthest point we wrote. - let reached = marker_at.or(end).max(self.end.take()); + let reached = match (marker_at.or(end), self.end.take()) { + (Some(bound), Some(written)) => Some(super::later(written, bound)), + (bound, written) => bound.or(written), + }; self.last_duration = None; self.previous_timestamp = None; let tail_end = marker_at.filter(|_| !reordered); - if let Err(err) = self.close_group(tail_end, reached) { + // Media ends whole where its last frame does, as the duration marker says. A data track's + // items have no cadence to estimate that from. + let estimated = tail_end.filter(|_| self.container.kind() != Kind::Data); + if let Err(err) = self.close_group(whole.or(end).or(estimated), tail_end, reached) { // A group that fails to close fails with the cause, rather than staying open for the next // write to continue. self.buffer.clear(); @@ -732,9 +790,11 @@ where self.publish_estimate() } - /// Flush the buffer into the open group and finish it, raising the live edge to `reached`. + /// Flush the buffer into the open group and finish it, raising the live edge to `reached`. With + /// a `bound`, the group ends whole there rather than only closing. fn close_group( &mut self, + bound: Option, tail_end: Option, reached: Option, ) -> crate::Result<()> { @@ -743,7 +803,10 @@ where self.container.finish_group(group, tail_end)?; } if let Some(group) = self.group.take() { - group.finish()?; + match bound.and_then(|bound| crate::start::end(&self.inner, group.start, bound)) { + Some(end) => group.finish(end)?, + None => group.close()?, + }; // Only a closed group reaches anything; a cut with none open names a point nothing reached. if let Some(end) = reached { self.raise_live_edge(end); @@ -768,11 +831,19 @@ where self.cadence = Some(delta); } self.previous_timestamp = Some(timestamp); + if self.newest.is_none_or(|newest| timestamp_lt(newest, timestamp)) { + self.newest = Some(timestamp); + } // Timestamp and duration can be at different scales, so add them in micros; the // sub-microsecond rounding that costs is far below a segment boundary. - let micros = timestamp.as_micros() + duration.map(|d| d.as_micros()).unwrap_or(0); - let Ok(micros) = u64::try_from(micros) else { return }; + let Ok(micros) = timestamp.as_micros() else { return }; + let Ok(extra) = duration.map_or(Ok(0), |duration| duration.as_micros()) else { + return; + }; + let Ok(micros) = u64::try_from(micros + extra) else { + return; + }; let Ok(end) = moq_net::Timestamp::from_micros(micros) else { return; }; @@ -794,13 +865,14 @@ where add_micros(last, self.cadence?) } - /// Close the current group (if any) and open the next group at the given sequence. + /// Close the current group (if any) and open the next group at the given start. /// - /// The next [`write`](Self::write) must be a keyframe and will land in a group with - /// `sequence`. Useful for joining mid-stream. - pub fn seek(&mut self, sequence: u64) -> crate::Result<()> { + /// The next [`write`](Self::write) must be a keyframe and will land in a group keyed `start`, + /// in the track's timescale, rather than at its own timestamp. The groups after it are keyed + /// by their keyframes again, which have to start after it. Useful for joining mid-stream. + pub fn seek(&mut self, start: u64) -> crate::Result<()> { self.cut(None)?; - self.pending_sequence = Some(sequence); + self.pending_start = Some(start); Ok(()) } @@ -812,7 +884,7 @@ where /// group looks exactly like the one that would have followed, and a consumer bounding a /// sample by the next group's first frame hands it the entire gap as its duration. That /// produced a 2405 second video sample out of a publisher that had been paused 40 minutes. - /// Consecutive sequence numbers can't rule a pause out, so this + /// Group starts alone can't rule a pause out, so this /// marker is the only thing that can say one happened. /// /// It also fixes what a subscriber joining mid-break sees. A subscription starts at the @@ -820,28 +892,66 @@ where /// the marker and waits for real media, instead of being served the group from *before* /// the break as though it were live. /// - /// Audio and video write one empty frame at the exclusive end of the previous epoch. - /// That object exists on moq-transport, so the live edge moves immediately; a consumer + /// Audio and video write one empty frame one unit of the track's timescale before the exclusive + /// end of the previous epoch, so what resumes at that end starts after it, but never below the + /// latest frame written: a frame without a duration ends where it starts, so what resumes then + /// has to start after that frame. That object exists on moq-transport, so the live edge moves immediately; a consumer /// MUST NOT submit it to a decoder. An empty payload is data on a data track, so its /// marker is one object with no bytes at all, not even the container's timestamp. The - /// next [`write`](Self::write) opens the group after the marker and may not start below - /// the last media group's start. + /// next [`write`](Self::write) opens the group after the marker, which it has to start after: + /// group starts strictly increase. + /// + /// Before anything is written there is no timeline to break, so it publishes nothing. /// /// To bound the closing group's final frame, [`cut(end)`](Self::cut) before calling this. /// Otherwise the open group closes without a duration marker: what resumes may land sooner /// than one estimated frame later (a capture that reopens at once), and a guessed end past /// it would read as a rewind to every consumer. pub fn discontinuity(&mut self) -> crate::Result<()> { - self.close(None, None)?; + // The live edge once the open group closes, which raises it to the furthest point written. + let edge = match (&self.group, self.end) { + (Some(_), Some(end)) if self.live_edge.is_none_or(|edge| !timestamp_lt(end, edge)) => Some(end), + _ => self.live_edge, + }; + // The marker starts one unit of the track's timescale before the live edge, so media + // resuming right at the edge still starts after it, but not below the latest frame, which + // it would otherwise claim to end before. Or one unit past the newest group when that + // doesn't pass the group's start. The open group ends whole there, so a reader can tell + // the marker follows it. + let marker = match (self.pending_start, edge) { + // A seek keyed the next group. Its marker is stamped at the live edge, or one unit past + // the newest group when the edge is unknown or doesn't pass it, since starts strictly + // increase. + (Some(start), edge) => { + let edge = edge.unwrap_or(moq_net::Timestamp::ZERO); + let timestamp = match self.inner.info().timescale { + Some(_) => crate::start::next(&self.inner, edge)?.1, + None => edge, + }; + Some((start, timestamp, false)) + } + // Nothing was written, so there is no timeline to break. + (None, None) => None, + (None, Some(edge)) => { + let newest = self.newest.unwrap_or(moq_net::Timestamp::ZERO); + let (start, timestamp) = crate::start::before(&self.inner, edge, newest)?; + Some((start, timestamp, true)) + } + }; + let whole = marker.and_then(|(_, timestamp, appended)| appended.then_some(timestamp)); + self.close(None, None, whole)?; // Nothing is measured across the break: the frames still open on this side have no end, and // the gap to the far side is not a frame duration. self.estimator.discontinuity(); self.cadence = None; - let mut group = match self.pending_sequence.take() { - Some(sequence) => self.inner.create_group(moq_net::group::Info { sequence })?, - None => self.inner.append_group()?, + let Some((start, timestamp, appended)) = marker else { + return Ok(()); + }; + self.pending_start = None; + let mut group = match appended { + true => self.inner.append_group(start)?, + false => self.inner.create_group(moq_net::group::Info { start })?, }; - let timestamp = self.live_edge.unwrap_or(moq_net::Timestamp::ZERO); match self.container.kind() { Kind::Data => group.write_frame(timestamp, Bytes::new())?, Kind::Audio | Kind::Video => self.container.write( @@ -854,7 +964,7 @@ where }], )?, } - group.finish()?; + group.close()?; Ok(()) } @@ -867,7 +977,7 @@ where let mut timestamps = self .buffer .iter() - .map(|frame| std::time::Duration::from(frame.timestamp)); + .filter_map(|frame| std::time::Duration::try_from(frame.timestamp).ok()); let Some(first) = timestamps.next() else { return std::time::Duration::ZERO; }; @@ -901,7 +1011,8 @@ where } let boundary = self.buffer.get(i + 1).map(|f| f.timestamp).or(next); if let Some(boundary) = boundary - && let Ok(boundary) = boundary.convert(self.buffer[i].timestamp.scale()) + && let Some(scale) = self.buffer[i].timestamp.scale() + && let Ok(boundary) = boundary.convert(scale) && let Ok(duration) = boundary.checked_sub(self.buffer[i].timestamp) { self.buffer[i].duration = Some(duration); @@ -1419,7 +1530,10 @@ mod tests { type Error = crate::Error; fn write(&self, _group: &mut moq_net::group::Producer, frames: &[Frame]) -> Result<(), Self::Error> { - match frames.iter().any(|f| f.timestamp.as_micros() == self.0 as u128) { + match frames + .iter() + .any(|f| f.timestamp.as_micros().unwrap() == self.0 as u128) + { true => Err(moq_net::Error::FrameTooLarge.into()), false => Ok(()), } @@ -1508,7 +1622,7 @@ mod tests { assert_eq!( groups[1], vec![(10_000, 0)], - "the marker is one empty frame at the exclusive end" + "the marker is one empty frame at the last frame, which has no duration to end later" ); assert_eq!(groups[2][0], (2_405_070_000, 2)); } @@ -1528,11 +1642,11 @@ mod tests { producer.discontinuity().unwrap(); let fresh = producer.track().largest().map(|largest| largest.group); - assert_eq!(fresh, stale.map(|s| s + 1), "the marker group is the live edge now"); + assert!(fresh > stale, "the marker group is the live edge now"); } - /// The marker sits at the duration marker a cut wrote, not at the last frame below it, - /// which a consumer bounding groups by their reach would read as a rewind. + /// The marker sits a unit before the duration marker a cut wrote, not at the last frame below + /// it, which a consumer bounding groups by their reach would read as a rewind. #[tokio::test] async fn discontinuity_after_a_cut_starts_at_the_duration_marker() { for (end, marker) in [(Some(25_000), 25_000), (None, 20_000)] { @@ -1555,7 +1669,7 @@ mod tests { Some(&(marker, 0)), "cut({end:?}) writes its duration marker" ); - assert_eq!(groups[1], vec![(marker, 0)], "cut({end:?}) then discontinuity"); + assert_eq!(groups[1], vec![(marker - 1, 0)], "cut({end:?}) then discontinuity"); } } @@ -1609,7 +1723,7 @@ mod tests { let mut frames = Vec::new(); while let Some(frame) = group.read_frame().await.unwrap() { let decoded = hang::container::Frame::decode(frame.payload).unwrap(); - frames.push((decoded.timestamp.as_micros(), decoded.payload.len())); + frames.push((decoded.timestamp.as_micros().unwrap(), decoded.payload.len())); } groups.push(frames); } @@ -1755,31 +1869,31 @@ mod tests { assert!(matches!(err, crate::Error::MissingKeyframe(_))); } - /// Drain all groups from a finished track, returning their sequence numbers. - async fn collect_sequences(mut consumer: moq_net::track::Subscriber) -> Vec { - let mut sequences = Vec::new(); + /// Drain all groups from a finished track, returning their starts. + async fn collect_starts(mut consumer: moq_net::track::Subscriber) -> Vec { + let mut starts = Vec::new(); while let Some(group) = consumer.recv_group().await.unwrap() { - sequences.push(group.sequence); + starts.push(group.start); } - sequences + starts } - /// `seek(n)` opens the next group at sequence `n`. + /// `seek(n)` opens the next group at start `n`. #[tokio::test] - async fn seek_uses_explicit_sequence() { + async fn seek_uses_explicit_start() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Data)); - producer.write(frame(0, true)).unwrap(); // seq 0 + producer.write(frame(0, true)).unwrap(); // start 0 producer.seek(42).unwrap(); - producer.write(frame(10_000, true)).unwrap(); // seq 42 + producer.write(frame(10_000, true)).unwrap(); // start 42 let _finished = producer.finish().unwrap(); - assert_eq!(collect_sequences(consumer).await, vec![0, 42]); + assert_eq!(collect_starts(consumer).await, vec![0, 42]); } - /// `seek` is consumed on the next group creation; subsequent groups auto-increment from there. + /// `seek` is consumed on the next group creation; subsequent groups start at their keyframes. #[tokio::test] async fn seek_clears_pending_after_use() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.video)); @@ -1787,11 +1901,27 @@ mod tests { let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Data)); producer.seek(5).unwrap(); - producer.write(frame(0, true)).unwrap(); // seq 5 - producer.write(frame(10_000, true)).unwrap(); // seq 6 (auto-incremented) + producer.write(frame(0, true)).unwrap(); // start 5 + producer.write(frame(10_000, true)).unwrap(); // start 10000, its keyframe + let _finished = producer.finish().unwrap(); + + assert_eq!(collect_starts(consumer).await, vec![5, 10_000]); + } + + /// A marker after a seek takes the seek's start, stamped past the groups before it even when + /// nothing says where their content ended. + #[tokio::test] + async fn discontinuity_after_a_seek_takes_its_start() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let consumer = track.subscribe(replay()); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Audio)); + + producer.write(frame(0, true)).unwrap(); + producer.seek(100_000).unwrap(); + producer.discontinuity().unwrap(); let _finished = producer.finish().unwrap(); - assert_eq!(collect_sequences(consumer).await, vec![5, 6]); + assert_eq!(collect_starts(consumer).await, vec![0, 100_000]); } /// Records the frames handed to each `write`, so tests can inspect the @@ -1964,7 +2094,8 @@ mod tests { hang::container::Frame::decode(frame.payload) .unwrap() .timestamp - .as_micros(), + .as_micros() + .unwrap(), ); } assert_eq!(timestamps, vec![20_000, 30_000, 35_000]); @@ -1997,13 +2128,29 @@ mod tests { assert_eq!(groups[1][..2], [(150_000, 2), (183_000, 2)]); } - /// A group may start exactly where the previous one did: only a start below it is a rewind. + /// Group starts strictly increase, so a keyframe at the open group's start continues that group + /// rather than opening one. #[tokio::test] - async fn a_keyframe_at_the_previous_group_start_is_accepted() { + async fn a_keyframe_at_the_previous_group_start_continues_it() { let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let consumer = track.subscribe(replay()); let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Audio)); producer.write(frame(100_000, true)).unwrap(); producer.write(frame(100_000, true)).unwrap(); + producer.write(frame(120_000, true)).unwrap(); + let _finished = producer.finish().unwrap(); + assert_eq!(collect_starts(consumer).await, vec![100_000, 120_000]); + } + + /// With no group open, a keyframe at the previous group's start can't open one. + #[tokio::test] + async fn a_keyframe_at_a_closed_group_start_is_refused() { + let track = track_producer("test", hang::container::track_info(hang::catalog::PRIORITY.audio)); + let mut producer = Producer::new(track, Container::Legacy(crate::container::Kind::Audio)); + producer.write(frame(100_000, true)).unwrap(); + producer.cut(None).unwrap(); + let err = producer.write(frame(100_000, true)).unwrap_err(); + assert!(matches!(err, crate::Error::TimestampRewind(_)), "{err:?}"); } /// A frame below the start of the group before its own reaches back past a whole group. @@ -2080,7 +2227,7 @@ mod tests { crate::container::Consumer::new(subscriber, Container::Legacy(crate::container::Kind::Video)); let mut timestamps = Vec::new(); while let Some(frame) = consumer.read_frame().await.unwrap() { - timestamps.push(frame.timestamp.as_micros()); + timestamps.push(frame.timestamp.as_micros().unwrap()); } assert_eq!(timestamps, [1_000_000, 1_040_000, 1_050_000]); } diff --git a/rs/moq-mux/src/container/source.rs b/rs/moq-mux/src/container/source.rs index e115b28da2d..b587746e209 100644 --- a/rs/moq-mux/src/container/source.rs +++ b/rs/moq-mux/src/container/source.rs @@ -65,6 +65,10 @@ enum SourceState { pending: Box>, media: HangContainer, live: bool, + /// Hand out only groups starting after this one. + after: Option, + /// Where the groups read before ended. + reached: Option, instance: Option, }, /// The resolved consumer, reading frames. Boxed because it's much larger than @@ -80,8 +84,10 @@ struct Start { media: HangContainer, /// Start at the newest group, rather than the oldest the max delay still reaches. live: bool, - /// Read only groups after this one, so a resubscription skips groups already read. + /// Read only groups starting after this one, so a resubscription skips groups already read. after: Option, + /// Where the groups read before ended, so the first group after them reads as continuing. + reached: Option, } /// A resolved subscription. @@ -119,6 +125,7 @@ impl Target { media, live: false, after: None, + reached: None, }, } } @@ -131,6 +138,8 @@ pub(crate) struct Resume { group: Option, /// The latest presentation time delivered. latest: Option, + /// Where the groups read through ended. + reached: Option, } /// What a source continuing another rendition's track skips: media up to what that track delivered. @@ -177,6 +186,8 @@ pub(crate) struct ExportSource { latest: Option, /// The group of the last frame delivered. delivered: Option, + /// Where the groups read through ended, here or by the source this one resumes. + reached: Option, /// Set while continuing another source, until the first sync frame past what it delivered. splice: Option, /// Counts the payload of its frames the export emits, once subscribed under a @@ -323,6 +334,7 @@ impl ExportSource { received: source.received(name), latest: None, delivered: None, + reached: None, splice: None, bill: Default::default(), } @@ -351,6 +363,7 @@ impl ExportSource { target: self.target.clone(), group: self.delivered, latest: self.latest, + reached: self.reached, } } @@ -364,6 +377,8 @@ impl ExportSource { pub(crate) fn resuming(mut self, resume: &Resume) -> Self { if resume.target == self.target { self.start().after = resume.group; + self.start().reached = resume.reached; + self.reached = resume.reached; } else { self.start().live = true; self.splice = resume.latest.map(|read| Splice { read, group: None }); @@ -390,7 +405,7 @@ impl ExportSource { } } - /// The group sequence of the last frame read, or None until the subscription resolves. + /// The group start of the last frame read, or None until the subscription resolves. /// Meaningful only once a frame was read. pub(crate) fn group(&self) -> Option { match &self.state { @@ -504,7 +519,7 @@ impl ExportSource { if !self.spliced(frame)? { continue; } - let at = Duration::from(frame.timestamp); + let at = Duration::try_from(frame.timestamp)?; self.latest = Some(self.latest.map_or(at, |latest| latest.max(at))); self.delivered = self.group(); } @@ -524,7 +539,7 @@ impl ExportSource { let Some(splice) = self.splice.as_mut() else { return Ok(true); }; - let at = Duration::from(frame.timestamp); + let at = Duration::try_from(frame.timestamp)?; if sync && at > splice.read { self.splice = None; return Ok(true); @@ -547,8 +562,13 @@ impl ExportSource { let broadcast = ready!(binding.poll_broadcast(waiter))?; let instance = binding.sibling().then(|| broadcast.clone()); let mut subscription = moq_net::track::Subscription::default().with_delay(self.delay); + // From the group starting at `after`: a time start names the group containing it, so the + // request reaches that group too and the reader below leaves it out. if let Some(after) = start.after { - subscription = subscription.with_range((Bound::Excluded(after), Bound::Unbounded)); + subscription = subscription.with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(after), + end: None, + }]); } let track = match broadcast.track(&self.target.track) { Ok(track) => track, @@ -563,6 +583,8 @@ impl ExportSource { pending, media: start.media, live: start.live, + after: start.after, + reached: start.reached, instance, }; } @@ -581,7 +603,12 @@ impl ExportSource { }; let priority = self.source.control(track.control())?; let SourceState::Subscribing { - media, live, instance, .. + media, + live, + after, + reached, + instance, + .. } = std::mem::replace(&mut self.state, SourceState::Ended) else { unreachable!("just matched Subscribing"); @@ -590,7 +617,11 @@ impl ExportSource { true => track.bill(), false => Default::default(), }; - let consumer = Consumer::new(track, media); + let track = match after { + Some(after) => track.with_range((Bound::Excluded(after), Bound::Unbounded)), + None => track, + }; + let consumer = Consumer::new(track, media).continuing(reached); let consumer = match live { true => consumer.live(), false => consumer, @@ -646,8 +677,8 @@ impl ExportSource { // Say what the export never receives and where. tracing::warn!( track = consumer.name(), - at = ?std::time::Duration::from(timestamp), - skipped = ?std::time::Duration::from(duration), + at = ?timestamp, + skipped = ?duration, "media skipped upstream; resuming at the next group" ); self.finish_ends(); @@ -664,8 +695,10 @@ impl ExportSource { } Some(Item::FrameEnd(end)) => return Poll::Ready(Ok(Some(Event::FrameEnd { end, opens: false }))), Some(Item::GroupEnd) => { + let reached = consumer.reached(); self.finish_ends(); self.open = false; + self.reached = reached; return Poll::Ready(Ok(Some(Event::GroupEnd))); } // The skip is counted with the gap the next group reads. @@ -972,7 +1005,7 @@ mod tests { kio::clock::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))) .await .ok() - .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros())) + .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros().unwrap())) }; let wire = || HangContainer::Legacy(crate::container::Kind::Audio); let info = || hang::container::track_info(hang::catalog::PRIORITY.audio); @@ -1042,7 +1075,7 @@ mod tests { // together. let settle = kio::clock::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))); assert!(settle.await.is_err(), "nothing is written yet"); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); let held = match failure { "malformed" => { // An empty Legacy audio frame is malformed. @@ -1112,7 +1145,7 @@ mod tests { .await .expect("the read settles") .unwrap() - .map(|frame| frame.timestamp.as_micros()) + .map(|frame| frame.timestamp.as_micros().unwrap()) }; assert_eq!(read().await, Some(0)); @@ -1171,10 +1204,10 @@ mod tests { .live(); let frame = kio::wait(|waiter| export.poll_read(waiter)).await.unwrap().unwrap(); - assert_eq!(frame.timestamp.as_micros(), 48_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 48_000); live.track.write(raw_frame(72_000, b"audio", true)).unwrap(); let frame = kio::wait(|waiter| export.poll_read(waiter)).await.unwrap().unwrap(); - assert_eq!(frame.timestamp.as_micros(), 72_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 72_000); assert_eq!(export.skips(), 0, "nothing below the start counts as skipped"); } /// A live source picks the newest group without narrowing what the publisher sends: a @@ -1220,16 +1253,16 @@ mod tests { // One 24 ms frame per group, as the TS import publishes MPEG-1 audio. let wire = HangContainer::Legacy(crate::container::Kind::Audio); let write = |sequence: u64| { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); wire.write(&mut group, &[raw_frame(sequence * 24_000, b"audio", true)]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; let mut read = async || { kio::clock::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))) .await .ok() - .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros())) + .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros().unwrap())) }; assert_eq!(read().await, None, "subscribed, with nothing published yet"); @@ -1303,10 +1336,10 @@ mod tests { let wire = HangContainer::Legacy(crate::container::Kind::Audio); let pes = |pes: u64| { for sequence in pes * 5..pes * 5 + 5 { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); wire.write(&mut group, &[raw_frame(sequence * 24_000, b"audio", true)]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } }; // About two seconds of it, so the budget's backlog reaches back about 450 ms. @@ -1328,7 +1361,7 @@ mod tests { kio::clock::timeout(Duration::from_millis(100), kio::wait(|waiter| export.poll_read(waiter))) .await .ok() - .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros())) + .map(|frame| frame.unwrap().map(|frame| frame.timestamp.as_micros().unwrap())) }; let mut timestamps = Vec::new(); // Only the backlog's oldest group has landed. @@ -1389,7 +1422,7 @@ mod tests { moving.b.write(marked_frame(i, 0xb1)).unwrap(); } match (poll().await.unwrap(), first) { - (Ok(Some(frame)), Some(first)) => assert_eq!(frame.timestamp.as_micros(), first), + (Ok(Some(frame)), Some(first)) => assert_eq!(frame.timestamp.as_micros().unwrap(), first), ( Err(crate::Error::RenditionRewound { rendition, @@ -1444,15 +1477,15 @@ mod tests { let (track, mut export) = audio_export().await; let wire = HangContainer::Legacy(crate::container::Kind::Audio); let group = |sequence: u64, timestamp: u64| { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); wire.write(&mut group, &[raw_frame(timestamp, b"audio", true)]).unwrap(); group }; // Group 1 is missing whole; group 3 is cut short after its first frame. - group(0, 0).finish().unwrap(); - group(2, 40_000).finish().unwrap(); + group(0, 0).close().unwrap(); + group(2, 40_000).close().unwrap(); let cut = group(3, 60_000); - group(4, 80_000).finish().unwrap(); + group(4, 80_000).close().unwrap(); let _track = track.finish().unwrap(); let waiter = kio::Waiter::noop(); @@ -1460,7 +1493,7 @@ mod tests { let Poll::Ready(Ok(Some(frame))) = export.poll_read(&waiter) else { panic!("frame ready") }; - (frame.timestamp.as_micros(), export.skips(), export.cuts()) + (frame.timestamp.as_micros().unwrap(), export.skips(), export.cuts()) }; assert_eq!(read(&mut export), (0, 0, 0)); assert_eq!(read(&mut export), (40_000, 1, 0), "group 1 is missing whole"); @@ -1477,9 +1510,9 @@ mod tests { let (track, mut export) = audio_export().await; let wire = HangContainer::Legacy(crate::container::Kind::Audio); for (sequence, timestamp, payload) in [(0, 0, b"audio".as_slice()), (2, 1_000_000, b"".as_slice())] { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); wire.write(&mut group, &[raw_frame(timestamp, payload, true)]).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let _track = track.finish().unwrap(); let waiter = kio::Waiter::noop(); @@ -1528,9 +1561,9 @@ mod tests { ), (2, vec![raw_frame(100_000, b"resumed", true)]), ] { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); wire.write(&mut group, &frames).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let _track = track.finish().unwrap(); let waiter = kio::Waiter::noop(); @@ -1538,7 +1571,7 @@ mod tests { let Poll::Ready(Ok(Some(frame))) = export.poll_read(&waiter) else { panic!("frame ready") }; - assert_eq!(frame.timestamp.as_micros(), timestamp); + assert_eq!(frame.timestamp.as_micros().unwrap(), timestamp); assert_eq!( (export.skips(), export.markers()), (generation, generation), diff --git a/rs/moq-mux/src/container/test_util.rs b/rs/moq-mux/src/container/test_util.rs index 4a50ec3b96d..68a0d8e4502 100644 --- a/rs/moq-mux/src/container/test_util.rs +++ b/rs/moq-mux/src/container/test_util.rs @@ -313,7 +313,7 @@ pub(crate) async fn published( .expect("the importer finished its tracks") .unwrap() { - timestamps.push(frame.timestamp.as_micros()); + timestamps.push(frame.timestamp.as_micros().unwrap()); } out.insert(name, timestamps); } diff --git a/rs/moq-mux/src/container/text.rs b/rs/moq-mux/src/container/text.rs index 755772a3a8f..74ab8e4ba73 100644 --- a/rs/moq-mux/src/container/text.rs +++ b/rs/moq-mux/src/container/text.rs @@ -11,8 +11,14 @@ pub(super) struct Cue<'a> { pub settings: &'a str, } -pub(super) fn stamp(value: Timestamp) -> String { - format_millis(value.as_micros() / 1000) +/// A cue time's microseconds, refused when [unscaled](Timestamp::unscaled): a scale-less track's +/// key is no time. +fn micros(value: Timestamp) -> Result { + value.as_micros().map_err(|_| "cue timestamp has no timescale") +} + +fn stamp(value: Timestamp) -> Result { + Ok(format_millis(micros(value)? / 1000)) } fn format_millis(millis: u128) -> String { @@ -75,8 +81,8 @@ pub(super) fn append(output: &mut String, cue: Cue<'_>) -> Result<(), &'static s output.push('\n'); } // WebVTT has millisecond precision: round the end up so a sub-millisecond cue stays positive. - let start = cue.start.as_micros() / 1000; - let end = cue.end.as_micros().div_ceil(1000).max(start + 1); + let start = micros(cue.start)? / 1000; + let end = micros(cue.end)?.div_ceil(1000).max(start + 1); output.push_str(&format_millis(start)); output.push_str(" --> "); output.push_str(&format_millis(end)); @@ -101,7 +107,8 @@ pub(super) fn has_timestamp(body: &str) -> bool { pub(super) fn rebase(body: &str, cue: Range, origin: Timestamp) -> Result { let start = cue.start; let end = cue.end; - let origin = origin.convert(start.scale()).map_err(|_| "inline timestamp overflow")?; + let scale = start.scale().ok_or("cue timestamp has no timescale")?; + let origin = origin.convert(scale).map_err(|_| "inline timestamp overflow")?; let mut output = String::new(); let mut remaining = body; let mut previous = start; @@ -112,9 +119,7 @@ pub(super) fn rebase(body: &str, cue: Range, origin: Timestamp) -> Re let close = remaining.find('>').ok_or("unterminated inline timestamp")?; let value = &remaining[..close]; let relative = timestamp(value)?; - let relative = relative - .convert(start.scale()) - .map_err(|_| "inline timestamp overflow")?; + let relative = relative.convert(scale).map_err(|_| "inline timestamp overflow")?; let relative = relative .checked_sub(origin) .map_err(|_| "inline timestamp before cue origin")?; @@ -123,7 +128,7 @@ pub(super) fn rebase(body: &str, cue: Range, origin: Timestamp) -> Re return Err("inline timestamp outside cue"); } previous = absolute; - output.push_str(&stamp(absolute)); + output.push_str(&stamp(absolute)?); remaining = &remaining[close..]; } } diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index 5dc4a02f46f..d397e5fc0a4 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -582,7 +582,9 @@ impl Track { pending.restart += 1; pending.skip += 1; } - let high = high.filter(|_| !stepped).map_or(pts, |high| high.max(pts)); + let high = high + .filter(|_| !stepped) + .map_or(pts, |high| crate::container::later(high, pts)); self.high = Some((pending.restart, high)); } @@ -618,6 +620,11 @@ impl Track { *input.trimmed += 1; continue; } + // Media is timed: a scale-less track's unscaled key has no time to mux at. Refused + // here, so every frame the exporter holds has one (see [`nanos`]). + if frame.timestamp.scale().is_none() { + return Err(moq_net::Error::TimestampMismatch.into()); + } let mut pending = Pending { frame, restart: self.base.0 + self.source.markers() + self.steps, @@ -779,7 +786,7 @@ impl Track { frame: &Frame, ) -> anyhow::Result<(Timestamp, Option)> { let lead = delay.saturating_sub(self.ahead(delay)); - let key = Duration::from(decode) + let key = Duration::try_from(decode)? .checked_add(lead) .context("MPEG-TS decode time past the end of the timeline")?; let mut key = Timestamp::try_from(key)?; @@ -789,7 +796,7 @@ impl Track { { key = match start.is_some() && shared == start { true => previous, - false => key.max(previous), + false => crate::container::later(key, previous), }; } self.released = Some((decode, key, start)); @@ -900,7 +907,7 @@ impl Track { // sent ahead by its send-ahead: judged as arriving that much earlier, as a live source // sends it, every track's first frames are on the clock whichever anchors it. let ahead = match jitter.replaying() { - true => (Duration::from(decode) + jitter.delay()).saturating_sub(Duration::from(key)), + true => (Duration::try_from(decode)? + jitter.delay()).saturating_sub(Duration::try_from(key)?), false => Duration::ZERO, }; // Logical clocks can begin at zero. Shift both readings by the same representable amount so @@ -910,7 +917,7 @@ impl Track { let arrival = Arrival { arrived: arrived.saturating_sub(ahead), read: read.saturating_sub(ahead), - decode: key.into(), + decode: key.try_into()?, restart, sync, leading, @@ -1286,7 +1293,7 @@ impl Pack { /// When the frame after the last presents, in microseconds, if the frames run on without /// a gap. fn end(&self) -> u128 { - self.unit.timestamp.as_micros() + u128::from(self.samples) * 1_000_000 / u128::from(self.rate) + nanos(self.unit.timestamp) / 1_000 + u128::from(self.samples) * 1_000_000 / u128::from(self.rate) } } @@ -1493,7 +1500,10 @@ impl SiTrack { let (promote, drop_pending) = match group.poll_read_frame(waiter) { Poll::Ready(Ok(Some(frame))) => { snapshot.apply(&frame.payload); - *timestamp = (*timestamp).max(frame.timestamp.unwrap_or(Timestamp::ZERO)); + // A scale-less track's frames carry no time, so its revision stays at zero. + if frame.timestamp.scale().is_some() { + *timestamp = crate::container::later(*timestamp, frame.timestamp); + } (false, false) } Poll::Ready(Ok(None)) => (true, false), @@ -1535,9 +1545,7 @@ impl SiTrack { self.poll(&kio::Waiter::noop()); let Some(revision) = &self.future else { break }; if !interval.is_zero() { - let stamped = Duration::from(revision.timestamp) - .as_nanos() - .div_ceil(interval.as_nanos()); + let stamped = nanos(revision.timestamp).div_ceil(interval.as_nanos()); let at = revision .after .map_or(stamped, |after| stamped.max(slot(after, interval) + 1)); @@ -1551,7 +1559,10 @@ impl SiTrack { self.dirty |= snapshot != self.active; self.active = snapshot; } - self.position = Some(self.position.map_or(timestamp, |position| position.max(timestamp))); + self.position = Some( + self.position + .map_or(timestamp, |position| crate::container::later(position, timestamp)), + ); } /// Take a due snapshot, keeping revisions and repeats on their respective cadence. @@ -1572,7 +1583,10 @@ impl SiTrack { self.dirty = false; // A zero interval can emit on a reordered PTS. Keep the anchor ahead of it, // or raising the interval later could credit that reorder span as elapsed time. - self.last_emit = Some(self.last_emit.map_or(timestamp, |last| last.max(timestamp))); + self.last_emit = Some( + self.last_emit + .map_or(timestamp, |last| crate::container::later(last, timestamp)), + ); Some(self.active.sections().cloned().collect()) } } @@ -1898,8 +1912,8 @@ where self.jitter.discard(&ready); } else { self.program.last_timestamp = Some(ready.item.frame.timestamp); - let decode = ready.item.decode.as_nanos(); - self.lay_before(ready.item.key.as_nanos())?; + let decode = nanos(ready.item.decode); + self.lay_before(nanos(ready.item.key))?; match self.list(&name)? { true => self.mux(&name, ready.item)?, false => self @@ -3026,7 +3040,7 @@ where if matches!(kind, Kind::Video(_)) { if self.program.video_out.is_none() { self.program.video_out = Some(queued.key); - self.program.video_start = Some(start.max(queued.frame.timestamp)); + self.program.video_start = Some(crate::container::later(start, queued.frame.timestamp)); } return false; } @@ -3045,9 +3059,9 @@ where .filter_map(|(_, track)| self.jitter.peek(&track.key)) .map(|next| match placed { true => next.key, - false => next.frame.timestamp.max(start), + false => crate::container::later(next.frame.timestamp, start), }) - .min(); + .reduce(crate::container::earlier); let at = match placed { true => queued.key, false => queued.frame.timestamp, @@ -3116,7 +3130,7 @@ where .values() .filter(|t| matches!(t.kind, Kind::Video(_))) .filter_map(|t| t.pending.as_ref().map(|p| p.frame.timestamp)) - .min() + .reduce(crate::container::earlier) } /// Whether `pid`'s next packet is its first since a switch to another instance, so it has @@ -3459,7 +3473,9 @@ where .iter() .filter_map(|(name, track)| Some((track.parked.front()?.key, name, track.pid))) .collect(); - waiting.sort(); + waiting.sort_by(|a, b| { + (a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)).then_with(|| (a.1, a.2).cmp(&(b.1, b.2))) + }); let mut claimed: Vec = self .program .tracks @@ -3687,7 +3703,7 @@ where _ => unreachable!("a placed track"), }; let pid = track.pid; - let place = frame.timestamp.as_nanos(); + let place = nanos(frame.timestamp); // Its first access unit decodes at the DTS, which a reordered picture carries ahead of its PTS. let decode = pes .as_ref() @@ -3761,8 +3777,8 @@ where pack.start == start && pack.rate == rate && pack.payload.len() + payload.len() <= size - && decode.as_nanos().saturating_sub(pack.decode.as_nanos()) <= budget - && pack.end().abs_diff(unit.timestamp.as_micros()) <= 12 + && nanos(decode).saturating_sub(nanos(pack.decode)) <= budget + && pack.end().abs_diff(nanos(unit.timestamp) / 1_000) <= 12 }); if !joins { self.flush(name)?; @@ -3780,7 +3796,7 @@ where samples: 0, rate, }); - pack.frames.push((decode.as_nanos(), payload.len())); + pack.frames.push((nanos(decode), payload.len())); pack.charges.push(charge); pack.payload.extend_from_slice(&payload); pack.samples += samples; @@ -3821,11 +3837,11 @@ where .iter() .filter_map(|(name, track)| { let pack = track.pack.as_ref()?; - let by = pack.decode.as_nanos().saturating_sub(track.drain().as_nanos()); + let by = nanos(pack.decode).saturating_sub(track.drain().as_nanos()); done(track, schedule::slot(by)).then(|| (pack.decode, name.clone())) }) .collect(); - due.sort(); + due.sort_by(|a, b| (a.0.partial_cmp(&b.0).unwrap_or(std::cmp::Ordering::Equal)).then_with(|| a.1.cmp(&b.1))); for (_, name) in due { self.flush(&name)?; } @@ -3869,13 +3885,13 @@ where PCR_INTERVAL * u32::try_from(slots).unwrap_or(u32::MAX), ); } - let by = decode.as_nanos().saturating_sub(track.drain().as_nanos()); + let by = nanos(decode).saturating_sub(track.drain().as_nanos()); self.lead_tables()?; self.program.schedule.push_frames( pid, super::schedule::Timing { by, - decode: decode.as_nanos(), + decode: nanos(decode), }, out, keyframe, @@ -3898,6 +3914,7 @@ where let (known, settled) = self.slot_settled()?; let settled = Timestamp::from_micros(u64::try_from(settled / 1_000).ok()?).ok()?; // The tail of the generation still going out keeps its own clock after the next is acquired. + let settled = jitter::Decode::try_from(settled).ok()?; Some((known, self.jitter.at(self.program.generation, settled)?)) } @@ -4446,7 +4463,7 @@ fn si_due(timestamp: Timestamp, last: Option, interval: Duration) -> }; // Reordered (B-frame) timestamps step backwards; saturate rather than wrap so a // dip never counts as elapsed time (a zero interval still means "every frame"). - Duration::from(timestamp).saturating_sub(Duration::from(last)) >= interval + nanos(timestamp).saturating_sub(nanos(last)) >= interval.as_nanos() } /// DVB's TDT and TOT: tables whose content is the current time, so a repeat of an @@ -4461,7 +4478,7 @@ fn is_clock_table(table_id: u8) -> bool { /// takes before it gets here. Coarser units would floor a sub-unit interval to zero and /// divide by it. fn slot(timestamp: Timestamp, interval: Duration) -> u128 { - Duration::from(timestamp).as_nanos() / interval.as_nanos() + nanos(timestamp) / interval.as_nanos() } /// A repetition slot's boundary as a media timestamp. @@ -4501,7 +4518,7 @@ fn adaptation_size(af: &AdaptationField) -> usize { /// AAC's 1,024 samples at 48 kHz. fn place(timestamp: Timestamp, samples: u64, rate: u32) -> u128 { let frame = 1_000_000 * u128::from(samples); - (timestamp.as_micros() * u128::from(rate) + frame / 2) / frame + (nanos(timestamp) / 1_000 * u128::from(rate) + frame / 2) / frame } /// The 33-bit wire timestamp field (90 kHz). DTS and PTS both wrap into it. @@ -4511,7 +4528,16 @@ const TS_TIMESTAMP_MASK: u64 = (1 << 33) - 1; /// this domain so it never wraps mid-stream (the source timestamps are already unwrapped); /// [`to_ts_timestamp`] masks to the 33-bit wire field only at emission. fn to_ticks(timestamp: Timestamp) -> u64 { - (timestamp.as_micros() * 90_000 / 1_000_000) as u64 + (nanos(timestamp) / 1_000 * 90_000 / 1_000_000) as u64 +} + +/// A media timestamp in nanoseconds. +/// +/// Every timestamp the exporter holds has a scale: a track refuses an unscaled frame as it reads +/// one ([`Track::fill`]), an SI revision keeps none, and the rest are stamped here. Zero stands in +/// for one that can't reach here rather than a panic. +fn nanos(timestamp: Timestamp) -> u128 { + timestamp.as_nanos().unwrap_or_default() } fn to_ts_timestamp(timestamp: Timestamp) -> anyhow::Result { @@ -5432,9 +5458,9 @@ mod tests { assert!(jitter.poll_next(&waiter).is_pending()); // The last available audio frame, including its scheduling lead, anchors both orders. let (decode, key, _) = tracks[1].released.unwrap(); - assert_eq!(decode.as_micros(), 800_000); + assert_eq!(decode.as_micros().unwrap(), 800_000); assert_eq!( - jitter.at(0, key), + jitter.at(0, key.try_into().unwrap()), Some(start.saturating_add(Duration::from_millis(500))), "{order:?}" ); @@ -5445,7 +5471,7 @@ mod tests { }) .find(|ready| ready.track.name == "video") .expect("a video start"); - assert_eq!(video.item.frame.timestamp.as_micros(), 600_000, "{order:?}"); + assert_eq!(video.item.frame.timestamp.as_micros().unwrap(), 600_000, "{order:?}"); } } @@ -5861,7 +5887,10 @@ mod tests { // One table per slot the stream covers, however often the reorder revisits a boundary. let first = slot(stamps[0], PSI_INTERVAL); - let last = slot(*stamps.iter().max().unwrap(), PSI_INTERVAL); + let last = slot( + stamps.iter().copied().reduce(crate::container::later).unwrap(), + PSI_INTERVAL, + ); assert_eq!(run_cadence(&stamps, PSI_INTERVAL), (last - first + 1) as usize); } diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 1f3c4430fb6..fd35db0f592 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -46,6 +46,17 @@ fn timed() -> moq_net::track::Info { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } +/// Append `payload` to a timed SI track as a group of its own, one unit past the newest start: +/// the export places tables by the media they ride, not their own timestamps, and starts have +/// to increase. +fn write_si(track: &mut moq_net::track::Producer, payload: impl Into) { + let start = track.largest().map_or(0, |largest| largest.group + 1); + let scale = track.info().timescale.expect("a timed track"); + track + .write_frame(Timestamp::new(start, scale).unwrap(), payload.into()) + .unwrap(); +} + const SC: &[u8] = &[0, 0, 0, 1]; // Reusable H.264 parameter-set and slice NALs (NAL type = first byte & 0x1f). const SPS: &[u8] = &[0x67, 0x42, 0xc0, 0x1f, 0xde]; @@ -241,9 +252,9 @@ async fn a_follower_keeps_msf_after_catalog_eof() { .create_track(moq_msf::DEFAULT_NAME, hang::Catalog::default_track_info()) .unwrap(); let msf = moq_msf::Catalog::<()>::new(Vec::new()).to_json().unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(moq_net::Timestamp::now(), msf).unwrap(); - group.finish().unwrap(); + let mut group = track.append_group(0).unwrap(); + group.write_frame(moq_net::Timestamp::ZERO, msf).unwrap(); + group.close().unwrap(); broadcast.announce().unwrap(); (broadcast, track) }; @@ -998,7 +1009,7 @@ async fn export_trims_leading_pictures_after_each_delivery_break() { (11, vec![200, 180, 190, 210]), (13, vec![400, 380, 390, 410]), ] { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); let frames: Vec<_> = stamps .into_iter() .enumerate() @@ -1014,7 +1025,7 @@ async fn export_trims_leading_pictures_after_each_delivery_break() { }) .collect(); format.write(&mut group, &frames).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let _track = track.finish().unwrap(); let mut export = open_export(crate::source::announced(&consumer).await, RECORDING_MAX_AGE) @@ -1302,9 +1313,7 @@ async fn export_declared(reorder: u8, groups: &[&[u64]]) -> (Export = stamps .iter() @@ -1321,7 +1330,7 @@ async fn export_declared(reorder: u8, groups: &[&[u64]]) -> (Export (u64, Vec) { let mut track = consumer.track(name).unwrap().subscribe(None).await.unwrap(); let mut group = track.recv_group().await.unwrap().expect("a snapshot group"); @@ -3263,7 +3273,7 @@ async fn newest_si_group(consumer: &moq_net::broadcast::Consumer, name: &str) -> while let Some(frame) = kio::wait(|waiter| group.poll_read_frame(waiter)).await.unwrap() { frames.push(frame.payload); } - (group.sequence, frames) + (group.start, frames) } /// The newest snapshot of an SI track, reduced to its sections. @@ -3421,9 +3431,13 @@ async fn service_layer_survives_roundtrip() { Some(Duration::from_secs(10)), "the DVB NIT interval was filled in" ); - let (nit_sequence, nit_frames) = newest_si_group(&consumer, &nit_entry.track).await; + // Both NIT packets and the SDT after them land at one program time. A group starts at its + // time, and a second group at the same time would start a unit later, so the NIT's only group + // starts where the SDT's does. + let (nit_start, nit_frames) = newest_si_group(&consumer, &nit_entry.track).await; + let (sdt_start, _) = newest_si_group(&consumer, &sdt_entry.track).await; assert_eq!( - nit_sequence, 0, + nit_start, sdt_start, "the repeated NIT committed once; a repetition cuts no group" ); assert_eq!( @@ -4196,11 +4210,16 @@ async fn discontinuity_re_emits_tables_and_resumes_the_clock() { assert!(!audio_pts.is_empty()); assert_eq!(export.discontinuity(), 1, "the marker was observed"); - let resume = marked.iter().position(|f| f.timestamp.as_micros() >= 605_000_000); + let resume = marked + .iter() + .position(|f| f.timestamp.as_micros().unwrap() >= 605_000_000); let resume = resume.unwrap_or_else(|| { panic!( "resumed media missing, marked timestamps {:?}", - marked.iter().map(|f| f.timestamp.as_micros()).collect::>() + marked + .iter() + .map(|f| f.timestamp.as_micros().unwrap()) + .collect::>() ) }); assert_eq!( @@ -5643,9 +5662,7 @@ async fn export_twice(with_video: bool, late_clock: bool) -> (Vec, Vec Vec<(usize, u64, u128)> { | (u64::from(packet[8]) << 9) | (u64::from(packet[9]) << 1) | u64::from(packet[10] >> 7); - pcrs.push((at, base, frame.timestamp.as_micros())); + pcrs.push((at, base, frame.timestamp.as_micros().unwrap())); } at += 1; } @@ -8534,7 +8535,7 @@ async fn a_skipped_group_flags_the_clock() { for sequence in 0..80 + 2 * missing { kio::clock::sleep_until(start.saturating_add(Duration::from_millis(sequence * 100))).await; if !(20..20 + missing).contains(&sequence) && !(40 + missing..40 + 2 * missing).contains(&sequence) { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); for k in 0..5u64 { let frame = Frame { timestamp: Timestamp::from_millis(60_000 + sequence * 100 + k * 20).unwrap(), @@ -8546,7 +8547,7 @@ async fn a_skipped_group_flags_the_clock() { .write(&mut group, &[frame]) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } out.extend(poll_frames(&mut export)); } @@ -8557,7 +8558,7 @@ async fn a_skipped_group_flags_the_clock() { assert_skip_discontinuity(&out, 2 * usize::from(missing == 60 && delay.is_zero())); let ts: Vec = out.iter().flat_map(|f| f.payload.iter().copied()).collect(); for pair in out.windows(2) { - let jumped = pair[1].timestamp.as_micros() > pair[0].timestamp.as_micros() + 25_000; + let jumped = pair[1].timestamp.as_micros().unwrap() > pair[0].timestamp.as_micros().unwrap() + 25_000; assert_eq!(count_discontinuity(&pair[1..]), usize::from(jumped)); } // Each flag lands right before the audio that resumes after its hole. @@ -8664,13 +8665,14 @@ async fn a_timeline_restarting_at_zero_fails_the_export() { .unwrap(); let start = kio::clock::now(); // One-second groups of 20 ms frames for 10 s, then a group back at zero. The track refuses - // a group whose first frame rewinds, so that group opens at the previous start. + // a group whose first frame doesn't pass the previous group's, so that group opens a + // millisecond past the previous start. for sequence in 0..11u64 { kio::clock::sleep_until(start.saturating_add(Duration::from_secs(sequence))).await; let base = if sequence < 10 { sequence * 1_000 } else { 0 }; - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); for k in 0..50u64 { - let millis = if sequence == 10 && k == 0 { 9_000 } else { base + k * 20 }; + let millis = if sequence == 10 && k == 0 { 9_001 } else { base + k * 20 }; let frame = Frame { timestamp: Timestamp::from_millis(millis).unwrap(), duration: None, @@ -8681,7 +8683,7 @@ async fn a_timeline_restarting_at_zero_fails_the_export() { .write(&mut group, &[frame]) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); if sequence < 10 { poll_frames(&mut export); } @@ -9160,7 +9162,7 @@ async fn the_same_instance_continues_after_a_drop() { same_instance_returns(false).await; } -/// A returning route serves the same publisher's catalog and monotonically numbered groups. +/// A returning route serves the same publisher's catalog and groups keyed by their times. #[tokio::test(start_paused = true)] async fn a_return_keeps_catalog_and_group_history() { let wall = kio::clock::Wall::MOQ_EPOCH + Duration::from_millis(86_400_000); @@ -9176,9 +9178,10 @@ async fn a_return_keeps_catalog_and_group_history() { let source = crate::source::resolved(&origin, "live").await; let old = source.broadcast(); let mut export = open_export(source, RECORDING_MAX_AGE).await.unwrap().with_replay(); - for (sequence, ms) in (0..200).step_by(20).enumerate() { + // Each group starts at its frame, in the track's microseconds. + for ms in (0..200).step_by(20) { write_aac(&mut track, ms); - assert_eq!(groups.recv_group().await.unwrap().unwrap().sequence, sequence as u64); + assert_eq!(groups.recv_group().await.unwrap().unwrap().start, (60_000 + ms) * 1_000); } let mut frames = drain_frames(&mut export).await; assert_eq!(pes_count(&frames), 10); @@ -9206,12 +9209,9 @@ async fn a_return_keeps_catalog_and_group_history() { let stream = crate::catalog::Consumer::new(&back, Default::default()).await.unwrap(); let mut export = export.follow(back, stream).unwrap(); let resumed = start.elapsed().as_millis() as u64; - for (sequence, ms) in (resumed..resumed + 200).step_by(20).enumerate() { + for ms in (resumed..resumed + 200).step_by(20) { write_aac(&mut track, ms); - assert_eq!( - groups.recv_group().await.unwrap().unwrap().sequence, - 10 + sequence as u64 - ); + assert_eq!(groups.recv_group().await.unwrap().unwrap().start, (60_000 + ms) * 1_000); } let _finished = track.finish().unwrap(); catalog.finish().unwrap(); @@ -10619,9 +10619,9 @@ async fn a_cancelled_replacement_after_the_follow_can_follow_again() { .broadcast() .create_track(name, hang::Catalog::default_track_info()) .unwrap(); - let mut group = old_catalog.append_group().unwrap(); - group.write_frame(Timestamp::now(), snapshot).unwrap(); - group.finish().unwrap(); + let mut group = old_catalog.append_group(0).unwrap(); + group.write_frame(Timestamp::ZERO, snapshot).unwrap(); + group.close().unwrap(); let source = crate::source::resolved(&origin, "live").await; let stream = source.catalog::<()>(format).await.unwrap(); let follow = Duration::from_secs(10); @@ -10853,7 +10853,7 @@ async fn a_repointed_si_entry_never_reads_a_replacement() { }; let si_track = |broadcast: &mut moq_net::broadcast::Producer, name: &str, sdt: &[u8]| { let mut track = broadcast.create_track(name, timed()).unwrap(); - track.write_frame(Timestamp::ZERO, Bytes::copy_from_slice(sdt)).unwrap(); + write_si(&mut track, Bytes::copy_from_slice(sdt)); track }; @@ -11418,7 +11418,7 @@ async fn placed_pes_survives_tune_in_beside_the_keyframe() { ); for (pes, ms) in out.iter().zip([1_500u128, 2_500]) { assert_eq!( - slices[pes.slice].timestamp.as_micros(), + slices[pes.slice].timestamp.as_micros().unwrap(), (ms * 1_000).div_ceil(PCR_INTERVAL.as_micros()) * PCR_INTERVAL.as_micros() + PCR_INTERVAL.as_micros(), "{ms} ms: rides the first slot opening after its placement" ); @@ -11553,7 +11553,9 @@ async fn placed_sections_keep_their_source_positions() { let name = scte_track(&catalog.snapshot()).unwrap(); let cues = read_cues(&consumer, &name).await; assert_eq!( - cues.iter().map(|(_, time)| time.as_micros() as u64).collect::>(), + cues.iter() + .map(|(_, time)| time.as_micros().unwrap() as u64) + .collect::>(), source ); let mut exporter = open_ts( diff --git a/rs/moq-mux/src/container/ts/export_timing_test.rs b/rs/moq-mux/src/container/ts/export_timing_test.rs index d73d186be3a..f554d1a4d4f 100644 --- a/rs/moq-mux/src/container/ts/export_timing_test.rs +++ b/rs/moq-mux/src/container/ts/export_timing_test.rs @@ -282,12 +282,13 @@ impl Live { match std::mem::take(&mut self.stalling) { true => self.stalled.push(group), false => { - group.finish().unwrap(); + group.close().unwrap(); } } } if keyframe { - self.group = Some(self.video.append_group().unwrap()); + let start = video.strict_add_signed(self.offset); + self.group = Some(self.video.append_group(start).unwrap()); } if stall(tick) { self.stalling = true; @@ -379,7 +380,7 @@ impl Live { /// End the media tracks while the catalog can still announce more. fn finish_media(&mut self) { if let Some(group) = self.group.take() { - group.finish().unwrap(); + group.close().unwrap(); } self.stalled.clear(); drop(self.video.clone().finish().unwrap()); @@ -519,7 +520,7 @@ async fn end_waits_for_catalog_and_media() { // Catalog EOF fixes the layout, but the named tracks can still publish media. live.run(3 * GOP, &mut [&mut leg], |_| false); // Close the groups so their tails flush, leaving both tracks open for more. - live.group.take().unwrap().finish().unwrap(); + live.group.take().unwrap().close().unwrap(); live.audio.cut(None).unwrap(); 3 * GOP as usize } else { @@ -979,15 +980,14 @@ async fn a_successor_settles_held_video_across_a_skip_or_restart() { live.run(8 * GOP + tail, &mut [&mut leg], |_| false); live.fall_silent(Duration::from_secs(3), &mut [&mut leg]); if restart { - live.group.take().unwrap().finish().unwrap(); - let mut marker = live.video.append_group().unwrap(); + live.group.take().unwrap().close().unwrap(); + // A microsecond before the keyframe that resumes: group starts strictly increase. + let at = live.tick * VIDEO_US - 1; + let mut marker = live.video.append_group(at).unwrap(); HangContainer::Legacy(crate::container::Kind::Video) - .write( - &mut marker, - &[crate::container::test_util::raw_frame(live.tick * VIDEO_US, b"", true)], - ) + .write(&mut marker, &[crate::container::test_util::raw_frame(at, b"", true)]) .unwrap(); - marker.finish().unwrap(); + marker.close().unwrap(); } else { // Leave the old group blocked so the successor makes the consumer skip it. live.stalling = true; @@ -1016,7 +1016,7 @@ async fn a_clean_group_end_settles_held_video_before_a_silence() { let mut leg = live.join(Duration::ZERO).await; let tail = if declared { 2 } else { 8 }; live.run(8 * GOP + tail, &mut [&mut leg], |_| false); - live.group.take().unwrap().finish().unwrap(); + live.group.take().unwrap().close().unwrap(); leg.poll_now(); live.fall_silent(Duration::from_secs(3), &mut [&mut leg]); live.run(live.tick + 8 * GOP, &mut [&mut leg], |_| false); @@ -1086,12 +1086,12 @@ async fn on_a_scaled_clock(scale: f64, ticks: u64) -> (Duration, u64, Stats) { let mut leg = live.join(Duration::ZERO).await; live.run(ticks, &mut [&mut leg], |_| false); - let first = leg.out.first().expect("output").1.timestamp.as_micros() as u64; + let first = leg.out.first().expect("output").1.timestamp.as_micros().unwrap() as u64; let held: Vec = leg .out .iter() - .filter(|(_, frame)| frame.timestamp.as_micros() as u64 >= first + 10_000_000) - .map(|(sent, frame)| sent.saturating_duration_since(live.sent(frame.timestamp.as_micros() as u64))) + .filter(|(_, frame)| frame.timestamp.as_micros().unwrap() as u64 >= first + 10_000_000) + .map(|(sent, frame)| sent.saturating_duration_since(live.sent(frame.timestamp.as_micros().unwrap() as u64))) .collect(); assert!(held.len() > 1_000, "too little output to judge: {}", held.len()); let spread = *held.iter().max().unwrap() - *held.iter().min().unwrap(); @@ -1153,7 +1153,7 @@ fn holds(live: &Live, leg: &Leg, from: Instant) -> Vec { leg.out .iter() .filter(|(sent, _)| *sent >= from) - .map(|(sent, frame)| sent.saturating_duration_since(live.sent(frame.timestamp.as_micros() as u64))) + .map(|(sent, frame)| sent.saturating_duration_since(live.sent(frame.timestamp.as_micros().unwrap() as u64))) .collect() } @@ -1245,7 +1245,7 @@ fn assert_same_packets(a: &Leg, b: &Leg, from: u64) { leg.out .iter() .map(|(_, frame)| frame.clone()) - .filter(|frame| frame.timestamp.as_micros() as u64 >= from) + .filter(|frame| frame.timestamp.as_micros().unwrap() as u64 >= from) .collect() }; let (a, b) = (pick(a), pick(b)); @@ -1292,7 +1292,14 @@ async fn pair_at(rate: u64, scale: f64, lag: i64, ticks: u64, stall: impl Fn(u64 let mut b = live.join(Duration::from_millis(100)).await; live.run(ticks, &mut [&mut a, &mut b], stall); live.finish(&mut [&mut a, &mut b]); - let joined = b.out.first().expect("the joiner rendered").1.timestamp.as_micros() as u64; + let joined = b + .out + .first() + .expect("the joiner rendered") + .1 + .timestamp + .as_micros() + .unwrap() as u64; (a, b, joined + 2_000_000) } @@ -1340,7 +1347,9 @@ async fn a_joiner_keeps_the_system_clock_in_tolerance() { // The instant each slot first went out, by its PCR time. let mut first = std::collections::BTreeMap::new(); for (sent, frame) in &leg.out { - first.entry(frame.timestamp.as_micros() as u64).or_insert(*sent); + first + .entry(frame.timestamp.as_micros().unwrap() as u64) + .or_insert(*sent); } let slots: Vec<(u64, Instant)> = first.into_iter().collect(); // (window end in seconds of PCR time, the system clock's offset in ppm) @@ -1389,7 +1398,7 @@ async fn a_joiner_runs_at_the_delay_from_its_first_output() { .out .iter() .map(|(sent, frame)| { - let media = frame.timestamp.as_micros() as u64; + let media = frame.timestamp.as_micros().unwrap() as u64; sent.saturating_duration_since(live.send_times(media, media).1) }) .collect(); diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 889b756470e..62b19008ec0 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -4895,7 +4895,8 @@ fn advance_pts(pts: Option, samples: u64, sample_rate: u32) -> anyhow // `pts` is a 90 kHz PES PTS; rescale the sample-rate advance to match before adding // (the scale-aware Timestamp rejects mixed scales). let advance = Timestamp::from_scale(samples, sample_rate as u64)?; - Ok(Some(pts.checked_add(advance.convert(pts.scale())?)?)) + let scale = pts.scale().ok_or(moq_net::TimeOverflow)?; + Ok(Some(pts.checked_add(advance.convert(scale)?)?)) } /// Tracks the wrap-around of the 33-bit, 90 kHz PTS field so timestamps stay @@ -5777,7 +5778,7 @@ pub(super) mod test { .decode(pes_packet(VIDEO_PID, PTS_SECS * 90_000).as_slice()) .unwrap(); - let now = catalog.clock().now().as_micros() / 1_000_000; + let now = catalog.clock().now().as_micros().unwrap() / 1_000_000; assert!( (PTS_SECS as u128..PTS_SECS as u128 + 5).contains(&now), "catalog clock reads {now}s, not the first video PTS" @@ -6711,7 +6712,7 @@ pub(super) mod test { assert_eq!( frames .iter() - .map(|frame| frame.timestamp.as_micros()) + .map(|frame| frame.timestamp.as_micros().unwrap()) .collect::>(), (0..10).map(|i| 1_000_000 + i * 72_000).collect::>() ); @@ -6752,7 +6753,7 @@ pub(super) mod test { assert_eq!( frames .iter() - .map(|frame| frame.timestamp.as_micros()) + .map(|frame| frame.timestamp.as_micros().unwrap()) .collect::>(), [1_000_000, 1_000_000 + step, 2_000_000], "{stream_type:?}" @@ -6789,7 +6790,7 @@ pub(super) mod test { assert_eq!( frames .iter() - .map(|frame| frame.timestamp.as_micros()) + .map(|frame| frame.timestamp.as_micros().unwrap()) .collect::>(), [1_000_000, 1_000_000 + step, 2_000_000, 3_000_000], "{stream_type:?}" @@ -6840,7 +6841,7 @@ pub(super) mod test { assert_eq!( frames .iter() - .map(|frame| frame.timestamp.as_micros()) + .map(|frame| frame.timestamp.as_micros().unwrap()) .collect::>(), [1_000_000, 1_000_000 + step, 1_500_000, 1_500_000 + step], "{stream_type:?}" @@ -6889,7 +6890,7 @@ pub(super) mod test { assert_eq!( frames .iter() - .map(|frame| frame.timestamp.as_micros()) + .map(|frame| frame.timestamp.as_micros().unwrap()) .collect::>(), [1_000_000, 1_000_000 + step], "{stream_type:?} carry={carry}" @@ -7404,7 +7405,8 @@ pub(super) mod test { .decode(audio_pes_packet(MP2_PID, 0, 90_000, &opening).as_slice()) .unwrap(); - import.seek(10).unwrap(); + // Two seconds in, in the track's microseconds: past the group already published. + import.seek(2_000_000).unwrap(); // Landing mid-frame after the seek, on a fragment that parses as a header but whose // declared frame ends inside a real one. @@ -7924,7 +7926,7 @@ pub(super) mod test { assert_ne!(frame.timestamp, Timestamp::ZERO, "cue must not stamp zero"); // Placement is normalized to microseconds, independently of the video PTS. assert_eq!( - Duration::from(frame.timestamp), + Duration::try_from(frame.timestamp).unwrap(), Duration::from_micros(expected), "cue stamped at its first section byte on the PCR clock" ); @@ -7963,7 +7965,10 @@ pub(super) mod test { let (consumer, catalog, _import) = import_all(&data).unwrap(); let name = track_on(&catalog.snapshot().ext.mpegts, 0x21); let frames = read_track(&consumer, &name, crate::container::Kind::Data).await; - let times: Vec<_> = frames.iter().map(|frame| frame.timestamp.as_micros() as u64).collect(); + let times: Vec<_> = frames + .iter() + .map(|frame| frame.timestamp.as_micros().unwrap() as u64) + .collect(); let clock = |at: u64, before: u64, ticks: u64, after: u64| { (ticks + (at - before) * 1_350_000 / (after - before)) / 27 }; @@ -8193,8 +8198,9 @@ pub(super) mod test { audio_pes_packet(pid, cc, pts, &payload) } - /// Read every retained frame of `name`, with the count of timeline breaks the consumer - /// crossed reading them. + /// Read every retained frame of `name`, with the count of declared timeline breaks the consumer + /// crossed reading them. Gaps are not counted: these streams carry a PES of two frames a second + /// apart, and the hole after each reads as a gap. async fn read_breaks( consumer: &moq_net::broadcast::Consumer, name: &str, @@ -8212,7 +8218,8 @@ pub(super) mod test { while let Ok(Ok(Some(read))) = kio::clock::timeout(std::time::Duration::from_millis(50), reader.read()).await { match read { crate::container::Read::Frame { frame, .. } => frames.push(frame), - crate::container::Read::Gap { .. } | crate::container::Read::End { .. } => breaks += 1, + crate::container::Read::End { .. } => breaks += 1, + crate::container::Read::Gap { .. } => {} } } (frames, breaks) @@ -8301,7 +8308,11 @@ pub(super) mod test { .all(|frame| frame.payload[4] == 0xCC || frame.payload[4] == 0xDD), "no program 1 frame leaks in" ); - assert_eq!(frames[0].timestamp.as_micros(), 3_601_000_000, "program 2's own PTS"); + assert_eq!( + frames[0].timestamp.as_micros().unwrap(), + 3_601_000_000, + "program 2's own PTS" + ); } #[test] @@ -8608,20 +8619,21 @@ pub(super) mod test { .as_slice(), ) .unwrap(); + // Each era resumes a second on, since a track's group starts only move forward. import.decode(pcr_packet(PCR_PID, 24_300_000).as_slice()).unwrap(); import.decode(packet(true, 0, 0, &CUE).as_slice()).unwrap(); import.decode(pes_packet(PCR_PID, 90_000).as_slice()).unwrap(); import.decode(pcr_packet(PCR_PID, 27_000_000).as_slice()).unwrap(); import.decode(clock_break_packet(PCR_PID).as_slice()).unwrap(); - import.decode(pcr_packet(PCR_PID, 24_300_000).as_slice()).unwrap(); + import.decode(pcr_packet(PCR_PID, 51_300_000).as_slice()).unwrap(); import.decode(packet(true, 1, 0, &CUE).as_slice()).unwrap(); - import.decode(pes_packet(PCR_PID, 90_000).as_slice()).unwrap(); - import.decode(pcr_packet(PCR_PID, 27_000_000).as_slice()).unwrap(); + import.decode(pes_packet(PCR_PID, 180_000).as_slice()).unwrap(); + import.decode(pcr_packet(PCR_PID, 54_000_000).as_slice()).unwrap(); import.decode(clock_break_packet(PCR_PID).as_slice()).unwrap(); - import.decode(pcr_packet(PCR_PID, 24_300_000).as_slice()).unwrap(); + import.decode(pcr_packet(PCR_PID, 78_300_000).as_slice()).unwrap(); import.decode(packet(true, 2, 0, &CUE).as_slice()).unwrap(); - import.decode(pes_packet(PCR_PID, 90_000).as_slice()).unwrap(); - import.decode(pcr_packet(PCR_PID, 27_000_000).as_slice()).unwrap(); + import.decode(pes_packet(PCR_PID, 270_000).as_slice()).unwrap(); + import.decode(pcr_packet(PCR_PID, 81_000_000).as_slice()).unwrap(); let _finished = import.finish().unwrap(); let name = track_on(&catalog.snapshot().ext.mpegts, 0x21); let (frames, breaks) = read_breaks(&consumer, &name, crate::container::Kind::Data).await; @@ -8852,7 +8864,7 @@ pub(super) mod test { assert_eq!(breaks, 1, "the timebase break did not reach this track"); let fills: Vec = frames.iter().map(|f| f.payload[4]).collect(); assert_eq!(fills, [0xAA, 0xBB, 0xCC, 0xDD], "media either side of the break"); - assert_eq!(frames[2].timestamp.as_micros(), 21_000_000, "the new timeline"); + assert_eq!(frames[2].timestamp.as_micros().unwrap(), 21_000_000, "the new timeline"); } } @@ -9086,7 +9098,7 @@ pub(super) mod test { let mut frames = BTreeMap::new(); for name in catalog.snapshot().audio.renditions.keys() { for frame in read_track(consumer, name, crate::container::Kind::Audio).await { - frames.insert(frame.payload[4], frame.timestamp.as_micros() as i64); + frames.insert(frame.payload[4], frame.timestamp.as_micros().unwrap() as i64); } } frames @@ -9558,7 +9570,7 @@ pub(super) mod test { fn assert_forward(frames: &[crate::container::Frame]) { let (mut start, mut floor) = (None, None); for frame in frames { - let ts = frame.timestamp.as_micros(); + let ts = frame.timestamp.as_micros().unwrap(); if frame.keyframe { assert!( start.is_none_or(|start| ts >= start), @@ -9590,7 +9602,10 @@ pub(super) mod test { let frames = video_frames(&data).await; assert_eq!(frames.len(), 1); assert_eq!(frames[0].payload.as_ref(), au); - assert_eq!(frames[0].timestamp.as_micros(), u128::from(FRAME) * 1_000_000 / 90_000); + assert_eq!( + frames[0].timestamp.as_micros().unwrap(), + u128::from(FRAME) * 1_000_000 / 90_000 + ); } #[test] @@ -9700,11 +9715,14 @@ pub(super) mod test { let frames = read_track(&consumer, &name, crate::container::Kind::Video).await; let observed: Vec<_> = frames .iter() - .map(|f| (f.timestamp.as_micros(), f.keyframe, f.payload.clone())) + .map(|f| (f.timestamp.as_micros().unwrap(), f.keyframe, f.payload.clone())) .collect(); assert_eq!(frames.len(), pictures.len(), "{kind:?} {layout}"); assert_eq!( - frames.iter().map(|f| f.timestamp.as_micros()).collect::>(), + frames + .iter() + .map(|f| f.timestamp.as_micros().unwrap()) + .collect::>(), pictures .iter() .map(|(pts, _)| u128::from(*pts) * 1_000_000 / 90_000) @@ -9733,7 +9751,7 @@ pub(super) mod test { assert_eq!( roundtrip .iter() - .map(|f| (f.timestamp.as_micros(), f.keyframe, f.payload.clone())) + .map(|f| (f.timestamp.as_micros().unwrap(), f.keyframe, f.payload.clone())) .collect::>(), observed, "export {kind:?} {layout}" @@ -9912,7 +9930,7 @@ pub(super) mod test { } assert!(frames[recovery].keyframe, "recovery starts at a keyframe"); assert_eq!( - frames[recovery].timestamp.as_micros(), + frames[recovery].timestamp.as_micros().unwrap(), u128::from(90_000 + 9 * FRAME) * 1_000_000 / 90_000 ); } @@ -10337,7 +10355,7 @@ pub(super) mod test { let mut frames = Vec::new(); while let Some(mut frame) = group.next_frame().await.unwrap() { let frame = hang::container::Frame::decode(frame.read_all().await.unwrap()).unwrap(); - frames.push((frame.timestamp.as_micros(), frame.payload.is_empty())); + frames.push((frame.timestamp.as_micros().unwrap(), frame.payload.is_empty())); } groups.push(frames); } @@ -10636,9 +10654,12 @@ pub(super) mod test { assert_forward(&frames); // The next GOP's cues follow the video rather than piling up on the edge. assert!( - frames[3].timestamp.as_micros() > frames[2].timestamp.as_micros(), + frames[3].timestamp.as_micros().unwrap() > frames[2].timestamp.as_micros().unwrap(), "the cues did not move with the video: {:?}", - frames.iter().map(|f| f.timestamp.as_micros()).collect::>() + frames + .iter() + .map(|f| f.timestamp.as_micros().unwrap()) + .collect::>() ); } @@ -11024,7 +11045,7 @@ pub(super) mod test { for (frames, breaks) in read_all_breaks(&consumer, &catalog).await { assert_eq!(breaks, 0, "a rollover is not a break"); - let stamps: Vec = frames.iter().map(|f| f.timestamp.as_micros()).collect(); + let stamps: Vec = frames.iter().map(|f| f.timestamp.as_micros().unwrap()).collect(); assert!( stamps.windows(2).all(|pair| pair[0] < pair[1]), "the rollover broke the timeline: {stamps:?}" diff --git a/rs/moq-mux/src/container/ts/oracle.rs b/rs/moq-mux/src/container/ts/oracle.rs index 7865c299ec0..361c3ab0357 100644 --- a/rs/moq-mux/src/container/ts/oracle.rs +++ b/rs/moq-mux/src/container/ts/oracle.rs @@ -94,7 +94,7 @@ impl<'a> Admitted<'a> { let mut matched = Vec::with_capacity(released.len()); let mut next = 0; for (i, (_, frame)) in released.iter().enumerate() { - let micros = frame.timestamp.as_micros() as u64; + let micros = frame.timestamp.as_micros().unwrap() as u64; let Some(found) = (next..self.objects.len()).find(|&j| self.objects[j].micros == micros) else { panic!("released object {i} at {micros} us is no published object after the one before it"); }; @@ -115,10 +115,10 @@ impl<'a> Admitted<'a> { self.objects[first].group ); - let origin = opened.timestamp.as_micros() as i128; + let origin = opened.timestamp.as_micros().unwrap() as i128; for ((at, frame), j) in released.iter().zip(&matched) { let out = at.duration_since(*start).as_micros() as i128; - let lag = out - (frame.timestamp.as_micros() as i128 - origin); + let lag = out - (frame.timestamp.as_micros().unwrap() as i128 - origin); assert!( lag.unsigned_abs() <= tolerance.as_micros(), "object {j} goes out {lag} us off its PCR time" diff --git a/rs/moq-mux/src/container/ts/passthrough_test.rs b/rs/moq-mux/src/container/ts/passthrough_test.rs index 08fc3c08e31..0b7ef3318ed 100644 --- a/rs/moq-mux/src/container/ts/passthrough_test.rs +++ b/rs/moq-mux/src/container/ts/passthrough_test.rs @@ -29,8 +29,11 @@ struct Object { struct Imported { catalog: hang::catalog::Catalog, objects: Vec, - /// The sequence of each break marker: a group of one object with no bytes. + /// The start of each break marker: a group of one object with no bytes. markers: Vec, + /// The runs between consecutive groups that the publisher did not declare skipped, where a + /// reader can't tell that no group lies. + holes: Vec>, } impl Imported { @@ -95,24 +98,33 @@ async fn import_onto( .await?; let mut objects = Vec::new(); let mut markers = Vec::new(); + let mut starts = Vec::new(); while let Some(mut group) = kio::clock::timeout(Duration::from_secs(1), track.recv_group()).await?? { + starts.push(group.start); while let Some(frame) = group.read_frame().await? { if frame.payload.is_empty() { - markers.push(group.sequence); + markers.push(group.start); continue; } let frame = hang::container::Frame::decode(frame.payload)?; objects.push(Object { - group: group.sequence, - micros: frame.timestamp.as_micros() as u64, + group: group.start, + micros: frame.timestamp.as_micros().unwrap() as u64, payload: frame.payload, }); } } + starts.sort_unstable(); + let holes = starts + .windows(2) + .map(|pair| pair[0] + 1..pair[1]) + .filter(|run| !track.is_skipped(run.clone())) + .collect(); Ok(Imported { catalog: catalog.snapshot(), objects, markers, + holes, }) } @@ -570,12 +582,21 @@ async fn a_flagged_jump_forward_starts_a_group_behind_a_break() { let after = groups[jump][0].micros; assert!((11_959_000..11_960_000).contains(&before), "{before}"); assert!((29_999_000..30_000_000).contains(&after), "{after}"); - assert_eq!( - imported.markers, - [groups[jump - 1][0].group + 1], + // Starts are micros, like the objects' timestamps. The marker lies between the two groups, + // not below the last object it ends, and the publisher declared every other start between + // them skipped. + let [marker] = imported.markers[..] else { + panic!("one marker, random access {random_access}: {:?}", imported.markers); + }; + assert!( + groups[jump - 1][0].group < marker && marker < groups[jump][0].group, "a marker group between them declares the break, random access {random_access}" ); - assert_eq!(groups[jump][0].group, groups[jump - 1][0].group + 2); + assert!( + marker >= before, + "the marker ends the content before it: {marker} {before}" + ); + assert_eq!(imported.holes, [], "random access {random_access}"); assert_eq!(imported.section().random_access, random_access); } } @@ -599,11 +620,11 @@ async fn an_import_shifts_onto_a_clock_already_in_use() { let clock = catalog.clock(); let fixed = catalog.snapshot().media.clock; tokio::time::advance(Duration::from_secs(5)).await; - let before = clock.now().as_micros() as u64; + let before = clock.now().as_micros().unwrap() as u64; let shifted = import_onto(broadcast, catalog.clone(), &data, data.len(), None) .await .unwrap(); - let after = clock.now().as_micros() as u64; + let after = clock.now().as_micros().unwrap() as u64; assert_eq!( catalog.snapshot().media.clock, @@ -618,14 +639,25 @@ async fn an_import_shifts_onto_a_clock_already_in_use() { let offset = first as i64 - verbatim.objects[0].micros as i64; assert!(offset < 0, "an hour in shifts back onto the capture's five seconds"); assert_eq!(shifted.objects.len(), verbatim.objects.len()); + // A group starts at its first object, in micros, so the offset moves the starts too. for (shifted, verbatim) in shifted.objects.iter().zip(&verbatim.objects) { - assert_eq!((shifted.group, &shifted.payload), (verbatim.group, &verbatim.payload)); + assert_eq!(&shifted.payload, &verbatim.payload); assert_eq!( - shifted.micros as i64 - verbatim.micros as i64, - offset, - "one offset for every object" + ( + shifted.micros as i64 - verbatim.micros as i64, + shifted.group as i64 - verbatim.group as i64 + ), + (offset, offset), + "one offset for every object and its group" ); } + let markers: Vec = shifted + .markers + .iter() + .zip(&verbatim.markers) + .map(|(shifted, verbatim)| *shifted as i64 - *verbatim as i64) + .collect(); + assert_eq!(markers, [offset], "the break moves with the timeline"); } /// A flagged step back is a restart: new content, which a timeline cannot step back into. diff --git a/rs/moq-mux/src/container/ts/pes_packet_test.rs b/rs/moq-mux/src/container/ts/pes_packet_test.rs index 62d46bbd0b5..34e14a70014 100644 --- a/rs/moq-mux/src/container/ts/pes_packet_test.rs +++ b/rs/moq-mux/src/container/ts/pes_packet_test.rs @@ -196,7 +196,7 @@ fn start(captured: bool) -> Imported { .unwrap(); // Paused, the clock reads the same until the first PTS anchors on it. let offset = match captured { - true => catalog.clock().now().as_micros() as i128, + true => catalog.clock().now().as_micros().unwrap() as i128, false => 0, }; let import = super::Import::new(broadcast, catalog.reserve()); @@ -299,7 +299,7 @@ async fn async_klv_takes_the_program_clock_at_its_first_byte() { .collect(); let got: Vec<_> = frames .iter() - .map(|frame| (frame.timestamp.as_micros() as u64, frame.payload.to_vec())) + .map(|frame| (frame.timestamp.as_micros().unwrap() as u64, frame.payload.to_vec())) .collect(); assert_eq!(got.len(), expected.len()); for (k, (got, expected)) in got.iter().zip(&expected).enumerate() { @@ -366,7 +366,7 @@ async fn sync_klv_rebases_its_pts_and_dts_onto_the_catalog_clock() { assert_eq!(frames.len(), feed.pes.len()); for (k, (frame, (at, pes))) in frames.iter().zip(&feed.pes).enumerate() { assert_eq!( - frame.timestamp.as_micros() as u64, + frame.timestamp.as_micros().unwrap() as u64, placed(feed.clock(*at).unwrap(), offset), "PES {k} timestamp" ); @@ -654,7 +654,7 @@ async fn a_data_only_program_preserves_placements_across_the_pcr_wrap() { for (frame, (at, pes)) in frames.iter().zip(&feed.pes) { assert_eq!(frame.payload.as_ref(), pes); assert_eq!( - frame.timestamp.as_micros(), + frame.timestamp.as_micros().unwrap(), u128::from(placed(feed.clock(*at).unwrap(), 0)) ); } @@ -695,7 +695,7 @@ async fn a_clock_break_places_each_side_on_its_own_timebase() { let (frames, err) = read(&imported.consumer, &name).await; assert!(err.is_none(), "{err:?}"); assert_eq!(frames.len(), 2); - assert_eq!(frames[0].timestamp.as_micros() as u64, placed(before, 0)); + assert_eq!(frames[0].timestamp.as_micros().unwrap() as u64, placed(before, 0)); assert!(frames[1].timestamp > frames[0].timestamp); assert_eq!(&frames[1].payload[..], &feed.pes[1].1[..]); } @@ -733,7 +733,10 @@ async fn a_pcr_pid_switch_places_each_side_on_its_own_timebase() { let (name, _) = imported.entry().unwrap(); let (frames, err) = read(&imported.consumer, &name).await; assert!(err.is_none(), "{err:?}"); - let got: Vec<_> = frames.iter().map(|frame| frame.timestamp.as_micros() as u64).collect(); + let got: Vec<_> = frames + .iter() + .map(|frame| frame.timestamp.as_micros().unwrap() as u64) + .collect(); assert_eq!(got, [placed(before, 0), placed(after, 0)]); assert_eq!(imported.stats().unplaced, 0); } @@ -777,7 +780,7 @@ async fn a_pes_header_split_by_a_pcr_is_placed_from_its_start_code() { let (frames, err) = read(&imported.consumer, &name).await; assert!(err.is_none(), "{err:?}"); assert_eq!(frames.len(), 1); - assert_eq!(frames[0].timestamp.as_micros() as u64, expected); + assert_eq!(frames[0].timestamp.as_micros().unwrap() as u64, expected); assert_eq!(&frames[0].payload[..], &feed.pes[1].1[..]); assert_eq!(imported.stats().unplaced, 1, "the PES begun before the first PCR"); } @@ -806,7 +809,7 @@ async fn a_pes_header_held_past_several_pcrs_takes_the_first_after_it() { let (frames, err) = read(&imported.consumer, &name).await; assert!(err.is_none(), "{err:?}"); assert_eq!(frames.len(), 1); - assert_eq!(frames[0].timestamp.as_micros() as u64, expected); + assert_eq!(frames[0].timestamp.as_micros().unwrap() as u64, expected); } /// A later PMT that names a PCR carries the private PES stream refused for want of one, and one @@ -913,7 +916,7 @@ async fn a_revised_private_stream_publishes_under_a_fresh_name() { let (frames, end) = old_reader.await.unwrap(); assert_eq!(end, Ok(true), "the old track finishes"); assert_eq!(frames.len(), 2); - assert_eq!(frames[1].timestamp.as_micros() as u64, held); + assert_eq!(frames[1].timestamp.as_micros().unwrap() as u64, held); let tracks = imported.catalog.snapshot().ext.mpegts.tracks; let listed: Vec<_> = tracks.iter().filter(|(_, track)| track.pid == DATA_PID).collect(); assert_eq!(listed.len(), 1, "one entry for the PID: {tracks:?}"); diff --git a/rs/moq-mux/src/container/ts/recovery_test.rs b/rs/moq-mux/src/container/ts/recovery_test.rs index cf09d932032..2b21c45c31a 100644 --- a/rs/moq-mux/src/container/ts/recovery_test.rs +++ b/rs/moq-mux/src/container/ts/recovery_test.rs @@ -457,11 +457,13 @@ fn splice(a: Vec, b: Vec, declare: bool) -> Clip { } } -/// One track as the import published it: each group's sequence and frames, in decode order. +/// One track as the import published it: each group's start and frames, in decode order. struct Recorded { name: String, video: bool, groups: Vec<(u64, Vec)>, + /// Where each group ended whole, when the import said. + ends: Vec>, } /// What the import of a clip published. @@ -505,6 +507,7 @@ async fn record(profile: Profile) -> Recording { .await .unwrap(); let mut groups = Vec::new(); + let mut ends = Vec::new(); while let Some(mut group) = kio::clock::timeout(Duration::from_secs(1), track.recv_group()) .await .unwrap() @@ -514,9 +517,19 @@ async fn record(profile: Profile) -> Recording { while let Some(frame) = group.read_frame().await.unwrap() { frames.push(frame); } - groups.push((group.sequence, frames)); + ends.push(group.end()); + groups.push((group.start, frames)); } - tracks.push(Recorded { name, video, groups }); + // The import appends each group, so nothing lies below its first or between two. + for run in skips(groups.iter().map(|(start, _)| *start)) { + assert!(track.is_skipped(run.clone()), "{name}: the import skips {run:?}"); + } + tracks.push(Recorded { + name, + video, + groups, + ends, + }); } assert_eq!( tracks.iter().map(|track| track.video).collect::>(), @@ -534,6 +547,17 @@ async fn record(profile: Profile) -> Recording { } } +/// The group starts skipped below the first of `starts` and between each two, which no group holds. +fn skips(starts: impl IntoIterator) -> Vec> { + let mut runs = Vec::new(); + let mut next = 0; + for start in starts { + runs.push(next..start); + next = start + 1; + } + runs +} + /// The presentation time of a published frame, in microseconds on the catalog clock, and /// whether it is the marker of a declared discontinuity. A whole video PES is stamped with its /// placement, so its own PTS says when it presents. @@ -544,7 +568,7 @@ fn decoded(frame: &moq_net::frame::Frame) -> (u64, bool) { let pts = super::psi::PesStart::parse(&frame.payload).unwrap().pts.unwrap(); pts * 100 / 9 } - _ => frame.timestamp.as_micros() as u64, + _ => frame.timestamp.as_micros().unwrap() as u64, }; (pts, frame.payload.is_empty()) } @@ -912,6 +936,16 @@ async fn run(recording: &Recording, arm: &Arm) -> Run { let consumer = broadcast.consume(); let groups: Vec)>> = recording.tracks.iter().map(|track| track.groups.clone()).collect(); + let ends: Vec>> = recording.tracks.iter().map(|track| track.ends.clone()).collect(); + // The import's skips, which a relay forwards with the first group past each: a group the path + // omits is not skipped, so the export sees it missing. + let mut skips: Vec<_> = (recording.tracks.iter()) + .map(|track| { + skips(track.groups.iter().map(|(start, _)| *start)) + .into_iter() + .peekable() + }) + .collect(); let start = Instant::now(); let publisher = tokio::spawn(async move { let _keep = &broadcast; @@ -933,20 +967,28 @@ async fn run(recording: &Recording, arm: &Arm) -> Run { let (sequence, frames) = &groups[t][g]; let Some(f) = step else { let group = open[t].take().unwrap(); - match kept[t] == frames.len() { - true => { - group.finish().unwrap(); + match (kept[t] == frames.len(), ends[t][g]) { + (true, Some(end)) => { + group.finish(end).unwrap(); } - false => group.abort(moq_net::Error::Old).unwrap(), + (true, None) => { + group.close().unwrap(); + } + (false, _) => group.abort(moq_net::Error::Old).unwrap(), } continue; }; + if open[t].is_none() { + while let Some(run) = skips[t].next_if(|run| run.end <= *sequence) { + producers[t].skip_groups(run); + } + } let group = open[t].get_or_insert_with(|| { kept[t] = 0; - let info = moq_net::group::Info { sequence: *sequence }; + let info = moq_net::group::Info { start: *sequence }; producers[t].create_group(info).unwrap() }); - assert_eq!(group.sequence, *sequence, "a schedule interleaves groups of track {t}"); + assert_eq!(group.start, *sequence, "a schedule interleaves groups of track {t}"); group .write_frame(frames[f].timestamp, frames[f].payload.clone()) .unwrap(); diff --git a/rs/moq-mux/src/container/ts/release.rs b/rs/moq-mux/src/container/ts/release.rs index f8538d4925c..4db7d842c3b 100644 --- a/rs/moq-mux/src/container/ts/release.rs +++ b/rs/moq-mux/src/container/ts/release.rs @@ -156,7 +156,7 @@ impl Release { let arrival = Arrival { arrived: arrived.max(self.subscribed), read: kio::clock::now(), - decode: frame.timestamp.into(), + decode: frame.timestamp.try_into()?, restart: track.markers(), // The output opens on a group, at a random access point; from then on every // object goes out on its own, so one dropped late is a gap of its packets only. diff --git a/rs/moq-mux/src/container/ts/release_test.rs b/rs/moq-mux/src/container/ts/release_test.rs index 887be35255c..0190a4d22c4 100644 --- a/rs/moq-mux/src/container/ts/release_test.rs +++ b/rs/moq-mux/src/container/ts/release_test.rs @@ -61,7 +61,7 @@ async fn import(data: &[u8]) -> Imported { if frame.payload.is_empty() { let micros = objects.last().map_or(0, |object: &Object| object.micros); objects.push(Object { - group: group.sequence, + group: group.start, micros, payload: Bytes::new(), }); @@ -69,12 +69,16 @@ async fn import(data: &[u8]) -> Imported { } let frame = hang::container::Frame::decode(frame.payload).unwrap(); objects.push(Object { - group: group.sequence, - micros: frame.timestamp.as_micros() as u64, + group: group.start, + micros: frame.timestamp.as_micros().unwrap() as u64, payload: frame.payload, }); } } + // The importer appends each group, so nothing lies below its first or between two. + for run in skips(&objects) { + assert!(track.is_skipped(run.clone()), "the importer skips {run:?}"); + } Imported { _passthrough: passthrough, _catalog: catalog, @@ -84,17 +88,33 @@ async fn import(data: &[u8]) -> Imported { } } +/// The group starts the importer skipped: those below its first group's and between each two, +/// which no group holds. +fn skips(objects: &[Object]) -> Vec> { + let mut starts: Vec = objects.iter().map(|object| object.group).collect(); + starts.dedup(); + let first = starts.first().map(|first| 0..*first); + first + .into_iter() + .chain(starts.windows(2).map(|pair| pair[0] + 1..pair[1])) + .collect() +} + /// Republish `objects` on a new broadcast's `m2ts` track, in the groups they were published in, /// each once `at` says it is due after the start and not before the one ahead of it, as a relay -/// delivering them would. +/// delivering them would, skipping the starts between them as the importer did. fn republish(objects: Vec, at: impl Fn(usize, &Object) -> Duration + Send + 'static) -> Republished { - republish_cut(objects, &[], at) + let skips = skips(&objects); + republish_cut(objects, skips, &[], at) } -/// [`republish`], aborting each group in `cut` once its last object in `objects` is out, as a -/// relay that evicts a group partway does. +/// [`republish`], skipping `skips`, and aborting each group in `cut` once its last object in +/// `objects` is out, as a relay that evicts a group partway does. Each run of `skips` goes out +/// with the first group past it, as a relay forwarding the publisher's skips would: a group the +/// relay never delivers is not skipped, so a reader sees it missing. fn republish_cut( objects: Vec, + skips: Vec>, cut: &[u64], at: impl Fn(usize, &Object) -> Duration + Send + 'static, ) -> Republished { @@ -113,6 +133,7 @@ fn republish_cut( let task = tokio::spawn(async move { let _keep = (&broadcast, &catalog); let mut group: Option<(u64, moq_net::group::Producer)> = None; + let mut skips = skips.into_iter().peekable(); let mut due = Duration::ZERO; for (i, object) in objects.into_iter().enumerate() { due = due.max(at(i, &object)); @@ -121,7 +142,10 @@ fn republish_cut( if let Some((sequence, group)) = group.take() { end(sequence, group, &cut); } - let info = moq_net::group::Info { sequence: object.group }; + while let Some(run) = skips.next_if(|run| run.end <= object.group) { + track.skip_groups(run); + } + let info = moq_net::group::Info { start: object.group }; group = Some((object.group, track.create_group(info).unwrap())); } let timestamp = moq_net::Timestamp::from_micros(object.micros).unwrap(); @@ -150,7 +174,7 @@ fn end(sequence: u64, group: moq_net::group::Producer, cut: &[u64]) { match cut.contains(&sequence) { true => group.abort(moq_net::Error::Old).unwrap(), false => { - group.finish().unwrap(); + group.close().unwrap(); } } } @@ -271,7 +295,7 @@ async fn a_flagged_jump_crosses_without_holding_the_output() { .map(|(i, pair)| { ( i, - pair[1].1.timestamp.as_micros() as i128 - pair[0].1.timestamp.as_micros() as i128, + pair[1].1.timestamp.as_micros().unwrap() as i128 - pair[0].1.timestamp.as_micros().unwrap() as i128, ) }) .max_by_key(|(_, step)| *step) @@ -355,13 +379,14 @@ async fn an_object_past_its_deadline_is_a_gap_in_that_release_only() { async fn release_omitting(cut: &[u64], omit: impl Fn(usize, &Object) -> bool) -> (Loss, Release) { let objects = import(KYRION).await.objects; let first = objects[0].micros; + let skips = skips(&objects); let delivered = objects .iter() .enumerate() .filter(|(i, object)| !omit(*i, object)) .map(|(_, object)| object.clone()) .collect(); - let republished = republish_cut(delivered, cut, move |_, object| { + let republished = republish_cut(delivered, skips, cut, move |_, object| { Duration::from_micros(object.micros - first) + Duration::from_millis(40) }); let (released, release) = spawn_release(&republished).await.unwrap(); @@ -423,7 +448,9 @@ async fn fixtures_are_released_at_their_exact_positions() { ]; for (name, data) in fixtures { let objects = import(&data).await.objects; - let groups = objects.last().unwrap().group - objects[0].group + 1; + let mut groups: Vec = objects.iter().map(|object| object.group).collect(); + groups.dedup(); + let groups = groups.len(); assert!( groups >= 5, "{name}: {groups} groups, one per random access point of program 1" @@ -452,7 +479,7 @@ async fn fixtures_are_released_at_their_exact_positions() { pids.contains(&0x0011) && pids.contains(&0x1abc), "{name}: the SDT and the unlisted PID" ); - let last = released.last().unwrap().1.timestamp.as_micros() as u64; + let last = released.last().unwrap().1.timestamp.as_micros().unwrap() as u64; assert!(last > fixture::WRAP / 27, "{name}: the release crosses the wrap"); } } diff --git a/rs/moq-mux/src/container/ts/si.rs b/rs/moq-mux/src/container/ts/si.rs index 064d8911c0b..1b345d313da 100644 --- a/rs/moq-mux/src/container/ts/si.rs +++ b/rs/moq-mux/src/container/ts/si.rs @@ -447,11 +447,14 @@ impl Entry { ) } }; - let mut group = track.append_group()?; + // A cut can land on the same millisecond as the last one (a retraction cuts at once, and SI + // without media PTS cuts at zero), so it is stamped a unit later instead of refused. + let (start, pts) = crate::start::next(track, pts)?; + let mut group = track.append_group(start)?; for payload in frames { group.write_frame(pts, payload)?; } - group.finish()?; + group.close()?; self.last_cut = Some(now); Ok(()) } @@ -800,9 +803,10 @@ mod tests { /// budget: at moq-net's zero default the superseded revision would be skipped at once. #[test] fn a_superseded_si_revision_stays_readable_within_the_export_budget() { + // Each revision's group starts at its time, in the track's milliseconds. for (info, expected) in [ - (track_info(Some(hang::container::MAX_AGE)), vec![0, 1]), - (track_info(Some(Duration::ZERO)), vec![1]), + (track_info(Some(hang::container::MAX_AGE)), vec![0, 100]), + (track_info(Some(Duration::ZERO)), vec![100]), ] { let broadcast = moq_net::broadcast::Info::new().produce(); let mut track = broadcast.create_track("0x0014-0x70.si", info).unwrap(); @@ -819,7 +823,7 @@ mod tests { let waiter = kio::Waiter::noop(); let mut read = Vec::new(); while let std::task::Poll::Ready(Ok(Some(group))) = reader.poll_next_group(&waiter) { - read.push(group.sequence); + read.push(group.start); } assert_eq!(read, expected); } diff --git a/rs/moq-mux/src/container/ts/video.rs b/rs/moq-mux/src/container/ts/video.rs index dc69f8389fd..7fdce207e84 100644 --- a/rs/moq-mux/src/container/ts/video.rs +++ b/rs/moq-mux/src/container/ts/video.rs @@ -495,7 +495,7 @@ mod tests { assert_eq!(units.len(), 1 + usize::from(lead)); let unit = units.last().unwrap(); let owner = if prefix.len() == 4 && cut == 1 { 2 } else { 1 }; - assert_eq!(unit.pts.unwrap().as_secs(), owner, "lead {lead} cut {cut}"); + assert_eq!(unit.pts.unwrap().as_secs().unwrap(), owner, "lead {lead} cut {cut}"); assert_eq!( unit.data.as_ref(), [START_CODE.as_ref(), &[0x65, 0x80, 0xab, 0xcd]].concat() @@ -555,10 +555,10 @@ mod tests { .push(pending(&staging, &[0, 0, 1, 0x65, 0x80], Some(270_000))) .unwrap(); let first = assembly.next(false).unwrap().unwrap(); - assert_eq!(first.pts.unwrap().as_secs(), 1); + assert_eq!(first.pts.unwrap().as_secs().unwrap(), 1); assert_eq!(&first.data[4..], &data[3..]); let second = assembly.next(true).unwrap().unwrap(); - assert_eq!(second.pts.unwrap().as_secs(), 3); + assert_eq!(second.pts.unwrap().as_secs().unwrap(), 3); drop((first, second)); assembly.reset(); assert_eq!(staging.bytes.load(Ordering::Relaxed), 0); diff --git a/rs/moq-mux/src/container/ts/video_timing.rs b/rs/moq-mux/src/container/ts/video_timing.rs index c823175741a..22e9ae86def 100644 --- a/rs/moq-mux/src/container/ts/video_timing.rs +++ b/rs/moq-mux/src/container/ts/video_timing.rs @@ -88,7 +88,11 @@ impl Clock { .ok_or_else(|| anyhow::anyhow!("TS video access unit has no timestamp anchor for its POC clock"))?; // Always measure from the explicit anchor, retaining fractional ticks rather // than accumulating the rounding error of a frame-by-frame increment. - let scale = i128::from(anchor.pts.scale().as_u64()); + let scale = anchor + .pts + .scale() + .ok_or_else(|| anyhow::anyhow!("TS video timestamp anchor has no timescale"))?; + let scale = i128::from(scale.as_u64()); let denominator = scale .checked_mul(rate.denominator.into()) .ok_or_else(|| anyhow::anyhow!("TS video timestamp overflow"))?; @@ -586,7 +590,7 @@ mod tests { for poc in [2, 7, 12, 60_006] { let got = clock.stamp(None, poc, rate, false).unwrap(); let expected = (123_123i128 * 2 + (i128::from(poc) - 6) * 3003) * 1_000_000 / 180_000; - assert_eq!(got.as_micros(), expected as u128); + assert_eq!(got.as_micros().unwrap(), expected as u128); } assert!( clock.stamp(None, 0, rate, true).is_err(), diff --git a/rs/moq-mux/src/container/ttml.rs b/rs/moq-mux/src/container/ttml.rs index c6ea7b9aaf5..519ae0c770f 100644 --- a/rs/moq-mux/src/container/ttml.rs +++ b/rs/moq-mux/src/container/ttml.rs @@ -43,10 +43,12 @@ pub(super) fn rewrite(document: &[u8], source: Timestamp, sample: Range bool { let Ok(wire) = crate::container::fmp4::Wire::from_init(init) else { return false; }; - let frame = moq_net::frame::Frame::new(None, Bytes::copy_from_slice(fragment)); + let frame = moq_net::frame::Frame::new(moq_net::Timestamp::unscaled(0), Bytes::copy_from_slice(fragment)); wire.decode(frame).is_ok() } @@ -206,7 +206,7 @@ pub fn catalog(data: &[u8]) -> bool { Box::new(move || consumer.poll_next(&waiter)) }; - let mut group = track.append_group().expect("a fresh track takes a group"); + let mut group = track.append_group(0).expect("a fresh track takes a group"); let mut accepted = false; for frame in data.split(|&byte| byte == 0) { if group diff --git a/rs/moq-mux/src/import/container.rs b/rs/moq-mux/src/import/container.rs index 00edc7dd508..e6a988dc1fd 100644 --- a/rs/moq-mux/src/import/container.rs +++ b/rs/moq-mux/src/import/container.rs @@ -143,9 +143,9 @@ impl Container { self.inner.cut() } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { - self.inner.seek(sequence) + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> Result<()> { + self.inner.seek(start) } } diff --git a/rs/moq-mux/src/import/track.rs b/rs/moq-mux/src/import/track.rs index b3f6df2b0c4..14d120de4f5 100644 --- a/rs/moq-mux/src/import/track.rs +++ b/rs/moq-mux/src/import/track.rs @@ -347,7 +347,8 @@ impl Track { } if pts .checked_sub(start) - .is_ok_and(|span| std::time::Duration::from(span) >= AUDIO_GROUP_WARN_AFTER) + .and_then(std::time::Duration::try_from) + .is_ok_and(|span| span >= AUDIO_GROUP_WARN_AFTER) { self.warned = true; tracing::warn!( @@ -492,11 +493,11 @@ impl Track { Ok(()) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> Result<()> { self.group_start = None; self.reset_split(); - import!(&mut self.kind, import => import.seek(sequence)) + import!(&mut self.kind, import => import.seek(start)) } /// Drop the partial access unit of a format split from a byte stream. @@ -796,11 +797,11 @@ impl TrackStream { stream!(&mut self.kind, _, import => import.cut(end)) } - /// Close the current group and open the next one at `sequence`. - pub fn seek(&mut self, sequence: u64) -> Result<()> { + /// Close the current group and open the next one at `start`. + pub fn seek(&mut self, start: u64) -> Result<()> { stream!(&mut self.kind, split, import => { split.reset(); - import.seek(sequence) + import.seek(start) }) } diff --git a/rs/moq-mux/src/jitter.rs b/rs/moq-mux/src/jitter.rs index a432e4da317..049ced2980d 100644 --- a/rs/moq-mux/src/jitter.rs +++ b/rs/moq-mux/src/jitter.rs @@ -335,8 +335,7 @@ impl Buffer { self.committed().map_or(&self.active, |committed| &committed.old) } - pub fn at(&self, generation: u64, decode: impl Into) -> Option { - let decode = decode.into(); + pub fn at(&self, generation: u64, decode: Decode) -> Option { self.committed() .and_then(|committed| committed.old.at(generation, decode)) .or_else(|| self.active.at(generation, decode)) @@ -382,9 +381,12 @@ impl Decode { } } -impl From for Decode { - fn from(timestamp: Timestamp) -> Self { - Self::from_nanos(timestamp.as_nanos() as i128) +impl TryFrom for Decode { + type Error = moq_net::TimeOverflow; + + /// Refused for an [unscaled](Timestamp::unscaled) timestamp, which has no time to decode at. + fn try_from(timestamp: Timestamp) -> Result { + Ok(Self::from_nanos(timestamp.as_nanos()? as i128)) } } @@ -1304,12 +1306,12 @@ impl Engine { /// When a frame of `generation` decoding at `decode` goes out: on the current clock, or on the /// previous one while that generation's tail is still going out. - pub fn at(&self, generation: u64, decode: impl Into) -> Option { + pub fn at(&self, generation: u64, decode: Decode) -> Option { let clock = [self.clock, self.previous] .into_iter() .flatten() .find(|clock| clock.generation == generation)?; - self.deadline(&clock, decode.into()) + self.deadline(&clock, decode) } /// When a frame decoding at `decode` goes out on `clock`, if any instant holds it. @@ -1795,7 +1797,7 @@ mod tests { buffer.cancel(); assert_eq!(buffer.reporting().dropped(), 2); assert_eq!( - buffer.at(0, ms(200)), + buffer.at(0, ms(200).try_into().unwrap()), Some(start.saturating_add(delay).saturating_add(Duration::from_millis(200))) ); buffer.push(0, arrival(kio::clock::now(), 200, "A resumed")).unwrap(); @@ -1917,7 +1919,9 @@ mod tests { let frame = Arrival { arrived: now, read: now, - decode: ms(u64::from(key) * 100_000 + (now - start).as_millis() as u64).into(), + decode: ms(u64::from(key) * 100_000 + (now - start).as_millis() as u64) + .try_into() + .unwrap(), restart: 0, sync: true, leading: false, @@ -1990,7 +1994,10 @@ mod tests { buffer.at(0, before_zero), Some(start.saturating_add(DELAY).saturating_sub(Duration::from_nanos(22_223))) ); - assert_eq!(buffer.at(0, ms(0)), Some(start.saturating_add(DELAY))); + assert_eq!( + buffer.at(0, ms(0).try_into().unwrap()), + Some(start.saturating_add(DELAY)) + ); kio::clock::sleep(DELAY).await; assert_eq!(due(&mut buffer), ["video", "audio"]); } @@ -2050,7 +2057,7 @@ mod tests { Arrival { arrived, read: arrived, - decode: ms(decode).into(), + decode: ms(decode).try_into().unwrap(), restart: 0, sync: true, leading: false, @@ -2117,7 +2124,7 @@ mod tests { } assert!(due(&mut buffer).is_empty()); assert_eq!( - buffer.at(0, ms(800)), + buffer.at(0, ms(800).try_into().unwrap()), Some(start.saturating_add(Duration::from_millis(500))) ); tokio::time::advance(Duration::from_millis(500)).await; @@ -2550,7 +2557,7 @@ mod tests { assert!(due(&mut buffer).is_empty()); let tail = start.saturating_add(Duration::from_millis(500)); assert_eq!( - buffer.at(1, ms(5_000)), + buffer.at(1, ms(5_000).try_into().unwrap()), Some(tail), "the export clock follows the queued tail" ); @@ -2842,7 +2849,10 @@ mod tests { let frame = Arrival { arrived, read: arrived, - decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap().into(), + decode: Timestamp::from_micros(decode.as_micros() as u64) + .unwrap() + .try_into() + .unwrap(), restart: 0, sync: true, leading: false, @@ -2862,8 +2872,13 @@ mod tests { } if hour >= settled && queued.is_zero() - && let Some(deadline) = buffer.at(0, Timestamp::from_micros(decode.as_micros() as u64).unwrap()) - { + && let Some(deadline) = buffer.at( + 0, + Timestamp::from_micros(decode.as_micros() as u64) + .unwrap() + .try_into() + .unwrap(), + ) { let slack = since(arrived, deadline) as f64 / 1e9; run.worst = run.worst.max((slack - delay.as_secs_f64()).abs()); } @@ -3030,7 +3045,10 @@ mod tests { let frame = Arrival { arrived, read: arrived, - decode: Timestamp::from_micros(decode.as_micros() as u64).unwrap().into(), + decode: Timestamp::from_micros(decode.as_micros() as u64) + .unwrap() + .try_into() + .unwrap(), restart: 0, sync: true, leading: false, @@ -3169,7 +3187,10 @@ mod tests { let frame = Arrival { arrived, read: arrived, - decode: Timestamp::from_micros((decode * 1e6).round() as u64).unwrap().into(), + decode: Timestamp::from_micros((decode * 1e6).round() as u64) + .unwrap() + .try_into() + .unwrap(), restart: 0, sync: true, leading: false, diff --git a/rs/moq-mux/src/json.rs b/rs/moq-mux/src/json.rs index b3c8daab62b..2e94601a9d5 100644 --- a/rs/moq-mux/src/json.rs +++ b/rs/moq-mux/src/json.rs @@ -80,7 +80,7 @@ use std::task::{Poll, ready}; -use moq_net::Timed; +use moq_json::Timed; use serde::Serialize; use serde::de::DeserializeOwned; @@ -218,7 +218,7 @@ impl Snapshot { /// The value is written at that timestamp and measures the entry's `jitter` and `delay`; one /// ahead of now measures as zero delay. An unchanged value writes nothing and measures nothing. pub fn update(&mut self, value: &T, timestamp: moq_net::Timestamp) -> crate::Result<()> { - match self.inner.update(Timed::from(value).at(timestamp))? { + match self.inner.update(Timed::new(value, timestamp))? { Some(size) => self.listing.record(size, Some(timestamp)), None => Ok(()), } @@ -299,7 +299,7 @@ impl Stream { /// error publishing the measured bitrate is returned after the record was written, so the track /// stays open and a retry would duplicate it. pub fn append(&mut self, value: &T, timestamp: moq_net::Timestamp) -> crate::Result<()> { - let size = match self.inner.append(Timed::from(value).at(timestamp)) { + let size = match self.inner.append(Timed::new(value, timestamp)) { Ok(size) => size, // Refused before anything was written, so the log and its entry carry on. Err( @@ -633,13 +633,13 @@ mod test { let mut stamps = Vec::new(); while let Poll::Ready(Ok(Some(mut group))) = subscriber.poll_recv_group(&waiter) { while let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) { - stamps.push(frame.timestamp.unwrap().as_millis()); + stamps.push(frame.timestamp.as_millis().unwrap()); } } assert_eq!(stamps.len(), 1); assert!( // The track stores milliseconds. - first.as_millis() <= stamps[0] && stamps[0] <= after.as_millis(), + first.as_millis().unwrap() <= stamps[0] && stamps[0] <= after.as_millis().unwrap(), "{first:?} {stamps:?} {after:?}" ); } @@ -664,9 +664,12 @@ mod test { let mut full_track = full.consume(); let mut delta_track = delta.consume(); - for value in [json!({ "n": 1 }), json!({ "n": 2 })] { - full.update(&value, catalog.clock().now()).unwrap(); - delta.update(&value, catalog.clock().now()).unwrap(); + // A millisecond apart: a group starts at its value's time, and starts strictly increase. + let now = catalog.clock().now(); + for (ms, value) in [json!({ "n": 1 }), json!({ "n": 2 })].into_iter().enumerate() { + let at = moq_net::Timestamp::from_micros(now.as_micros().unwrap() as u64 + ms as u64 * 1_000).unwrap(); + full.update(&value, at).unwrap(); + delta.update(&value, at).unwrap(); } full.finish().unwrap(); delta.finish().unwrap(); diff --git a/rs/moq-mux/src/lib.rs b/rs/moq-mux/src/lib.rs index 02928ab45c8..1ffed526d58 100644 --- a/rs/moq-mux/src/lib.rs +++ b/rs/moq-mux/src/lib.rs @@ -60,6 +60,7 @@ mod pace; pub mod rate; pub mod select; mod source; +mod start; mod stats; pub mod sync; diff --git a/rs/moq-mux/src/linger.rs b/rs/moq-mux/src/linger.rs index a73df4eb9af..536144e6c7e 100644 --- a/rs/moq-mux/src/linger.rs +++ b/rs/moq-mux/src/linger.rs @@ -98,9 +98,12 @@ mod tests { let info = moq_net::track::Info::default().with_max_age(hang::container::MAX_AGE); let track = route.broadcast().create_track("video", info).unwrap(); for _ in 0..3 { - let mut group = track.append_group().unwrap(); - group.write_frame(None, bytes::Bytes::from_static(b"frame")).unwrap(); - group.finish().unwrap(); + let key = track.largest().map_or(0, |largest| largest.group + 1); + let mut group = track.append_group(key).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(key), bytes::Bytes::from_static(b"frame")) + .unwrap(); + group.close().unwrap(); } let track = track.finish().unwrap(); (route.broadcast().clone(), (route, track)) diff --git a/rs/moq-mux/src/pace.rs b/rs/moq-mux/src/pace.rs index 70e4e3783d4..a35fe7cb483 100644 --- a/rs/moq-mux/src/pace.rs +++ b/rs/moq-mux/src/pace.rs @@ -95,8 +95,13 @@ impl Pacer { /// `now` by more than the configured lead (see the type docs). The result is /// never later than `now + lead + delay`, but may be arbitrarily far in the /// past for a frame that arrived late. + /// + /// An [unscaled](Timestamp::unscaled) `ts` has no media time to pace, so it goes out at + /// `now + delay` and leaves the anchor alone. pub fn pace(&mut self, ts: Timestamp, now: Instant) -> Instant { - let nanos = ts.as_nanos(); + let Ok(nanos) = ts.as_nanos() else { + return self.delayed(now); + }; let (anchor, base) = *self.anchor.get_or_insert((now, nanos)); // The lead is compared against the undelayed instant. Folding the delay in @@ -121,10 +126,11 @@ impl Pacer { /// and folds that arrival into the anchor. This is the read-only form, for a /// player that queues frames on arrival and asks again at presentation time: /// the anchor may have moved since, and a stale instant would present the - /// whole queue late by however far it moved. + /// whole queue late by however far it moved. `None` too for an + /// [unscaled](Timestamp::unscaled) `ts`, which has no media time. pub fn due(&self, ts: Timestamp) -> Option { let (anchor, base) = self.anchor?; - Some(self.delayed(send_at(anchor, base, ts.as_nanos())?)) + Some(self.delayed(send_at(anchor, base, ts.as_nanos().ok()?)?)) } /// Hold an undelayed instant for the configured delay, saturating at the far @@ -170,9 +176,13 @@ impl Pacer { /// `pace`'s. A sleeping sink's sleeps push the `now` it paces with forward, so /// a backlog can stay within the lead of every individual call while total /// delivery lag grows; such a caller measures lag against when the frame could - /// have arrived and hurries when that overshoots. + /// have arrived and hurries when that overshoots. An [unscaled](Timestamp::unscaled) `ts` + /// has no media time to pin, so the anchor stays. pub fn hurry(&mut self, ts: Timestamp, now: Instant) -> Instant { - self.anchor = Some((now, ts.as_nanos())); + let Ok(nanos) = ts.as_nanos() else { + return self.delayed(now); + }; + self.anchor = Some((now, nanos)); // A fresh pin has no lag absorbed into it yet, so the budget starts over. self.slack = Duration::ZERO; // `now` is the live edge, and the delay is measured from it: a re-anchor diff --git a/rs/moq-mux/src/source.rs b/rs/moq-mux/src/source.rs index 5e10b3c681f..8d79e9d6a7f 100644 --- a/rs/moq-mux/src/source.rs +++ b/rs/moq-mux/src/source.rs @@ -653,7 +653,10 @@ mod tests { let track = broadcast.create_track("audio", None).unwrap(); let preferences = moq_net::track::Subscription::default() .with_delay(moq_net::track::Delay::default().with_max(std::time::Duration::from_secs(3))) - .with_range(5..=10); + .with_ranges([moq_net::track::Range { + start: moq_net::track::Anchor::Time(5), + end: Some(moq_net::track::Anchor::Time(10)), + }]); let first = track.subscribe(preferences.clone()); let held = source.control(first.control()).unwrap(); let other = track.subscribe(preferences.clone()); diff --git a/rs/moq-mux/src/start.rs b/rs/moq-mux/src/start.rs new file mode 100644 index 00000000000..2a274630a05 --- /dev/null +++ b/rs/moq-mux/src/start.rs @@ -0,0 +1,85 @@ +//! Where a group starts on a timed track: its first frame's timestamp in the track's timescale. +//! +//! Starts strictly increase, so a group has to start after every group the track holds. + +/// The start of a group whose first frame is at `timestamp`, its timescale, and the newest start +/// on `track` when one is held: `timestamp` in the track's timescale. Refused with +/// [`TimestampMismatch`](moq_net::Error::TimestampMismatch) on a scale-less track or past the +/// timescale's range. +fn of( + track: &moq_net::track::Producer, + timestamp: moq_net::Timestamp, +) -> crate::Result<(moq_net::Timestamp, moq_net::Timescale, Option)> { + let start = track.info().rescale(timestamp)?; + let scale = start.scale().ok_or(moq_net::Error::TimestampMismatch)?; + Ok((start, scale, track.largest().map(|largest| largest.group))) +} + +/// The start of a group stamped at `timestamp` on `track`, and the timestamp to stamp its first +/// frame with: `timestamp` in the track's timescale, or one unit past the newest start when it +/// doesn't pass that. +/// +/// For writes stamped from a clock rather than media, such as a catalog or a table revision, which +/// can land twice in one unit of the track's timescale: the later one is stamped a unit later +/// rather than refused. +pub(crate) fn next( + track: &moq_net::track::Producer, + timestamp: moq_net::Timestamp, +) -> crate::Result<(u64, moq_net::Timestamp)> { + match of(track, timestamp)? { + (start, scale, Some(newest)) if start.value() <= newest => { + let bumped = newest.checked_add(1).ok_or(moq_net::Error::TimestampMismatch)?; + let at = moq_net::Timestamp::new(bumped, scale).map_err(|_| moq_net::Error::TimestampMismatch)?; + Ok((bumped, at)) + } + (start, _, _) => Ok((start.value(), timestamp)), + } +} + +/// [`next`] for a group stamped one unit of the track's timescale before `timestamp`, so a group +/// whose first frame is at `timestamp` can still follow it, but never below `floor`. +pub(crate) fn before( + track: &moq_net::track::Producer, + timestamp: moq_net::Timestamp, + floor: moq_net::Timestamp, +) -> crate::Result<(u64, moq_net::Timestamp)> { + let (start, scale, _) = of(track, timestamp)?; + let (floor, _, _) = of(track, floor)?; + let below = start.value().saturating_sub(1).max(floor.value()); + let below = moq_net::Timestamp::new(below, scale).map_err(|_| moq_net::Error::TimestampMismatch)?; + next(track, below) +} + +/// The start of a group whose first frame is at `timestamp` on `track`: `timestamp` in the track's +/// timescale, or `None` when that doesn't pass the newest start, such as a keyframe repeating the +/// last one's timestamp. +/// +/// For groups keyed by media, whose timestamps a publisher can't move. +pub(crate) fn opens(track: &moq_net::track::Producer, timestamp: moq_net::Timestamp) -> crate::Result> { + let (start, _, newest) = of(track, timestamp)?; + Ok(newest + .is_none_or(|newest| start.value() > newest) + .then_some(start.value())) +} + +/// [`opens`], refusing a start that doesn't pass the newest with +/// [`TimestampRewind`](crate::container::TimestampRewind). +pub(crate) fn after(track: &moq_net::track::Producer, timestamp: moq_net::Timestamp) -> crate::Result { + match of(track, timestamp)? { + (start, scale, Some(newest)) if start.value() <= newest => { + let floor = moq_net::Timestamp::new(newest, scale).unwrap_or(start); + Err(crate::container::TimestampRewind { timestamp, floor }.into()) + } + (start, _, _) => Ok(start.value()), + } +} + +/// Where a group ends whole on `track` when its content stops at `timestamp`: `timestamp` in the +/// track's timescale, or `None` when that doesn't pass `start` or won't convert. +pub(crate) fn end(track: &moq_net::track::Producer, start: u64, timestamp: moq_net::Timestamp) -> Option { + let end = track.info().rescale(timestamp).ok()?; + // A scale-less track's key is no time. + end.scale()?; + let end = end.value(); + (end > start).then_some(end) +} diff --git a/rs/moq-mux/src/stats.rs b/rs/moq-mux/src/stats.rs index 4bc39b73ae7..b0f641a74db 100644 --- a/rs/moq-mux/src/stats.rs +++ b/rs/moq-mux/src/stats.rs @@ -67,14 +67,18 @@ impl Received { pub(crate) fn frame(&self, timestamp: moq_net::Timestamp, bytes: usize) { self.frames.fetch_add(1, Relaxed); self.bytes.fetch_add(bytes as u64, Relaxed); - // A clock before the MoQ epoch has no wire encoding, so such an arrival is never recorded. + // A clock before the MoQ epoch has no wire encoding, so such an arrival is never recorded, + // and neither is a scale-less track's unscaled key, which has no media time. let Some(wall) = crate::clock::host_wall() else { return; }; + let Ok(micros) = timestamp.as_micros() else { + return; + }; let wall = wall .checked_duration_since(Wall::UNIX_EPOCH) .map_or(0, |since| u64::try_from(since.as_millis()).unwrap_or(0)); - let micros = u64::try_from(timestamp.as_micros()).unwrap_or(u64::MAX); + let micros = u64::try_from(micros).unwrap_or(u64::MAX); self.timestamp.store(micros, Relaxed); self.wall.store(wall, Relaxed); } @@ -283,7 +287,7 @@ mod tests { let report = ®istry.received()["hd"]; assert_eq!((report.frames, report.bytes), (2, 150)); let arrival = report.arrival.as_ref().unwrap(); - assert_eq!(arrival.timestamp.as_millis(), 80); + assert_eq!(arrival.timestamp.as_millis().unwrap(), 80); assert_eq!(arrival.wall, wall); } diff --git a/rs/moq-mux/src/sync.rs b/rs/moq-mux/src/sync.rs index a311453d443..2f28607c857 100644 --- a/rs/moq-mux/src/sync.rs +++ b/rs/moq-mux/src/sync.rs @@ -242,10 +242,11 @@ impl State { /// Fold an arrival into `kind`'s timeline, reporting whether that moved the schedule. fn received(&mut self, kind: Kind, timestamp: Timestamp, now: Instant) -> bool { - let arrival = Anchor { - at: now, - media: Duration::from(timestamp), + // An unscaled key has no media time to anchor on. + let Ok(media) = Duration::try_from(timestamp) else { + return false; }; + let arrival = Anchor { at: now, media }; let before = self.reference(); let half = self.delay() / 2; let part = &mut self.parts[kind as usize]; @@ -315,7 +316,7 @@ impl State { /// When `timestamp` should be presented, or `None` before any frame has /// anchored the clock. `None` means as soon as possible: there is nothing - /// left to wait for. + /// left to wait for, as for an unscaled timestamp, which has no media time. fn due(&self, timestamp: Timestamp) -> Option { self.schedule().due(timestamp) } @@ -346,9 +347,10 @@ impl Schedule { self.delay } - /// When `timestamp` is due on this schedule, or `None` before the first anchor. + /// When `timestamp` is due on this schedule, or `None` before the first anchor or for an + /// unscaled timestamp, which has no media time. pub fn due(&self, timestamp: Timestamp) -> Option { - let at = self.reference?.at(Duration::from(timestamp))?; + let at = self.reference?.at(Duration::try_from(timestamp).ok()?)?; Some(at.checked_add(self.delay).unwrap_or(at)) } } @@ -390,7 +392,7 @@ pub struct Mapping { impl Mapping { fn map(self, source: Timestamp) -> Result { - let source = Duration::from(source); + let source = Duration::try_from(source).map_err(|_| Error::Unscaled)?; let target = if source >= self.source { self.target.checked_add(source - self.source).ok_or(Error::Overflow)? } else { @@ -429,6 +431,8 @@ impl Track { /// A frame arriving earlier than the schedule predicted pulls playback toward /// live; a late one leaves it alone. A move makes every queued frame due /// earlier, so a presenter has to recompute the deadline it is asleep on. + /// An [unscaled](Timestamp::unscaled) timestamp has no media time, so it + /// moves nothing. pub fn received(&self, timestamp: Timestamp, now: Instant) -> bool { let held = self.mapping.lock().unwrap().held; let now = now.checked_add(held).unwrap_or(now); @@ -487,7 +491,8 @@ impl Track { } } - /// Map a source timestamp onto the shared broadcast timeline. + /// Map a source timestamp onto the shared broadcast timeline. Refuses an + /// [unscaled](Timestamp::unscaled) one with [`Error::Unscaled`]. pub fn map(&self, source: Timestamp) -> Result { self.mapping.lock().unwrap().mapping.map(source) } @@ -542,23 +547,25 @@ impl Track { let mapping = self.mapping.lock().unwrap().mapping; self.clock.state.lock().unwrap().ends[self.role as usize] = Some(Boundary { starts: mapping.starts, - end: Duration::from(mapping.map(source)?), + end: Duration::try_from(mapping.map(source)?).expect("a mapped timestamp is in microseconds"), }); self.clock.wake(); Ok(()) } /// Record the first replacement media for `transition`, as it arrives. + /// + /// An [unscaled](Timestamp::unscaled) `source` has no media time to plan + /// from, so it records nothing and this role maps alone. pub fn plan(&self, transition: Transition, source: Timestamp, delay: Option) { + let Ok(source) = Duration::try_from(source) else { + return; + }; let arrived = kio::clock::now(); let mut state = self.clock.state.lock().unwrap(); let delay = delay.unwrap_or(state.delay()); if let Some(pending) = state.transition(transition) { - pending.plans[self.role as usize] = Some(Plan { - source: Duration::from(source), - delay, - arrived, - }); + pending.plans[self.role as usize] = Some(Plan { source, delay, arrived }); } drop(state); self.clock.wake(); @@ -598,7 +605,7 @@ impl Track { .map(media(plan.source)) .ok() .zip(reference) - .and_then(|(mapped, reference)| reference.at(Duration::from(mapped))) + .and_then(|(mapped, reference)| reference.at(Duration::try_from(mapped).ok()?)) { held = scheduled.saturating_duration_since(plan.arrived); } @@ -720,7 +727,7 @@ impl Resume<'_> { let (source, delay, arrived) = match paired { Some(origin) => (origin.source, origin.delay, Some(origin.arrived)), None => ( - Duration::from(self.source), + Duration::try_from(self.source).map_err(|_| Error::Unscaled)?, self.delay.unwrap_or(state.delay()), own_plan.map(|plan| plan.arrived), ), @@ -764,6 +771,9 @@ pub enum Error { /// The requested delay exceeds the system clock. #[error("playback delay exceeds the system clock")] DelayOverflow, + /// The timestamp is [unscaled](Timestamp::unscaled), a scale-less track's key with no media time. + #[error("timestamp has no timescale")] + Unscaled, } /// A result from mapping media onto the playout clock. @@ -1234,7 +1244,7 @@ mod tests { video.resume(transition, ms(40), None).ready().await.unwrap(), ); // The data track has no replacement yet, so the others map alone. - assert_eq!(video.map(ms(40)).unwrap().as_millis(), 10_100); + assert_eq!(video.map(ms(40)).unwrap().as_millis().unwrap(), 10_100); data.plan(transition, ms(20), None); data.apply(transition, data.resume(transition, ms(20), None).ready().await.unwrap()); assert_eq!(data.map(ms(40)).unwrap(), video.map(ms(40)).unwrap()); @@ -1256,8 +1266,8 @@ mod tests { transition, audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), ); - assert_eq!(audio.map(ms(0)).unwrap().as_millis(), 10_120); - assert_eq!(video.map(ms(10_500)).unwrap().as_millis(), 10_500); + assert_eq!(audio.map(ms(0)).unwrap().as_millis().unwrap(), 10_120); + assert_eq!(video.map(ms(10_500)).unwrap().as_millis().unwrap(), 10_500); assert_eq!(clock.state.lock().unwrap().due(ms(10_500)), unchanged); } @@ -1377,7 +1387,7 @@ mod tests { video.resume(transition, ms(40), None).ready().await.unwrap(), ); assert_eq!( - video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + video.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), 40 ); } @@ -1408,7 +1418,7 @@ mod tests { audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), ); assert_eq!( - video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + video.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), 40 ); } @@ -1437,7 +1447,7 @@ mod tests { audio.resume(transition, ms(0), Some(DELAY)).ready().await.unwrap(), ); assert_eq!( - first.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + first.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), 40 ); } @@ -1634,7 +1644,7 @@ mod tests { .unwrap(), ); assert_eq!( - video.map(ms(40)).unwrap().as_millis() - audio.map(ms(0)).unwrap().as_millis(), + video.map(ms(40)).unwrap().as_millis().unwrap() - audio.map(ms(0)).unwrap().as_millis().unwrap(), 40 ); assert_eq!( diff --git a/rs/moq-mux/tests/legacy_audio.rs b/rs/moq-mux/tests/legacy_audio.rs new file mode 100644 index 00000000000..7a8c2ed4812 --- /dev/null +++ b/rs/moq-mux/tests/legacy_audio.rs @@ -0,0 +1,171 @@ +//! Legacy audio read across a session, on wires that can and can't say the starts between two +//! groups were skipped. + +use std::time::Duration; + +use bytes::Bytes; +use moq_mux::catalog::hang::Container; +use moq_mux::container::Container as _; +use moq_mux::container::{Consumer, Frame, Producer, Read}; + +/// How long a step may take on the paused clock before the test fails. +const TIMEOUT: Duration = Duration::from_secs(10); + +/// One 20 ms CELT packet, which states its own duration in its TOC byte. +const PACKET: [u8; 2] = [31 << 3, 0]; + +fn origin() -> moq_net::origin::Producer { + let (origin, driver) = moq_net::origin::Producer::new(Default::default()); + tokio::spawn(moq_net::time::run(driver)); + origin +} + +/// A publisher's origin and a subscriber's origin connected over one session on `version`. +async fn connect( + version: moq_net::Version, +) -> ( + moq_net::origin::Producer, + moq_net::origin::Producer, + [moq_net::Session; 2], +) { + let publisher = origin(); + let subscriber = origin(); + let mut transport = moq_net::transport::memory::Config::default(); + transport.protocol = Some(version.alpn().to_string()); + let (client_transport, server_transport) = moq_net::transport::memory::pair(transport); + let client = moq_net::Client::new() + .with_versions(version.into()) + .with_subscriber(subscriber.clone()); + let server = moq_net::Server::new() + .with_versions(version.into()) + .with_publisher(publisher.consume()); + let (client, server) = tokio::join!( + async { + let (session, driver) = client + .connect(kio::clock::now(), client_transport) + .await + .expect("client session"); + tokio::spawn(moq_net::time::run(driver)); + session + }, + async { + let (session, driver) = server + .accept(kio::clock::now(), server_transport) + .await + .expect("server session"); + tokio::spawn(moq_net::time::run(driver)); + session + } + ); + (publisher, subscriber, [client, server]) +} + +/// An encoder's one-packet groups follow each other at their starts, and the publisher +/// declares the starts between them skipped. moq-lite-07 says so; earlier wires report the +/// run as dropped, so only the packets' codec durations show nothing was lost. +#[tokio::test(start_paused = true)] +async fn one_packet_opus_groups_read_on_without_a_gap() { + for version in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { + let (publisher, subscriber, _sessions) = connect(version.parse().unwrap()).await; + let config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + let route = publisher.route("live", Default::default()).expect("route"); + let track = route + .broadcast() + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .expect("track"); + let mut producer = Producer::new(track, Container::try_from(&config).unwrap()); + + let remote = kio::clock::timeout(TIMEOUT, subscriber.consume().follow("live", None).unwrap().available()) + .await + .expect("announced") + .expect("remote broadcast") + .unwrap(); + let track = remote.track("audio").expect("track"); + let subscription = moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(5))); + let subscribed = kio::clock::timeout(TIMEOUT, track.subscribe(subscription)) + .await + .expect("subscribed") + .expect("subscription"); + let mut reader = Consumer::new(subscribed, Container::try_from(&config).unwrap()); + + for n in 0..10u64 { + let start = moq_net::Timestamp::from_micros(n * 20_000).unwrap(); + let end = moq_net::Timestamp::from_micros((n + 1) * 20_000).unwrap(); + producer + .write(Frame { + timestamp: start, + payload: Bytes::from_static(&PACKET), + keyframe: true, + duration: None, + }) + .unwrap(); + producer.cut(Some(end)).unwrap(); + let read = kio::clock::timeout(TIMEOUT, reader.read()) + .await + .unwrap_or_else(|_| panic!("{version}: packet {n} never read")) + .unwrap() + .expect("a read"); + match read { + Read::Frame { frame, .. } => assert_eq!(frame.timestamp, start, "{version}"), + other => panic!("{version}: packet {n} read as {other:?}"), + } + kio::clock::sleep(Duration::from_millis(20)).await; + } + } +} + +/// A run reported dropped, as moq-lite-05 and 06 report a skip, still waits for the group past +/// it to show where it starts: stepping before that group's first packet lands would call a +/// contiguous timeline a loss. +#[tokio::test(start_paused = true)] +async fn a_dropped_run_waits_for_the_next_groups_first_packet() { + for version in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { + let (publisher, subscriber, _sessions) = connect(version.parse().unwrap()).await; + let config = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + let container = Container::try_from(&config).unwrap(); + let route = publisher.route("live", Default::default()).expect("route"); + let track = route + .broadcast() + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .expect("track"); + + let remote = kio::clock::timeout(TIMEOUT, subscriber.consume().follow("live", None).unwrap().available()) + .await + .expect("announced") + .expect("remote broadcast") + .unwrap(); + let subscription = moq_net::track::Subscription::default() + .with_delay(moq_net::track::Delay::default().with_max(Duration::from_secs(5))); + let subscribed = kio::clock::timeout(TIMEOUT, remote.track("audio").expect("track").subscribe(subscription)) + .await + .expect("subscribed") + .expect("subscription"); + let mut reader = Consumer::new(subscribed, Container::try_from(&config).unwrap()); + + for n in 0..4u64 { + let start = n * 20_000; + // Appending declares every start since the previous group skipped, before the packet. + let mut group = track.append_group(start).unwrap(); + let early = kio::clock::timeout(Duration::from_millis(50), reader.read()).await; + assert!(early.is_err(), "{version}: read {early:?} before packet {n} landed"); + let packet = Frame { + timestamp: moq_net::Timestamp::from_micros(start).unwrap(), + payload: Bytes::from_static(&PACKET), + keyframe: true, + duration: None, + }; + container.write(&mut group, &[packet]).unwrap(); + group.finish(start + 20_000).unwrap(); + let read = kio::clock::timeout(TIMEOUT, reader.read()) + .await + .unwrap_or_else(|_| panic!("{version}: packet {n} never read")) + .unwrap() + .expect("a read"); + assert!( + matches!(read, Read::Frame { .. }), + "{version}: packet {n} read as {read:?}" + ); + } + } +} diff --git a/rs/moq-net-tests/tests/integration/ietf_conformance.rs b/rs/moq-net-tests/tests/integration/ietf_conformance.rs index 57b56cf64a1..a9e3d04130f 100644 --- a/rs/moq-net-tests/tests/integration/ietf_conformance.rs +++ b/rs/moq-net-tests/tests/integration/ietf_conformance.rs @@ -71,12 +71,14 @@ async fn routing(version: &str, cluster: bool) { .unwrap(); let mut subscription = remote.track("test").unwrap().subscribe(None).await.unwrap(); crate::support::harness::subscribed(&track).await; - let mut group = track.append_group().unwrap(); - group.write_frame(None, &b"conformance"[..]).unwrap(); - group.finish().unwrap(); + let mut group = track.append_group(0).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(0), &b"conformance"[..]) + .unwrap(); + group.close().unwrap(); let track = track.finish().unwrap(); let mut received = subscription.recv_group().await.unwrap().unwrap(); - assert_eq!(received.sequence, 0); + assert_eq!(received.start, 0); assert_eq!( received.read_frame().await.unwrap().unwrap().payload.as_ref(), b"conformance" diff --git a/rs/moq-net-tests/tests/integration/refresh.rs b/rs/moq-net-tests/tests/integration/refresh.rs index 0b46cb80487..fc76703babe 100644 --- a/rs/moq-net-tests/tests/integration/refresh.rs +++ b/rs/moq-net-tests/tests/integration/refresh.rs @@ -14,8 +14,9 @@ fn origin(hop: u64) -> origin::Producer { /// complete snapshot. #[kio::test] async fn json_snapshot_refresh_crosses_a_two_relay_cluster() { + use moq_json::Timed; use moq_json::snapshot; - use moq_net::{Timed, Timescale, Timestamp}; + use moq_net::{Timescale, Timestamp}; use serde_json::{Value, json}; for ratio in [1, 2] { @@ -57,21 +58,23 @@ async fn json_snapshot_refresh_crosses_a_two_relay_cluster() { subscribed(&track).await; let at = |n| Timestamp::from_millis(100 + n).unwrap(); + // A group starts at its first frame's time, in the track's milliseconds. + let key = |n| 100 + n; let value = |n| json!({ "n": n, "p": "" }); - let snapshot_bytes = producer.update(Timed::from(&value(0)).at(at(0))).unwrap().unwrap(); + let snapshot_bytes = producer.update(Timed::new(&value(0), at(0))).unwrap().unwrap(); let mut original = baseline.recv_group().await.unwrap().unwrap(); - assert_eq!(original.sequence, 0); + assert_eq!(original.start, key(0)); let first = original.read_frame().await.unwrap().unwrap(); - assert_eq!(first.timestamp, Some(at(0))); + assert_eq!(first.timestamp, at(0)); assert_eq!(first.payload.len(), snapshot_bytes); assert_eq!(serde_json::from_slice::(&first.payload).unwrap(), value(0)); let mut delta_bytes = 0; let mut joins = Vec::new(); for n in 1..=2 * u64::from(ratio) { - delta_bytes += producer.update(Timed::from(&value(n)).at(at(n))).unwrap().unwrap(); + delta_bytes += producer.update(Timed::new(&value(n), at(n))).unwrap().unwrap(); let delta = original.read_frame().await.unwrap().unwrap(); - assert_eq!(delta.timestamp, Some(at(n))); + assert_eq!(delta.timestamp, at(n)); assert_eq!( serde_json::from_slice::(&delta.payload).unwrap(), json!({ "n": n }) @@ -79,7 +82,7 @@ async fn json_snapshot_refresh_crosses_a_two_relay_cluster() { if n == 1 || n == 2 * u64::from(ratio) { let mut join = snapshot::Consumer::::new(remote.clone(), Default::default()); let received = join.latest().await.unwrap().unwrap(); - assert_eq!((received.value, received.at), (value(n), Some(at(n)))); + assert_eq!((received.value, received.at), (value(n), at(n))); assert!(producer.poll_refresh(&kio::Waiter::noop()).is_pending()); assert_eq!(track.demand().wanted(), Some(0)); joins.push(join); @@ -92,7 +95,7 @@ async fn json_snapshot_refresh_crosses_a_two_relay_cluster() { ); let n = 2 * u64::from(ratio) + 1; - delta_bytes += producer.update(Timed::from(&value(n)).at(at(n))).unwrap().unwrap(); + delta_bytes += producer.update(Timed::new(&value(n), at(n))).unwrap().unwrap(); original.read_frame().await.unwrap().unwrap(); assert!(delta_bytes > ratio as usize * snapshot_bytes); // Simulated time passing cannot roll a group, even with deltas over the ratio. @@ -114,7 +117,7 @@ async fn json_snapshot_refresh_crosses_a_two_relay_cluster() { }) .await .unwrap(); - assert_eq!(cached.sequence, 0, "the cached group is not withheld"); + assert_eq!(cached.start, key(0), "the cached group is not withheld"); let mut cached_bytes = 0; for _ in 0..=n { cached_bytes += cached.read_frame().await.unwrap().unwrap().payload.len(); @@ -124,24 +127,24 @@ async fn json_snapshot_refresh_crosses_a_two_relay_cluster() { producer.poll_refresh(&kio::Waiter::noop()).is_pending(), "answered once" ); - assert_eq!(fresh.sequence, 1, "new content gets a new group identity"); + assert_eq!(fresh.start, key(n), "new content gets a new group identity"); let frame = fresh.read_frame().await.unwrap().unwrap(); - assert_eq!(frame.timestamp, Some(at(n)), "refresh preserves the newest source time"); + assert_eq!(frame.timestamp, at(n), "refresh preserves the newest source time"); assert_eq!(serde_json::from_slice::(&frame.payload).unwrap(), value(n)); assert_eq!(frame.payload.len(), snapshot_bytes); assert!(fresh.poll_read_frame(&kio::Waiter::noop()).is_pending()); - assert_eq!(baseline.recv_group().await.unwrap().unwrap().sequence, 1); + assert_eq!(baseline.recv_group().await.unwrap().unwrap().start, key(n)); - producer.update(Timed::from(&value(n + 1)).at(at(n + 1))).unwrap(); + producer.update(Timed::new(&value(n + 1), at(n + 1))).unwrap(); let delta = fresh.read_frame().await.unwrap().unwrap(); - assert_eq!(delta.timestamp, Some(at(n + 1))); + assert_eq!(delta.timestamp, at(n + 1)); assert_eq!( serde_json::from_slice::(&delta.payload).unwrap(), json!({ "n": n + 1 }) ); let mut current = snapshot::Consumer::::new(remote.clone(), Default::default()); let received = current.latest().await.unwrap().unwrap(); - assert_eq!((received.value, received.at), (value(n + 1), Some(at(n + 1)))); + assert_eq!((received.value, received.at), (value(n + 1), at(n + 1))); assert!(producer.poll_refresh(&kio::Waiter::noop()).is_pending()); let _finished = producer.finish().unwrap(); drop((current, fresh, late, joins, original, baseline, links)); @@ -180,16 +183,16 @@ async fn ratio_joins_cross_two_relays() { let mut baseline = remote.subscribe(None).await.unwrap(); assert_eq!(baseline.info().refresh, (name == "moq-lite-07-wip").then_some(1)); crate::support::harness::subscribed(&producer).await; - let mut group = producer.append_group().unwrap(); - group.write_frame(None, &b"key"[..]).unwrap(); - group.write_frame(None, &b"abc"[..]).unwrap(); + let mut group = producer.append_group(0).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), &b"key"[..]).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), &b"abc"[..]).unwrap(); let mut received = baseline.recv_group().await.unwrap().unwrap(); assert_eq!(received.read_frame().await.unwrap().unwrap().payload.len(), 3); assert_eq!(received.read_frame().await.unwrap().unwrap().payload.len(), 3); let mut under = remote.subscribe(None).await.unwrap(); let _under_group = under.recv_group().await.unwrap().unwrap(); assert_eq!(producer.demand().wanted(), Some(0), "ratio equality does not refresh"); - group.write_frame(None, &b"d"[..]).unwrap(); + group.write_frame(moq_net::Timestamp::unscaled(0), &b"d"[..]).unwrap(); assert_eq!(received.read_frame().await.unwrap().unwrap().payload.len(), 1); let mut joins = Vec::new(); for _ in 0..8 { diff --git a/rs/moq-net-tests/tests/integration/restart.rs b/rs/moq-net-tests/tests/integration/restart.rs index 0033f6a3413..e2615cff19f 100644 --- a/rs/moq-net-tests/tests/integration/restart.rs +++ b/rs/moq-net-tests/tests/integration/restart.rs @@ -8,6 +8,7 @@ //! instance's cached groups. use crate::support; +use support::AppendNext; use std::{cell::RefCell, rc::Rc, time::Duration}; @@ -95,11 +96,12 @@ async fn quiet(version: Version, announced: &mut origin::Routes) { } } -/// Write one group of one frame tagged `tag`. +/// Write the next group of one frame tagged `tag`, starting at its key in milliseconds. fn write(track: &track::Producer, tag: &str) { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, tag.as_bytes().to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + let start = Timestamp::from_millis(group.start).unwrap(); + group.write_frame(start, tag.as_bytes().to_vec()).unwrap(); + group.close().unwrap(); } /// The next frame a subscription delivers, as text. @@ -598,7 +600,7 @@ async fn unanswered_takeover(version: &str) { Err(err) => assert!(matches!(err, moq_net::Error::Unroutable), "{version}: {err}"), Ok(group) => panic!( "{version}: the replaced instance's SUBSCRIBE went on: {:?}", - group.map(|group| group.sequence) + group.map(|group| group.start) ), } } diff --git a/rs/moq-net-tests/tests/integration/serve.rs b/rs/moq-net-tests/tests/integration/serve.rs index 06511d7c119..93517c30b37 100644 --- a/rs/moq-net-tests/tests/integration/serve.rs +++ b/rs/moq-net-tests/tests/integration/serve.rs @@ -37,9 +37,9 @@ async fn first_subscribe(version: &str, publish: bool) { track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = held_track.append_group().unwrap(); + let mut group = held_track.append_group(0).unwrap(); group.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let handler = async { if publish { let prepared = requests @@ -55,9 +55,9 @@ async fn first_subscribe(version: &str, publish: bool) { track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); prepared.announce().unwrap(); Some((prepared, track)) } else { diff --git a/rs/moq-net-tests/tests/integration/snapshot.rs b/rs/moq-net-tests/tests/integration/snapshot.rs index a5c8b8d24d2..4c1119dce5f 100644 --- a/rs/moq-net-tests/tests/integration/snapshot.rs +++ b/rs/moq-net-tests/tests/integration/snapshot.rs @@ -20,12 +20,15 @@ fn reads_skip_an_owed_hole() { let mut latest = snapshot::Consumer::::new(track.consume(), Default::default()); let mut next = snapshot::Consumer::::new(track.consume(), Default::default()); let waiter = kio::Waiter::noop(); - let put = |sequence: u64| { - let mut group = track.create_group(group::Info { sequence }).unwrap(); + let put = |start: u64| { + let mut group = track.create_group(group::Info { start }).unwrap(); group - .write_frame(None, serde_json::to_vec(&json!({ "n": sequence })).unwrap()) + .write_frame( + moq_net::Timestamp::unscaled(start), + serde_json::to_vec(&json!({ "n": start })).unwrap(), + ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; put(0); diff --git a/rs/moq-net/benches/backing.rs b/rs/moq-net/benches/backing.rs index f27e93ff7c5..8cf99ee61fa 100644 --- a/rs/moq-net/benches/backing.rs +++ b/rs/moq-net/benches/backing.rs @@ -305,13 +305,15 @@ impl Cache { if let Some((recv, bytes)) = slot.take() { self.groups[frame.track].push((recv.finish().unwrap(), bytes)); } - *slot = Some((GroupRecv::new(self.tracks[frame.track].append_group().unwrap()), 0)); + // A group starts at its first frame's timestamp, in the track's milliseconds. + let group = self.tracks[frame.track].append_group(frame.timestamp).unwrap(); + *slot = Some((GroupRecv::new(group), 0)); } let (recv, bytes) = slot.as_mut().expect("every track opens with a group"); let chunks: Vec<_> = frame.chunks.iter().map(|chunk| driver.chunk(media, *chunk)).collect(); *bytes += chunks.iter().map(Bytes::len).sum::(); let timestamp = Timestamp::new(frame.timestamp, Timescale::MILLI).unwrap(); - recv.frame(Some(timestamp), chunks).unwrap(); + recv.frame(timestamp, chunks).unwrap(); } for (track, recv) in open.into_iter().enumerate() { if let Some((recv, bytes)) = recv { diff --git a/rs/moq-net/benches/group.rs b/rs/moq-net/benches/group.rs index 75292684cc0..91211f4e8f8 100644 --- a/rs/moq-net/benches/group.rs +++ b/rs/moq-net/benches/group.rs @@ -69,6 +69,11 @@ fn timed() -> track::Info { .with_timescale(moq_net::Timescale::MILLI) } +/// Where the group at `key` starts on a [`timed`] track: groups start strictly in key order. +fn at(key: u64) -> Timestamp { + Timestamp::from_millis(key).unwrap() +} + /// A small, fixed payload shared across every frame. Cloning a `Bytes` is a /// refcount bump (no allocation), so the benchmark isolates the per-frame control /// overhead rather than payload allocation. @@ -100,7 +105,7 @@ struct Ctx { fn fresh_group() -> Ctx { let broadcast = broadcast::Producer::new(broadcast::Info::default()); let track = broadcast.create_track("bench", timed()).unwrap(); - let group = track.append_group().unwrap(); + let group = track.append_group(0).unwrap(); Ctx { _broadcast: broadcast, _track: track, @@ -163,7 +168,7 @@ fn filled_group(n: usize, payload: &Bytes) -> (Ctx, group::Cons for _ in 0..n { group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let consumer = group.consume(); ( Ctx { @@ -256,10 +261,10 @@ struct TrackCtx { fn filled_track(n: usize, payload: &Bytes) -> TrackCtx { let broadcast = broadcast::Producer::new(broadcast::Info::default()); let track = broadcast.create_track("bench", timed()).unwrap(); - for _ in 0..n { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - group.finish().unwrap(); + for key in 0..n as u64 { + let mut group = track.append_group(key).unwrap(); + group.write_frame(at(key), payload.clone()).unwrap(); + group.close().unwrap(); } TrackCtx { _broadcast: broadcast, @@ -271,14 +276,17 @@ fn filled_track(n: usize, payload: &Bytes) -> TrackCtx { fn replay() -> track::Subscription { track::Subscription::default() .with_delay(track::Delay::default().with_max(Duration::MAX)) - .with_range(0..) + .with_ranges([track::Range { + start: track::Anchor::Time(0), + end: None, + }]) } -/// Drain N cached groups from a track, in arrival order and in sequence order. +/// Drain N cached groups from a track, in arrival order and in key order. /// /// The two differ in how they find the next group: `recv_group` walks an arrival -/// index, while `next_group` seeks by sequence. Both should be flat in the cache -/// depth; a per-delivery scan over the cache makes the sequence-ordered arm +/// index, while `next_group` seeks by key. Both should be flat in the cache +/// depth; a per-delivery scan over the cache makes the key-ordered arm /// quadratic in N. fn bench_track_recv(c: &mut Criterion) { let payload = Bytes::from(vec![0u8; PAYLOAD]); @@ -302,7 +310,7 @@ fn bench_track_recv(c: &mut Criterion) { BatchSize::SmallInput, ); }); - g.bench_with_input(BenchmarkId::new("sequence", n), &n, |b, &n| { + g.bench_with_input(BenchmarkId::new("key", n), &n, |b, &n| { b.iter_batched( || { let ctx = filled_track(n, &payload); @@ -326,14 +334,14 @@ fn bench_track_recv(c: &mut Criterion) { let ctx = filled_track(n, &payload); let mut subscriber = ctx.track.subscribe(None); let edge = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(edge.sequence, n as u64 - 1); + assert_eq!(edge.start, n as u64 - 1); (ctx, subscriber) }, |(ctx, mut subscriber)| { - for _ in 0..LIVE_BATCH { - let mut group = ctx.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - group.finish().unwrap(); + for key in (n..n + LIVE_BATCH).map(|key| key as u64) { + let mut group = ctx.track.append_group(key).unwrap(); + group.write_frame(at(key), payload.clone()).unwrap(); + group.close().unwrap(); let group = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); black_box(group); } @@ -359,7 +367,7 @@ fn bench_roundtrip(c: &mut Criterion) { for _ in 0..n { ctx.group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); } - let group = ctx.group.finish().unwrap(); + let group = ctx.group.close().unwrap(); let mut consumer = group.consume(); for _ in 0..n { let frame = consumer.read_frame().now_or_never().unwrap().unwrap(); @@ -445,10 +453,10 @@ fn bench_stamp(c: &mut Criterion) { } let batch = left.min(n as u64); let start = Instant::now(); - for _ in 0..batch { - let mut group = ctx.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - group.finish().unwrap(); + for key in n as u64..n as u64 + batch { + let mut group = ctx.track.append_group(key).unwrap(); + group.write_frame(at(key), payload.clone()).unwrap(); + group.close().unwrap(); } total += start.elapsed(); left -= batch; @@ -465,11 +473,12 @@ fn bench_stamp(c: &mut Criterion) { let mut left = iters; while left > 0 { let ctx = filled_track(0, &payload); - let mut groups: Vec<_> = (0..n).map(|_| ctx.track.append_group().unwrap()).collect(); + let mut groups: Vec<_> = (0..n as u64).map(|key| ctx.track.append_group(key).unwrap()).collect(); let batch = left.min(n as u64); let start = Instant::now(); for group in groups.iter_mut().take(batch as usize) { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let start = at(group.start); + group.write_frame(start, payload.clone()).unwrap(); } total += start.elapsed(); left -= batch; @@ -503,13 +512,13 @@ fn bench_inflate(c: &mut Criterion) { let track = broadcast .create_track("bench", timed().with_compression(moq_net::track::Compression::Deflate)) .unwrap(); - let consumers: Vec = (0..groups) - .flat_map(|_| { - let mut group = track.append_group().unwrap(); + let consumers: Vec = (0..groups as u64) + .flat_map(|key| { + let mut group = track.append_group(key).unwrap(); for payload in &payloads { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(at(key), payload.clone()).unwrap(); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); (0..readers).map(move |_| group.consume()).collect::>() }) .collect(); @@ -552,24 +561,29 @@ fn bench_compressed_updates(c: &mut Criterion) { track::Info::default().with_compression(moq_net::track::Compression::Deflate), ) .unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(None, payload.clone()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_group(0).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload.clone()) + .unwrap(); + group.close().unwrap(); track }) .collect(); (broadcast, producers, Vec::with_capacity(concurrent)) }, |(broadcast, producers, mut groups)| { - for _ in 0..UPDATES { + for update in 0..UPDATES { for track in &producers { - for _ in 0..concurrent { - let mut group = track.append_group().unwrap(); - group.write_frame(None, payload.clone()).unwrap(); + for index in 0..concurrent { + let key = (1 + update * concurrent + index) as u64; + let mut group = track.append_group(key).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload.clone()) + .unwrap(); groups.push(group); } for group in groups.drain(..) { - group.finish().unwrap(); + group.close().unwrap(); } } } @@ -726,7 +740,8 @@ fn contended( let track = broadcast .create_track("bench", track::Info::default().with_max_age(None)) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut key = 0; + let mut group = track.append_group(key).unwrap(); // The group readers move to once theirs ends. let next = Arc::new(Mutex::new(group.consume())); let mut sets: Vec, group::Consumer)>> = (0..threads).map(|_| Vec::new()).collect(); @@ -735,7 +750,7 @@ fn contended( true => { let mut subscriber = track.subscribe(None); let group = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(group.sequence, 0, "the subscriber starts at the live group"); + assert_eq!(group.start, 0, "the subscriber starts at the live group"); (Some(subscriber), group) } false => (None, group.consume()), @@ -781,14 +796,17 @@ fn contended( let start = Instant::now(); for _ in 0..iters { if written + burst > ROLL { - let rolled = track.append_group().unwrap(); + key += 1; + let rolled = track.append_group(key).unwrap(); *next.lock().unwrap() = rolled.consume(); let done = std::mem::replace(&mut group, rolled); - done.finish().unwrap(); + done.close().unwrap(); written = 0; } for _ in 0..burst { - group.write_frame(None, payload.clone()).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload.clone()) + .unwrap(); } written += burst; crew.run(); @@ -837,11 +855,12 @@ fn streaming(b: &mut criterion::Bencher, chunk: &Bytes, arrivals: usize, threads let track = broadcast .create_track("bench", track::Info::default().with_max_age(None)) .unwrap(); - let info = frame::Info { + let info = |key| frame::Info { size: (arrivals * chunk.len()) as u64, - timestamp: None, + timestamp: moq_net::Timestamp::unscaled(key), }; - let group = track.append_group().unwrap(); + let mut key = 0; + let group = track.append_group(key).unwrap(); // The group readers move to once theirs ends. let next = Arc::new(Mutex::new(group.consume())); let mut sets: Vec)>> = (0..threads).map(|_| Vec::new()).collect(); @@ -887,7 +906,7 @@ fn streaming(b: &mut criterion::Bencher, chunk: &Bytes, arrivals: usize, threads let crew = Crew::new(work); let mut frames = 0; - let mut writing = Some(group.create_frame(info).unwrap()); + let mut writing = Some(group.create_frame(info(key)).unwrap()); b.iter_custom(|iters| { let start = Instant::now(); for _ in 0..iters { @@ -898,13 +917,14 @@ fn streaming(b: &mut criterion::Bencher, chunk: &Bytes, arrivals: usize, threads let group = frame.finish().unwrap(); frames += 1; if frames == ROLL { - let rolled = track.append_group().unwrap(); + key += 1; + let rolled = track.append_group(key).unwrap(); *next.lock().unwrap() = rolled.consume(); - group.finish().unwrap(); + group.close().unwrap(); frames = 0; - rolled.create_frame(info).unwrap() + rolled.create_frame(info(key)).unwrap() } else { - group.create_frame(info).unwrap() + group.create_frame(info(key)).unwrap() } } _ => frame, @@ -927,7 +947,7 @@ fn pooled_group(capacity: Option) -> Ctx { } let broadcast = broadcast::Producer::new(info); let track = broadcast.create_track("bench", timed()).unwrap(); - let group = track.append_group().unwrap(); + let group = track.append_group(0).unwrap(); Ctx { _broadcast: broadcast, _track: track, @@ -1003,7 +1023,7 @@ fn bench_paced(c: &mut Criterion) { let mut parked = 0u64; let next = moq_net::frame::Info { size: payload.len() as u64, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }; b.iter(|| { parked += ctx.group.poll_ready(&writer, &next).is_pending() as u64; @@ -1041,11 +1061,12 @@ fn shed_live_group(payloads: &[Bytes]) -> Flights { let broadcast = broadcast::Producer::new(info); let track = broadcast.create_track("bench", track::Info::default()).unwrap(); let requests = track.requests(); - let mut live = track.append_group().unwrap(); + let mut live = track.append_group(0).unwrap(); let mut reader = live.consume(); let waiter = kio::Waiter::noop(); for payload in payloads { - live.write_frame(None, payload.clone()).unwrap(); + live.write_frame(moq_net::Timestamp::unscaled(live.start), payload.clone()) + .unwrap(); while let Poll::Ready(Ok(Some(_))) = reader.poll_read_frame(&waiter) {} } Flights { @@ -1112,7 +1133,10 @@ fn replay_flights(ctx: &Flights, payloads: &[Bytes], readers: usize, apart: usiz while let Some(payload) = payloads.get(flight.next) && flight.group.poll_ready(&waiter, &frame_info(payload)).is_ready() { - flight.group.write_frame(None, payload.clone()).unwrap(); + flight + .group + .write_frame(moq_net::Timestamp::unscaled(flight.group.start), payload.clone()) + .unwrap(); flight.next += 1; written += 1; } @@ -1122,10 +1146,11 @@ fn replay_flights(ctx: &Flights, payloads: &[Bytes], readers: usize, apart: usiz (served, written, peak) } +/// The header admission weighs: only its size. fn frame_info(payload: &Bytes) -> frame::Info { frame::Info { size: payload.len() as u64, - timestamp: None, + timestamp: moq_net::Timestamp::unscaled(0), } } diff --git a/rs/moq-net/benches/memory.rs b/rs/moq-net/benches/memory.rs index 1f830dc3894..f3ddc709278 100644 --- a/rs/moq-net/benches/memory.rs +++ b/rs/moq-net/benches/memory.rs @@ -105,6 +105,8 @@ struct Link { tracks: Vec, subscribers: Vec, payload: Bytes, + /// The next group's start on every track, in milliseconds. + next: u64, _broadcast: origin::Route, _sessions: (moq_net::Session, moq_net::Session), _origins: (origin::Producer, origin::Producer), @@ -185,6 +187,7 @@ impl Link { tracks, subscribers, payload: Bytes::from(vec![0; size]), + next: 0, _broadcast: broadcast, _sessions: sessions, _origins: (publisher, subscriber), @@ -200,12 +203,15 @@ impl Link { /// Write a group on every track, then read each back, returning the payload bytes /// read so skipped work can't look like a speedup. async fn round(&mut self) -> usize { + let key = self.next; + self.next += 1; + let timestamp = Timestamp::from_millis(key).unwrap(); for track in &self.tracks { - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(key).unwrap(); for _ in 0..FRAMES { - group.write_frame(Timestamp::ZERO, self.payload.clone()).unwrap(); + group.write_frame(timestamp, self.payload.clone()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let mut bytes = 0; diff --git a/rs/moq-net/benches/origin.rs b/rs/moq-net/benches/origin.rs index 77c1b5ccecd..49514f7158b 100644 --- a/rs/moq-net/benches/origin.rs +++ b/rs/moq-net/benches/origin.rs @@ -60,6 +60,11 @@ fn timed() -> track::Info { .with_timescale(moq_net::Timescale::MILLI) } +/// Where the group at `key` starts on a [`timed`] track: groups start strictly in key order. +fn at(key: u64) -> Timestamp { + Timestamp::from_millis(key).unwrap() +} + /// `(publishers, subscribers)` shapes for the fan-out benchmarks. const SHAPES: [(usize, usize); 3] = [(100, 10), (1_000, 100), (1_000, 1_000)]; @@ -188,12 +193,12 @@ fn bench_pending_churn(c: &mut Criterion) { }) .collect(); let mut cursors: Vec<_> = (0..readers).map(|_| producer.consume().routes()).collect(); - let mut sequence = 0u64; + let mut key = 0u64; b.iter(|| { - sequence += 1; + key += 1; drop( producer - .route(format!("a/{sequence}"), moq_net::origin::RouteOptions::default()) + .route(format!("a/{key}"), moq_net::origin::RouteOptions::default()) .unwrap(), ); }); @@ -1110,9 +1115,9 @@ fn bench_handoff(c: &mut Criterion) { .route("room/live", moq_net::origin::RouteOptions::default()) .unwrap(); let track = incumbent.broadcast().create_track("video", timed()).unwrap(); - let mut first = track.append_group().unwrap(); + let mut first = track.append_group(0).unwrap(); first.write_frame(Timestamp::ZERO, b"one".as_ref()).unwrap(); - first.finish().unwrap(); + first.close().unwrap(); let resolved = consumer.request("room/live", None).await.unwrap(); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); @@ -1123,9 +1128,9 @@ fn bench_handoff(c: &mut Criterion) { .route("room/live", moq_net::origin::RouteOptions::hidden()) .unwrap(); let track = standby.broadcast().create_track("video", timed()).unwrap(); - let mut second = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); - second.write_frame(Timestamp::ZERO, b"two".as_ref()).unwrap(); - second.finish().unwrap(); + let mut second = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); + second.write_frame(at(1), b"two".as_ref()).unwrap(); + second.close().unwrap(); // Announcing is what makes the standby a route the front can take. standby.announce().unwrap(); let replaced = consumer.request("room/live", None).await.unwrap(); @@ -1133,7 +1138,7 @@ fn bench_handoff(c: &mut Criterion) { let group = replacement.recv_group().await.unwrap().expect("standby group"); total += started.elapsed(); // Rejoining the incumbent would replay its cached group 0 instead. - assert_eq!(group.sequence, 1, "standby group"); + assert_eq!(group.start, 1, "standby group"); drop(subscription); drop(replacement); @@ -1162,6 +1167,8 @@ struct Watching { pair: MockPair, tracks: Vec, subscriptions: Vec, + /// The next group's key on every track. + next: u64, _broadcasts: Vec, _origins: [origin::Producer; 2], } @@ -1215,6 +1222,7 @@ async fn watching(routes: usize, subscribers: usize) -> Watching { pair, tracks, subscriptions, + next: 0, _broadcasts: broadcasts, _origins: [relay, received], }; @@ -1225,10 +1233,12 @@ async fn watching(routes: usize, subscribers: usize) -> Watching { impl Watching { /// Every subscribed track writes a group, and every subscription reads it. async fn round(&mut self) { + let key = self.next; + self.next += 1; for track in &self.tracks[..self.subscriptions.len()] { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"frame".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_group(key).unwrap(); + group.write_frame(at(key), b"frame".as_ref()).unwrap(); + group.close().unwrap(); } for subscription in &mut self.subscriptions { subscription.recv_group().await.unwrap().expect("track ended"); @@ -1344,16 +1354,19 @@ fn bench_relay(c: &mut Criterion) { subscriptions }); + let mut next = 0; b.iter_custom(|iterations| { runtime.block_on(async { let started = std::time::Instant::now(); for _ in 0..iterations { + let key = next; + next += 1; for source in &sources { - let mut group = source.append_group().unwrap(); + let mut group = source.append_group(key).unwrap(); for _ in 0..RELAY_FRAMES { - group.write_frame(Timestamp::ZERO, b"frame".as_ref()).unwrap(); + group.write_frame(at(key), b"frame".as_ref()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } for subscription in &mut subscriptions { let mut group = subscription.recv_group().await.unwrap().expect("group"); @@ -1403,15 +1416,18 @@ fn bench_passthrough(c: &mut Criterion) { }); let mut held: Vec> = (0..readers).map(|_| Vec::with_capacity(open)).collect(); + let mut next = 0; b.iter_custom(|iterations| { runtime.block_on(async { let started = std::time::Instant::now(); for _ in 0..iterations { // Received as each is written: a reader behind the live edge skips ahead. for _ in 0..open { - let mut group = source.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"frame".as_ref()).unwrap(); - group.finish().unwrap(); + let key = next; + next += 1; + let mut group = source.append_group(key).unwrap(); + group.write_frame(at(key), b"frame".as_ref()).unwrap(); + group.close().unwrap(); for (subscription, held) in subscriptions.iter_mut().zip(&mut held) { held.push(subscription.recv_group().await.unwrap().expect("group")); } @@ -1456,7 +1472,7 @@ fn bench_parked(c: &mut Criterion) { .unwrap() }) .collect(); - let _writers: Vec<_> = sources.iter().map(|source| source.append_group().unwrap()).collect(); + let _writers: Vec<_> = sources.iter().map(|source| source.append_group(0).unwrap()).collect(); let (subscriptions, mut groups) = runtime.block_on(async { let resolved = producer.consume().request("room/live", None).await.unwrap(); let mut subscriptions = Vec::new(); @@ -1514,9 +1530,9 @@ impl Generation { let mut held = Vec::with_capacity(tracks); for i in 0..tracks { let track = broadcast.broadcast().create_track(format!("{i}"), timed()).unwrap(); - let mut written = track.append_group().unwrap(); + let mut written = track.append_group(0).unwrap(); written.write_frame(Timestamp::ZERO, b"f".as_ref()).unwrap(); - written.finish().unwrap(); + written.close().unwrap(); held.push(track); } Self { @@ -1681,14 +1697,14 @@ impl ResumeOverlap { enum ResumeReader { Arrival(track::Subscriber), - Sequence(track::Ordered), + Key(track::Ordered), } impl ResumeReader { fn poll_group(&mut self, waiter: &kio::Waiter) -> Poll>> { match self { Self::Arrival(reader) => reader.poll_recv_group(waiter), - Self::Sequence(reader) => reader.poll_next_group(waiter), + Self::Key(reader) => reader.poll_next_group(waiter), } } } @@ -1718,14 +1734,10 @@ async fn resume_overlap(shape: ResumeShape, ordered: bool) -> ResumeOverlap { ) .unwrap(); let track = broadcast.broadcast().create_track("0", timed()).unwrap(); - for sequence in order { - let mut group = track - .create_group(moq_net::group::Info { - sequence: sequence as u64, - }) - .unwrap(); - group.write_frame(Timestamp::ZERO, b"f".as_ref()).unwrap(); - group.finish().unwrap(); + for key in order { + let mut group = track.create_group(moq_net::group::Info { start: key as u64 }).unwrap(); + group.write_frame(at(key as u64), b"f".as_ref()).unwrap(); + group.close().unwrap(); } broadcast.announce().unwrap(); (broadcast, track) @@ -1775,7 +1787,7 @@ async fn resume_overlap(shape: ResumeShape, ordered: bool) -> ResumeOverlap { .into_iter() .map(|subscriber| subscriber.with_range(..)) .map(|subscriber| match ordered { - true => ResumeReader::Sequence(subscriber.ordered()), + true => ResumeReader::Key(subscriber.ordered()), false => ResumeReader::Arrival(subscriber), }) .collect(); @@ -1794,9 +1806,9 @@ fn bench_resume_overlap(c: &mut Criterion) { group.sample_size(10); for (order, ordered, waiting) in [ ("arrival", false, false), - ("sequence", true, false), + ("key", true, false), ("arrival_waiting", false, true), - ("sequence_waiting", true, true), + ("key_waiting", true, true), ] { for (readers, retained, routes) in RESUME_OVERLAP { if waiting && !(readers == 1 && retained == 64 && routes >= 2) { diff --git a/rs/moq-net/benches/ranges.rs b/rs/moq-net/benches/ranges.rs index f4a3e52f994..f2ea9e247e2 100644 --- a/rs/moq-net/benches/ranges.rs +++ b/rs/moq-net/benches/ranges.rs @@ -13,12 +13,12 @@ //! - `ranges_fills_out`: ranges below where the feed starts, asked of a handler that never //! answers, beside one live range. A live group lands and the reader drains it while every //! fill is still out. -//! - `ranges_sparse`: a feed numbered sparsely, as a moq-transport publisher may, with the -//! reader's ranges in the gaps between its groups. A group lands past them, and the reader -//! polls with nothing in range. -//! - `ranges_time_gaps`: time ranges each ending between two groups of a sparsely numbered -//! history, so each end waits on the missing sequences of a gap of its own, beside one time -//! end past the live edge whose cap moves twice per group, as in `ranges_time_cap`. +//! - `ranges_sparse`: a feed keyed sparsely, as a moq-transport publisher may number it, with +//! each of the reader's ranges inside one of its groups. A group lands past them, and the +//! reader polls with nothing in range. +//! - `ranges_time_gaps`: time ranges each ending between two groups of a sparsely keyed +//! history, so each end waits on the keys of a gap of its own that may still arrive, beside +//! one time end past the live edge whose cap moves twice per group, as in `ranges_time_cap`. //! //! Run with `cargo bench -p moq-net --features fuzz --bench ranges`. @@ -49,11 +49,19 @@ fn millis(millis: u64) -> Timestamp { Timestamp::from_millis(millis).unwrap() } -/// Write a one-frame group at `sequence`, presented at `at` milliseconds. -fn land(track: &track::Producer, sequence: u64, at: u64) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(millis(at), Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); +/// Write a one-frame group starting at `key` milliseconds. +fn land(track: &track::Producer, key: u64) { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(millis(key), Bytes::from_static(b"x")).unwrap(); + group.close().unwrap(); +} + +/// The groups from the one containing `start` through the one containing `end`. +fn span(start: u64, end: u64) -> track::Range { + track::Range { + start: track::Anchor::Time(start), + end: Some(track::Anchor::Time(end)), + } } /// Hand out everything the reader has now, returning how many groups. @@ -79,14 +87,11 @@ impl TimeEnds { pub(crate) fn new(ranges: u64) -> Self { let broadcast = broadcast::Info::default().produce(); let track = broadcast.create_track("bench", timed()).unwrap(); - land(&track, 0, 0); + land(&track, 0); // Each range starts and ends far past the live edge, at a time of its own. let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(BUDGET)) - .with_ranges((0..ranges).map(|i| track::Range { - start: track::Anchor::Time(millis(LIVE + i * 2)), - end: Some(track::Anchor::Time(millis(LIVE + i * 2 + 1))), - })); + .with_ranges((0..ranges).map(|i| span(LIVE + i * 2, LIVE + i * 2 + 1))); let mut reader = track.subscribe(subscription); drain(&mut reader); Self { @@ -98,7 +103,7 @@ impl TimeEnds { } pub(crate) fn step(&mut self) { - land(&self.track, self.next, self.next); + land(&self.track, self.next); self.next += 1; assert_eq!(drain(&mut self.reader), 1, "every end is past the group"); } @@ -116,17 +121,11 @@ impl TimeCap { pub(crate) fn new(ranges: u64) -> Self { let broadcast = broadcast::Info::default().produce(); let track = broadcast.create_track("bench", timed()).unwrap(); - land(&track, 0, 0); + land(&track, 0); // The time end, then disjoint groups far past anything the run lands. let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(BUDGET)) - .with_ranges( - std::iter::once(track::Range { - start: track::Anchor::Time(millis(0)), - end: Some(track::Anchor::Time(millis(LIVE))), - }) - .chain((1..ranges).map(|i| track::Range::groups(LIVE + i * 2..LIVE + i * 2 + 1))), - ); + .with_ranges(std::iter::once(span(0, LIVE)).chain((1..ranges).map(|i| span(LIVE + i * 2, LIVE + i * 2)))); let mut reader = track.subscribe(subscription); drain(&mut reader); Self { @@ -138,10 +137,10 @@ impl TimeCap { } pub(crate) fn step(&mut self) { - let mut group = self.track.create_group(group::Info { sequence: self.next }).unwrap(); + let mut group = self.track.create_group(group::Info { start: self.next }).unwrap(); let mut handed = drain(&mut self.reader); group.write_frame(millis(self.next), Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); handed += drain(&mut self.reader); self.next += 1; assert_eq!(handed, 1, "the group is in range"); @@ -163,18 +162,17 @@ impl FillsOut { let mut track = broadcast.create_track("bench", timed()).unwrap(); // Held and never polled, so every fill stays out. let requests = track.requests(); - // The feed starts past every range, with a sequence between each that no range asks + // The feed starts past every range, with a key between each that no range asks // for, so each range is a fill of its own. let start = ranges * 2; track.start_at(start).unwrap(); - land(&track, start, start); + land(&track, start); let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(BUDGET)) - .with_ranges( - (0..ranges) - .map(|i| track::Range::groups(i * 2..i * 2 + 1)) - .chain([track::Range::groups(start..)]), - ); + .with_ranges((0..ranges).map(|i| span(i * 2, i * 2)).chain([track::Range { + start: track::Anchor::Time(start), + end: None, + }])); let mut reader = track.subscribe(subscription); drain(&mut reader); Self { @@ -187,13 +185,14 @@ impl FillsOut { } pub(crate) fn step(&mut self) { - land(&self.track, self.next, self.next); + land(&self.track, self.next); self.next += 1; black_box(drain(&mut self.reader)); } } -/// One reader whose ranges sit in the gaps of a sparsely numbered feed; see `ranges_sparse`. +/// One reader whose ranges each sit inside a group of a sparsely keyed feed; see +/// `ranges_sparse`. pub(crate) struct Sparse { _broadcast: broadcast::Producer, track: track::Producer, @@ -201,7 +200,7 @@ pub(crate) struct Sparse { next: u64, } -/// The step between a sparse feed's sequences. +/// The step between a sparse feed's keys. const STRIDE: u64 = 4; impl Sparse { @@ -209,18 +208,20 @@ impl Sparse { let broadcast = broadcast::Info::default().produce(); let track = broadcast.create_track("bench", timed()).unwrap(); // The feed's groups at every multiple of the stride, and the reader's ranges one past - // each, which no group fills, above group 0, the one group in range. + // each, which no group starts at, so each holds the group before it. for i in 0..=ranges { - land(&track, i * STRIDE, i); + land(&track, i * STRIDE); } let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(BUDGET)) - .with_ranges( - std::iter::once(track::Range::groups(0..1)) - .chain((1..=ranges).map(|i| track::Range::groups(i * STRIDE + 1..i * STRIDE + 2))), - ); + .with_ranges(std::iter::once(span(0, 0)).chain((1..=ranges).map(|i| span(i * STRIDE + 1, i * STRIDE + 1)))); let mut reader = track.subscribe(subscription); - assert_eq!(drain(&mut reader), 1, "group 0 is the one in range"); + let groups = usize::try_from(ranges).unwrap() + 1; + assert_eq!( + drain(&mut reader), + groups, + "each range holds the group it starts inside" + ); Self { _broadcast: broadcast, track, @@ -230,48 +231,39 @@ impl Sparse { } pub(crate) fn step(&mut self) { - land(&self.track, self.next * STRIDE, self.next); + land(&self.track, self.next * STRIDE); self.next += 1; assert_eq!(drain(&mut self.reader), 0, "nothing lands in range"); } } -/// One reader whose time ends wait in the gaps of a sparsely numbered feed; see +/// One reader whose time ends wait in the gaps of a sparsely keyed feed; see /// `ranges_time_gaps`. pub(crate) struct TimeGaps { _broadcast: broadcast::Producer, track: track::Producer, reader: track::Subscriber, - /// Where the live feed went on from the history. - live: u64, next: u64, } +/// The milliseconds between a sparse history's groups. +const GAP: u64 = 10; + impl TimeGaps { pub(crate) fn new(ranges: u64) -> Self { let broadcast = broadcast::Info::default().produce(); let track = broadcast.create_track("bench", timed()).unwrap(); - // The history's groups at every multiple of the stride, ten milliseconds apart, and - // each range from just after one starts until halfway to the next: it ends before the - // next group, but the sequences between may still arrive inside it. The live feed goes - // on from the last, one sequence and millisecond at a time, read by a range from there. + // The history's groups ten milliseconds apart, and each range from just after one + // starts until halfway to the next: it ends before the next group, but the keys + // between may still arrive inside it. The live feed goes on from the last, one + // millisecond at a time, read by a range from there. for i in 0..=ranges { - land(&track, i * STRIDE, i * 10); + land(&track, i * GAP); } - let live = track::Range { - start: track::Anchor::Time(millis(ranges * 10)), - end: Some(track::Anchor::Time(millis(LIVE))), - }; + let live = span(ranges * GAP, LIVE); let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(BUDGET)) - .with_ranges( - (0..ranges) - .map(|i| track::Range { - start: track::Anchor::Time(millis(i * 10 + 1)), - end: Some(track::Anchor::Time(millis(i * 10 + 5))), - }) - .chain([live]), - ); + .with_ranges((0..ranges).map(|i| span(i * GAP + 1, i * GAP + 5)).chain([live])); let mut reader = track.subscribe(subscription); let groups = usize::try_from(ranges).unwrap() + 1; assert_eq!(drain(&mut reader), groups, "each range's own group, and the live edge"); @@ -279,18 +271,16 @@ impl TimeGaps { _broadcast: broadcast, track, reader, - live: ranges * STRIDE, - next: ranges * STRIDE + 1, + next: ranges * GAP + 1, } } pub(crate) fn step(&mut self) { // Each a millisecond after the last, from where the history ended. - let at = self.next - self.live + self.live / STRIDE * 10; - let mut group = self.track.create_group(group::Info { sequence: self.next }).unwrap(); + let mut group = self.track.create_group(group::Info { start: self.next }).unwrap(); let mut handed = drain(&mut self.reader); - group.write_frame(millis(at), Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + group.write_frame(millis(self.next), Bytes::from_static(b"x")).unwrap(); + group.close().unwrap(); handed += drain(&mut self.reader); self.next += 1; assert_eq!(handed, 1, "the live group is in range"); diff --git a/rs/moq-net/benches/refresh.rs b/rs/moq-net/benches/refresh.rs index f39ad727710..80c499cbf40 100644 --- a/rs/moq-net/benches/refresh.rs +++ b/rs/moq-net/benches/refresh.rs @@ -20,12 +20,16 @@ use moq_net::{broadcast, track}; const TRACKS: [usize; 3] = [1, 100, 1_000]; const SUBSCRIBERS: [usize; 3] = [1, 10, 100]; -/// Append a group whose delta outweighs its first frame. -fn refreshable(track: &track::Producer) { - let mut group = track.append_group().unwrap(); - group.write_frame(None, Bytes::from_static(b"key")).unwrap(); - group.write_frame(None, Bytes::from_static(b"delta")).unwrap(); - group.finish().unwrap(); +/// Append group `key`, whose delta outweighs its first frame. +fn refreshable(track: &track::Producer, key: u64) { + let mut group = track.append_group(key).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), Bytes::from_static(b"key")) + .unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), Bytes::from_static(b"delta")) + .unwrap(); + group.close().unwrap(); } fn join(c: &mut Criterion) { @@ -43,7 +47,7 @@ fn join(c: &mut Criterion) { .collect(); let track = &producers[tracks / 2]; let demand = track.demand(); - refreshable(track); + refreshable(track, 0); let mut latest = 0; group.bench_function(BenchmarkId::new(format!("{tracks}x"), subscribers), |b| { b.iter_custom(|iterations| { @@ -55,8 +59,8 @@ fn join(c: &mut Criterion) { elapsed += start.elapsed(); assert_eq!(wanted, Some(latest + 1)); drop(joined); - refreshable(track); latest += 1; + refreshable(track, latest); } elapsed }) diff --git a/rs/moq-net/benches/schedule.rs b/rs/moq-net/benches/schedule.rs index db5867b5dbc..923e8e04d71 100644 --- a/rs/moq-net/benches/schedule.rs +++ b/rs/moq-net/benches/schedule.rs @@ -5,11 +5,11 @@ //! queue work, then criterion times the run. The four schedulers: //! //! - `groups`: one send group per subscription at its priority, streams ordered by group -//! sequence inside it (what native QUIC uses). +//! key inside it (what native QUIC uses). //! - `queue`: the publishers' `PriorityQueue`, ranking every in-flight group on the session //! into one `u8` send order, re-applied to each stream whose rank moves. -//! - `scalar-seq`: a 32-bit `[priority][group sequence]` key, which leaves 24 bits for the -//! sequence and compares sequences across subscriptions. +//! - `scalar-seq`: a 32-bit `[priority][group key]` key, which leaves 24 bits for the +//! key and compares keys across subscriptions. //! - `scalar`: the priority alone, so equal subscriptions share by stream count. //! //! `fanout` sweeps subscriptions against groups in flight per subscription, timing one frame, @@ -100,7 +100,7 @@ fn workloads() -> Vec { capacity: 20_000, ticks: 3_000, }, - // Group sequences crossing 2^24, where the scalar's 24 bits wrap. + // Group keys crossing 2^24, where the scalar's 24 bits wrap. Workload { name: "wrap", tracks: vec![ diff --git a/rs/moq-net/benches/session/mod.rs b/rs/moq-net/benches/session/mod.rs index 193750f4b44..491eefe409a 100644 --- a/rs/moq-net/benches/session/mod.rs +++ b/rs/moq-net/benches/session/mod.rs @@ -33,7 +33,7 @@ //! `session_delivery_datagram` writes to a datagram track per broadcast instead of groups: //! one datagram per track per round across publisher and viewer axes, then larger payloads //! and cadences, on moq-lite and moq-transport-22. Throughput counts delivered datagrams, -//! each checked for its sequence and payload, so a lost one fails. +//! each checked for its key and payload, so a lost one fails. //! Delivery uses expiry-only pools: history expires between rounds outside timing, //! so byte pressure cannot shed unread frames during a batch. //! `SESSION_EXPIRY_ONLY=1` advances the cache clock by a second instead of the 30s @@ -516,13 +516,35 @@ impl MeasurementClock { } } +/// Append the next group on a [`timed`] track, one millisecond past its largest start or at +/// 0 on an empty track, so groups keep the key order a counter would give them. +fn append_next(track: &track::Producer) -> group::Producer { + track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap() +} + +/// Where a group on a [`timed`] track starts, as its frames' timestamp. +fn at(group: &group::Producer) -> Timestamp { + Timestamp::from_millis(group.start).unwrap() +} + +/// The groups from the one starting at `key` onward. +fn from(key: u64) -> track::Range { + track::Range { + start: track::Anchor::Time(key), + end: None, + } +} + fn write_group(track: &track::Producer, payload: &Bytes, frames: usize) -> u64 { - let mut group = track.append_group().unwrap(); + let mut group = append_next(track); + let timestamp = at(&group); for _ in 0..frames { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(timestamp, payload.clone()).unwrap(); } - let group = group.finish().unwrap(); - group.sequence + let group = group.close().unwrap(); + group.start } /// Read one whole group, returning its payload bytes so skipped work can't look @@ -1044,8 +1066,8 @@ impl Room { let mut bytes = 0; for (broadcast, track) in &viewer.fetches { let newest = newest[*broadcast]; - for sequence in newest + 1 - burst..=newest { - let mut group = track.fetch_group(sequence, None).await.unwrap(); + for key in newest + 1 - burst..=newest { + let mut group = track.fetch_group(key, None).await.unwrap(); while let Some(frame) = group.read_frame().await.unwrap() { bytes += frame.payload.len(); } @@ -1085,7 +1107,7 @@ impl Room { } } - /// Line every viewer up on the same next sequence before the strict rounds. + /// Line every viewer up on the same next key before the strict rounds. /// /// A consumer is live once its track's info arrives, but on lite-05/06 the SUBSCRIBE /// that carries the datagrams follows that answer, so a datagram may still miss it. @@ -1167,10 +1189,10 @@ impl Room { self.written += 1; for (writing, &broadcast) in self.writing.iter_mut().zip(&self.watched) { - let group = writing.get_or_insert_with(|| self.cluster.tracks[broadcast].append_group().unwrap()); - group.write_frame(Timestamp::ZERO, self.payload.clone()).unwrap(); + let group = writing.get_or_insert_with(|| append_next(&self.cluster.tracks[broadcast])); + group.write_frame(at(group), self.payload.clone()).unwrap(); if last { - writing.take().expect("written above").finish().unwrap(); + writing.take().expect("written above").close().unwrap(); } } if last { @@ -1377,7 +1399,7 @@ pub(crate) fn range_updates(c: &mut Criterion) { &[TRACK], Some( track::Subscription::default() - .with_range(span..) + .with_ranges([from(span)]) .with_delay(track::Delay::default().with_max(None)), ), ) @@ -1397,7 +1419,7 @@ pub(crate) fn range_updates(c: &mut Criterion) { viewer.subscribers[0] .update( track::Subscription::default() - .with_range(future..) + .with_ranges([from(future)]) .with_delay(track::Delay::default().with_max(None)), ) .unwrap(); @@ -1407,9 +1429,10 @@ pub(crate) fn range_updates(c: &mut Criterion) { assert!(viewer.subscribers[0].poll_recv_group(waiter).is_pending()); } loop { - if cluster.tracks[0].subscription().is_some_and(|subscription| { - subscription.ranges == [track::Range::groups(future..)] - }) { + if cluster.tracks[0] + .subscription() + .is_some_and(|subscription| subscription.ranges == [from(future)]) + { return std::task::Poll::Ready(()); } if cluster.tracks[0].poll_subscription_changed(waiter).is_pending() { @@ -1418,15 +1441,15 @@ pub(crate) fn range_updates(c: &mut Criterion) { } }) .await; - for sequence in next..future { - assert_eq!(write_group(&cluster.tracks[0], &payload, 1), sequence); + for key in next..future { + assert_eq!(write_group(&cluster.tracks[0], &payload, 1), key); } let started = Instant::now(); for viewer in &mut viewers { viewer.subscribers[0] .update( track::Subscription::default() - .with_range(future - 1..) + .with_ranges([from(future - 1)]) .with_delay(track::Delay::default().with_max(None)), ) .unwrap(); @@ -1461,14 +1484,14 @@ fn join(c: &mut Criterion, name: &str, shapes: impl IntoIterator::new(version, *shape, false).await; let payload = Bytes::from(vec![0; shape.frame]); for track in &cluster.tracks { - for sequence in 0..shape.history { - let mut group = track.append_group().unwrap(); + for key in 0..shape.history as u64 { + let mut group = track.append_group(key).unwrap(); for _ in 0..shape.group_frames { group - .write_frame(Timestamp::from_millis(sequence as u64).unwrap(), payload.clone()) + .write_frame(Timestamp::from_millis(key).unwrap(), payload.clone()) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } } cluster diff --git a/rs/moq-net/benches/tail.rs b/rs/moq-net/benches/tail.rs index faa5425f642..ba9ad24c939 100644 --- a/rs/moq-net/benches/tail.rs +++ b/rs/moq-net/benches/tail.rs @@ -45,8 +45,8 @@ impl Held { moq_net::fuzz::received(&track); track.finish_at(end, None).unwrap(); // Group 0 never arrives: the hole that holds the reader. - for sequence in 1..=groups as u64 { - track.create_group(group::Info { sequence }).unwrap().finish().unwrap(); + for key in 1..=groups as u64 { + track.create_group(group::Info { start: key }).unwrap().close().unwrap(); } for _ in 0..groups { assert!(matches!(reader.poll_recv_group(&waiter), Poll::Ready(Ok(Some(_))))); diff --git a/rs/moq-net/benches/track.rs b/rs/moq-net/benches/track.rs index 849496bcfb1..165534618b1 100644 --- a/rs/moq-net/benches/track.rs +++ b/rs/moq-net/benches/track.rs @@ -30,7 +30,7 @@ //! each append must wake only the reads it can expire, so waking the backlog shows up //! as a slope over it. //! -//! `track_sparse_read` hands out groups from a feed with every other sequence missing, +//! `track_sparse_read` hands out groups from a feed with every other key missing, //! holding a fixed number of gaps the reader still remembers: each hand-out must cost the //! same however many gaps it holds, so a walk over them shows up as a slope. //! `track_sparse_poll` repeats a parked read over a feed holding deferred gaps, without @@ -38,7 +38,7 @@ //! repeated read revisits them again. //! //! `track_idle_poll` repeats every parked read over readers sharing one track, in arrival -//! and sequence order, with nothing new: a repeated read parks again without walking the +//! and key order, with nothing new: a repeated read parks again without walking the //! read path, so each costs a small constant, and a slope over the readers, or a jump in //! that constant, means the early exit stopped applying. //! @@ -100,7 +100,7 @@ //! cached, and the runs it skipped: a widening costs the runs it re-offers, so a scan of the //! cache or the audience shows up as a slope. //! -//! `track_time_end` lands groups one at a time behind a sequence that never arrives, each +//! `track_time_end` lands groups one at a time behind a key that never arrives, each //! drained by readers whose range ends at a time no group reaches, swept over the groups and //! the readers: an unsettled time end walks on from where it stopped, so a rescan of the //! groups behind the gap on every read shows up as a slope over them. @@ -146,6 +146,27 @@ fn timed() -> track::Info { .with_timescale(moq_net::Timescale::MILLI) } +/// Where the group at `key` starts on a [`timed`] track: groups start strictly in key order. +fn at(key: u64) -> Timestamp { + Timestamp::from_millis(key).unwrap() +} + +/// The groups from the one containing `start` through the one containing `end`. +fn span(start: u64, end: u64) -> track::Range { + track::Range { + start: track::Anchor::Time(start), + end: Some(track::Anchor::Time(end)), + } +} + +/// The groups from the one containing `start` onward. +fn from(start: u64) -> track::Range { + track::Range { + start: track::Anchor::Time(start), + end: None, + } +} + /// Fanout sizes spanning a direct viewer, a small room, and a large room. const FANOUT: [usize; 4] = [1, 8, 64, 512]; @@ -217,17 +238,12 @@ impl FetchReaders { fn new(readers: usize, history: usize) -> Self { let broadcast = broadcast::Producer::new(broadcast::Info::default()); let track = broadcast.create_track("bench", timed()).unwrap(); - let mut open = track.append_group().unwrap(); + let mut open = track.append_group(0).unwrap(); open.write_frame(Timestamp::ZERO, Bytes::from_static(b"f")).unwrap(); - for sequence in 1..=history { - let mut group = track.append_group().unwrap(); - group - .write_frame( - Timestamp::from_millis(sequence as u64).unwrap(), - Bytes::from_static(b"f"), - ) - .unwrap(); - group.finish().unwrap(); + for key in 1..=history as u64 { + let mut group = track.append_group(key).unwrap(); + group.write_frame(at(key), Bytes::from_static(b"f")).unwrap(); + group.close().unwrap(); } let consumer = track.consume(); @@ -266,6 +282,8 @@ struct Fanout { subscribers: Vec, waiters: Vec, payload: Bytes, + /// The next group's key. + next: u64, } impl Fanout { @@ -290,14 +308,22 @@ impl Fanout { subscribers, waiters, payload: Bytes::from(vec![0; PAYLOAD]), + next: 0, } } + /// Append the next group, whose frames start at its key. + fn append(&mut self) -> moq_net::group::Producer { + let group = self.track.append_group(self.next).unwrap(); + self.next += 1; + group + } + /// Append one finished group and deliver its handle to every subscriber. fn cycle(&mut self) { - let mut group = self.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, self.payload.clone()).unwrap(); - group.finish().unwrap(); + let mut group = self.append(); + group.write_frame(at(group.start), self.payload.clone()).unwrap(); + group.close().unwrap(); for (subscriber, waiter) in self.subscribers.iter_mut().zip(&self.waiters) { let group = match subscriber.poll_recv_group(waiter) { @@ -325,10 +351,10 @@ impl AbortedScan { .unwrap(); let mut stale = Vec::with_capacity(aborted); - for _ in 0..aborted { - stale.push(track.append_group().unwrap()); + for key in 0..aborted as u64 { + stale.push(track.append_group(key).unwrap()); } - track.append_group().unwrap().finish().unwrap(); + track.append_group(aborted as u64).unwrap().close().unwrap(); for group in stale { group.abort(Error::Cancel).unwrap(); } @@ -372,9 +398,9 @@ impl Swept { let tracks = (0..tracks) .map(|i| { let track = broadcast.create_track(format!("bench{i}"), timed()).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); track }) .collect(); @@ -456,12 +482,15 @@ impl Aged { let mut sub = track.subscribe( track::Subscription::default().with_delay(track::Delay::default().with_max(AGED_BUDGET)), ); - let mut held = track.append_group().unwrap(); - held.write_frame(None, Bytes::from_static(b"x")).unwrap(); - for _ in 1..groups.max(2) { - let mut group = track.append_group().unwrap(); - group.write_frame(None, Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + let mut held = track.append_group(0).unwrap(); + held.write_frame(moq_net::Timestamp::unscaled(held.start), Bytes::from_static(b"x")) + .unwrap(); + for key in 1..groups.max(2) as u64 { + let mut group = track.append_group(key).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), Bytes::from_static(b"x")) + .unwrap(); + group.close().unwrap(); } let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(mut read))) = sub.poll_recv_group(&waiter) else { @@ -610,9 +639,9 @@ fn bench_evict_fanout(c: &mut Criterion) { for _ in 0..iterations { let start = Instant::now(); for _ in 0..evicted { - let mut group = fanout.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, fanout.payload.clone()).unwrap(); - group.finish().unwrap(); + let mut group = fanout.append(); + group.write_frame(at(group.start), fanout.payload.clone()).unwrap(); + group.close().unwrap(); } elapsed += start.elapsed(); for (subscriber, waiter) in fanout.subscribers.iter_mut().zip(&fanout.waiters) { @@ -671,9 +700,9 @@ fn bench_fanout_frame(c: &mut Criterion) { for _ in 0..iterations { if written == FRAMES_PER_GROUP { if let Some(open) = open.take() { - open.finish().unwrap(); + open.close().unwrap(); } - open = Some(fanout.track.append_group().unwrap()); + open = Some(fanout.append()); written = 0; for (subscriber, waiter) in fanout.subscribers.iter_mut().zip(&waiters) { assert!(matches!(subscriber.poll_recv_group(waiter), Poll::Ready(Ok(Some(_))))); @@ -682,8 +711,9 @@ fn bench_fanout_frame(c: &mut Criterion) { queue.lock().unwrap().clear(); } let group = open.as_mut().unwrap(); + let timestamp = at(group.start); let start = Instant::now(); - group.write_frame(Timestamp::ZERO, fanout.payload.clone()).unwrap(); + group.write_frame(timestamp, fanout.payload.clone()).unwrap(); // A woken reader re-polls, as its task would. let woken = std::mem::take(&mut *queue.lock().unwrap()); for index in woken { @@ -716,16 +746,13 @@ impl FetchBudgets { fn new(readers: usize, history: usize) -> Self { let broadcast = broadcast::Producer::new(broadcast::Info::default()); let track = broadcast.create_track("bench", timed()).unwrap(); - for second in 0..4u64 { - let mut group = track.append_group().unwrap(); - group - .write_frame(Timestamp::from_secs(second).unwrap(), Bytes::from_static(b"f")) - .unwrap(); - group.finish().unwrap(); + for key in (0..4u64).map(|second| second * 1000) { + let mut group = track.append_group(key).unwrap(); + group.write_frame(at(key), Bytes::from_static(b"f")).unwrap(); + group.close().unwrap(); } - let mut open = track.append_group().unwrap(); - open.write_frame(Timestamp::from_secs(4).unwrap(), Bytes::from_static(b"f")) - .unwrap(); + let mut open = track.append_group(4000).unwrap(); + open.write_frame(at(4000), Bytes::from_static(b"f")).unwrap(); let mut fetched = Self { _broadcast: broadcast, @@ -741,13 +768,13 @@ impl FetchBudgets { fetched } - /// Fetch group 2 with budget `i`, distinct from every other, returning the waiter that - /// keeps its deadlines registered. + /// Fetch the group at 2 seconds with budget `i`, distinct from every other, returning the + /// waiter that keeps its deadlines registered. fn fetch(&self, i: usize) -> kio::Waiter { let waiter = kio::Waiter::noop(); let budget = Duration::from_millis(1_500) + Duration::from_micros(i as u64); let options = moq_net::group::Fetch::default().with_max_delay(budget); - let mut pending = self.consumer.fetch_group(2, options); + let mut pending = self.consumer.fetch_group(2000, options); assert!(matches!(kio::Task::poll(&mut *pending, &waiter), Poll::Ready(Ok(_)))); waiter } @@ -835,10 +862,10 @@ fn parallel_write(pool: &cache::Pool, writers: usize, iterations: u64) -> Durati // Arrive once setup is done, then park until the main thread has stamped the clock. barrier.wait(); barrier.wait(); - for _ in 0..iterations { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - group.finish().unwrap(); + for key in 0..iterations { + let mut group = track.append_group(key).unwrap(); + group.write_frame(at(key), payload.clone()).unwrap(); + group.close().unwrap(); } }) }) @@ -985,14 +1012,14 @@ fn bench_subscriber_join(c: &mut Criterion) { /// Each held reader asks for distinct values, so the aggregate's tables grow with the audience. fn bench_subscriber_preferences(c: &mut Criterion) { let ordinary = track::Subscription::default() - .with_range(100..200) + .with_ranges([span(100, 199)]) .with_order(track::Order::Ascending); let held = |i: usize| { let i = i as u64; track::Subscription::default() .with_priority(((i % 255) as i16 - 128) as i8) .with_delay(track::Delay::default().with_max(Duration::from_micros(i))) - .with_range(100 + i..200 + i) + .with_ranges([span(100 + i, 199 + i)]) .with_order(track::Order::Ascending) }; let extreme = track::Subscription::default() @@ -1048,7 +1075,7 @@ fn bench_range_demand(c: &mut Criterion) { // Far past every held range, so each move adds one run and removes another. let away = |offset: u64| { let group = (1 << 40) + offset; - track::Subscription::default().with_range(group..=group) + track::Subscription::default().with_ranges([span(group, group)]) }; let mut group = c.benchmark_group("track_range_demand"); for steady in STEADY { @@ -1060,7 +1087,7 @@ fn bench_range_demand(c: &mut Criterion) { // Overlapping, so the held ranges union into one run. let start = (i % distinct) as u64 * 10; subscriber - .update(track::Subscription::default().with_range(start..start + 1000)) + .update(track::Subscription::default().with_ranges([span(start, start + 999)])) .unwrap(); } let mut viewer = setup.track.subscribe(away(0)); @@ -1079,7 +1106,7 @@ fn bench_range_demand(c: &mut Criterion) { fn bench_subscription_ranges(c: &mut Criterion) { let mut group = c.benchmark_group("track_subscription_ranges"); - for span in SPAN { + for width in SPAN { for ranges in RANGES { let broadcast = broadcast::Info::default().produce(); let track = broadcast @@ -1088,14 +1115,14 @@ fn bench_subscription_ranges(c: &mut Criterion) { // A gap between neighbors keeps every range distinct in the union. let viewers: Vec<_> = (0..ranges) .map(|i| { - let start = i * (span + 1); - track.subscribe(track::Subscription::default().with_range(start..start + span)) + let start = i * (width + 1); + track.subscribe(track::Subscription::default().with_ranges([span(start, start + width - 1)])) }) .collect(); assert_eq!(track.subscription().unwrap().ranges.len(), ranges as usize); group.throughput(Throughput::Elements(ranges)); - group.bench_function(BenchmarkId::new(format!("span_{span}"), ranges), |b| { + group.bench_function(BenchmarkId::new(format!("span_{width}"), ranges), |b| { b.iter(|| black_box(track.subscription())); }); drop((viewers, track, broadcast)); @@ -1106,27 +1133,29 @@ fn bench_subscription_ranges(c: &mut Criterion) { fn bench_range_serving(c: &mut Criterion) { let mut group = c.benchmark_group("track_range_serving"); - for span in SERVED_SPAN { + for width in SERVED_SPAN { for ranges in SERVED_RANGES { let broadcast = broadcast::Info::default().produce(); let track = broadcast .create_track("bench", moq_net::track::Info::default().with_max_age(None)) .unwrap(); // A gap between neighbors keeps every range distinct, and its group cached. - for _ in 0..ranges * (span + 1) { - let mut group = track.append_group().unwrap(); - group.write_frame(None, Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + for key in 0..ranges * (width + 1) { + let mut group = track.append_group(key).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), Bytes::from_static(b"x")) + .unwrap(); + group.close().unwrap(); } let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(AGED_BUDGET)) .with_ranges((0..ranges).map(|i| { - let start = i * (span + 1); - track::Range::groups(start..start + span) + let start = i * (width + 1); + span(start, start + width - 1) })); - group.throughput(Throughput::Elements(ranges * span)); - group.bench_function(BenchmarkId::new(format!("span_{span}"), ranges), |b| { + group.throughput(Throughput::Elements(ranges * width)); + group.bench_function(BenchmarkId::new(format!("span_{width}"), ranges), |b| { b.iter_batched( || track.subscribe(subscription.clone()), |mut reader| { @@ -1136,7 +1165,7 @@ fn bench_range_serving(c: &mut Criterion) { black_box(group); served += 1; } - assert_eq!(served, ranges * span); + assert_eq!(served, ranges * width); reader }, BatchSize::SmallInput, @@ -1169,6 +1198,8 @@ impl std::task::Wake for Woken { /// cost that grows with the parked reads shows up as a slope. fn bench_parked_read(c: &mut Criterion) { const MICROS_PER_GROUP: u64 = 2500; + // A group's start on the millisecond track, from its first frame's microseconds. + let millis = |micros: u64| micros / 1000; let mut group = c.benchmark_group("track_parked_read"); group.throughput(Throughput::Elements(1)); for parked in [8, 64, 512] { @@ -1192,7 +1223,7 @@ fn bench_parked_read(c: &mut Criterion) { let queue = Arc::new(std::sync::Mutex::new(Vec::new())); let mut reads: Vec<_> = (0..parked) .map(|read| { - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(millis(micros)).unwrap(); group .write_frame(Timestamp::from_micros(micros).unwrap(), Bytes::from_static(b"x")) .unwrap(); @@ -1227,10 +1258,10 @@ fn bench_parked_read(c: &mut Criterion) { let start = WallInstant::now(); for _ in 0..iterations { - let mut next = track.append_group().unwrap(); + let mut next = track.append_group(millis(micros)).unwrap(); next.write_frame(Timestamp::from_micros(micros).unwrap(), Bytes::from_static(b"x")) .unwrap(); - next.finish().unwrap(); + next.close().unwrap(); micros += MICROS_PER_GROUP; repoll(&mut reads); } @@ -1281,11 +1312,14 @@ impl Sparse { sparse } - fn hand_out(&mut self, sequence: u64) { - let timestamp = self.timed.then(|| Timestamp::from_micros(sequence).unwrap()); - let mut group = self.track.create_group(moq_net::group::Info { sequence }).unwrap(); + fn hand_out(&mut self, key: u64) { + let timestamp = match self.timed { + true => at(key), + false => moq_net::Timestamp::unscaled(key), + }; + let mut group = self.track.create_group(moq_net::group::Info { start: key }).unwrap(); group.write_frame(timestamp, Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let Poll::Ready(Ok(Some(read))) = self.sub.poll_recv_group(&kio::Waiter::noop()) else { panic!("the group is cached"); }; @@ -1366,7 +1400,7 @@ impl Idle { fn new(readers: usize, ordered: bool) -> Self { let broadcast = broadcast::Info::default().produce(); let track = broadcast.create_track("bench", timed()).unwrap(); - let mut open = track.append_group().unwrap(); + let mut open = track.append_group(0).unwrap(); open.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap(); let waiter = kio::Waiter::noop(); let (mut recv, mut next) = (Vec::new(), Vec::new()); @@ -1494,7 +1528,6 @@ struct GapFlush { _broadcast: broadcast::Producer, track: track::Producer, sub: track::Subscriber, - gaps: u64, } impl GapFlush { @@ -1504,20 +1537,18 @@ impl GapFlush { let mut sub = track.subscribe( track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(3600))), ); - let mut hand_out = |sequence: u64, millis: u64| { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); - group - .write_frame(Timestamp::from_millis(millis).unwrap(), Bytes::from_static(b"x")) - .unwrap(); + let mut hand_out = |key: u64| { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); + group.write_frame(at(key), Bytes::from_static(b"x")).unwrap(); let Poll::Ready(Ok(Some(read))) = sub.poll_recv_group(&kio::Waiter::noop()) else { panic!("the group is cached"); }; black_box(read); group }; - // Every odd sequence is missing. - let handed: Vec<_> = (0..gaps).map(|gap| hand_out(gap * 2, gap)).collect(); - hand_out(gaps * 2, gaps).finish().unwrap(); + // Every odd key is missing. + let handed: Vec<_> = (0..gaps).map(|gap| hand_out(gap * 2)).collect(); + hand_out(gaps * 2).close().unwrap(); for group in handed { group.abort(Error::Cancel).unwrap(); } @@ -1525,7 +1556,6 @@ impl GapFlush { _broadcast: broadcast, track, sub, - gaps, } } @@ -1533,15 +1563,12 @@ impl GapFlush { // A profiling boundary excludes publishing the groups and their aborts. #[inline(never)] fn flush(&mut self) { + const TWO_HOURS: u64 = 7_200_000; let mut group = self .track - .create_group(moq_net::group::Info { - sequence: self.gaps * 2 + 1, - }) - .unwrap(); - group - .write_frame(Timestamp::from_millis(7_200_000).unwrap(), Bytes::from_static(b"x")) + .create_group(moq_net::group::Info { start: TWO_HOURS }) .unwrap(); + group.write_frame(at(TWO_HOURS), Bytes::from_static(b"x")).unwrap(); let Poll::Ready(Ok(Some(read))) = self.sub.poll_recv_group(&kio::Waiter::noop()) else { panic!("the group is cached"); }; @@ -1591,26 +1618,25 @@ impl EmptyRun { fn new(run: u64, readers: usize, mode: Run) -> Self { let broadcast = broadcast::Info::default().produce(); let track = broadcast.create_track("bench", timed()).unwrap(); - let append = |millis: Option| { - let mut group = track.append_group().unwrap(); - if let Some(millis) = millis { - group - .write_frame(Timestamp::from_millis(millis).unwrap(), Bytes::from_static(b"x")) - .unwrap(); + // A group at `key` milliseconds, with a frame there or none. + let append = |key: u64, frame: bool| { + let mut group = track.append_group(key).unwrap(); + if frame { + group.write_frame(at(key), Bytes::from_static(b"x")).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); }; - append(Some(0)); - for _ in 0..run { - append(None); + append(0, true); + for key in 1..=run { + append(key, false); } const HOUR: u64 = 3_600_000; - let after: &[u64] = match mode { - Run::Stale => &[2 * HOUR, 4 * HOUR], - Run::Fresh | Run::Retired => &[5], + let after = match mode { + Run::Stale => vec![2 * HOUR, 4 * HOUR], + Run::Fresh | Run::Retired => vec![run + 5], }; - for millis in after { - append(Some(*millis)); + for key in after { + append(key, true); } let (budget, handed) = match mode { // The run's last group is written off, but two groups follow it. @@ -1680,18 +1706,18 @@ fn bench_range_widening(c: &mut Criterion) { let track = broadcast .create_track("bench", moq_net::track::Info::default().with_max_age(None)) .unwrap(); - for _ in 0..cached { - let mut group = track.append_group().unwrap(); - group.write_frame(None, Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + for key in 0..cached { + let mut group = track.append_group(key).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), Bytes::from_static(b"x")) + .unwrap(); + group.close().unwrap(); } let narrow = track::Subscription::default() .with_delay(track::Delay::default().with_max(AGED_BUDGET)) - .with_range(0..1); + .with_ranges([span(0, 0)]); let mid = cached / 2; - let wide = narrow - .clone() - .with_ranges([track::Range::groups(0..1), track::Range::groups(mid..mid + WIDENED)]); + let wide = narrow.clone().with_ranges([span(0, 0), span(mid, mid + WIDENED - 1)]); let waiter = kio::Waiter::noop(); let readers: Vec<_> = (0..readers) .map(|_| { @@ -1740,16 +1766,11 @@ impl BudgetWidening { let mut reader = track.subscribe(narrow.clone()); let beside = (0..readers).map(|_| track.subscribe(narrow.clone())).collect(); let waiter = kio::Waiter::noop(); - let mut write = |sequences: std::ops::Range| { - for sequence in sequences { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group - .write_frame( - Timestamp::from_millis(100 * sequence).unwrap(), - Bytes::from_static(b"x"), - ) - .unwrap(); - group.finish().unwrap(); + let mut write = |keys: std::ops::Range| { + for key in keys { + let mut group = track.create_group(group::Info { start: 100 * key }).unwrap(); + group.write_frame(at(100 * key), Bytes::from_static(b"x")).unwrap(); + group.close().unwrap(); } while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&waiter) { black_box(group); @@ -1761,8 +1782,8 @@ impl BudgetWidening { for run in 0..runs { write(3 * run + 1..3 * run + 4); } - for sequence in 3 * runs + 1..cached { - write(sequence..sequence + 1); + for key in 3 * runs + 1..cached { + write(key..key + 1); } Self { _broadcast: broadcast, @@ -1810,7 +1831,7 @@ fn bench_budget_widening(c: &mut Criterion) { } /// Readers whose range ends at a time no group reaches yet, as groups land behind a missing -/// sequence; see [`bench_time_end`]. +/// key; see [`bench_time_end`]. struct TimeEnd { _broadcast: broadcast::Producer, track: track::Producer, @@ -1822,17 +1843,14 @@ impl TimeEnd { fn new(readers: usize, groups: u64) -> Self { let broadcast = broadcast::Info::default().produce(); let track = broadcast.create_track("bench", timed()).unwrap(); - let mut zero = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut zero = track.create_group(group::Info { start: 0 }).unwrap(); zero.write_frame(Timestamp::from_millis(0).unwrap(), Bytes::from_static(b"x")) .unwrap(); - zero.finish().unwrap(); + zero.close().unwrap(); // Group 1 never arrives, and the budget outlasts the run, so the end never settles. let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(AGED_BUDGET)) - .with_ranges([track::Range { - start: track::Anchor::Absolute(0), - end: Some(track::Anchor::Time(Timestamp::from_secs(u32::MAX.into()).unwrap())), - }]); + .with_ranges([span(0, u64::from(u32::MAX) * 1000)]); let waiter = kio::Waiter::noop(); let readers = (0..readers) .map(|_| { @@ -1851,12 +1869,12 @@ impl TimeEnd { fn run(&mut self) { let waiter = kio::Waiter::noop(); - for sequence in 2..self.groups + 2 { - let mut group = self.track.create_group(group::Info { sequence }).unwrap(); + for key in 2..self.groups + 2 { + let mut group = self.track.create_group(group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence).unwrap(), Bytes::from_static(b"x")) + .write_frame(Timestamp::from_millis(key).unwrap(), Bytes::from_static(b"x")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); for reader in &mut self.readers { let mut handed = 0; while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&waiter) { @@ -1869,7 +1887,7 @@ impl TimeEnd { } } -/// Groups landing one at a time behind a missing sequence, each drained by readers whose +/// Groups landing one at a time behind a missing key, each drained by readers whose /// time end it cannot settle, swept over the groups and the readers. Each group must cost /// each reader the same, so a rescan behind the gap shows up as a slope over the groups. fn bench_time_end(c: &mut Criterion) { @@ -1901,18 +1919,16 @@ impl Fills { let broadcast = broadcast::Info::default().produce(); let mut track = broadcast.create_track("bench", timed()).unwrap(); let dynamic = track.requests(); - // The feed starts past every run, so each is a miss asked at once. + // The feed starts past every run, so each is a miss. let start = runs * 2; track.start_at(start).unwrap(); - let mut group = track.create_group(group::Info { sequence: start }).unwrap(); + let mut group = track.create_group(group::Info { start }).unwrap(); group .write_frame(Timestamp::from_millis(start).unwrap(), Bytes::from_static(b"x")) .unwrap(); - group.finish().unwrap(); - // One range per run, with a sequence between each that no reader asks for. - let ranges = (0..runs) - .map(|run| track::Range::groups(run * 2..run * 2 + 1)) - .chain([track::Range::groups(start..)]); + group.close().unwrap(); + // One range per run, with a key between each that no reader asks for. + let ranges = (0..runs).map(|run| span(run * 2, run * 2)).chain([from(start)]); let subscription = track::Subscription::default() .with_ranges(ranges) .with_delay(track::Delay::default().with_max(budget)); @@ -1927,8 +1943,10 @@ impl Fills { } /// Every reader reads what is cached and asks for its misses, then raises its priority, - /// which reaches every fill it waits on. The handler drops each fill they share, and - /// every reader settles the drops. + /// which reaches every fill it waits on. A run past the first starts at a time below the + /// feed's start, which each reader looks up first: the handler can't resolve times, so it + /// drops each lookup, and the readers then ask that run as a miss. The handler drops each + /// fill they share, and every reader settles the drops. fn run(&mut self) { let waiter = kio::Waiter::noop(); for reader in &mut self.readers { @@ -1940,16 +1958,30 @@ impl Fills { reader.update(reader.subscription().with_priority(1)).unwrap(); assert!(reader.poll_recv_group(&waiter).is_pending()); } - let mut fills = 0; - while let Poll::Ready(Ok(fill)) = self.dynamic.poll_requested_fill(&waiter) { - assert_eq!(fill.priority(), 1, "the update reached the fill"); - fill.drop_groups(fill.start().group..fill.end()); - fills += 1; + let (mut lookups, mut fills) = (0, 0); + // The lookups go out first, and the misses they leave behind them. + for _ in 0..2 { + while let Poll::Ready(Ok(fill)) = self.dynamic.poll_requested_fill(&waiter) { + assert_eq!(fill.priority(), 1, "the update reached the fill"); + match fill.time() { + Some(_) => lookups += 1, + None => { + fill.drop_groups(fill.start().group..fill.end()); + fills += 1; + } + } + } + for reader in &mut self.readers { + assert!(reader.poll_recv_group(&waiter).is_pending()); + } } + let readers = self.readers.len() as u64; + assert_eq!( + lookups, + (self.runs - 1) * readers, + "each reader looks up each later run" + ); assert_eq!(fills, self.runs, "readers share each run's fill"); - for reader in &mut self.readers { - assert!(reader.poll_recv_group(&waiter).is_pending()); - } } } @@ -1987,24 +2019,24 @@ struct RangeFills { } impl RangeFills { - fn new(readers: usize, ranges: u64, span: u64) -> Self { + fn new(readers: usize, ranges: u64, width: u64) -> Self { let broadcast = broadcast::Info::default().produce(); let mut track = broadcast.create_track("bench", timed()).unwrap(); let dynamic = track.requests(); - // The feed starts past every range, so each is a miss asked at once. A gap between - // neighbors keeps every range its own fill. - let start = ranges * (span + 1); + // The feed starts past every range, so each is a miss. A gap before each range keeps + // it its own fill, and has every range start at a time no group held places. + let start = ranges * (width + 1) + 1; track.start_at(start).unwrap(); - let mut group = track.create_group(group::Info { sequence: start }).unwrap(); + let mut group = track.create_group(group::Info { start }).unwrap(); group .write_frame(Timestamp::from_millis(start).unwrap(), Bytes::from_static(b"x")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let subscription = track::Subscription::default() .with_delay(track::Delay::default().with_max(AGED_BUDGET)) .with_ranges((0..ranges).map(|i| { - let first = i * (span + 1); - track::Range::groups(first..first + span) + let first = i * (width + 1) + 1; + span(first, first + width - 1) })); let readers = (0..readers).map(|_| track.subscribe(subscription.clone())).collect(); Self { @@ -2012,34 +2044,40 @@ impl RangeFills { _track: track, dynamic, readers, - groups: ranges * span, + groups: ranges * width, } } /// The readers ask for their misses, the handler accepts every group of every fill once, - /// and each reader drains what they filled. + /// and each reader drains what they filled. Each range starts at a time below the feed's + /// start, which each reader looks up first: the handler can't resolve times, so it drops + /// each lookup, and the readers then ask that range as a miss. fn run(&mut self) { let waiter = kio::Waiter::noop(); for reader in &mut self.readers { assert!(reader.poll_recv_group(&waiter).is_pending(), "everything is a miss"); } - while let Poll::Ready(Ok(fill)) = self.dynamic.poll_requested_fill(&waiter) { - while let Some(sequence) = fill.wanted() { - let mut group = fill.accept(sequence, None).unwrap(); - group - .write_frame(Timestamp::from_millis(sequence).unwrap(), Bytes::from_static(b"x")) - .unwrap(); - group.finish().unwrap(); + let mut filled = vec![0; self.readers.len()]; + // The lookups go out first, and the misses they leave behind them. + for _ in 0..2 { + while let Poll::Ready(Ok(fill)) = self.dynamic.poll_requested_fill(&waiter) { + // A lookup wants no group, so it is dropped unanswered. + while let Some(key) = fill.wanted() { + let mut group = fill.accept(key, None).unwrap(); + group + .write_frame(Timestamp::from_millis(key).unwrap(), Bytes::from_static(b"x")) + .unwrap(); + group.close().unwrap(); + } } - } - for reader in &mut self.readers { - let mut filled = 0; - while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&waiter) { - black_box(group); - filled += 1; + for (reader, filled) in self.readers.iter_mut().zip(&mut filled) { + while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&waiter) { + black_box(group); + *filled += 1; + } } - assert_eq!(filled, self.groups); } + assert!(filled.iter().all(|filled| *filled == self.groups)); } } @@ -2095,21 +2133,21 @@ impl RefusedFill { let mut track = broadcast.create_track("bench", timed()).unwrap(); let dynamic = track.requests(); track.start_at(Self::START).unwrap(); - let mut group = track.create_group(group::Info { sequence: Self::START }).unwrap(); + let mut group = track.create_group(group::Info { start: Self::START }).unwrap(); group.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let waiter = kio::Waiter::noop(); let readers = (0..readers) .map(|_| { let mut reader = track.subscribe( track::Subscription::default() - .with_range(0..) + .with_ranges([from(0)]) .with_delay(track::Delay::default().with_max(AGED_BUDGET)), ); let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&waiter) else { panic!("the live group is cached"); }; - assert_eq!(group.sequence, Self::START); + assert_eq!(group.start, Self::START); assert!(reader.poll_recv_group(&waiter).is_pending()); reader }) @@ -2133,8 +2171,8 @@ impl RefusedFill { fn run(&mut self, refusals: u64) { let waiter = kio::Waiter::noop(); for refused in 1..=refusals { - let sequence = self.fill.wanted().unwrap(); - self.fill.drop_groups(sequence..sequence + 1); + let key = self.fill.wanted().unwrap(); + self.fill.drop_groups(key..key + 1); for reader in &mut self.readers { assert!(reader.poll_recv_group(&waiter).is_pending()); } diff --git a/rs/moq-net/benches/viewers.rs b/rs/moq-net/benches/viewers.rs index 0be9c30317c..e9771ce59df 100644 --- a/rs/moq-net/benches/viewers.rs +++ b/rs/moq-net/benches/viewers.rs @@ -81,9 +81,9 @@ impl Room { .route(format!("room/{i}"), moq_net::origin::RouteOptions::default()) .unwrap(); let track = broadcast.broadcast().create_track("video", timed()).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(Timestamp::ZERO, b"frame".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); (broadcast, track) }) .collect(); diff --git a/rs/moq-net/src/coding/reader.rs b/rs/moq-net/src/coding/reader.rs index 376bd959393..95d8ce2fac3 100644 --- a/rs/moq-net/src/coding/reader.rs +++ b/rs/moq-net/src/coding/reader.rs @@ -739,12 +739,12 @@ mod tests { kio::Waiter, Arc, ) { - let group = crate::group::Info { sequence: 0 }.produce(); + let group = crate::group::Info { start: 0 }.produce(); let mut consumer = group.consume(); let frame = group .create_frame(crate::frame::Info { size: size as u64, - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::ZERO, }) .unwrap(); @@ -982,7 +982,7 @@ mod tests { /// A group 17 MB past an idle reader, so its writer is paced. fn paced_group() -> (crate::group::Producer, crate::group::Consumer) { - let mut group = crate::group::Info { sequence: 0 }.produce(); + let mut group = crate::group::Info { start: 0 }.produce(); let idle = group.consume(); let payload = Bytes::from(vec![0u8; 1 << 20]); for _ in 0..17 { @@ -1002,7 +1002,7 @@ mod tests { let group = crate::recv::Group::new(group); let header = crate::frame::Header::from(crate::frame::Info { size: 1, - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::ZERO, }); let mut reader = queue(&[], crate::lite::Version::Lite05); assert!(reader.poll_admit(noop, &group, &header).is_pending(), "paused to admit"); @@ -1013,7 +1013,7 @@ mod tests { let mut frame = group .create_frame(crate::frame::Info { size: 1, - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::ZERO, }) .unwrap(); let mut reader = queue(&[b"x"], crate::lite::Version::Lite05); diff --git a/rs/moq-net/src/error.rs b/rs/moq-net/src/error.rs index 15f9268519e..1a95d70a295 100644 --- a/rs/moq-net/src/error.rs +++ b/rs/moq-net/src/error.rs @@ -387,9 +387,9 @@ pub enum Error { #[error("duplicate")] Duplicate, - /// A group was created at a sequence its track declared skipped: a skip says the - /// sequence will never be produced, so it is final. See `track::Producer::skip_groups`. - #[error("sequence was skipped")] + /// A group was created at a key its track declared skipped: a skip says the + /// key will never be produced, so it is final. See `track::Producer::skip_groups`. + #[error("key was skipped")] Skipped, /// Nobody is reading any more, so the producer stopped; or the session closed cleanly, diff --git a/rs/moq-net/src/fuzz.rs b/rs/moq-net/src/fuzz.rs index 8133d0b3d25..fa3fc0d6952 100644 --- a/rs/moq-net/src/fuzz.rs +++ b/rs/moq-net/src/fuzz.rs @@ -50,7 +50,7 @@ pub struct DropSet { } impl DropSet { - /// `runs` disjoint dropped runs of `span` sequences each, read by `readers` wire cursors + /// `runs` disjoint dropped runs of `span` keys each, read by `readers` wire cursors /// whose ranges hold the newest `reported` of them, each already reported. pub fn new(readers: usize, runs: u64, reported: u64, span: u64) -> Self { let broadcast = broadcast::Producer::new(broadcast::Info::default()); @@ -132,7 +132,7 @@ pub fn object_group(frames: usize, properties: bool) -> (broadcast::Producer, tr ..Default::default() }; let track = broadcast.create_track("objects", info).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); let payload = Bytes::from_static(&[0; 64]); for id in 0..frames { let fields = properties.then(|| { @@ -154,7 +154,7 @@ pub fn object_group(frames: usize, properties: bool) -> (broadcast::Producer, tr let header = frame::Header { info: frame::Info { size: payload.len() as u64, - timestamp: None, + timestamp: crate::Timestamp::unscaled(group.start), }, fields, inflated: None, @@ -163,7 +163,7 @@ pub fn object_group(frames: usize, properties: bool) -> (broadcast::Producer, tr frame.write(payload.clone()).unwrap(); group = frame.finish().unwrap(); } - (broadcast, track, group.finish().unwrap()) + (broadcast, track, group.close().unwrap()) } /// A private frame batch for the model's batch read and write benchmarks. @@ -1382,7 +1382,7 @@ impl LiteSample { } Self::Group => lite::Group { subscribe: 3, - sequence: 1_234, + key: 1_234, frame_start: 0, } .encode(w, version) @@ -1532,7 +1532,7 @@ where /// /// Byte 0 picks the version, which is the whole point: lite-01 to lite-06 use the QUIC /// two-bit length tag, while lite-07 and the moq-transport drafts count leading ones, and -/// the two disagree about which byte sequences are even legal. +/// the two disagree about which byte keys are even legal. /// /// The leading-ones form spans the full `u64`, while the QUIC form stops at /// `2^62 - 1`; a value always re-encodes in the form it was read in. @@ -1629,7 +1629,7 @@ impl Default for Messages { }, lite_group: lite::Group { subscribe: 7, - sequence: 123_456, + key: 123_456, frame_start: 0, }, ietf_subscribe: ietf::Subscribe { @@ -2401,10 +2401,11 @@ impl FrameRecv { let streams = (0..streams) .map(|_| { - let group = track.append_group().unwrap(); + let group = track.append_next().unwrap(); + // Each group starts at its own key, so the starts increase. let info = frame::Info { size: size as u64, - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::from_millis(group.start).unwrap(), }; let frame = group.create_frame(info).unwrap(); let stream = Chunks { @@ -2463,7 +2464,7 @@ impl FrameRecv { res.unwrap(); let stream = slot.take().unwrap(); let group = stream.frame.finish().unwrap(); - group.finish().unwrap(); + group.close().unwrap(); open -= 1; } } @@ -2545,11 +2546,7 @@ impl GroupRecv { } /// Receive one frame whose payload is `chunks`, in order, and commit it. - pub fn frame( - &mut self, - timestamp: Option, - chunks: impl IntoIterator, - ) -> crate::Result<()> { + pub fn frame(&mut self, timestamp: crate::Timestamp, chunks: impl IntoIterator) -> crate::Result<()> { let mut size = 0; self.queue .borrow_mut() @@ -2570,7 +2567,7 @@ impl GroupRecv { /// Finish the group, which stays cached until dropped or aborted. pub fn finish(mut self) -> crate::Result { - self.group.take().ok_or(crate::Error::Closed)?.finish() + self.group.take().ok_or(crate::Error::Closed)?.close() } } diff --git a/rs/moq-net/src/fuzz/ingest.rs b/rs/moq-net/src/fuzz/ingest.rs index 20a0d2a38d4..e499f4148c9 100644 --- a/rs/moq-net/src/fuzz/ingest.rs +++ b/rs/moq-net/src/fuzz/ingest.rs @@ -24,7 +24,7 @@ const MAX_VARINT: usize = 9; /// One frame as a relay reads it back out of the group. #[derive(Debug, PartialEq)] struct Seen { - timestamp: Option, + timestamp: crate::Timestamp, payload: Bytes, /// The Object ID and property block the frame kept, if it was not plain. fields: Option<(u64, Option)>, @@ -133,7 +133,7 @@ fn check(session: Session, info: &track::Info, body: &[u8], ingested: &Ingested) let mut payload = 0; for (index, frame) in ingested.frames.iter().enumerate() { assert_eq!( - frame.timestamp.is_some(), + frame.timestamp.scale().is_some(), info.timescale.is_some(), "frame {index} does not match its track's timeline" ); @@ -219,7 +219,7 @@ fn run(session: Session, info: &track::Info, body: &[u8], chunk: usize) -> Optio frame_start, } => { let mut reader = reader(Bytes::copy_from_slice(body), chunk, version); - let producer = track.append_group().ok()?; + let producer = track.append_next().ok()?; let consumer = producer.consume(); let mut viewer = stored(producer.consume(), info.compressed()); let mut group = crate::recv::Group::new(producer); @@ -235,11 +235,7 @@ fn run(session: Session, info: &track::Info, body: &[u8], chunk: usize) -> Optio Session::Ietf { version } => { let (header, used) = ietf::GroupHeader::decode_slice(body, version).ok()?; let mut reader = reader(Bytes::copy_from_slice(&body[used..]), chunk, version); - let producer = track - .create_group(group::Info { - sequence: header.group_id, - }) - .ok()?; + let producer = track.create_group(group::Info { start: header.group_id }).ok()?; producer .set_wire(group::Wire { sparse: false, diff --git a/rs/moq-net/src/fuzz/session.rs b/rs/moq-net/src/fuzz/session.rs index d4670925c0e..4323c626819 100644 --- a/rs/moq-net/src/fuzz/session.rs +++ b/rs/moq-net/src/fuzz/session.rs @@ -475,20 +475,21 @@ impl Bystander { } /// Write the next group: two frames of the same size every time, so groups charge the - /// cache alike, holding the sequence so a misplaced one shows. + /// cache alike, holding the key so a misplaced one shows. fn publish(&self) { let mut sent = self.sent.borrow_mut(); - let sequence = sent.len() as u64; - let mut group = self.track.append_group().expect("the bystander's group"); + let key = sent.len() as u64; + let mut group = self.track.append_next().expect("the bystander's group"); let mut frames = Vec::new(); for frame in 0..2u8 { - let mut payload = sequence.to_be_bytes().repeat(8); + let mut payload = key.to_be_bytes().repeat(8); payload.push(frame); let payload = Bytes::from(payload); - group.write_frame(None, payload.clone()).expect("the bystander's frame"); + let key = crate::Timestamp::unscaled(group.start); + group.write_frame(key, payload.clone()).expect("the bystander's frame"); frames.push(payload); } - group.finish().expect("the bystander's group finish"); + group.close().expect("the bystander's group finish"); sent.push(frames); } @@ -777,9 +778,10 @@ fn record(version: Version, scenario: Scenario) -> Vec { route.announce().expect("the client's announce"); let subscribed = async { while let Ok(None) = track.subscription_changed().await {} }; let _ = kio::clock::timeout(timeout, subscribed).await; - let mut group = track.append_group().expect("the client's group"); - group.write_frame(None, Bytes::from_static(b"frame")).expect("a frame"); - group.finish().expect("the client's group finish"); + let mut group = track.append_next().expect("the client's group"); + let key = crate::Timestamp::unscaled(group.start); + group.write_frame(key, Bytes::from_static(b"frame")).expect("a frame"); + group.close().expect("the client's group finish"); kio::clock::sleep(timeout).await; } Scenario::Subscribe => { diff --git a/rs/moq-net/src/ietf/group.rs b/rs/moq-net/src/ietf/group.rs index a9e3eed088c..d1285b43afa 100644 --- a/rs/moq-net/src/ietf/group.rs +++ b/rs/moq-net/src/ietf/group.rs @@ -345,7 +345,7 @@ mod tests { let decoded = decode_time(&buf, Timescale::MICRO, Version::Draft18).unwrap(); assert_eq!(decoded.value(), 96_000); - assert_eq!(decoded.scale(), Timescale::MICRO); + assert_eq!(decoded.scale(), Some(Timescale::MICRO)); } /// The value on the wire is in the track's units, not the frame's. @@ -358,7 +358,7 @@ mod tests { let decoded = decode_time(&buf, Timescale::MICRO, Version::Draft18).unwrap(); assert_eq!(decoded.value(), 2_000_000); - assert_eq!(decoded.scale(), Timescale::MICRO); + assert_eq!(decoded.scale(), Some(Timescale::MICRO)); } /// An object-scope Timescale (which LOC permits) is ignored: the track's units apply. @@ -376,7 +376,7 @@ mod tests { let decoded = decode_time(&buf, Timescale::MICRO, Version::Draft18).unwrap(); assert_eq!(decoded.value(), 42); - assert_eq!(decoded.scale(), Timescale::MICRO); + assert_eq!(decoded.scale(), Some(Timescale::MICRO)); } /// The track's timescale supplies the units. @@ -386,7 +386,7 @@ mod tests { let decoded = decode_time(&buf, Timescale::MILLI, Version::Draft18).unwrap(); assert_eq!(decoded.value(), 1234); - assert_eq!(decoded.scale(), Timescale::MILLI); + assert_eq!(decoded.scale(), Some(Timescale::MILLI)); } /// No Timestamp property at all yields None. diff --git a/rs/moq-net/src/ietf/object.rs b/rs/moq-net/src/ietf/object.rs index f88d6f3a015..cfbf17cc278 100644 --- a/rs/moq-net/src/ietf/object.rs +++ b/rs/moq-net/src/ietf/object.rs @@ -337,10 +337,12 @@ pub(crate) fn egress_time( return Ok(None); }; if let Some(sealed) = properties.sealed_time { - if timescale != Some(sealed.scale()) { + if timescale != sealed.scale() { return Err(Error::Unsupported); } - if frame.convert(sealed.scale()).is_ok_and(|frame| frame == sealed) { + if let Some(scale) = sealed.scale() + && frame.convert(scale).is_ok_and(|frame| frame == sealed) + { return Ok(None); } } @@ -799,11 +801,11 @@ mod compression_tests { crate::track::Info::default().with_compression(crate::track::Compression::Deflate), ) .unwrap(); - let group = track.append_group().unwrap(); + let group = track.append_next().unwrap(); let header = crate::frame::Header { info: crate::frame::Info { size: 1, - timestamp: None, + timestamp: crate::Timestamp::unscaled(group.start), }, fields: None, inflated: inflated(Some(crate::group::MAX_CACHE_BYTES + 1), true).unwrap(), diff --git a/rs/moq-net/src/ietf/properties.rs b/rs/moq-net/src/ietf/properties.rs index a8519fe40e6..0295ddd6c97 100644 --- a/rs/moq-net/src/ietf/properties.rs +++ b/rs/moq-net/src/ietf/properties.rs @@ -1,7 +1,7 @@ /// Track Properties: relay-visible metadata attached to tracks. /// /// SUBSCRIBE_OK, PUBLISH, and FETCH_OK carry Track Properties. -/// They appear after the message parameters as a sequence of Key-Value-Pairs +/// They appear after the message parameters as a key of Key-Value-Pairs /// (same delta-encoded format) until the end of the message. /// /// Unlike Message Parameters which have a count prefix, Track Properties diff --git a/rs/moq-net/src/ietf/publisher.rs b/rs/moq-net/src/ietf/publisher.rs index 296a8179116..4bafca066a2 100644 --- a/rs/moq-net/src/ietf/publisher.rs +++ b/rs/moq-net/src/ietf/publisher.rs @@ -161,17 +161,17 @@ struct FetchRange { } impl FetchRange { - /// The first Object ID read from group `sequence`. - fn skip(&self, sequence: u64) -> u64 { - match sequence == self.start.group { + /// The first Object ID read from group `key`. + fn skip(&self, key: u64) -> u64 { + match key == self.start.group { true => self.start.object, false => 0, } } - /// The exclusive Object ID bound in group `sequence`, or `None` to read it whole. - fn until(&self, sequence: u64) -> Option { - (sequence == self.end.group && self.end.object > 0).then_some(self.end.object) + /// The exclusive Object ID bound in group `key`, or `None` to read it whole. + fn until(&self, key: u64) -> Option { + (key == self.end.group && self.end.object > 0).then_some(self.end.object) } } @@ -274,9 +274,9 @@ impl PrepareFetch { // The cache's newest object is Largest Object while a live feed writes the track, or once // the track ended at its newest group. A fetch-only copy retains its publisher's report // independently. A joining FETCH ends where its subscription starts instead. - let finished = track.final_sequence(); - let known = track.is_live() - || finished.is_some_and(|end| Some(end) == track.latest_sequence().map(|latest| latest + 1)); + let finished = track.final_key(); + let known = + track.is_live() || finished.is_some_and(|end| Some(end) == track.latest_key().map(|latest| latest + 1)); let largest = match !self.joined && (known || track.reported_object().is_some()) { true => live_edge(track).largest, false => None, @@ -363,7 +363,7 @@ fn poll_settle( group: &group::Consumer, joined: bool, ) -> Poll, (Error, String)>> { - let sequence = range.start.group; + let key = range.start.group; match group.poll_closed(waiter) { // FETCH objects carry no status, so a status object only bounds the frames before it. Poll::Ready(Ok(())) => { @@ -377,7 +377,7 @@ fn poll_settle( Poll::Pending => {} } let edge = group.last_object().map_or(0, |last| last.saturating_add(1)); - if range.until(sequence).is_some_and(|until| until <= edge) { + if range.until(key).is_some_and(|until| until <= edge) { return Poll::Ready(Ok(None)); } // A joining FETCH ends where its subscription starts, at the saved Largest Object, so @@ -385,7 +385,7 @@ fn poll_settle( if joined { return Poll::Pending; } - if track.is_live() && track.latest_sequence().is_none_or(|latest| sequence >= latest) { + if track.is_live() && track.latest_key().is_none_or(|latest| key >= latest) { return Poll::Ready(Err(( Error::Unsupported, "FETCH of a live group must end below its live edge".into(), @@ -395,7 +395,7 @@ fn poll_settle( } struct FetchObjectInfo<'a> { - sequence: u64, + key: u64, object: u64, prior: FetchPrior, /// The track's Publisher Priority on the wire, which the first object states for all. @@ -420,13 +420,13 @@ fn fetch_prior(prior: Option, object: u64) -> Result { Ok(FetchPrior::Same((step != 1).then_some(step))) } -/// Finds group `sequence` of a track for a FETCH from object `skip`, without reading it. +/// Finds group `key` of a track for a FETCH from object `skip`, without reading it. /// /// This is a [`track::Consumer::fetch_group`], so a relay fetches a miss upstream. The /// track's info resolves first, as it would for a SUBSCRIBE; `ahead` is the whole group /// already asked for, used for a miss. struct FindFetch { - sequence: u64, + key: u64, skip: u64, priority: i8, compression: bool, @@ -450,14 +450,14 @@ enum FindFetchState { impl FindFetch { fn new( track: &track::Consumer, - sequence: u64, + key: u64, skip: u64, priority: i8, compression: bool, ahead: Option, ) -> Self { Self { - sequence, + key, skip, priority, compression, @@ -488,7 +488,7 @@ impl FindFetch { match &mut self.state { FindFetchState::Info(resolving) => { let info = ready!(resolving.poll_ok(waiter))?; - let cached = track.peek_group(self.sequence); + let cached = track.peek_group(self.key); let wire = cached.as_ref().and_then(|group| group.wire()); if wire.as_ref().is_some_and(|wire| !wire.fetchable) { return Poll::Ready(Err(Error::Unsupported)); @@ -507,7 +507,7 @@ impl FindFetch { Some(Ahead::Fetched(result)) => return Poll::Ready(result.map(|group| (info, group))), Some(Ahead::Fetching(fetching)) => fetching, None => track.fetch_group( - self.sequence, + self.key, group::Fetch { priority: self.priority, frame_start, @@ -1990,7 +1990,7 @@ impl SubscribeTrack { // bound the subscription before subscribing so the ratio never judges it. From the // start of its group, so a relay's copy holds the group's head for its other readers, // as a live join would; the resolved range trims below. - let fixed: ServeRange = filter_range(msg.filter, LiveEdge::default()); + let fixed: ServeRange = filter_range(msg.filter, LiveEdge::default(), |_, _| None); if let Some(start) = fixed.start { ServeRange { start: Some(Location { object: 0, ..start }), @@ -2034,7 +2034,7 @@ impl SubscribeTrack { // current-group join (Next Object plus a StartGroup=1 fill) cover the group with no gap // and no overlap. let edge = live_edge(track); - let mut range = filter_range(msg.filter, edge); + let mut range = filter_range(msg.filter, edge, |latest, back| track.group_back(latest, back)); let fill = msg .fill .filter(|_| msg.forward && Filter::is_draft20(publisher.version)) @@ -2682,8 +2682,8 @@ impl RunFill { let priority: i8 = *ranking.priority.read(); let mut rank: Option>> = match self.fill { FillServe::Empty => return Poll::Ready(()), - FillServe::Group { sequence, skip, .. } => Some(ranking.rank(Location { - group: sequence, + FillServe::Group { key, skip, .. } => Some(ranking.rank(Location { + group: key, object: skip, })), // Only the refusal goes out on its stream, so it competes with nothing. @@ -2727,17 +2727,17 @@ impl RunFill { Ok(info) => super::priority::to_wire(info.priority), Err(err) => return self.fail(writer, err), }; - let FillServe::Group { sequence, skip, until } = self.fill else { + let FillServe::Group { key, skip, until } = self.fill else { return self.fail(writer, Error::Unsupported); }; let fetching: kio::Pending = self.track.fetch_group( - sequence, + key, group::Fetch { priority, ..Default::default() }, ); - let filled: u64 = sequence; + let filled: u64 = key; let slice: GroupSlice = GroupSlice { skip, until }; let fetched = wait!(Fetch => kio::Task::poll(&mut **fetching, waiter)); let group: group::Consumer = match fetched { @@ -3451,7 +3451,7 @@ impl Publisher { header: FetchObjectInfo<'_>, ) -> Result<(), Error> { let FetchObjectInfo { - sequence, + key, object, prior, priority, @@ -3489,7 +3489,7 @@ impl Publisher { // track's, as a subgroup header's Publisher Priority is. FetchPrior::None => ietf::FetchObject::Object { subgroup: ietf::FetchSubgroup::Zero, - group: Some(sequence), + group: Some(key), object: Some(object), priority: Some(priority), properties, @@ -4388,7 +4388,7 @@ struct FillTail { /// The stream serving a subscription's highest group, as it was opened. struct LiveGroup { - sequence: u64, + key: u64, slice: GroupSlice, /// Set once a narrowing gave the stream up before its FIN. retired: Arc, @@ -5101,8 +5101,8 @@ impl FetchOpen { group, end, } = prepared; - let sequence = range.start.group; - let (skip, until) = (range.skip(sequence), range.until(sequence)); + let key = range.start.group; + let (skip, until) = (range.skip(key), range.until(key)); // Only the TIMESCALE in FETCH_OK opts the answer into timestamps, so a requester that // opted out of the properties gets an untimed track. let timescale = info.timescale.filter(|_| self.properties_wanted); @@ -5113,7 +5113,7 @@ impl FetchOpen { let draft20 = Filter::is_draft20(version); // Only a finished track's final object ends it, never a live track's Largest Object. let complete = end.is_some_and(|end| until.is_none_or(|until| until >= end)); - let end_of_track = !joined && complete && track.final_sequence() == sequence.checked_add(1); + let end_of_track = !joined && complete && track.final_key() == key.checked_add(1); // One past the last object the answer carries; settling bounds a live group's range. let stop = match (end, until) { (Some(end), until) => until.map_or(end, |until| until.min(end)), @@ -5136,14 +5136,11 @@ impl FetchOpen { Some(until) => until - 1, None => stop.saturating_sub(1).max(skip), }; - Location { - group: sequence, - object, - } + Location { group: key, object } } else if end_of_track && range.end.object > 0 { // A bounded response cannot promise objects past the track's end. Location { - group: sequence, + group: key, object: stop, } } else { @@ -5157,7 +5154,7 @@ impl FetchOpen { until: until.filter(|until| end.is_none_or(|end| *until < end)), }; let priority = super::priority::to_wire(info.priority); - let write = match FetchGroupWrite::new(group, sequence, slice, timescale, version, priority) { + let write = match FetchGroupWrite::new(group, key, slice, timescale, version, priority) { Ok(write) => write, Err(err) => return self.reject(&err, &err.to_string()), }; @@ -6722,7 +6719,7 @@ impl TrackServe { ) -> Self { let floor = match range.start { Some(start) => start.group, - None => track.latest_sequence().unwrap_or(0), + None => track.latest_key().unwrap_or(0), }; let end = range.end.map_or(Bound::Unbounded, |end| Bound::Included(end.group)); let track = track.with_range((Bound::Included(floor), end)); @@ -6809,7 +6806,7 @@ impl TrackServe { start: Some(Location { group: 0, object: 0 }), end: None, }, - _ => filter_range(filter, edge), + _ => filter_range(filter, edge, |latest, back| cache.group_back(latest, back)), }; } let mut range = self.requested; @@ -6829,10 +6826,10 @@ impl TrackServe { // A stream still serving this group from the fill's edge keeps what it covers, so // the tail resumes where that stream stops and nothing goes out twice. let live = self.live.as_ref().filter(|live| { - live.sequence == next.group + live.key == next.group && live.slice.skip <= next.object && !live.retired.load(Ordering::Relaxed) - && !range.excludes(live.sequence, live.slice) + && !range.excludes(live.key, live.slice) }); let start = match live { Some(live) => live.slice.until.map(|until| Location { @@ -6909,7 +6906,7 @@ impl TrackServe { /// ratio rather than forwarded as is: every hop's cache holds the same group. One /// naming a group already held asks nothing, since that group is on its way. fn request_new_group(&self, value: u64) { - if value != 0 && self.track.latest_sequence().is_some_and(|latest| value <= latest) { + if value != 0 && self.track.latest_key().is_some_and(|latest| value <= latest) { return; } self.track.join(); @@ -6963,10 +6960,10 @@ impl TrackServe { fn serve_group(&mut self, group: group::Consumer, slice: GroupSlice) -> GroupServe { let group = if self.compression { group.compressed() } else { group }; - let sequence = group.sequence; + let key = group.start; let msg = ietf::GroupHeader { track_alias: self.request_id.0, - group_id: sequence, + group_id: key, sub_group_id: 0, // The publisher's own ranking of its tracks, which is what a relay // prefers when it can't pick between its subscribers' priorities. The @@ -6982,13 +6979,13 @@ impl TrackServe { }; let rank = self.ranking.rank(Location { - group: sequence, + group: key, object: slice.skip, }); let retired = Arc::new(AtomicBool::new(false)); - if self.live.as_ref().is_none_or(|live| sequence >= live.sequence) { + if self.live.as_ref().is_none_or(|live| key >= live.key) { self.live = Some(LiveGroup { - sequence, + key, slice, retired: retired.clone(), }); @@ -7075,10 +7072,10 @@ impl TrackServe { } match self.track.poll_recv_group(waiter) { Poll::Ready(Ok(Some(group))) => { - let sequence = group.sequence; - tracing::debug!(subscribe = %self.request_id, track = %self.track.name(), sequence, "serving group"); + let key = group.start; + tracing::debug!(subscribe = %self.request_id, track = %self.track.name(), key, "serving group"); - let slice = self.range.slice(sequence); + let slice = self.range.slice(key); if slice.until.is_some_and(|until| until <= slice.skip) { continue; } @@ -7190,7 +7187,7 @@ fn emit_datagram( ) -> bool { let mut properties = Vec::new(); if let Some(timescale) = timescale { - let Some(timestamp) = datagram.timestamp else { + let Some(timestamp) = datagram.timestamp.timed() else { tracing::debug!(sequence = datagram.sequence, "dropping a datagram without a timestamp"); return false; }; @@ -7405,7 +7402,7 @@ impl crate::transport::poll::SendStream f /// status field. struct FetchGroupWrite { group: group::Consumer, - sequence: u64, + key: u64, slice: GroupSlice, timescale: Option, version: Version, @@ -7444,7 +7441,7 @@ enum FetchGroupState { impl FetchGroupWrite { fn new( mut group: group::Consumer, - sequence: u64, + key: u64, slice: GroupSlice, timescale: Option, version: Version, @@ -7458,7 +7455,7 @@ impl FetchGroupWrite { // A FETCH's group may be found from its first object in range. index: group.index(), group, - sequence, + key, slice, timescale, version, @@ -7486,11 +7483,11 @@ impl FetchGroupWrite { fn object<'a>(&self, frame: &'a frame::Object) -> Result, Error> { Ok(FetchObjectInfo { - sequence: self.sequence, + key: self.key, object: self.index, prior: fetch_prior(self.prior, self.index)?, priority: self.priority, - timestamp: frame.timestamp, + timestamp: frame.timestamp.timed(), inflated: frame.inflated, timescale: self.timescale, version: self.version, @@ -7558,11 +7555,11 @@ impl FetchGroupWrite { continue; } let header = FetchObjectInfo { - sequence: self.sequence, + key: self.key, object: self.index, prior: fetch_prior(self.prior, self.index)?, priority: self.priority, - timestamp: frame.timestamp, + timestamp: frame.timestamp.timed(), inflated: frame.inflated, timescale: self.timescale, version: self.version, @@ -7720,7 +7717,7 @@ impl EndOfTrack { } /// A group stream's place in the session's send order: its subscription's priority, then -/// its sequence in the subscription's GROUP_ORDER. +/// its key in the subscription's GROUP_ORDER. enum Rank { /// In the request's send group, at a place fixed for the stream's life: the request's /// priority moves the whole group. @@ -8178,7 +8175,7 @@ impl GroupServe { writer, Object { delta, - timestamp: batched.timestamp, + timestamp: batched.timestamp.timed(), size: batched.payload.len() as u64, inflated: batched.inflated, status: 0, @@ -8226,7 +8223,7 @@ impl GroupServe { writer, Object { delta: object_delta(self.prior_object, id)?, - timestamp: next.timestamp, + timestamp: next.timestamp.timed(), size: next.size, inflated: next.inflated, status: 0, @@ -8331,7 +8328,7 @@ impl GroupServe { } Poll::Pending => return Poll::Pending, }; - let sequence = self.msg.group_id; + let key = self.msg.group_id; self.state = GroupState::Done; // Only an acknowledged marker spares the standalone one: a stream reset // after its FIN may never deliver it. @@ -8341,7 +8338,7 @@ impl GroupServe { terminal.sent.store(true, Ordering::Relaxed); } return Poll::Ready(res.map(|()| { - tracing::debug!(sequence, "finished group"); + tracing::debug!(key, "finished group"); })); } GroupState::Done => return Poll::Ready(Ok(())), @@ -8577,9 +8574,9 @@ mod group_priority_test { "test", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 5 }).unwrap(); + let mut group = track.create_group(group::Info { start: 5 }).unwrap(); for _ in 0..5 { - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } let mut serve = TrackServe::new( crate::driver::Transport::new(SinkSession::new(log.clone())), @@ -8592,13 +8589,13 @@ mod group_priority_test { assert!(serve.poll(&kio::Waiter::noop()).is_pending()); let mut group = Some(group); if finished { - group.take().unwrap().finish().unwrap(); + group.take().unwrap().close().unwrap(); assert!(serve.poll(&kio::Waiter::noop()).is_pending()); assert!(log.trail().iter().any(|(_, event)| matches!(event, Sent::Finish))); } track.finish_at(track::Position { group: 5, frame: 5 }, None).unwrap(); if let Some(group) = group { - group.finish().unwrap(); + group.close().unwrap(); } kio::wait(|waiter| serve.poll(waiter)).await.unwrap(); if let Some(end) = serve.end() { @@ -8646,9 +8643,9 @@ mod group_priority_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut old = track.append_group().unwrap(); - old.write_frame(crate::Timestamp::ZERO, b"old".as_slice()).unwrap(); - old.finish().unwrap(); + let mut old = track.append_next().unwrap(); + old.write_frame(old.key_time(), b"old".as_slice()).unwrap(); + old.close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("old group"); // Set directly rather than through the track, which a later group expires. @@ -8688,10 +8685,10 @@ mod group_priority_test { assert!(!terminal.sent.load(Ordering::Relaxed), "not until acknowledged"); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(crate::Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(futures::poll!(serving.as_mut()), Poll::Ready(Err(Error::Old)))); assert!( !terminal.sent.load(Ordering::Relaxed), @@ -8725,11 +8722,11 @@ mod group_priority_test { "test", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); group .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"hello".as_slice()) .unwrap(); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let msg = ietf::GroupHeader { track_alias: 0, @@ -8870,11 +8867,11 @@ mod group_priority_test { let mut appended = 0; let produce = move || { for _ in 0..BATCH.min(GROUPS - appended) { - let mut group = producer.create_group(group::Info { sequence: appended }).unwrap(); + let mut group = producer.create_group(group::Info { start: appended }).unwrap(); group .write_frame(crate::Timestamp::from_millis(appended).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); appended += 1; } appended == GROUPS @@ -8890,7 +8887,7 @@ mod group_priority_test { } /// On a transport with send groups, a subscription opens one at its priority and its - /// group streams join it, ranked newest first by sequence alone: the session-wide queue + /// group streams join it, ranked newest first by key alone: the session-wide queue /// takes no part. A priority update moves the group once, not each stream. #[test] fn grouped_streams_rank_within_their_subscription() { @@ -8914,12 +8911,12 @@ mod group_priority_test { ServeRange::default(), None, ); - for sequence in 0..2 { - let mut group = track.create_group(group::Info { sequence }).unwrap(); + for key in 0..2 { + let mut group = track.create_group(group::Info { start: key }).unwrap(); group - .write_frame(crate::Timestamp::from_millis(sequence).unwrap(), b"x".as_slice()) + .write_frame(crate::Timestamp::from_millis(key).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let waiter = kio::Waiter::noop(); @@ -8962,13 +8959,13 @@ mod group_priority_test { track::Subscription::default() .with_delay(track::Delay::default().with_max(std::time::Duration::from_secs(3600))), ); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); for millis in 0..FRAMES as u64 { group .write_frame(crate::Timestamp::from_millis(millis).unwrap(), vec![0u8; PAYLOAD]) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let serve = TrackServe::new( crate::driver::Transport::new(session.clone()), subscriber, @@ -9013,12 +9010,14 @@ mod group_priority_test { None, ); - let group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let group = track.create_group(group::Info { start: 0 }).unwrap(); assert!(futures::poll!(kio::wait(|waiter| serve.poll(waiter))).is_pending()); // A later group is served meanwhile, its stream independent of the one waiting. - let mut later = track.create_group(group::Info { sequence: 1 }).unwrap(); - later.write_frame(None, b"later".as_slice()).unwrap(); - later.finish().unwrap(); + let mut later = track.create_group(group::Info { start: 1 }).unwrap(); + later + .write_frame(crate::Timestamp::unscaled(later.start), b"later".as_slice()) + .unwrap(); + later.close().unwrap(); for _ in 0..8 { assert!(futures::poll!(kio::wait(|waiter| serve.poll(waiter))).is_pending()); kio::clock::sim::yield_now().await; @@ -9041,7 +9040,7 @@ mod group_priority_test { .create_object(frame::Header { info: frame::Info { size: 1, - timestamp: None, + timestamp: crate::Timestamp::unscaled(0), }, fields: Some(std::sync::Arc::new(crate::model::object::Fields { id: 3, @@ -9051,7 +9050,7 @@ mod group_priority_test { }) .unwrap(); frame.write(b"x".as_slice()).unwrap(); - frame.finish().unwrap().finish().unwrap(); + frame.finish().unwrap().close().unwrap(); let _finished = track.finish().unwrap(); kio::wait(|waiter| serve.poll(waiter)).await.unwrap(); @@ -9081,9 +9080,9 @@ mod group_priority_test { let track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "test", info); let subscriber = track.subscribe(None); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"hello".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"hello".as_slice()).unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let mut serve = TrackServe::new( @@ -9153,17 +9152,17 @@ mod group_priority_test { }) .collect(); - for sequence in 0..groups { + for key in 0..groups { for (track, serve) in &mut serves { - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group - .write_frame(crate::Timestamp::from_millis(sequence).unwrap(), b"x".as_slice()) + .write_frame(crate::Timestamp::from_millis(key).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); settle_serve(serve).await; } let held = serves.iter().map(|(_, serve)| serve.machines() as u64).sum::(); - assert_eq!(held, tracks, "{tracks} tracks after {} groups each", sequence + 1); + assert_eq!(held, tracks, "{tracks} tracks after {} groups each", key + 1); } *gate.write().ok().unwrap() = true; @@ -9197,8 +9196,8 @@ mod group_priority_test { Some(Timescale::MILLI), ); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"x".as_slice()).unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); settle_serve(&mut serve).await; assert_eq!(serve.machines(), 1, "group 0 waits for credit"); @@ -9206,10 +9205,10 @@ mod group_priority_test { settle_serve(&mut serve).await; assert_eq!(serve.machines(), 0, "the evicted group leaves without credit"); - let mut next = track.append_group().unwrap(); + let mut next = track.append_next().unwrap(); next.write_frame(crate::Timestamp::from_millis(1).unwrap(), b"x".as_slice()) .unwrap(); - next.finish().unwrap(); + next.close().unwrap(); settle_serve(&mut serve).await; assert_eq!(serve.machines(), 1, "group 1 takes its place"); @@ -9243,7 +9242,7 @@ mod group_priority_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(crate::Timestamp::ZERO, vec![b'x'; FRAME]).unwrap(); let meter = registry.publisher_meter(crate::Path::from_static("demo"), &crate::stats::Tier::default()); let consumer = subscriber @@ -9311,9 +9310,9 @@ mod group_priority_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut old = track.append_group().unwrap(); - old.write_frame(crate::Timestamp::ZERO, b"old".as_slice()).unwrap(); - old.finish().unwrap(); + let mut old = track.append_next().unwrap(); + old.write_frame(old.key_time(), b"old".as_slice()).unwrap(); + old.close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("old group"); let mut serve = GroupServe::new( @@ -9338,10 +9337,10 @@ mod group_priority_test { ); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(crate::Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(serving.await, Err(Error::Old))); } @@ -9357,7 +9356,7 @@ mod group_priority_test { track::Info::default(), ); let mut subscriber = track.subscribe(None); - let _stalled = track.create_group(group::Info { sequence: 0 }).unwrap(); + let _stalled = track.create_group(group::Info { start: 0 }).unwrap(); let group = subscriber.recv_group().await.unwrap().expect("stalled group"); let mut serve = GroupServe::new( @@ -9381,9 +9380,10 @@ mod group_priority_test { "the relay state is unsettled" ); - let mut edge = track.create_group(group::Info { sequence: 1 }).unwrap(); - edge.write_frame(None, b"edge".as_slice()).unwrap(); - edge.finish().unwrap(); + let mut edge = track.create_group(group::Info { start: 1 }).unwrap(); + edge.write_frame(crate::Timestamp::unscaled(edge.start), b"edge".as_slice()) + .unwrap(); + edge.close().unwrap(); assert!(matches!(serving.await, Err(Error::Old))); } @@ -9399,12 +9399,12 @@ mod group_priority_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut producer = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut producer = track.create_group(group::Info { start: 0 }).unwrap(); producer.start_at(1).unwrap(); producer .write_frame(crate::Timestamp::ZERO, b"tail".as_slice()) .unwrap(); - producer.finish().unwrap(); + producer.close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("the group"); let mut serve = GroupServe::new( @@ -9439,9 +9439,9 @@ mod group_priority_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut old = track.append_group().unwrap(); - old.write_frame(crate::Timestamp::ZERO, b"old".as_slice()).unwrap(); - old.finish().unwrap(); + let mut old = track.append_next().unwrap(); + old.write_frame(old.key_time(), b"old".as_slice()).unwrap(); + old.close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("old group"); let mut serve = GroupServe::new( @@ -9467,10 +9467,10 @@ mod group_priority_test { assert!(log.resets().is_empty()); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(crate::Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(futures::poll!(serving.as_mut()), Poll::Ready(Err(Error::Old)))); assert_eq!(log.resets(), vec![crate::StreamError::Cancel.to_code()]); @@ -9486,10 +9486,10 @@ mod group_priority_test { "test", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"done".as_slice()).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"done".as_slice()).unwrap(); let consumer = group.consume(); - group.finish().unwrap(); + group.close().unwrap(); // Another subscription's group at 22 sits in the same queue, so each update // moves this stream across it. @@ -9538,10 +9538,10 @@ mod group_priority_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let old = track.append_group().unwrap(); + let old = track.append_next().unwrap(); let mut frame = old .create_frame(frame::Info { - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::ZERO, size: 2, }) .unwrap(); @@ -9572,17 +9572,17 @@ mod group_priority_test { *open = false; drop(open); frame.write(b"b".as_slice()).unwrap(); - frame.finish().unwrap().finish().unwrap(); + frame.finish().unwrap().close().unwrap(); assert!( futures::poll!(serving.as_mut()).is_pending(), "the final byte is transport-blocked" ); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(crate::Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(serving.await, Err(Error::Old))); } @@ -9606,12 +9606,10 @@ mod subscribe_cursor_test { "video", crate::track::Info::timed(), ); - for sequence in 0..4 { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group - .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"frame".as_slice()) - .unwrap(); - group.finish().unwrap(); + for key in 0..4 { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); } let subscriber = track.subscribe(None); @@ -9865,9 +9863,9 @@ mod serve_tests { "{version}" ); if !super::super::request_stream::fin_cancels(version) { - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"after FIN".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"after FIN".as_slice()).unwrap(); + group.close().unwrap(); assert!(futures::poll!(run.as_mut()).is_pending()); assert_eq!(occurrences(&h.log, b"after FIN"), 1, "{version}"); } @@ -10046,8 +10044,8 @@ mod serve_tests { )); // An unfinished group keeps its stream open. - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"open".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"open".as_slice()).unwrap(); assert!(futures::poll!(run.as_mut()).is_pending()); let before = h.log.priorities(); assert_eq!(before, [254], "below the other subscription"); @@ -10213,8 +10211,8 @@ mod serve_tests { async fn publish_done_after_a_track_error_follows_every_stream_close() { let version = Version::Draft22; let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"head".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"head".as_slice()).unwrap(); let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) .await @@ -10274,8 +10272,8 @@ mod serve_tests { async fn publish_done_at_the_track_end_counts_and_follows_every_stream() { let version = Version::Draft22; let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"head".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"head".as_slice()).unwrap(); let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) .await @@ -10287,7 +10285,7 @@ mod serve_tests { }) .await; - group.finish().unwrap(); + group.close().unwrap(); h.track.finish_shared().unwrap(); kio::clock::timeout(std::time::Duration::from_secs(10), serve) .await @@ -10328,9 +10326,9 @@ mod serve_tests { // Frozen timestamps, so only the wall clock since group 1 arrived convicts group 0. let append = || { - let mut group = h.track.append_group().unwrap(); - group.write_frame(timestamp(), b"x".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.append_next().unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); + group.close().unwrap(); }; let finished = || { data_streams(&h.log) @@ -10433,10 +10431,10 @@ mod serve_tests { /// inside it and the group stream waits for object 1. async fn fail_update_mid_flight(version: Version) -> Wire { let h = serve(version); - let group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); + let group = h.track.create_group(group::Info { start: 0 }).unwrap(); let mut frame = group .create_frame(frame::Info { - timestamp: Some(timestamp()), + timestamp: timestamp(), size: 4, }) .unwrap(); @@ -10555,11 +10553,11 @@ mod serve_tests { .count(); (ok, answers) }; - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); let mut paused = 0; if !start_paused { serve_until(serve.as_mut(), "SUBSCRIBE_OK", || answers().0).await; - group.write_frame(timestamp(), b"before".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"before".as_slice()).unwrap(); serve_until(serve.as_mut(), "the object", || sent(b"before") == 1).await; request.push(&forward_update(version, 2, false)); paused = 1; @@ -10569,13 +10567,13 @@ mod serve_tests { }) .await; - group.write_frame(timestamp(), b"paused".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"paused".as_slice()).unwrap(); request.push(&forward_update(version, 4, true)); serve_until(serve.as_mut(), "the resume", || { answers().1 == paused + 1 && h.track.subscription().is_some() }) .await; - group.write_frame(timestamp(), b"after".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"after".as_slice()).unwrap(); serve_until(serve.as_mut(), "the object after", || sent(b"after") == 1).await; assert_eq!(sent(b"paused"), 0, "{version}: served what it missed"); } @@ -10587,9 +10585,11 @@ mod serve_tests { let broadcast = crate::broadcast::Producer::new(Default::default()); let track = broadcast.create_track("audio", track::Info::default()).unwrap(); let subscriber = track.subscribe(Subscription::default().with_start(0).with_end(1)); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(None, &b"bounded"[..]).unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"bounded"[..]) + .unwrap(); + group.close().unwrap(); let mut datagrams = broadcast.create_datagrams("audio", Default::default()).unwrap(); let consumer = broadcast.consume().datagrams("audio").unwrap(); let session = ScriptedSession::new(Vec::new()).with_datagrams(1200); @@ -10613,7 +10613,7 @@ mod serve_tests { serving.track.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Ok(None)) )); - datagrams.append(None, &b"live"[..]).unwrap(); + datagrams.append(crate::Timestamp::unscaled(0), &b"live"[..]).unwrap(); assert!(serving.poll(&kio::Waiter::noop()).is_pending()); assert_eq!(session.log.datagrams.lock().unwrap().len(), 1); } @@ -10643,7 +10643,7 @@ mod serve_tests { let mut serve = std::pin::pin!(h.publisher.clone().run_subscribe_stream(stream, msg)); serve_until(serve.as_mut(), "SUBSCRIBE_OK", || !h.log.trail().is_empty()).await; if !start_paused { - datagrams.append(None, &b"before"[..]).unwrap(); + datagrams.append(crate::Timestamp::unscaled(0), &b"before"[..]).unwrap(); serve_until(serve.as_mut(), "the datagram", || { h.log.datagrams.lock().unwrap().len() == 1 }) @@ -10652,10 +10652,10 @@ mod serve_tests { } serve_until(serve.as_mut(), "demand released", || datagrams.priority().is_none()).await; let sent = h.log.datagrams.lock().unwrap().len(); - datagrams.append(None, &b"paused"[..]).unwrap(); + datagrams.append(crate::Timestamp::unscaled(1), &b"paused"[..]).unwrap(); request.push(&forward_update(version, 4, true)); serve_until(serve.as_mut(), "resumed demand", || datagrams.priority().is_some()).await; - datagrams.append(None, &b"after"[..]).unwrap(); + datagrams.append(crate::Timestamp::unscaled(2), &b"after"[..]).unwrap(); serve_until(serve.as_mut(), "the resumed datagram", || { h.log.datagrams.lock().unwrap().len() == sent + 1 }) @@ -10677,8 +10677,8 @@ mod serve_tests { async fn a_joining_fetch_after_a_resume_is_served() { let version = Version::Draft18; let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"head".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"head".as_slice()).unwrap(); let mut request = ScriptedSession::new(Vec::new()); request.log = h.log.clone(); let stream = Stream::open(&crate::driver::Transport::new(request.clone()), version) @@ -10781,9 +10781,9 @@ mod serve_tests { /// Group 4 of `h`'s track: a 3 byte first frame and a 5 byte delta, over a refresh /// ratio of 1. fn refresh_group(h: &Serve) -> group::Producer { - let mut group = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - group.write_frame(Timestamp::ZERO, b"key".as_slice()).unwrap(); - group.write_frame(Timestamp::ZERO, b"delta".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 4 }).unwrap(); + group.write_frame(group.key_time(), b"key".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"delta".as_slice()).unwrap(); group } @@ -11901,9 +11901,9 @@ mod serve_tests { for version in ALL_VERSIONS { for wanted in [true, false] { let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - group.write_frame(timestamp(), b"a".as_slice()).unwrap(); - group.write_frame(timestamp(), b"b".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 4 }).unwrap(); + group.write_frame(group.key_time(), b"a".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"b".as_slice()).unwrap(); kio::clock::sleep(std::time::Duration::from_millis(1)).await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) @@ -12162,9 +12162,9 @@ mod serve_tests { for wanted in [true, false] { let mut h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); group.write_frame(stamp, b"stamped".as_slice()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut msg = subscribe(Filter::Unfiltered, None); msg.properties_wanted = wanted; @@ -12462,15 +12462,15 @@ mod serve_tests { async fn canonical_join_serves_the_head_on_a_fetch_stream() { for filter in [Filter::NextObject, Filter::Unfiltered] { let mut h = serve(Version::Draft22); - let mut earlier = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - earlier.write_frame(timestamp(), b"earlier".as_slice()).unwrap(); - earlier.finish().unwrap(); + let mut earlier = h.track.create_group(group::Info { start: 0 }).unwrap(); + earlier.write_frame(earlier.key_time(), b"earlier".as_slice()).unwrap(); + earlier.close().unwrap(); - let mut group = h.track.create_group(group::Info { sequence: 1 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 1 }).unwrap(); for payload in [b"head-0", b"head-1", b"head-2"] { - group.write_frame(timestamp(), payload.as_slice()).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); run_live( &mut h, @@ -12520,8 +12520,10 @@ mod serve_tests { for (start, end, wanted) in narrowed { let version = Version::Draft22; let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 1 }).unwrap(); - group.write_frame(timestamp(), b"before-update".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 1 }).unwrap(); + group + .write_frame(group.key_time(), b"before-update".as_slice()) + .unwrap(); kio::clock::sim::yield_now().await; let mut request = ScriptedSession::new(Vec::new()); request.log = h.log.clone(); @@ -12556,11 +12558,11 @@ mod serve_tests { Wire::read(&h.log, version).sent(ietf::RequestOk::ID).is_some() }) .await; - group.write_frame(timestamp(), b"after-update".as_slice()).unwrap(); - group.finish().unwrap(); - let mut later = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - later.write_frame(timestamp(), b"wanted".as_slice()).unwrap(); - later.finish().unwrap(); + group.write_frame(group.key_time(), b"after-update".as_slice()).unwrap(); + group.close().unwrap(); + let mut later = h.track.create_group(group::Info { start: 4 }).unwrap(); + later.write_frame(later.key_time(), b"wanted".as_slice()).unwrap(); + later.close().unwrap(); h.track.finish_shared().unwrap(); serving.await.unwrap(); assert_eq!( @@ -12601,11 +12603,13 @@ mod serve_tests { for (filter, narrowing) in cases { let version = Version::Draft22; let h = serve(version); - let mut history = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - history.write_frame(timestamp(), b"history".as_slice()).unwrap(); - history.finish().unwrap(); - let mut group = h.track.create_group(group::Info { sequence: 1 }).unwrap(); - group.write_frame(timestamp(), b"before-update".as_slice()).unwrap(); + let mut history = h.track.create_group(group::Info { start: 0 }).unwrap(); + history.write_frame(history.key_time(), b"history".as_slice()).unwrap(); + history.close().unwrap(); + let mut group = h.track.create_group(group::Info { start: 1 }).unwrap(); + group + .write_frame(group.key_time(), b"before-update".as_slice()) + .unwrap(); kio::clock::sim::yield_now().await; let mut request = ScriptedSession::new(Vec::new()); request.log = h.log.clone(); @@ -12655,9 +12659,9 @@ mod serve_tests { .await; } for payload in [b"after-1", b"after-2"] { - group.write_frame(timestamp(), payload.as_slice()).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); h.track.finish_shared().unwrap(); serving.await.unwrap(); assert_eq!( @@ -12687,8 +12691,8 @@ mod serve_tests { let credit = kio::Producer::new(false); let session = ScriptedSession::per_stream(vec![Vec::new()]).with_open_uni_gate(credit.consume()); let h = serve_session(version, crate::track::Info::timed(), session); - let mut group = h.track.create_group(group::Info { sequence: 1 }).unwrap(); - group.write_frame(timestamp(), b"head".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 1 }).unwrap(); + group.write_frame(group.key_time(), b"head".as_slice()).unwrap(); kio::clock::sim::yield_now().await; let mut request = ScriptedSession::new(Vec::new()); request.log = h.log.clone(); @@ -12748,8 +12752,8 @@ mod serve_tests { Wire::read(&h.log, version).sent(ietf::RequestOk::ID).is_some() }) .await; - group.write_frame(timestamp(), b"tail".as_slice()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"tail".as_slice()).unwrap(); + group.close().unwrap(); serving.await.unwrap(); assert_eq!(occurrences(&h.log, b"tail"), 1, "the update's tail is served"); } @@ -12760,11 +12764,11 @@ mod serve_tests { async fn a_fill_only_update_keeps_the_live_join_floor() { let version = Version::Draft22; let h = serve(version); - let mut history = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - history.write_frame(timestamp(), b"history".as_slice()).unwrap(); - history.finish().unwrap(); - let mut group = h.track.create_group(group::Info { sequence: 1 }).unwrap(); - group.write_frame(timestamp(), b"live".as_slice()).unwrap(); + let mut history = h.track.create_group(group::Info { start: 0 }).unwrap(); + history.write_frame(history.key_time(), b"history".as_slice()).unwrap(); + history.close().unwrap(); + let mut group = h.track.create_group(group::Info { start: 1 }).unwrap(); + group.write_frame(group.key_time(), b"live".as_slice()).unwrap(); kio::clock::sim::yield_now().await; let mut request = ScriptedSession::new(Vec::new()); request.log = h.log.clone(); @@ -12806,7 +12810,7 @@ mod serve_tests { Wire::read(&h.log, version).sent(ietf::RequestOk::ID).is_some() }) .await; - group.finish().unwrap(); + group.close().unwrap(); h.track.finish_shared().unwrap(); serving.await.unwrap(); assert_eq!(occurrences(&h.log, b"history"), 0, "the historical fill is refused"); @@ -12819,8 +12823,8 @@ mod serve_tests { for end in [None, Some(EndLocation { group: 0, object: None })] { let version = Version::Draft22; let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 1 }).unwrap(); - group.write_frame(timestamp(), b"history".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 1 }).unwrap(); + group.write_frame(group.key_time(), b"history".as_slice()).unwrap(); kio::clock::sim::yield_now().await; let mut request = ScriptedSession::new(Vec::new()); request.log = h.log.clone(); @@ -12870,12 +12874,14 @@ mod serve_tests { Wire::read(&h.log, version).sent(ietf::RequestOk::ID).is_some() }) .await; - group.write_frame(timestamp(), b"later-current".as_slice()).unwrap(); - group.finish().unwrap(); - for (sequence, payload) in [(2, b"excluded".as_slice()), (4, b"wanted".as_slice())] { - let mut group = h.track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(timestamp(), payload).unwrap(); - group.finish().unwrap(); + group + .write_frame(group.key_time(), b"later-current".as_slice()) + .unwrap(); + group.close().unwrap(); + for (key, payload) in [(2, b"excluded".as_slice()), (4, b"wanted".as_slice())] { + let mut group = h.track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), payload).unwrap(); + group.close().unwrap(); } h.track.finish_shared().unwrap(); serving.await.unwrap(); @@ -12896,11 +12902,11 @@ mod serve_tests { async fn relative_one_replays_the_current_group() { let mut h = serve(Version::Draft22); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); for payload in [b"head-0", b"head-1", b"head-2"] { - group.write_frame(timestamp(), payload.as_slice()).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); run_live(&mut h, subscribe(Filter::Relative(1), None)).await; @@ -12916,11 +12922,11 @@ mod serve_tests { async fn next_object_does_not_replay_the_head() { let mut h = serve(Version::Draft22); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); for payload in [b"head-0", b"head-1", b"head-2"] { - group.write_frame(timestamp(), payload.as_slice()).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); run_live(&mut h, subscribe(Filter::NextObject, None)).await; @@ -12945,10 +12951,10 @@ mod serve_tests { for version in [Version::Draft18, Version::Draft20, Version::Draft22] { for (filter, history) in [(Filter::Unfiltered, 0), (zero, 1)] { let mut h = serve(version); - for (sequence, payload) in [(0, b"history"), (1, b"latest!")] { - let mut group = h.track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(timestamp(), payload.as_slice()).unwrap(); - group.finish().unwrap(); + for (key, payload) in [(0, b"history"), (1, b"latest!")] { + let mut group = h.track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); + group.close().unwrap(); } run_live(&mut h, subscribe(filter, None)).await; @@ -12980,10 +12986,10 @@ mod serve_tests { .broadcast() .create_track("video", crate::track::Info::timed().with_max_age(None)) .unwrap(); - for (sequence, payload) in [(0, b"history"), (1, b"latest!")] { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(timestamp(), payload.as_slice()).unwrap(); - group.finish().unwrap(); + for (key, payload) in [(0, b"history"), (1, b"latest!")] { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); + group.close().unwrap(); } let (relay, _sessions) = crate::lite::ietf::test_hop(&upstream, "room", [false, false]).await; @@ -13010,7 +13016,7 @@ mod serve_tests { .expect("no group") .unwrap() .expect("track finished"); - assert_eq!(group.sequence, expected, "{version} {filter:?}"); + assert_eq!(group.start, expected, "{version} {filter:?}"); } let session = ScriptedSession::per_stream(vec![Vec::new()]); @@ -13046,10 +13052,10 @@ mod serve_tests { async fn a_multi_group_fill_is_refused_without_ending_the_subscription() { let mut h = serve(Version::Draft22); - for sequence in 0..2 { - let mut group = h.track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + for key in 0..2 { + let mut group = h.track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); } run_live( @@ -13090,8 +13096,8 @@ mod serve_tests { #[kio::test] async fn a_fill_ranks_as_the_group_it_fills() { let h = serve(Version::Draft22); - let mut group = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - group.write_frame(timestamp(), b"fill".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 4 }).unwrap(); + group.write_frame(group.key_time(), b"fill".as_slice()).unwrap(); let queue = h.publisher.priority.clone(); let priority = kio::Producer::new(PRIORITY); @@ -13110,7 +13116,7 @@ mod serve_tests { let opened = AtomicU64::new(0); let fill = FillServe::Group { - sequence: 4, + key: 4, skip: 0, until: None, }; @@ -13142,7 +13148,7 @@ mod serve_tests { "a REQUEST_UPDATE moved it to the top" ); - group.finish().unwrap(); + group.close().unwrap(); fill.await; assert_eq!(occurrences(&h.log, b"fill"), 1); assert!(h.log.resets().is_empty(), "a served fill must not reset"); @@ -13153,8 +13159,8 @@ mod serve_tests { #[kio::test] async fn a_fill_outranks_the_rest_of_its_group() { let h = serve(Version::Draft22); - let mut group = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - group.write_frame(timestamp(), b"head".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 4 }).unwrap(); + group.write_frame(group.key_time(), b"head".as_slice()).unwrap(); let ranking = Ranking { queue: h.publisher.priority.clone(), @@ -13173,7 +13179,7 @@ mod serve_tests { let opened = AtomicU64::new(0); let fill = FillServe::Group { - sequence: 4, + key: 4, skip: 0, until: Some(1), }; @@ -13270,10 +13276,10 @@ mod serve_tests { /// Fill the track with finished groups 0..=`latest`, so the live edge is a group past 0 /// and a `{0, 0}` End Location would be below the subscriber's own Start. fn publish_groups(h: &mut Serve, latest: u64) { - for sequence in 0..=latest { - let mut group = h.track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + for key in 0..=latest { + let mut group = h.track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); } } @@ -13356,7 +13362,7 @@ mod serve_tests { } }) .await; - let group = h.track.create_group(group::Info { sequence: 3 }).unwrap(); + let group = h.track.create_group(group::Info { start: 3 }).unwrap(); group .set_wire(group::Wire { sparse: true, @@ -13374,7 +13380,7 @@ mod serve_tests { .create_object(frame::Header { info: frame::Info { size: payload.len() as u64, - timestamp: Some(timestamp()), + timestamp: timestamp(), }, fields: Some(std::sync::Arc::new(crate::model::object::Fields { id: 5, @@ -13387,7 +13393,7 @@ mod serve_tests { if streamed { assert!(futures::poll!(serving.as_mut()).is_pending()); } - frame.finish().unwrap().finish().unwrap(); + frame.finish().unwrap().close().unwrap(); h.track.finish_shared().unwrap(); served(&h, serving.as_mut()).await; assert_eq!(occurrences(&h.log, payload), 1, "{version} streamed={streamed}"); @@ -13452,10 +13458,10 @@ mod serve_tests { for version in JOINING_DRAFTS { let mut h = serve(version); publish_groups(&mut h, LATEST - 1); - let mut group = h.track.create_group(group::Info { sequence: LATEST }).unwrap(); + let mut group = h.track.create_group(group::Info { start: LATEST }).unwrap(); let payloads: &[&[u8]] = &[b"first-object", b"", b"third-object"]; for payload in payloads { - group.write_frame(timestamp(), *payload).unwrap(); + group.write_frame(group.key_time(), *payload).unwrap(); } settle().await; @@ -13476,7 +13482,7 @@ mod serve_tests { 0, "subscription replayed the prefix" ); - group.write_frame(timestamp(), b"new-object".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"new-object".as_slice()).unwrap(); let mark = h.log.writes.lock().unwrap().len(); let response = joining_fetch(&h, mark).await.unwrap(); @@ -13674,10 +13680,10 @@ mod serve_tests { async fn a_joining_fetch_refuses_a_missing_prefix_before_fetch_ok() { let version = Version::Draft18; let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 5 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 5 }).unwrap(); group.start_at(1).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) .await @@ -13710,8 +13716,8 @@ mod serve_tests { async fn a_joining_fetch_refuses_a_prefix_group_it_lost() { let version = Version::Draft18; let h = serve(version); - let mut prefix = h.track.create_group(group::Info { sequence: 5 }).unwrap(); - prefix.write_frame(timestamp(), b"frame".as_slice()).unwrap(); + let mut prefix = h.track.create_group(group::Info { start: 5 }).unwrap(); + prefix.write_frame(prefix.key_time(), b"frame".as_slice()).unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) .await @@ -13831,9 +13837,9 @@ mod serve_tests { for version in JOINING_DRAFTS { let mut h = serve(version); publish_groups(&mut h, 4); - let mut group = h.track.create_group(group::Info { sequence: 5 }).unwrap(); - group.write_frame(timestamp(), b"the-head".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 5 }).unwrap(); + group.write_frame(group.key_time(), b"the-head".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) @@ -13928,9 +13934,9 @@ mod serve_tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".to_vec()).unwrap(); + group.close().unwrap(); settle().await; let bi = kio::Producer::new(true); @@ -14150,8 +14156,8 @@ mod serve_tests { serve_until(serving.as_mut(), "SUBSCRIBE_OK was sent", || !h.log.trail().is_empty()).await; // A group left open, so its stream is still serving when the grant ends. - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); serve_until(serving.as_mut(), "the group stream opened", || { !data_streams(&h.log).is_empty() }) @@ -14296,14 +14302,14 @@ mod serve_tests { /// Groups `0..count`, each holding `g-0` and `g-1`, skipping `hole`. fn publish_pairs(h: &mut Serve, count: u64, hole: Option) { - for sequence in (0..count).filter(|sequence| Some(*sequence) != hole) { - let mut group = h.track.create_group(group::Info { sequence }).unwrap(); + for key in (0..count).filter(|key| Some(*key) != hole) { + let mut group = h.track.create_group(group::Info { start: key }).unwrap(); for object in 0..2 { group - .write_frame(timestamp(), format!("{sequence}-{object}").into_bytes()) + .write_frame(group.key_time(), format!("{key}-{object}").into_bytes()) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } } @@ -14320,11 +14326,11 @@ mod serve_tests { async fn a_cached_standalone_fetch_yields_while_writing() { for version in FETCH_DRAFTS { let h = serve(version); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); for _ in 0..256 { - group.write_frame(timestamp(), vec![7; 1024]).unwrap(); + group.write_frame(group.key_time(), vec![7; 1024]).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); settle().await; let done = std::rc::Rc::new(std::cell::Cell::new(false)); @@ -14719,9 +14725,9 @@ mod serve_tests { for wanted in [Some(true), opt_out].into_iter().flatten() { let h = serve(version); let track = h._broadcast.broadcast().create_track("timed", info.clone()).unwrap(); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); group.write_frame(stamp, b"stamped".as_slice()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); settle().await; let range = fetch_range( @@ -14812,9 +14818,9 @@ mod serve_tests { ..track::Info::timed() }, ); - let mut group = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - group.write_frame(timestamp(), b"fill".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 4 }).unwrap(); + group.write_frame(group.key_time(), b"fill".as_slice()).unwrap(); + group.close().unwrap(); let ranking = Ranking { queue: h.publisher.priority.clone(), @@ -14827,7 +14833,7 @@ mod serve_tests { }; let opened = AtomicU64::new(0); let fill = FillServe::Group { - sequence: 4, + key: 4, skip: 0, until: None, }; @@ -14880,9 +14886,9 @@ mod serve_tests { ); let h = serve_track(version, info.clone()); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"a".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"a".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let range = fetch_range( version, @@ -15100,7 +15106,7 @@ mod serve_tests { h.track.finish_shared().unwrap(); None } - Largest::Live => Some(h.track.create_group(group::Info { sequence: 5 }).unwrap()), + Largest::Live => Some(h.track.create_group(group::Info { start: 5 }).unwrap()), Largest::Idle => { h.track.set_idle(); None @@ -15116,8 +15122,8 @@ mod serve_tests { None } Largest::Aborted => { - let mut newer = h.track.create_group(group::Info { sequence: 5 }).unwrap(); - newer.write_frame(timestamp(), b"5-0".to_vec()).unwrap(); + let mut newer = h.track.create_group(group::Info { start: 5 }).unwrap(); + newer.write_frame(newer.key_time(), b"5-0".to_vec()).unwrap(); newer.abort(Error::Cancel).unwrap(); None } @@ -15180,8 +15186,8 @@ mod serve_tests { let version = Version::Draft22; let mut h = serve(version); publish_pairs(&mut h, 5, None); - let empty = h.track.create_group(group::Info { sequence: 5 }).unwrap(); - empty.finish().unwrap(); + let empty = h.track.create_group(group::Info { start: 5 }).unwrap(); + empty.close().unwrap(); settle().await; let buf = standalone_fetch( @@ -15244,9 +15250,14 @@ mod serve_tests { let fetching = request.consume().fetch_group(4, None); let mut fetched = dynamic.requested_fill_group().await.unwrap().accept(None).unwrap(); for object in 0..2 { - fetched.write_frame(None, format!("4-{object}").into_bytes()).unwrap(); + fetched + .write_frame( + crate::Timestamp::unscaled(fetched.start), + format!("4-{object}").into_bytes(), + ) + .unwrap(); } - fetched.finish().unwrap(); + fetched.close().unwrap(); fetching.await.unwrap(); let fetch_type = fetch_range( @@ -15406,9 +15417,9 @@ mod serve_tests { for version in FETCH_DRAFTS { let mut h = serve(version); publish_pairs(&mut h, 4, None); - let mut live = h.track.create_group(group::Info { sequence: 4 }).unwrap(); + let mut live = h.track.create_group(group::Info { start: 4 }).unwrap(); for object in 0..2 { - live.write_frame(timestamp(), format!("4-{object}").into_bytes()) + live.write_frame(live.key_time(), format!("4-{object}").into_bytes()) .unwrap(); } settle().await; @@ -15444,18 +15455,18 @@ mod serve_tests { for version in FETCH_DRAFTS { let mut h = serve(version); publish_pairs(&mut h, 4, None); - let mut older = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - older.write_frame(timestamp(), b"4-0".to_vec()).unwrap(); - let mut newer = h.track.create_group(group::Info { sequence: 5 }).unwrap(); - newer.write_frame(timestamp(), b"5-0".to_vec()).unwrap(); + let mut older = h.track.create_group(group::Info { start: 4 }).unwrap(); + older.write_frame(older.key_time(), b"4-0".to_vec()).unwrap(); + let mut newer = h.track.create_group(group::Info { start: 5 }).unwrap(); + newer.write_frame(newer.key_time(), b"5-0".to_vec()).unwrap(); settle().await; let whole = Location { group: 4, object: 0 }; let fetch = standalone_fetch(&h, whole, whole, GroupOrder::Ascending); let finish = async { settle().await; - older.write_frame(timestamp(), b"4-1".to_vec()).unwrap(); - older.finish().unwrap(); + older.write_frame(older.key_time(), b"4-1".to_vec()).unwrap(); + older.close().unwrap(); }; let (buf, ()) = futures::join!(fetch, finish); let (ok, objects) = fetch_answer(buf, version); @@ -15550,7 +15561,7 @@ mod serve_tests { for version in FETCH_DRAFTS { // The group is still being written, so the fetch waits on it when the grant narrows. let h = serve(version); - let _group = h.track.clone().create_group(group::Info { sequence: 0 }).unwrap(); + let _group = h.track.clone().create_group(group::Info { start: 0 }).unwrap(); settle().await; let auth = crate::auth::Link::new(false); let mut h = h; @@ -15589,9 +15600,9 @@ mod serve_tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"0-0".to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"0-0".to_vec()).unwrap(); + group.close().unwrap(); settle().await; // Request streams flow, but no uni stream credit is ever granted. @@ -15688,8 +15699,8 @@ mod serve_tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), PAYLOAD.to_vec()).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), PAYLOAD.to_vec()).unwrap(); settle().await; // Request streams flow, but no uni stream credit is granted until the end. @@ -15822,8 +15833,8 @@ mod serve_tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), PAYLOAD.to_vec()).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), PAYLOAD.to_vec()).unwrap(); settle().await; let bi = kio::Producer::new(true); @@ -15998,9 +16009,9 @@ mod serve_tests { let h = serve_on(VERSION, auth, session); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), ANSWERED_HEAD.to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), ANSWERED_HEAD.to_vec()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -16165,9 +16176,9 @@ mod serve_tests { match parked { // One published group, so the subscription parks at the live edge. Parked::Acking => { - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".to_vec()).unwrap(); + group.close().unwrap(); } // An empty finished track: nothing to serve but the marker. _ => { @@ -16734,8 +16745,8 @@ mod serve_tests { for version in JOINING_DRAFTS { let mut h = serve(version); publish_pairs(&mut h, 4, None); - let mut group = h.track.create_group(group::Info { sequence: 4 }).unwrap(); - group.write_frame(timestamp(), b"4-0".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 4 }).unwrap(); + group.write_frame(group.key_time(), b"4-0".as_slice()).unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) @@ -16865,11 +16876,11 @@ mod serve_tests { let mut h = serve(version); // A live edge in the middle of group 5: objects 0 through 3. - let mut group = h.track.create_group(group::Info { sequence: 5 }).unwrap(); + let mut group = h.track.create_group(group::Info { start: 5 }).unwrap(); for payload in [b"5-0", b"5-1", b"5-2", b"5-3"] { - group.write_frame(timestamp(), payload.as_slice()).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); run_live(&mut h, subscribe(Filter::Unfiltered, None)).await; @@ -16958,12 +16969,12 @@ mod serve_tests { "test", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); for payload in [b"aa", b"bb", b"cc", b"dd"] { - group.write_frame(timestamp(), payload.as_slice()).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } let consumer = group.consume(); - group.finish().unwrap(); + group.close().unwrap(); let mut serve = GroupServe::new( crate::driver::Transport::new(session.clone()), @@ -17019,11 +17030,11 @@ mod serve_tests { ); let subscriber = track.subscribe(None); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); for payload in [b"aa", b"bb", b"cc"] { - group.write_frame(crate::Timestamp::ZERO, payload.as_slice()).unwrap(); + group.write_frame(group.key_time(), payload.as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let range = ServeRange { @@ -17222,9 +17233,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, subscribe_update_with_token(VERSION, true).await); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17423,9 +17434,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, script); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17498,9 +17509,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, script); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17553,9 +17564,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, script); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17600,9 +17611,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, script); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17647,9 +17658,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, script); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17718,9 +17729,9 @@ mod serve_tests { let h = serve_with_auth(version, auth, script); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), version) @@ -17787,9 +17798,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, script); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17844,9 +17855,9 @@ mod serve_tests { cancel: impl FnOnce(), label: &str, ) { - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let renewal = std::sync::Arc::new(std::sync::Mutex::new(None)); @@ -17917,9 +17928,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, subscribe_update_with_token(VERSION, true).await); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -17991,9 +18002,9 @@ mod serve_tests { let h = serve_with_auth(VERSION, auth, subscribe_update_with_token(VERSION, true).await); let rt = h.publisher.runtime.clone(); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -18096,9 +18107,9 @@ mod serve_tests { auth.handle().authorize(&crate::auth::Grant::default()); let h = serve_with_auth(VERSION, auth, Vec::new()); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -18150,9 +18161,9 @@ mod serve_tests { auth.handle().authorize(limit); } let h = serve_with_auth(VERSION, auth.clone(), Vec::new()); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) @@ -18354,8 +18365,8 @@ mod serve_tests { let unlimited = crate::auth::Link::new(false); let h = serve_with_auth(VERSION, unlimited, Vec::new()); - let mut group = h.track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(timestamp(), b"frame".as_slice()).unwrap(); + let mut group = h.track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"frame".as_slice()).unwrap(); settle().await; let stream = Stream::open(&crate::driver::Transport::new(h.session.clone()), VERSION) .await @@ -18644,11 +18655,11 @@ mod tests { crate::track::Info::timed(), ); for millis in [0, 1000] { - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); group .write_frame(crate::Timestamp::from_millis(millis).unwrap(), b"frame".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let subscription = serving_subscription(128); @@ -18657,14 +18668,14 @@ mod tests { "no Max Age leaves the publisher's max age alone" ); let mut subscriber = producer.subscribe(subscription); - for sequence in [0, 1] { + for key in [0, 1] { let group = subscriber .recv_group() .now_or_never() .expect("retained group should be ready") .unwrap() .expect("track should remain open"); - assert_eq!(group.sequence, sequence); + assert_eq!(group.start, key); } } @@ -21147,7 +21158,7 @@ mod tests { /// floor, the fill cap, and the advertised LARGEST_OBJECT all agree on where it is. #[derive(Default, Clone, Copy, Debug, PartialEq, Eq)] struct LiveEdge { - /// The newest group sequence, `None` before any group exists. + /// The newest group key, `None` before any group exists. latest: Option, /// The precise Largest Object: the newest object held, skipping a newest group whose /// frames cannot be read. `None` when no object is held, in which case nothing is @@ -21185,12 +21196,12 @@ fn live_edge(track: &track::Consumer) -> LiveEdge { /// The live objects immediately readable in this cache. fn cached_edge(track: &track::Consumer) -> LiveEdge { - let Some(latest) = track.latest_sequence() else { + let Some(latest) = track.latest_key() else { return LiveEdge::default(); }; match track.peek_latest() { - Some(group) if group.sequence == latest => { + Some(group) if group.start == latest => { let largest = match group.last_object() { Some(object) => Some(Location { group: latest, object }), // A group with no frames yet has no objects, so the largest sits in an @@ -21234,22 +21245,22 @@ fn cached_edge(track: &track::Consumer) -> LiveEdge { } } -/// The last object below `sequence`: the nearest earlier cached group that has started a +/// The last object below `key`: the nearest earlier cached group that has started a /// frame, walked in cache order so legal gaps in the group numbering are crossed. Empty /// groups exist for at most the instant between creation and first frame, so the walk is /// one step in practice. A group evicted from the cache is not visible, which is fine: /// Largest Object is the track from this publisher's perspective, and that is the cache. -fn largest_before(track: &track::Consumer, sequence: u64) -> Option { - let mut sequence = sequence; +fn largest_before(track: &track::Consumer, key: u64) -> Option { + let mut key = key; loop { - let group = track.peek_before(sequence)?; + let group = track.peek_before(key)?; if let Some(object) = group.last_object() { return Some(Location { - group: group.sequence, + group: group.start, object, }); } - sequence = group.sequence; + key = group.start; } } @@ -21282,13 +21293,13 @@ impl ServeRange { return None; } let (start, end) = match fill? { - FillServe::Group { sequence, skip, until } => ( + FillServe::Group { key, skip, until } => ( Location { - group: sequence, + group: key, object: skip, }, EndLocation { - group: sequence, + group: key, object: until.and_then(|until| until.checked_sub(1)), }, ), @@ -21300,26 +21311,26 @@ impl ServeRange { .then_some(next) } - /// Whether this range drops objects a stream serving `slice` of group `sequence` would send. - fn excludes(&self, sequence: u64, slice: GroupSlice) -> bool { - let ours = self.slice(sequence); - self.start.is_some_and(|start| sequence < start.group) - || self.end.is_some_and(|end| sequence > end.group) + /// Whether this range drops objects a stream serving `slice` of group `key` would send. + fn excludes(&self, key: u64, slice: GroupSlice) -> bool { + let ours = self.slice(key); + self.start.is_some_and(|start| key < start.group) + || self.end.is_some_and(|end| key > end.group) || ours.skip > slice.skip || ours .until .is_some_and(|until| slice.until.is_none_or(|theirs| until < theirs)) } - /// The objects of group `sequence` this range selects; interior groups are served whole. - fn slice(&self, sequence: u64) -> GroupSlice { + /// The objects of group `key` this range selects; interior groups are served whole. + fn slice(&self, key: u64) -> GroupSlice { GroupSlice { skip: match self.start { - Some(start) if start.group == sequence => start.object, + Some(start) if start.group == key => start.object, _ => 0, }, until: match self.end { - Some(end) if end.group == sequence => end.object.map(|object| object.saturating_add(1)), + Some(end) if end.group == key => end.object.map(|object| object.saturating_add(1)), _ => None, }, } @@ -21344,7 +21355,10 @@ struct GroupSlice { } /// The Locations a single Location Filter selects, resolved against the live edge. -fn filter_range(filter: Filter, edge: LiveEdge) -> ServeRange { +/// +/// `back` counts groups back from a key, as [`track::Consumer::group_back`] does: keys are +/// group starts, so subtracting from one counts no groups. +fn filter_range(filter: Filter, edge: LiveEdge, back: impl Fn(u64, u64) -> Option) -> ServeRange { match filter { // No restriction. moq-lite starts at the beginning of the latest group, which is // the join point it is built around; a subscription passes objects as they are @@ -21359,17 +21373,20 @@ fn filter_range(filter: Filter, edge: LiveEdge) -> ServeRange { start: edge.next, end: None, }, - // `{Largest.Group + 1 - groups, 0}`: 0 is the next group and 1 is the current one. - // Counted from `Largest.Group`, which sits below the newest group while that - // group has no objects yet; only with no largest at all does the newest group - // stand in for it. + // `{Largest.Group + 1 - groups, 0}`, counted in groups: 0 is the next group and 1 is + // the current one. Counted from `Largest.Group`, which sits below the newest group + // while that group has no objects yet; only with no largest at all does the newest + // group stand in for it. One counting back past every group held starts at the oldest. Filter::Relative(groups) => ServeRange { start: edge .largest .map(|largest| largest.group) .or(edge.latest) .map(|group| Location { - group: group.saturating_add(1).saturating_sub(groups), + group: match groups.checked_sub(1) { + None => group.saturating_add(1), + Some(count) => back(group, count).unwrap_or(0), + }, object: 0, }), end: None, @@ -21389,11 +21406,7 @@ enum FillServe { /// A single group served from the cache: `skip` frames dropped from the front, and /// delivery stopping before `until` when set (the current group is capped at the /// Largest Object snapshot; a whole past group reads to its end). - Group { - sequence: u64, - skip: u64, - until: Option, - }, + Group { key: u64, skip: u64, until: Option }, /// Object filters that this publisher cannot serve. Unsupported, } @@ -21448,7 +21461,7 @@ fn fill_range(fill: ietf::Fill, subscription: Filter, largest: Option) } FillServe::Group { - sequence: start.group, + key: start.group, skip: start.object, until: end.object.map(|object| object.saturating_add(1)), } @@ -21479,6 +21492,11 @@ mod range_tests { } } + /// Counts groups back over contiguous keys, as a track holding every group does. + fn contiguous(latest: u64, back: u64) -> Option { + latest.checked_sub(back) + } + /// A live edge of group 100 whose current group has objects 0 through 4. const EDGE: LiveEdge = LiveEdge { latest: Some(100), @@ -21495,16 +21513,8 @@ mod range_tests { "video", crate::track::Info::timed(), ); - track - .create_group(group::Info { sequence: 7 }) - .unwrap() - .finish() - .unwrap(); - track - .create_group(group::Info { sequence: 8 }) - .unwrap() - .finish() - .unwrap(); + track.create_group(group::Info { start: 7 }).unwrap().close().unwrap(); + track.create_group(group::Info { start: 8 }).unwrap().close().unwrap(); // Next Group against a live edge of 8 asks for 9, which does not exist yet. let mut subscriber = track.subscribe(None); @@ -21515,13 +21525,9 @@ mod range_tests { ); // The group it asked for is what it gets once published. - track - .create_group(group::Info { sequence: 9 }) - .unwrap() - .finish() - .unwrap(); + track.create_group(group::Info { start: 9 }).unwrap().close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("group 9"); - assert_eq!(group.sequence, 9); + assert_eq!(group.start, 9); } /// An absolute start and end retain their bounds on every draft. @@ -21532,7 +21538,7 @@ mod range_tests { end: Some(EndLocation { group: 9, object: None }), }); assert_eq!( - filter_range(msg.filter, EDGE), + filter_range(msg.filter, EDGE, contiguous), ServeRange { start: Some(Location { group: 4, object: 0 }), end: Some(EndLocation { group: 9, object: None }) @@ -21545,7 +21551,7 @@ mod range_tests { #[test] fn an_unfiltered_subscription_stays_live() { let msg = subscribe(Filter::Unfiltered); - assert_eq!(filter_range(msg.filter, EDGE), ServeRange::default()); + assert_eq!(filter_range(msg.filter, EDGE, contiguous), ServeRange::default()); } /// An absolute start from group 0 is the request for history the absent filter is not. @@ -21556,7 +21562,7 @@ mod range_tests { end: None, }); assert_eq!( - filter_range(msg.filter, EDGE), + filter_range(msg.filter, EDGE, contiguous), ServeRange { start: Some(Location::default()), end: None @@ -21570,7 +21576,7 @@ mod range_tests { fn next_object_starts_past_the_largest_object() { let msg = subscribe(Filter::NextObject); assert_eq!( - filter_range(msg.filter, EDGE), + filter_range(msg.filter, EDGE, contiguous), ServeRange { start: Some(Location { group: 100, object: 5 }), end: None, @@ -21589,7 +21595,7 @@ mod range_tests { }; let msg = subscribe(Filter::NextObject); assert_eq!( - filter_range(msg.filter, edge), + filter_range(msg.filter, edge, contiguous), ServeRange { start: Some(Location { group: 101, object: 0 }), end: None, @@ -21604,7 +21610,7 @@ mod range_tests { for (groups, expected) in [(0, 101), (1, 100), (2, 99), (5, 96)] { let msg = subscribe(Filter::Relative(groups)); assert_eq!( - filter_range(msg.filter, EDGE), + filter_range(msg.filter, EDGE, contiguous), ServeRange { start: Some(Location { group: expected, @@ -21628,7 +21634,7 @@ mod range_tests { }; let msg = subscribe(Filter::Relative(1)); assert_eq!( - filter_range(msg.filter, edge), + filter_range(msg.filter, edge, contiguous), ServeRange { start: Some(Location { group: 0, object: 0 }), end: None, @@ -21641,7 +21647,7 @@ mod range_tests { fn relative_saturates_at_the_start() { let msg = subscribe(Filter::Relative(500)); assert_eq!( - filter_range(msg.filter, EDGE), + filter_range(msg.filter, EDGE, contiguous), ServeRange { start: Some(Location { group: 0, object: 0 }), end: None, @@ -21653,7 +21659,10 @@ mod range_tests { #[test] fn relative_without_an_edge_stays_live() { let msg = subscribe(Filter::Relative(3)); - assert_eq!(filter_range(msg.filter, LiveEdge::default()), ServeRange::default()); + assert_eq!( + filter_range(msg.filter, LiveEdge::default(), contiguous), + ServeRange::default() + ); } /// A group created but not yet written has no objects, so the largest sits in an @@ -21668,13 +21677,13 @@ mod range_tests { "video", crate::track::Info::timed(), ); - let mut first = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut first = track.create_group(group::Info { start: 0 }).unwrap(); for _ in 0..3 { first .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"frame".as_slice()) .unwrap(); } - let _open = track.create_group(group::Info { sequence: 1 }).unwrap(); + let _open = track.create_group(group::Info { start: 1 }).unwrap(); let edge = live_edge(&track.consume()); assert_eq!(edge.latest, Some(1)); @@ -21696,7 +21705,7 @@ mod range_tests { /// arrived is stale on arrival. #[test] fn received_objects_outdate_the_report_after_eviction() { - for (sequence, frames, reported_first) in [(3, 6, true), (3, 6, false), (9, 1, true), (9, 1, false)] { + for (key, frames, reported_first) in [(3, 6, true), (3, 6, false), (9, 1, true), (9, 1, false)] { let track = track::Producer::new( std::sync::Arc::new(crate::broadcast::Info::default()), "video", @@ -21705,7 +21714,7 @@ mod range_tests { if reported_first { track.report_object(3, 4); } - let mut newer = track.create_group(group::Info { sequence }).unwrap(); + let mut newer = track.create_group(group::Info { start: key }).unwrap(); for _ in 0..frames { newer .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"frame".as_slice()) @@ -21717,11 +21726,8 @@ mod range_tests { newer.abort(Error::Evicted).unwrap(); let edge = live_edge(&track.consume()); - assert_eq!(edge.latest, Some(sequence)); - assert_eq!( - edge.largest, None, - "group {sequence}: the stale report is not the largest" - ); + assert_eq!(edge.latest, Some(key)); + assert_eq!(edge.largest, None, "group {key}: the stale report is not the largest"); } } @@ -21742,21 +21748,21 @@ mod range_tests { }) }); let staged = request.stage(crate::track::Info::timed()); - for sequence in 0..3 { - let mut group = staged.create_group(group::Info { sequence }).unwrap(); + for key in 0..3 { + let mut group = staged.create_group(group::Info { start: key }).unwrap(); for _ in 0..2 { group - .write_frame(crate::Timestamp::from_millis(sequence).unwrap(), b"frame".as_slice()) + .write_frame(crate::Timestamp::from_millis(key).unwrap(), b"frame".as_slice()) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let _track = staged.commit(); assert_eq!(snapshot.join().unwrap(), Some(Location { group: 2, object: 1 })); } } - /// Group numbering may legally skip sequences, so the walk follows the cache's own + /// Group numbering may legally skip keys, so the walk follows the cache's own /// order rather than decrementing by one. #[test] fn the_walkback_crosses_a_gap_in_the_numbering() { @@ -21765,13 +21771,13 @@ mod range_tests { "video", crate::track::Info::timed(), ); - let mut first = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut first = track.create_group(group::Info { start: 0 }).unwrap(); first .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"frame".as_slice()) .unwrap(); - first.finish().unwrap(); - // Sequence 1 never exists; the newest group is empty. - let _open = track.create_group(group::Info { sequence: 2 }).unwrap(); + first.close().unwrap(); + // Key 1 never exists; the newest group is empty. + let _open = track.create_group(group::Info { start: 2 }).unwrap(); let edge = live_edge(&track.consume()); assert_eq!(edge.latest, Some(2)); @@ -21789,17 +21795,15 @@ mod range_tests { "video", crate::track::Info::timed(), ); - let mut first = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut first = track.create_group(group::Info { start: 0 }).unwrap(); for _ in 0..2 { first .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"frame".as_slice()) .unwrap(); } - first.finish().unwrap(); - let mut newest = track.create_group(group::Info { sequence: 1 }).unwrap(); - newest - .write_frame(crate::Timestamp::from_millis(0).unwrap(), b"frame".as_slice()) - .unwrap(); + first.close().unwrap(); + let mut newest = track.create_group(group::Info { start: 1 }).unwrap(); + newest.write_frame(newest.key_time(), b"frame".as_slice()).unwrap(); newest.abort(Error::Cancel).unwrap(); let edge = live_edge(&track.consume()); @@ -21820,7 +21824,7 @@ mod range_tests { }), }); assert_eq!( - filter_range(msg.filter, EDGE), + filter_range(msg.filter, EDGE, contiguous), ServeRange { start: Some(Location { group: 4, object: 3 }), end: Some(EndLocation { @@ -21856,7 +21860,7 @@ mod fill_range_tests { assert_eq!( fill_range(fill(Filter::Relative(1)), Filter::NextObject, LARGEST), FillServe::Group { - sequence: 100, + key: 100, skip: 0, until: Some(5), } @@ -21922,7 +21926,7 @@ mod fill_range_tests { LARGEST ), FillServe::Group { - sequence: 100, + key: 100, skip: 2, until: Some(4), } @@ -21945,7 +21949,7 @@ mod fill_range_tests { LARGEST ), FillServe::Group { - sequence: 100, + key: 100, skip: 0, until: Some(5), } @@ -21994,7 +21998,7 @@ mod fill_range_tests { assert_eq!( fill_range(empty, Filter::Relative(1), LARGEST), FillServe::Group { - sequence: 100, + key: 100, skip: 0, until: Some(5), } @@ -22030,7 +22034,7 @@ mod fill_range_tests { Some(Location { group: 0, object: 9 }) ), FillServe::Group { - sequence: 0, + key: 0, skip: 0, until: Some(10), } @@ -22111,11 +22115,11 @@ mod compression_tests { ); let subscriber = track.subscribe(None); let frames = [vec![b'a'; 1024], vec![b'b'; 1024], Vec::new()]; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); for payload in &frames { - group.write_frame(crate::Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(group.key_time(), payload.clone()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); track.finish_shared().unwrap(); let response = ietf::SubscribeOk { track_alias: 0, diff --git a/rs/moq-net/src/ietf/subscriber.rs b/rs/moq-net/src/ietf/subscriber.rs index d92bdf31914..eb1ada63d65 100644 --- a/rs/moq-net/src/ietf/subscriber.rs +++ b/rs/moq-net/src/ietf/subscriber.rs @@ -317,7 +317,7 @@ enum Fill { /// A fetch stream is writing the head. A second one answers no request of ours. Active, - /// The head is written: `sequence` holds objects up to but excluding `next`, and its + /// The head is written: `key` holds objects up to but excluding `next`, and its /// producer is waiting for the live tail to claim it. /// /// The tail is what ends the group, and a publisher serving the subscription's range @@ -330,7 +330,7 @@ enum Fill { /// tail is coming, but streams are independent: the tail's own can still be behind it. /// Finishing the head on that guess drops the tail when it lands. Ready { - sequence: u64, + key: u64, next: u64, producer: group::Producer, }, @@ -340,11 +340,11 @@ enum Fill { /// since only a requested head says where its group begins. Done, - /// The head of `sequence` was requested and is never coming: the fill was refused, + /// The head of `key` was requested and is never coming: the fill was refused, /// failed, or ended without it. That group's mid-group stream has a hole at the front, /// so it gets dropped whether it arrives before this settles or after, which degrades /// the join to the next group boundary. - Failed { sequence: u64 }, + Failed { key: u64 }, /// The publisher ended the track, so no tail is coming. Unlike /// [`Fill::Done`], a head still being written is the whole group once it lands: a @@ -364,27 +364,27 @@ impl Fill { /// it would have headed. Without one, the fill was owed nothing. fn failed(largest: Option) -> Self { match largest { - Some(sequence) => Fill::Failed { sequence }, + Some(key) => Fill::Failed { key }, None => Fill::Done, } } - /// Whether a stream for `sequence` starting mid-group is missing the head it was owed. - fn headless(&self, sequence: u64) -> bool { - matches!(*self, Fill::Failed { sequence: s } if s == sequence) + /// Whether a stream for `key` starting mid-group is missing the head it was owed. + fn headless(&self, key: u64) -> bool { + matches!(*self, Fill::Failed { key: s } if s == key) } - /// Take the head for `sequence`, if this is one and it ends where the tail begins. + /// Take the head for `key`, if this is one and it ends where the tail begins. /// /// `start` is the Object ID the tail stream starts at, or `None` for a tail with no /// objects of its own, which takes the head whatever it ends at. - fn claim(&mut self, sequence: u64, start: Option) -> Result, Error> { + fn claim(&mut self, key: u64, start: Option) -> Result, Error> { match *self { - Fill::Ready { sequence: s, next, .. } if s == sequence => { + Fill::Ready { key: s, next, .. } if s == key => { if start.is_some_and(|start| start != next) { // A head that stops somewhere other than where the tail starts leaves a // hole the model cannot express, so neither half of the group is usable. - tracing::warn!(sequence, next, start, "the fill does not meet the live tail"); + tracing::warn!(key, next, start, "the fill does not meet the live tail"); self.cancel(); return Err(Error::Unsupported); } @@ -426,7 +426,7 @@ impl Fill { /// group: it starts at the group's first object and has no holes. fn release(&mut self) { if let Fill::Ready { producer, .. } = std::mem::replace(self, Fill::Done) { - let _ = producer.finish(); + let _ = producer.close(); } } @@ -464,7 +464,7 @@ impl Fill { let _ = producer.abort(err); } Fill::Ended => *self = Fill::Ended, - Fill::Failed { sequence } => *self = Fill::Failed { sequence }, + Fill::Failed { key } => *self = Fill::Failed { key }, _ => {} } } @@ -546,8 +546,8 @@ impl RangeWriter { } /// The group being written. - fn sequence(&self) -> Option { - self.current.as_ref().map(|(sequence, _)| *sequence) + fn key(&self) -> Option { + self.current.as_ref().map(|(key, _)| *key) } /// The current group's producer, `None` while its objects are read past. @@ -555,39 +555,39 @@ impl RangeWriter { self.current.as_mut()?.1.as_mut() } - /// The exclusive Object ID bounding group `sequence`, if FETCH_OK's End Location falls + /// The exclusive Object ID bounding group `key`, if FETCH_OK's End Location falls /// inside it. - fn bound(&self, sequence: u64) -> Option { - self.fetch.end.filter(|_| sequence == self.fetch.last) + fn bound(&self, key: u64) -> Option { + self.fetch.end.filter(|_| key == self.fetch.last) } - /// Finish the current group and open group `sequence`, returning the first Object ID it - /// starts at. The sequences between them do not exist. A group cached since it was asked + /// Finish the current group and open group `key`, returning the first Object ID it + /// starts at. The keys between them do not exist. A group cached since it was asked /// is read past, as is a datagram, which a stream track never holds. - fn open(&mut self, sequence: u64, datagram: bool) -> Result { - if sequence < self.next || sequence > self.fetch.last { + fn open(&mut self, key: u64, datagram: bool) -> Result { + if key < self.next || key > self.fetch.last { tracing::warn!( - sequence, + key, "a group fetch's group is out of order or past FETCH_OK's End Location" ); return Err(Error::ProtocolViolation); } self.finish()?; let fill = &self.fetch.fill; - fill.drop_groups(self.next..sequence); - self.next = sequence + 1; + fill.drop_groups(self.next..key); + self.next = key + 1; - let start = match sequence == self.fetch.first { + let start = match key == self.fetch.first { true => self.fetch.start, false => 0, }; let producer = match datagram { true => { - tracing::debug!(sequence, "dropping a datagram group from a fetch"); - fill.drop_groups(sequence..sequence + 1); + tracing::debug!(key, "dropping a datagram group from a fetch"); + fill.drop_groups(key..key + 1); None } - false => match fill.accept(sequence, None) { + false => match fill.accept(key, None) { Ok(producer) => Some(crate::recv::Group::new(producer)), // Delivered since it was asked: this fetch's copy is a duplicate. Err(Error::Duplicate) => None, @@ -602,7 +602,7 @@ impl RangeWriter { *state = GroupFetch::Receiving(Some(handle)); } } - self.current = Some((sequence, producer)); + self.current = Some((key, producer)); if let Some(producer) = self.producer() { // The objects keep the IDs they have in the group rather than restarting at 0. producer.start_at(start)?; @@ -780,7 +780,7 @@ impl Target { return None; }; let wanted = idle.demand.wanted()?; - let latest = idle.track.latest_sequence()?; + let latest = idle.track.latest_key()?; (wanted > latest).then_some(wanted) } @@ -842,19 +842,29 @@ fn refused(properties: &ietf::Properties, version: Version, negotiated: bool) -> /// The Largest Group a subscription knows: its SUBSCRIBE_OK's, or a newer one since. fn largest_group(track: &track::Producer, answer: Option) -> Option { - answer.map(|largest| largest.group).max(track.latest_sequence()) + answer.map(|largest| largest.group).max(track.latest_key()) } /// Whether `subscription` needs a live feed for a copy whose publisher reported its Largest /// Object in group `edge`: a reader asked for it ([`track::Subscription::feed`]), or one of /// its ranges is open, counts back from the live edge, or reaches the group the publisher -/// may still be writing. Ranges wholly below it read only history, which fills serve. +/// may still be writing. Ranges wholly below it read only history, which fills serve when +/// each names one key. A wider one subscribes instead: its fill would FETCH every key in it, +/// spanning groups the copy cannot tell apart, and our publishers answer only a one-group +/// FETCH. fn wants_feed(subscription: &track::Subscription, edge: Option) -> bool { if subscription.ranges.is_empty() { return false; } let history = subscription.start().is_some() && subscription.end().zip(edge).is_some_and(|(end, edge)| end <= edge); - subscription.feed || !history + // An empty range, ending before it starts, fetches nothing either. + let one_key = subscription.ranges.iter().all(|range| { + matches!( + (range.start, range.end), + (track::Anchor::Time(start), Some(track::Anchor::Time(end))) if end <= start + ) + }); + subscription.feed || !history || !one_key } /// A copy with no subscription upstream, kept for fetches and a returning subscriber. @@ -3728,11 +3738,8 @@ impl Plan { .transpose(); // A relative anchor the copy cannot settle yet counts back from the Largest Object, so // the live join that names it goes first and the answer resolves the anchor. No Group - // ID arithmetic spells it upstream. A time anchor names no group the answer settles. - let deferred = matches!(subscription, Err(Error::Unsupported)) - && target - .subscription() - .is_some_and(|subscription| !has_time(&subscription)); + // ID arithmetic spells it upstream. + let deferred = matches!(subscription, Err(Error::Unsupported)); let subscription = match subscription { Ok(subscription) => subscription, Err(_) if deferred => None, @@ -4618,7 +4625,8 @@ impl SubscribeOnce { } }; track.received(); - track.anchored_cached(); + track.anchored(); + track.starts_at_keys(); // Object IDs can start past zero or skip, while readers see frame ordinals, so only // the group carries over: its first frame is the edge a reader can count on. if let Some(largest) = largest { @@ -4957,8 +4965,8 @@ impl SubscribeOnce { if let Some(mut filling) = filling { // A handler may already have the old header and be waiting on its // answer. It must keep the old state, never claim the new request. - if let Fill::Failed { sequence } = *filling { - held.headless_through = held.headless_through.max(Some(sequence)); + if let Fill::Failed { key } = *filling { + held.headless_through = held.headless_through.max(Some(key)); } filling.cancel(); drop(filling); @@ -5222,8 +5230,10 @@ impl Established { // track may run past: a later reader subscribes afresh to the same // name, and its groups continue. match end { - // A no-op once an END_OF_TRACK declared the end, whose readers - // wait for a missing group only as long as the grace. + // Once an END_OF_TRACK declared the end at the newest group, this + // ends the copy's wait for the keys below it that never arrived: + // every data stream is accounted for, so with sparse starts they + // never existed. Ok(()) => { let _ = self.track.finish_shared(); } @@ -6039,7 +6049,8 @@ impl LearnTrackStatus { // Marked received before any reader sees it, so none resolves an anchor this // wire dropped. track.received(); - track.anchored_cached(); + track.anchored(); + track.starts_at_keys(); if let Some(largest) = ok.largest { track.report_object(largest.group, largest.object); } @@ -6151,6 +6162,8 @@ impl Subscriber { ); return Ok(None); } + // A scale-less datagram's time is its sequence. + let timestamp = crate::Timestamp::received(timestamp, datagram.group_id); if let Err(err) = producer.insert(crate::datagram::Datagram { sequence: datagram.group_id, timestamp, @@ -6411,21 +6424,21 @@ impl RecvGroup { | Error::FrameTooLarge | Error::GroupTooLarge), ) => { - tracing::warn!(%err, group = %producer.sequence, "refusing group"); + tracing::warn!(%err, group = %producer.key, "refusing group"); let _ = producer.abort(err.clone()); // Return the refusal to the dispatcher so it sends STOP_SENDING. Err(err) } // The track is malformed, not just this group, so no later group is trusted. Err(Error::MalformedTrack) => { - tracing::warn!(group = %producer.sequence, "malformed track"); + tracing::warn!(group = %producer.key, "malformed track"); let _ = producer.abort(Error::MalformedTrack); let _ = track.abort(Error::MalformedTrack); Err(Error::MalformedTrack) } // The peer reset the stream, the transport failed, or the track went away. Err(err) => { - tracing::debug!(%err, group = %producer.sequence, "group error"); + tracing::debug!(%err, group = %producer.key, "group error"); let _ = producer.abort(err); Ok(()) } @@ -6497,10 +6510,10 @@ impl OpenGroup { header: &ietf::GroupHeader, reading: &mut Reading, ) -> Poll> { - let sequence = header.group_id; + let key = header.group_id; // Stats (groups/frames/bytes) are counted in the model as the group is written, // through the tagged `track::Producer`. - let create = |track: &mut track::Producer| track.create_group(group::Info { sequence }); + let create = |track: &mut track::Producer| track.create_group(group::Info { start: key }); loop { match self.step { @@ -6527,9 +6540,7 @@ impl OpenGroup { }; } OpenStep::ClaimEarly => { - if let Some((producer, next)) = - ready!(poll_claim_fill(fill, track, sequence, None, reading, waiter))? - { + if let Some((producer, next)) = ready!(poll_claim_fill(fill, track, key, None, reading, waiter))? { return Poll::Ready(Ok(Opened::Group(producer, next))); } self.step = OpenStep::PeekFirst; @@ -6574,14 +6585,10 @@ impl OpenGroup { // unless it claims FIRST_OBJECT: a group may start past object 0. if !fill.read().outstanding() { let start = first.map_or(0, |first| first.id); - if start != 0 && !header.flags.first_object && (self.headless || fill.read().headless(sequence)) - { - tracing::warn!(sequence, start, "no fill to stitch a mid-group stream onto"); + if start != 0 && !header.flags.first_object && (self.headless || fill.read().headless(key)) { + tracing::warn!(key, start, "no fill to stitch a mid-group stream onto"); // Never arriving whole, the group is dropped so readers stop owing it. - track.give_up( - sequence..sequence + 1, - crate::track::Hole::drop(crate::StreamError::NotFound), - ); + track.give_up(key..key + 1, crate::track::Hole::drop(crate::StreamError::NotFound)); return Poll::Ready(Err(Error::Unsupported)); } return Poll::Ready(Ok(Opened::Group(create(track)?, start))); @@ -6590,13 +6597,13 @@ impl OpenGroup { // The first object's ID delta is its absolute Object ID (see `next_object_id`). match first.map(|first| first.id) { // A group delivered from its start stands alone, unless the fill already - // headed this very sequence: the publisher then served those objects + // headed this very key: the publisher then served those objects // twice, and the model has one producer per group. Refuse both unfinished // pieces rather than cache a cut group as complete. Some(0) => { - let headed = matches!(*fill.read(), Fill::Ready { sequence: s, .. } if s == sequence); + let headed = matches!(*fill.read(), Fill::Ready { key: s, .. } if s == key); if headed { - tracing::warn!(sequence, "a whole group arrived for one the fill already headed"); + tracing::warn!(key, "a whole group arrived for one the fill already headed"); if let Ok(mut state) = fill.write() { state.cancel(); } @@ -6619,15 +6626,12 @@ impl OpenGroup { return Poll::Ready(Ok(Opened::EndOfTrack { object, properties })); } OpenStep::ClaimTail(start) => { - let claimed = ready!(poll_claim_fill(fill, track, sequence, start, reading, waiter))?; + let claimed = ready!(poll_claim_fill(fill, track, key, start, reading, waiter))?; return Poll::Ready(match (start, claimed) { (Some(start), Some((producer, _))) => Ok(Opened::Group(producer, start)), (Some(start), None) => { - tracing::warn!(sequence, start, "no fill to stitch a mid-group stream onto"); - track.give_up( - sequence..sequence + 1, - crate::track::Hole::drop(crate::StreamError::NotFound), - ); + tracing::warn!(key, start, "no fill to stitch a mid-group stream onto"); + track.give_up(key..key + 1, crate::track::Hole::drop(crate::StreamError::NotFound)); Err(Error::Unsupported) } (None, Some((producer, next))) => Ok(Opened::Group(producer, next)), @@ -6639,7 +6643,7 @@ impl OpenGroup { } } -/// Take the head the fill fetch stream delivered for `sequence`, once it has finished +/// Take the head the fill fetch stream delivered for `key`, once it has finished /// writing it. /// /// The model has one producer per group, so this is the handoff: the fill owns the @@ -6655,7 +6659,7 @@ impl OpenGroup { fn poll_claim_fill( fill: &kio::Producer, track: &track::Producer, - sequence: u64, + key: u64, start: Option, reading: &mut Reading, waiter: &kio::Waiter, @@ -6687,7 +6691,7 @@ fn poll_claim_fill( debug_assert!(false, "fill closed while claiming its head"); Error::Cancel })? - .claim(sequence, start), + .claim(key, start), ) } @@ -6905,7 +6909,7 @@ impl RecvFill { } }; - if matches!(head, Fill::Ready { sequence, .. } if largest.is_some_and(|largest| sequence < largest)) { + if matches!(head, Fill::Ready { key, .. } if largest.is_some_and(|largest| key < largest)) { head.release(); } // A fill that ended without reaching the Largest Group still owed that group's head. @@ -6996,8 +7000,8 @@ impl RunFill { ) -> Poll> { Poll::Ready(match ready!(self.poll_objects(waiter, stream, track)) { Ok(()) => Ok(match self.head.take() { - Some((sequence, next, producer)) => Fill::Ready { - sequence, + Some((key, next, producer)) => Fill::Ready { + key, next, producer: producer.into_inner(), }, @@ -7067,8 +7071,8 @@ impl RunFill { { return Err(Error::ProtocolViolation); } - if let Some(sequence) = group { - self.prior_group = Some(sequence); + if let Some(key) = group { + self.prior_group = Some(key); } // A datagram is never a stream track's, so a FETCH stream has no group to rebuild. @@ -7082,11 +7086,11 @@ impl RunFill { let expected = self .head .as_ref() - .filter(|(sequence, _, _)| group.is_none_or(|group| group == *sequence)) + .filter(|(key, _, _)| group.is_none_or(|group| group == *key)) .map(|(_, next, _)| *next) .or_else(|| { self.skipped - .filter(|(sequence, _)| group.is_none_or(|group| group == *sequence)) + .filter(|(key, _)| group.is_none_or(|group| group == *key)) .map(|(_, next)| next) }) .unwrap_or(0); @@ -7108,8 +7112,8 @@ impl RunFill { let current = self .head .as_ref() - .map(|(sequence, _, _)| *sequence) - .or_else(|| self.skipped.map(|(sequence, _)| sequence)); + .map(|(key, _, _)| *key) + .or_else(|| self.skipped.map(|(key, _)| key)); if current.is_some() && group.is_some_and(|group| Some(group) != current) && !self.multiple { return Err(Error::Unsupported); } @@ -7118,12 +7122,12 @@ impl RunFill { producer.finish()?; } self.skipped = None; - let sequence = group.ok_or(Error::ProtocolViolation)?; - match open_fill_group(track, &mut self.head, sequence) { + let key = group.ok_or(Error::ProtocolViolation)?; + match open_fill_group(track, &mut self.head, key) { Ok(()) => {} // Widening a range can fill groups the copy already owns. Drain their // objects without replacing their producer or cutting the rest of the fill. - Err(Error::Duplicate) if self.multiple => self.skipped = Some((sequence, 0)), + Err(Error::Duplicate) if self.multiple => self.skipped = Some((key, 0)), Err(err) => return Err(err), } } @@ -7718,10 +7722,10 @@ impl FetchRun { // The publisher knows where the track ends, which a range FETCH downstream needs. if ok.end_of_track { - let Some(final_sequence) = end.group.checked_add(u64::from(whole_last || end.object > 0)) else { + let Some(final_key) = end.group.checked_add(u64::from(whole_last || end.object > 0)) else { return self.refuse(req, run, Error::ProtocolViolation); }; - self.fill.finish_track_at(final_sequence); + self.fill.finish_track_at(final_key); } // Before draft-22, an empty answer opens no fetch stream at all. @@ -7959,7 +7963,7 @@ impl GroupFetchObjects { ) -> Poll> { let timescale = writer.fetch.timescale; loop { - if let Some((sequence, id, payload)) = &mut self.payload { + if let Some((key, id, payload)) = &mut self.payload { match payload { FetchPayloadRead::Write(recv) => { let producer = writer.producer().expect("the object's group is being written"); @@ -7967,11 +7971,11 @@ impl GroupFetchObjects { } FetchPayloadRead::Skip(skip) => ready!(skip.poll(waiter, stream))?, } - let (sequence, id) = (*sequence, *id); + let (key, id) = (*key, *id); self.payload = None; self.next = id.checked_add(1).ok_or(Error::ProtocolViolation)?; - if writer.bound(sequence).is_some_and(|end| self.next > end) { - tracing::warn!(sequence, "a group fetch continued past FETCH_OK's End Location"); + if writer.bound(key).is_some_and(|end| self.next > end) { + tracing::warn!(key, "a group fetch continued past FETCH_OK's End Location"); return Poll::Ready(Err(Error::ProtocolViolation)); } } @@ -7990,9 +7994,9 @@ impl GroupFetchObjects { self.prior_group = Some(group); } // The first object always names its group, so a later one leaving it out stays in it. - let sequence = group.or(writer.sequence()).ok_or(Error::ProtocolViolation)?; - if writer.sequence() != Some(sequence) { - self.next = writer.open(sequence, object.datagram)?; + let key = group.or(writer.key()).ok_or(Error::ProtocolViolation)?; + if writer.key() != Some(key) { + self.next = writer.open(key, object.datagram)?; } else if object.datagram { // A datagram group has one object, so it never shares one with a stream. return Poll::Ready(Err(Error::ProtocolViolation)); @@ -8020,7 +8024,7 @@ impl GroupFetchObjects { Some(_) => FetchPayloadRead::Write(FetchPayloadRecv::new(FetchPayload { decoded, timescale, id })), None => FetchPayloadRead::Skip(SkipPayload::new()), }; - self.payload = Some((sequence, id, payload)); + self.payload = Some((key, id, payload)); } // A clean FIN short of the promised end would otherwise cache a truncated group as @@ -8029,11 +8033,11 @@ impl GroupFetchObjects { // stream must reach the bounded group at all, not just its objects. if !Filter::is_draft20(self.version) && let Some(end) = writer.fetch.end - && (writer.sequence() != Some(writer.fetch.last) || self.next < end) + && (writer.key() != Some(writer.fetch.last) || self.next < end) { - let sequence = writer.sequence(); + let key = writer.key(); let next = self.next; - tracing::warn!(?sequence, next, "a group fetch ended before FETCH_OK's End Location"); + tracing::warn!(?key, next, "a group fetch ended before FETCH_OK's End Location"); return Poll::Ready(Err(Error::ProtocolViolation)); } @@ -8115,7 +8119,7 @@ impl FetchPayloadRecv { let size = wait!(Size => stream.poll_varint(waiter))?; let FetchPayload { decoded, timescale, id } = payload; super::object::check_frame(&decoded.properties, timescale)?; - let timestamp = object_time(timescale, decoded.timestamp)?; + let timestamp = object_time(timescale, decoded.timestamp, producer.key)?; let plain = id == producer.frame_count() as u64 && decoded.properties.is_empty(); let fields = (!plain).then(|| { std::sync::Arc::new(crate::model::object::Fields { @@ -8216,9 +8220,9 @@ fn resolve_fetch_group(prior: Option, wire: Option) -> Result, - sequence: u64, + key: u64, ) -> Result<(), Error> { - let producer = track.create_group(group::Info { sequence })?; + let producer = track.create_group(group::Info { start: key })?; producer.set_wire(group::Wire { sparse: false, first_object: true, @@ -8228,7 +8232,7 @@ fn open_fill_group( fetchable: true, last: None, })?; - *head = Some((sequence, 0, crate::recv::Group::new(producer))); + *head = Some((key, 0, crate::recv::Group::new(producer))); Ok(()) } @@ -8326,15 +8330,16 @@ fn first_piece(run: std::ops::Range, group: u64, order: track::Order) -> st piece.expect("the run holds the group") } -/// A Normal object's timestamp: none on an untimed track, and required on a timed one, -/// where an object without a Timestamp is malformed. +/// A Normal object's timestamp: required on a timed track, where an object without a +/// Timestamp is malformed, and its group's `key` on a scale-less one, which carries none. fn object_time( timescale: Option, timestamp: Option, -) -> Result, Error> { + key: u64, +) -> Result { match (timescale, timestamp) { (Some(_), None) => Err(Error::MalformedTrack), - (_, timestamp) => Ok(timestamp), + (_, timestamp) => Ok(crate::Timestamp::received(timestamp, key)), } } @@ -8411,7 +8416,7 @@ impl GroupIngest { self.phase = IngestPhase::Status { timestamp: *timestamp }; continue; } - let timestamp = object_time(self.timescale, *timestamp)?; + let timestamp = object_time(self.timescale, *timestamp, group.start)?; let header = self.header(group, frame::Info { size, timestamp })?; self.phase = IngestPhase::Admit { header }; } @@ -8434,7 +8439,7 @@ impl GroupIngest { IngestPhase::Status { timestamp } => { let status = ready!(reader.poll_varint(waiter))?; if status == 0 { - let timestamp = object_time(self.timescale, *timestamp)?; + let timestamp = object_time(self.timescale, *timestamp, group.start)?; let header = self.header(group, frame::Info { size: 0, timestamp })?; self.phase = IngestPhase::Admit { header }; continue; @@ -14566,25 +14571,18 @@ fn settle_bounds( ) -> Result, Error> { match resolved.resolve(track, subscription) { Ok(subscription) => Ok(Some(wire_bounds(&subscription))), - Err(Error::Unsupported) if !has_time(subscription) => Ok(None), + Err(Error::Unsupported) => Ok(None), Err(err) => Err(err), } } -fn has_time(subscription: &track::Subscription) -> bool { - subscription - .ranges - .iter() - .any(|range| matches!(range.start, track::Anchor::Time(_)) || matches!(range.end, Some(track::Anchor::Time(_)))) -} - fn needs_resolution(subscription: &track::Subscription) -> bool { subscription.ranges.iter().any(|range| { let start = match range.start { - track::Anchor::Oldest | track::Anchor::Absolute(_) | track::Anchor::Relative(0) => false, - track::Anchor::Relative(_) | track::Anchor::Time(_) => true, + track::Anchor::Oldest | track::Anchor::Time(_) | track::Anchor::Relative(0) => false, + track::Anchor::Relative(_) => true, }; - start || matches!(range.end, Some(track::Anchor::Time(_) | track::Anchor::Relative(_))) + start || matches!(range.end, Some(track::Anchor::Relative(_))) }) } @@ -14597,7 +14595,7 @@ fn wire_bounds(subscription: &track::Subscription) -> WireBounds { subscription .ranges .iter() - .any(|range| matches!(range.start, track::Anchor::Absolute(_))) + .any(|range| matches!(range.start, track::Anchor::Time(_))) .then_some(0) }); (start, subscription.end()) @@ -14810,7 +14808,7 @@ mod stitch_tests { const VERSION: Version = Version::Draft22; const ALIAS: u64 = 7; const REQUEST: RequestId = RequestId(1); - const SEQUENCE: u64 = 4; + const KEY: u64 = 4; /// A distinct timestamp per object, so a stitched group's frames can be told apart. fn timestamp(index: usize) -> Timestamp { @@ -14819,8 +14817,8 @@ mod stitch_tests { /// A publisher's fill fetch stream: a FETCH_HEADER, then one object per payload /// numbered from the group's first. - fn fill_stream(sequence: u64, payloads: &[&[u8]]) -> Vec { - fill_stream_for(REQUEST, &[(sequence, payloads)]) + fn fill_stream(key: u64, payloads: &[&[u8]]) -> Vec { + fill_stream_for(REQUEST, &[(key, payloads)]) } /// A joining FETCH stream named by its own request id, possibly spanning groups. The @@ -14836,7 +14834,7 @@ mod stitch_tests { let mut object_index = 0usize; let mut prev_group = None; - for &(sequence, payloads) in groups { + for &(key, payloads) in groups { for (index, payload) in payloads.iter().enumerate() { let payload = payload.as_ref(); let mut properties = Vec::new(); @@ -14850,8 +14848,8 @@ mod stitch_tests { let first = index == 0; let group = match (first, prev_group) { (false, _) => None, - (true, None) => Some(sequence), - (true, Some(prev)) => Some(sequence.checked_sub(prev + 1).expect("ascending groups")), + (true, None) => Some(key), + (true, Some(prev)) => Some(key.checked_sub(prev + 1).expect("ascending groups")), }; ietf::FetchObject::Object { subgroup: ietf::FetchSubgroup::Zero, @@ -14869,7 +14867,7 @@ mod stitch_tests { buf.extend_from_slice(payload); object_index += 1; } - prev_group = Some(sequence); + prev_group = Some(key); } buf.to_vec() @@ -14878,11 +14876,11 @@ mod stitch_tests { /// The subscription's own subgroup stream, starting at `start` because a strict /// publisher delivers nothing before it: that head is the fill's job. Each object /// carries the Timestamp of its Object ID. - pub(super) fn tail_stream(sequence: u64, start: u64, payloads: &[&[u8]]) -> Vec { + pub(super) fn tail_stream(key: u64, start: u64, payloads: &[&[u8]]) -> Vec { let mut buf = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: sequence, + group_id: key, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags { @@ -14928,12 +14926,12 @@ mod stitch_tests { /// object's absolute ID. A strict publisher sets the bit exactly when `start` is 0; /// one that is out of spec can leave it clear and still start there. Each object /// carries the Timestamp of its Object ID, which the harness's timed track requires. - fn draft18_subgroup(sequence: u64, first_object: bool, start: u64, payloads: &[&[u8]]) -> Vec { + fn draft18_subgroup(key: u64, first_object: bool, start: u64, payloads: &[&[u8]]) -> Vec { let version = Version::Draft18; let mut buf = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: sequence, + group_id: key, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags { @@ -15065,15 +15063,15 @@ mod stitch_tests { self } - /// Name `sequence` as the Largest Group the fill answers, so it owes that group's head. - fn with_largest(self, sequence: u64) -> Self { + /// Name `key` as the Largest Group the fill answers, so it owes that group's head. + fn with_largest(self, key: u64) -> Self { self.subscriber .state .lock() .subscribes .get_mut(&REQUEST) .unwrap() - .fill_largest = Some(sequence); + .fill_largest = Some(key); self } @@ -15109,20 +15107,20 @@ mod stitch_tests { } /// Every frame of the next group, once it finishes. - async fn read_group(subscriber: &mut track::Subscriber) -> (u64, Vec<(Option, Vec)>) { + async fn read_group(subscriber: &mut track::Subscriber) -> (u64, Vec<(Timestamp, Vec)>) { let mut group = subscriber .recv_group() .await .expect("track aborted") .expect("track finished"); - let sequence = group.sequence; + let key = group.start; let mut frames = Vec::new(); while let Some(frame) = group.read_frame().await.expect("group aborted") { frames.push((frame.timestamp, frame.payload.to_vec())); } - (sequence, frames) + (key, frames) } /// Only the stream's end may follow an END_OF_GROUP or END_OF_TRACK status: an object @@ -15131,15 +15129,15 @@ mod stitch_tests { async fn an_object_after_an_end_status_ends_the_track() { use futures::FutureExt; - let header = tail_stream(SEQUENCE, 0, &[]).len(); - let object = &tail_stream(SEQUENCE, 0, &[b"past"])[header..]; + let header = tail_stream(KEY, 0, &[]).len(); + let object = &tail_stream(KEY, 0, &[b"past"])[header..]; for (payloads, status) in [ (&[&b"last"[..]][..], END_OF_GROUP), (&[&b"last"[..]][..], END_OF_TRACK), // A standalone END_OF_TRACK, which opens no group. (&[][..], END_OF_TRACK), ] { - let mut bytes = end_marker(tail_stream(SEQUENCE, 0, payloads), status); + let mut bytes = end_marker(tail_stream(KEY, 0, payloads), status); bytes.extend_from_slice(object); let h = Harness::new(Fill::Done, vec![bytes]); @@ -15164,8 +15162,8 @@ mod stitch_tests { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), - tail_stream(SEQUENCE, 2, &[b"tail-2"]), + fill_stream(KEY, &[b"head-0", b"head-1"]), + tail_stream(KEY, 2, &[b"tail-2"]), ], ); let mut consumer = h.track.subscribe(None); @@ -15179,14 +15177,14 @@ mod stitch_tests { head.expect("fill"); tail.expect("tail"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!( frames, vec![ - (Some(timestamp(0)), b"head-0".to_vec()), - (Some(timestamp(1)), b"head-1".to_vec()), - (Some(timestamp(2)), b"tail-2".to_vec()), + (timestamp(0), b"head-0".to_vec()), + (timestamp(1), b"head-1".to_vec()), + (timestamp(2), b"tail-2".to_vec()), ] ); assert!(matches!(*h.fill.read(), Fill::Done), "the head was claimed"); @@ -15201,8 +15199,8 @@ mod stitch_tests { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - fill_stream_for(REQUEST, &[(SEQUENCE - 2, history), (SEQUENCE, head)]), - tail_stream(SEQUENCE, 1, &[b"tail"]), + fill_stream_for(REQUEST, &[(KEY - 2, history), (KEY, head)]), + tail_stream(KEY, 1, &[b"tail"]), ], ); let mut consumer = h @@ -15220,17 +15218,14 @@ mod stitch_tests { let (head, tail) = futures::join!(receiving.recv_fill(&mut fill), tailing.recv_group(&mut tail)); head.expect("fill"); tail.expect("tail"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE - 2); - assert_eq!(frames, vec![(Some(timestamp(0)), b"history".to_vec())]); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY - 2); + assert_eq!(frames, vec![(timestamp(0), b"history".to_vec())]); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!( frames, - vec![ - (Some(timestamp(1)), b"head".to_vec()), - (Some(timestamp(1)), b"tail".to_vec()) - ] + vec![(timestamp(1), b"head".to_vec()), (timestamp(1), b"tail".to_vec())] ); assert!(matches!(*h.fill.read(), Fill::Done)); } @@ -15243,15 +15238,15 @@ mod stitch_tests { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - tail_stream(SEQUENCE, 0, &[b"initial"]), - tail_stream(SEQUENCE + 1, 1, &[b"tail"]), - tail_stream(SEQUENCE + 2, 0, &[b"next"]), + tail_stream(KEY, 0, &[b"initial"]), + tail_stream(KEY + 1, 1, &[b"tail"]), + tail_stream(KEY + 2, 0, &[b"next"]), ], ) .with_update(UPDATE); let mut initial = h.stream().await; h.subscriber.clone().recv_group(&mut initial).await.unwrap(); - let mut bytes = fill_stream_for(UPDATE, &[(SEQUENCE + 1, &[b"unfinished".as_slice()])]); + let mut bytes = fill_stream_for(UPDATE, &[(KEY + 1, &[b"unfinished".as_slice()])]); bytes.pop(); let pending = ScriptedSession::new(bytes); let (_, recv) = pending.open_bi(Default::default()).await.unwrap(); @@ -15269,8 +15264,8 @@ mod stitch_tests { let mut next = h.stream().await; h.subscriber.clone().recv_group(&mut next).await.unwrap(); let mut reader = h.track.subscribe(None); - let (sequence, frames) = read_group(&mut reader).await; - assert_eq!(sequence, SEQUENCE + 2); + let (key, frames) = read_group(&mut reader).await; + assert_eq!(key, KEY + 2); assert_eq!(frames[0].1, b"next"); assert!(h.session.log.closes().is_empty()); } @@ -15283,8 +15278,8 @@ mod stitch_tests { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), - tail_stream(SEQUENCE, 2, &[b"tail-2"]), + fill_stream(KEY, &[b"head-0", b"head-1"]), + tail_stream(KEY, 2, &[b"tail-2"]), ], ); let mut fill = h.stream().await; @@ -15426,11 +15421,11 @@ mod stitch_tests { track: "video".into(), filter: Filter::Absolute { start: ietf::Location { - group: SEQUENCE, + group: KEY, object: start, }, end: Some(ietf::EndLocation { - group: SEQUENCE, + group: KEY, object: Some(end - 1), }), }, @@ -15440,11 +15435,11 @@ mod stitch_tests { namespace: Path::from_static("room"), track: "video".into(), start: ietf::Location { - group: SEQUENCE, + group: KEY, object: start, }, end: ietf::Location { - group: SEQUENCE, + group: KEY, object: end, }, } @@ -15500,7 +15495,7 @@ mod stitch_tests { .into_values() .find(|bytes| { ietf::GroupHeader::decode(&mut Decoder::new(bytes, egress.into()), egress) - .is_ok_and(|header| header.group_id == SEQUENCE) + .is_ok_and(|header| header.group_id == KEY) }) .expect("the relay emitted the group") }; @@ -15521,7 +15516,7 @@ mod stitch_tests { let mut script = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: SEQUENCE, + group_id: KEY, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags { @@ -15580,7 +15575,7 @@ mod stitch_tests { let mut bytes = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: SEQUENCE, + group_id: KEY, sub_group_id: 0, publisher_priority: 128, flags: ietf::GroupFlags { @@ -15745,7 +15740,7 @@ mod stitch_tests { let properties = vec![0x3e, if index == 0 { 6 } else { 1 }]; ietf::FetchObject::Object { subgroup: ietf::FetchSubgroup::Zero, - group: (index == 0).then_some(SEQUENCE), + group: (index == 0).then_some(KEY), object: Some(if version == Version::Draft22 && index > 0 { 2 } else { @@ -15925,7 +15920,7 @@ mod stitch_tests { let mut script = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: SEQUENCE, + group_id: KEY, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags::default(), @@ -15943,13 +15938,16 @@ mod stitch_tests { let mut consumer = h.track.subscribe(None); let mut stream = h.stream().await.with_version(version); h.subscriber.clone().recv_group(&mut stream).await.unwrap(); - assert_eq!(read_group(&mut consumer).await, (SEQUENCE, vec![(None, vec![42]); 3])); + assert_eq!( + read_group(&mut consumer).await, + (KEY, vec![(Timestamp::unscaled(KEY), vec![42]); 3]) + ); } } #[kio::test] async fn an_overflowing_object_id_reaches_session_teardown() { - let mut bytes = tail_stream(SEQUENCE, 0, &[b"head"]); + let mut bytes = tail_stream(KEY, 0, &[b"head"]); crate::coding::Encoder::new(&mut bytes, VERSION.into()) .varint(u64::MAX) .unwrap(); @@ -15968,7 +15966,7 @@ mod stitch_tests { let mut refused = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: SEQUENCE, + group_id: KEY, sub_group_id: 1, publisher_priority: 0, flags: ietf::GroupFlags { @@ -15980,7 +15978,7 @@ mod stitch_tests { .encode(&mut crate::coding::Encoder::new(&mut refused, VERSION.into()), VERSION) .unwrap(); - let h = Harness::new(Fill::Done, vec![refused, tail_stream(SEQUENCE, 0, &[b"ok"])]); + let h = Harness::new(Fill::Done, vec![refused, tail_stream(KEY, 0, &[b"ok"])]); let mut consumer = h.track.subscribe(None); let mut stream = h.stream().await; @@ -15989,8 +15987,8 @@ mod stitch_tests { let mut stream = h.stream().await; h.subscriber.clone().recv_group(&mut stream).await.unwrap(); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 1); assert_eq!(frames[0].1, b"ok"); assert!(h.session.log.closes().is_empty()); @@ -16003,11 +16001,11 @@ mod stitch_tests { #[kio::test] async fn a_compact_non_zero_subgroup_is_refused() { for version in [Version::Draft22] { - let compact = |sequence: u64, first: Option| { + let compact = |key: u64, first: Option| { let mut bytes = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: sequence, + group_id: key, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags { @@ -16025,13 +16023,13 @@ mod stitch_tests { } bytes }; - let mut truncated = compact(SEQUENCE + 3, None); + let mut truncated = compact(KEY + 3, None); Encoder::new(&mut truncated, version.into()).varint(2u64).unwrap(); let scripts = vec![ - compact(SEQUENCE, Some(2)), + compact(KEY, Some(2)), truncated, - compact(SEQUENCE + 1, Some(0)), - compact(SEQUENCE + 2, None), + compact(KEY + 1, Some(0)), + compact(KEY + 2, None), ]; let mut h = Harness::with_timescale(Fill::Done, scripts, None); h.subscriber.version = version; @@ -16041,7 +16039,7 @@ mod stitch_tests { let result = h.subscriber.clone().recv_group(&mut stream).await; assert!(matches!(result, Err(Error::Unsupported)), "{version}: {result:?}"); assert_eq!( - h.track.latest_sequence(), + h.track.latest_key(), None, "{version}: the refused subgroup opened a group" ); @@ -16050,29 +16048,32 @@ mod stitch_tests { let result = h.subscriber.clone().recv_group(&mut stream).await; assert!(matches!(result, Err(Error::Unsupported)), "{version}: {result:?}"); assert_eq!( - h.track.latest_sequence(), + h.track.latest_key(), None, "{version}: a truncated subgroup opened a group" ); let mut stream = h.stream().await.with_version(version); h.subscriber.clone().recv_group(&mut stream).await.unwrap(); - assert_eq!(read_group(&mut consumer).await, (SEQUENCE + 1, vec![(None, vec![42])])); + assert_eq!( + read_group(&mut consumer).await, + (KEY + 1, vec![(Timestamp::unscaled(KEY + 1), vec![42])]) + ); let mut stream = h.stream().await.with_version(version); h.subscriber.clone().recv_group(&mut stream).await.unwrap(); - assert_eq!(read_group(&mut consumer).await, (SEQUENCE + 2, vec![])); + assert_eq!(read_group(&mut consumer).await, (KEY + 2, vec![])); assert!(h.session.log.closes().is_empty(), "{version}"); } } - /// A subgroup stream for `sequence` without extension headers, starting at object 0: the + /// A subgroup stream for `key` without extension headers, starting at object 0: the /// header, then `varints`, then `payload` verbatim. - fn raw_stream(sequence: u64, varints: &[u64], payload: &[u8]) -> Vec { + fn raw_stream(key: u64, varints: &[u64], payload: &[u8]) -> Vec { let mut buf = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: sequence, + group_id: key, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags { @@ -16094,7 +16095,7 @@ mod stitch_tests { #[kio::test] async fn a_malformed_extension_block_is_refused() { // Delta 0, then a 1-byte extension block holding half of a 2-byte varint. - let mut stream = tail_stream(SEQUENCE, 0, &[b"ok"]); + let mut stream = tail_stream(KEY, 0, &[b"ok"]); stream.extend_from_slice(&[0, 1, 0x40]); let h = Harness::new(Fill::Done, vec![stream]); h.dispatch_refused().await; @@ -16103,7 +16104,7 @@ mod stitch_tests { /// A FIN before the declared payload ends. #[kio::test] async fn a_payload_cut_short_is_refused() { - let h = Harness::with_timescale(Fill::Done, vec![raw_stream(SEQUENCE, &[0, 4], &[1, 2])], None); + let h = Harness::with_timescale(Fill::Done, vec![raw_stream(KEY, &[0, 4], &[1, 2])], None); h.dispatch_refused().await; } @@ -16111,14 +16112,14 @@ mod stitch_tests { #[kio::test] async fn an_oversized_frame_is_refused() { let size = crate::group::MAX_CACHE_BYTES + 1; - let h = Harness::with_timescale(Fill::Done, vec![raw_stream(SEQUENCE, &[0, size], &[])], None); + let h = Harness::with_timescale(Fill::Done, vec![raw_stream(KEY, &[0, size], &[])], None); h.dispatch_refused().await; } /// A second stream for a group the track already holds. #[kio::test] async fn a_second_stream_for_a_group_is_refused() { - let whole = raw_stream(SEQUENCE, &[0, 1], &[42]); + let whole = raw_stream(KEY, &[0, 1], &[42]); let h = Harness::with_timescale(Fill::Done, vec![whole.clone(), whole], None); let mut stream = h.stream().await; h.subscriber.clone().recv_group(&mut stream).await.unwrap(); @@ -16132,7 +16133,7 @@ mod stitch_tests { let mut script = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: SEQUENCE, + group_id: KEY, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags { @@ -16212,7 +16213,7 @@ mod stitch_tests { .unwrap(); ietf::FetchObject::Object { subgroup: ietf::FetchSubgroup::Zero, - group: Some(SEQUENCE), + group: Some(KEY), object: Some(0), priority: Some(0), properties: None, @@ -16243,7 +16244,7 @@ mod stitch_tests { let mut script = Vec::new(); ietf::GroupHeader { track_alias: ALIAS, - group_id: SEQUENCE, + group_id: KEY, sub_group_id: 0, publisher_priority: 0, flags: ietf::GroupFlags { @@ -16271,19 +16272,13 @@ mod stitch_tests { async fn an_end_of_track_after_a_group_ends_the_track_after_it() { let h = Harness::new( Fill::Done, - vec![end_marker(tail_stream(SEQUENCE, 0, &[b"last"]), END_OF_TRACK)], + vec![end_marker(tail_stream(KEY, 0, &[b"last"]), END_OF_TRACK)], ); let mut consumer = h.track.subscribe(None); let mut stream = h.stream().await; h.subscriber.clone().recv_group(&mut stream).await.unwrap(); - assert_eq!( - h.track.finished(), - Some(track::Position { - group: SEQUENCE, - frame: 1 - }) - ); + assert_eq!(h.track.finished(), Some(track::Position { group: KEY, frame: 1 })); let mut group = consumer.recv_group().await.unwrap().expect("the group arrives"); assert_eq!(group.read_frame().await.unwrap().unwrap().payload.as_ref(), b"last"); @@ -16301,11 +16296,7 @@ mod stitch_tests { let mut stream = h.stream().await; h.subscriber.clone().recv_group(&mut stream).await.unwrap(); assert_eq!(h.track.finished(), Some(track::Position { group: 5, frame: 5 })); - assert_eq!( - h.track.latest_sequence(), - None, - "a terminal marker creates no payload group" - ); + assert_eq!(h.track.latest_key(), None, "a terminal marker creates no payload group"); } /// A group at or past the end an END_OF_TRACK declared contradicts that end, which no @@ -16314,8 +16305,8 @@ mod stitch_tests { async fn a_group_past_the_declared_end_aborts_the_track() { use futures::FutureExt; - let mut h = Harness::new(Fill::Done, vec![tail_stream(SEQUENCE, 0, &[b"late"])]); - h.track.finish_at(track::Position::group(SEQUENCE), None).unwrap(); + let mut h = Harness::new(Fill::Done, vec![tail_stream(KEY, 0, &[b"late"])]); + h.track.finish_at(track::Position::group(KEY), None).unwrap(); let mut stream = h.stream().await; let res = h.subscriber.clone().recv_group(&mut stream).await; @@ -16330,15 +16321,12 @@ mod stitch_tests { /// and no group is created for it. #[kio::test] async fn an_end_of_track_at_object_zero_creates_no_group() { - let h = Harness::new( - Fill::Done, - vec![end_marker(tail_stream(SEQUENCE, 0, &[]), END_OF_TRACK)], - ); + let h = Harness::new(Fill::Done, vec![end_marker(tail_stream(KEY, 0, &[]), END_OF_TRACK)]); let mut stream = h.stream().await; h.subscriber.clone().recv_group(&mut stream).await.unwrap(); - assert_eq!(h.track.finished(), Some(track::Position::group(SEQUENCE))); - assert_eq!(h.track.latest_sequence(), None, "no group was created"); + assert_eq!(h.track.finished(), Some(track::Position::group(KEY))); + assert_eq!(h.track.latest_key(), None, "no group was created"); } /// The group ended exactly where we joined it, so the subscription's stream carries no @@ -16347,10 +16335,7 @@ mod stitch_tests { async fn an_empty_tail_finishes_the_filled_group() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![ - fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), - tail_stream(SEQUENCE, 2, &[]), - ], + vec![fill_stream(KEY, &[b"head-0", b"head-1"]), tail_stream(KEY, 2, &[])], ); let mut consumer = h.track.subscribe(None); @@ -16360,8 +16345,8 @@ mod stitch_tests { h.subscriber.clone().recv_fill(&mut fill).await.expect("fill"); h.subscriber.clone().recv_group(&mut tail).await.expect("tail"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 2, "the head is the whole group"); } @@ -16374,23 +16359,23 @@ mod stitch_tests { Fill::Serving(Some(Timescale::MICRO)), vec![ fill_stream_for::<&[u8]>(REQUEST, &[]), - tail_stream(SEQUENCE, 2, &[b"tail-2"]), - tail_stream(SEQUENCE + 1, 0, &[b"next-0"]), + tail_stream(KEY, 2, &[b"tail-2"]), + tail_stream(KEY + 1, 0, &[b"next-0"]), ], ) - .with_largest(SEQUENCE); + .with_largest(KEY); let mut consumer = h.track.subscribe(None); let mut fill = h.stream().await; h.subscriber.clone().recv_fill(&mut fill).await.expect("fill"); - assert!(matches!(*h.fill.read(), Fill::Failed { sequence: SEQUENCE })); + assert!(matches!(*h.fill.read(), Fill::Failed { key: KEY })); h.dispatch_refused().await; let mut next = h.stream().await; h.subscriber.clone().recv_group(&mut next).await.expect("next group"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE + 1, "the headless group was dropped"); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY + 1, "the headless group was dropped"); assert_eq!(frames.len(), 1); } @@ -16402,11 +16387,11 @@ mod stitch_tests { Fill::Serving(Some(Timescale::MICRO)), vec![ fill_stream_for::<&[u8]>(REQUEST, &[]), - tail_stream(SEQUENCE, 2, &[b"tail-2"]), - tail_stream(SEQUENCE + 1, 0, &[b"next-0"]), + tail_stream(KEY, 2, &[b"tail-2"]), + tail_stream(KEY + 1, 0, &[b"next-0"]), ], ) - .with_largest(SEQUENCE); + .with_largest(KEY); let mut consumer = h.track.subscribe(None); let mut fill = h.stream().await; @@ -16419,8 +16404,8 @@ mod stitch_tests { let mut next = h.stream().await; h.subscriber.clone().recv_group(&mut next).await.expect("next group"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE + 1, "the headless group was dropped"); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY + 1, "the headless group was dropped"); assert_eq!(frames.len(), 1); } @@ -16428,13 +16413,13 @@ mod stitch_tests { /// still stands alone. #[kio::test] async fn a_mid_group_stream_without_a_fill_stands_alone() { - let h = Harness::new(Fill::Done, vec![tail_stream(SEQUENCE, 2, &[b"tail-2"])]); + let h = Harness::new(Fill::Done, vec![tail_stream(KEY, 2, &[b"tail-2"])]); let mut consumer = h.track.subscribe(None); let mut tail = h.stream().await; h.subscriber.clone().recv_group(&mut tail).await.expect("tail"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 1); } @@ -16449,14 +16434,14 @@ mod stitch_tests { "video", track::Info::default().with_timescale(Timescale::MICRO), ); - let mut producer = track.create_group(group::Info { sequence: SEQUENCE }).unwrap(); + let mut producer = track.create_group(group::Info { start: KEY }).unwrap(); producer.write_frame(timestamp(0), b"head-0".as_slice()).unwrap(); let mut group = producer.consume(); // `remove_subscribe` got there first. let mut fill = Fill::Done; fill.install(Fill::Ready { - sequence: SEQUENCE, + key: KEY, next: 1, producer, }); @@ -16475,20 +16460,17 @@ mod stitch_tests { "video", track::Info::default().with_timescale(Timescale::MICRO), ); - let mut producer = track.create_group(group::Info { sequence: SEQUENCE }).unwrap(); + let mut producer = track.create_group(group::Info { start: KEY }).unwrap(); producer.write_frame(timestamp(0), b"head-0".as_slice()).unwrap(); let mut group = producer.consume(); - let mut fill = Fill::Failed { sequence: SEQUENCE }; + let mut fill = Fill::Failed { key: KEY }; fill.install(Fill::Ready { - sequence: SEQUENCE, + key: KEY, next: 1, producer, }); - assert!( - matches!(fill, Fill::Failed { sequence: SEQUENCE }), - "Failed is terminal" - ); + assert!(matches!(fill, Fill::Failed { key: KEY }), "Failed is terminal"); assert!(matches!(group.read_frame().await, Err(Error::Cancel))); } @@ -16503,18 +16485,15 @@ mod stitch_tests { "video", track::Info::default().with_timescale(Timescale::MICRO), ); - let mut producer = track.create_group(group::Info { sequence: SEQUENCE }).unwrap(); + let mut producer = track.create_group(group::Info { start: KEY }).unwrap(); producer.write_frame(timestamp(0), b"head-0".as_slice()).unwrap(); let mut group = producer.consume(); - let mut fill = Fill::Failed { sequence: SEQUENCE }; + let mut fill = Fill::Failed { key: KEY }; fill.end(); - assert!( - matches!(fill, Fill::Failed { sequence: SEQUENCE }), - "the end keeps it failed" - ); + assert!(matches!(fill, Fill::Failed { key: KEY }), "the end keeps it failed"); fill.install(Fill::Ready { - sequence: SEQUENCE, + key: KEY, next: 1, producer, }); @@ -16526,33 +16505,30 @@ mod stitch_tests { /// Object ID is past zero, so it is delivered rather than dropped as a cut tail. #[kio::test] async fn a_headless_group_claiming_its_first_object_is_delivered() { - let h = Harness::new(Fill::Done, vec![draft18_subgroup(SEQUENCE, true, 6, &[b"six"])]); + let h = Harness::new(Fill::Done, vec![draft18_subgroup(KEY, true, 6, &[b"six"])]); h.subscriber .state .lock() .subscribes .get_mut(&REQUEST) .unwrap() - .headless_through = Some(SEQUENCE); + .headless_through = Some(KEY); let mut consumer = h.track.subscribe(None); let mut stream = h.stream().await; h.subscriber.clone().recv_group(&mut stream).await.expect("group"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 1); } - /// A refused headless group is reported dropped, so a reader in sequence order, as a + /// A refused headless group is reported dropped, so a reader in key order, as a /// `track` socket session reads, moves on to the next group rather than wait on it. #[kio::test] async fn a_refused_headless_group_does_not_stall_an_ordered_reader() { let h = Harness::new( Fill::Done, - vec![ - tail_stream(SEQUENCE, 2, &[b"tail-2"]), - tail_stream(SEQUENCE + 1, 0, &[b"next-0"]), - ], + vec![tail_stream(KEY, 2, &[b"tail-2"]), tail_stream(KEY + 1, 0, &[b"next-0"])], ); // Subscribed before the answer, as the socket does. let mut ordered = h @@ -16565,15 +16541,15 @@ mod stitch_tests { .with_range(0..) .ordered(); h.track.received(); - h.track.anchored_cached(); - h.track.clone().start_at(SEQUENCE).unwrap(); + h.track.anchored(); + h.track.clone().start_at(KEY).unwrap(); h.subscriber .state .lock() .subscribes .get_mut(&REQUEST) .unwrap() - .headless_through = Some(SEQUENCE); + .headless_through = Some(KEY); h.dispatch_refused().await; @@ -16581,10 +16557,10 @@ mod stitch_tests { h.subscriber.clone().recv_group(&mut next).await.expect("next group"); let group = match ordered.poll_next_group(&kio::Waiter::noop()) { Poll::Ready(Ok(Some(group))) => group, - Poll::Ready(other) => panic!("no next group: {:?}", other.map(|group| group.map(|g| g.sequence))), + Poll::Ready(other) => panic!("no next group: {:?}", other.map(|group| group.map(|g| g.start))), Poll::Pending => panic!("the ordered reader still waits on the refused group"), }; - assert_eq!(group.sequence, SEQUENCE + 1); + assert_eq!(group.start, KEY + 1); } /// Widening updates replaced fills that failed to head this group and the next, and the @@ -16594,10 +16570,7 @@ mod stitch_tests { async fn a_headless_tail_after_its_fill_was_replaced_is_dropped() { let h = Harness::new( Fill::Done, - vec![ - tail_stream(SEQUENCE, 2, &[b"tail-2"]), - tail_stream(SEQUENCE + 1, 0, &[b"next-0"]), - ], + vec![tail_stream(KEY, 2, &[b"tail-2"]), tail_stream(KEY + 1, 0, &[b"next-0"])], ); h.subscriber .state @@ -16605,15 +16578,15 @@ mod stitch_tests { .subscribes .get_mut(&REQUEST) .unwrap() - .headless_through = Some(SEQUENCE + 1); + .headless_through = Some(KEY + 1); let mut consumer = h.track.subscribe(None); h.dispatch_refused().await; let mut next = h.stream().await; h.subscriber.clone().recv_group(&mut next).await.expect("next group"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE + 1, "the headless group was dropped"); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY + 1, "the headless group was dropped"); assert_eq!(frames.len(), 1); } @@ -16624,14 +16597,14 @@ mod stitch_tests { async fn an_unclaimed_head_is_cancelled_when_the_subscription_goes() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![fill_stream(SEQUENCE, &[b"head-0"])], + vec![fill_stream(KEY, &[b"head-0"])], ); let mut consumer = h.track.subscribe(None); let mut fill = h.stream().await; h.subscriber.clone().recv_fill(&mut fill).await.expect("fill"); let mut group = consumer.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, SEQUENCE); + assert_eq!(group.start, KEY); h.subscriber.remove_subscribe(REQUEST).expect("subscribed"); assert!(matches!(group.read_frame().await, Err(Error::Cancel))); @@ -16642,7 +16615,7 @@ mod stitch_tests { async fn an_unclaimed_head_is_cancelled_when_the_session_closes() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![fill_stream(SEQUENCE, &[b"head-0"])], + vec![fill_stream(KEY, &[b"head-0"])], ); let mut consumer = h.track.subscribe(None); let mut fill = h.stream().await; @@ -16660,16 +16633,16 @@ mod stitch_tests { ); } - /// A publisher that serves a head and then opens a whole group for the same sequence + /// A publisher that serves a head and then opens a whole group for the same key /// has contradicted its own fill. The model holds one producer per group, so the /// duplicate stream and unfinished head are refused without caching a cut group. #[kio::test] - async fn a_whole_group_for_a_headed_sequence_is_refused() { + async fn a_whole_group_for_a_headed_key_is_refused() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), - tail_stream(SEQUENCE, 0, &[b"again-0"]), + fill_stream(KEY, &[b"head-0", b"head-1"]), + tail_stream(KEY, 0, &[b"again-0"]), ], ); let mut consumer = h.track.subscribe(None); @@ -16694,7 +16667,7 @@ mod stitch_tests { async fn a_cancelled_session_aborts_a_waiting_fill_head() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![fill_stream(SEQUENCE, &[b"head-0"])], + vec![fill_stream(KEY, &[b"head-0"])], ); let mut consumer = h.track.subscribe(None); @@ -16720,7 +16693,7 @@ mod stitch_tests { async fn session_death_keeps_a_waiting_fill_heads_resume_position() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![fill_stream(SEQUENCE, &[b"head-0"])], + vec![fill_stream(KEY, &[b"head-0"])], ); let track = h.track.consume(); let mut consumer = h.track.subscribe(None); @@ -16728,13 +16701,7 @@ mod stitch_tests { h.subscriber.clone().recv_fill(&mut fill).await.expect("fill"); let mut group = consumer.recv_group().await.unwrap().expect("the fill head arrived"); let resume = track.resume_position(); - assert_eq!( - resume, - Some(track::Position { - group: SEQUENCE, - frame: 1 - }) - ); + assert_eq!(resume, Some(track::Position { group: KEY, frame: 1 })); let err = Error::Session(crate::SessionError::App(7)); h.subscriber.abort(&err); assert_eq!(track.resume_position(), resume); @@ -16746,15 +16713,15 @@ mod stitch_tests { /// The same contradiction as above, with the streams the other way round: the whole /// group lands before the fill has written its head. The model holds one producer per - /// live sequence, so the fill loses the race to create it and gives up, rather than a + /// live key, so the fill loses the race to create it and gives up, rather than a /// second producer appearing and the objects being delivered twice. #[kio::test] - async fn a_whole_group_that_precedes_the_head_wins_the_sequence() { + async fn a_whole_group_that_precedes_the_head_wins_the_key() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - tail_stream(SEQUENCE, 0, &[b"whole-0"]), - fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), + tail_stream(KEY, 0, &[b"whole-0"]), + fill_stream(KEY, &[b"head-0", b"head-1"]), ], ); let mut consumer = h.track.subscribe(None); @@ -16773,15 +16740,15 @@ mod stitch_tests { .await .expect("overlapping fill is drained without another producer"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 1, "the group is whatever one producer wrote, not both"); } /// A standalone subgroup tail keeps its identity while plain readers see its payload. #[kio::test] async fn a_tail_without_a_fill_is_preserved() { - let h = Harness::new(Fill::Done, vec![tail_stream(SEQUENCE, 2, &[b"tail-2"])]); + let h = Harness::new(Fill::Done, vec![tail_stream(KEY, 2, &[b"tail-2"])]); let mut consumer = h.track.subscribe(None); let mut tail = h.stream().await; h.subscriber.clone().recv_group(&mut tail).await.unwrap(); @@ -16793,7 +16760,7 @@ mod stitch_tests { /// group. The publisher is out of spec on the bit, not missing a head. #[kio::test] async fn a_clear_first_object_at_zero_delivers_the_group() { - let bytes = draft18_subgroup(SEQUENCE, false, 0, &[b"whole", b"next"]); + let bytes = draft18_subgroup(KEY, false, 0, &[b"whole", b"next"]); assert_eq!( u64::from(bytes[0]) & ietf::GroupFlags::FIRST_OBJECT_BIT, 0, @@ -16807,8 +16774,8 @@ mod stitch_tests { h.subscriber.clone().recv_group(&mut stream).await.expect("the group"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 2); assert_eq!(frames[0].1, b"whole"); assert_eq!(frames[1].1, b"next"); @@ -16821,7 +16788,7 @@ mod stitch_tests { use futures::FutureExt; let payloads: [&[u8]; 5] = [b"o0", b"o1", b"o2", b"o3", b"o4"]; - let bytes = end_marker(draft18_subgroup(SEQUENCE, true, 0, &payloads), END_OF_GROUP); + let bytes = end_marker(draft18_subgroup(KEY, true, 0, &payloads), END_OF_GROUP); let flags = ietf::GroupFlags::decode(u64::from(bytes[0])).unwrap(); assert!(flags.has_end, "the header marks the group's end"); @@ -16834,7 +16801,7 @@ mod stitch_tests { // The stream was read to its end, so everything is already delivered. let mut group = consumer.recv_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(group.sequence, SEQUENCE); + assert_eq!(group.start, KEY); for payload in payloads { let frame = group.read_frame().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), payload); @@ -16846,8 +16813,8 @@ mod stitch_tests { /// FIRST_OBJECT and the starting ID are independent relay metadata. #[kio::test] async fn a_clear_first_object_past_zero_is_preserved() { - let partial = draft18_subgroup(SEQUENCE, false, 3, &[b"tail-3"]); - let whole = draft18_subgroup(SEQUENCE + 1, true, 0, &[b"next"]); + let partial = draft18_subgroup(KEY, false, 3, &[b"tail-3"]); + let whole = draft18_subgroup(KEY + 1, true, 0, &[b"next"]); assert_eq!(u64::from(partial[0]) & ietf::GroupFlags::FIRST_OBJECT_BIT, 0); let mut h = Harness::new(Fill::Done, vec![partial, whole]); @@ -16856,8 +16823,8 @@ mod stitch_tests { let mut partial = read_draft18(&h.session).await; h.subscriber.clone().recv_group(&mut partial).await.unwrap(); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames[0].1, b"tail-3"); let mut whole = read_draft18(&h.session).await; @@ -16867,8 +16834,8 @@ mod stitch_tests { .await .expect("the next group"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE + 1); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY + 1); assert_eq!(frames[0].1, b"next"); assert!(h.session.log.closes().is_empty()); } @@ -16879,15 +16846,15 @@ mod stitch_tests { async fn a_received_rewind_is_accepted() { use futures::FutureExt; - let mut h = Harness::new(Fill::Done, vec![draft18_subgroup(SEQUENCE, true, 0, &[b"rewound"])]); + let mut h = Harness::new(Fill::Done, vec![draft18_subgroup(KEY, true, 0, &[b"rewound"])]); h.subscriber.version = Version::Draft18; // Starts past the incoming group's first object, at `timestamp(0)`. - let mut before = h.track.create_group((SEQUENCE - 1).into()).unwrap(); + let mut before = h.track.create_group((KEY - 1).into()).unwrap(); before.write_frame(timestamp(4000), b"before".as_slice()).unwrap(); - before.finish().unwrap(); + before.close().unwrap(); let mut consumer = h.track.subscribe(None); - assert_eq!(read_group(&mut consumer).await.0, SEQUENCE - 1); + assert_eq!(read_group(&mut consumer).await.0, KEY - 1); let mut stream = read_draft18(&h.session).await; h.subscriber.clone().recv_group(&mut stream).await.unwrap(); @@ -16898,7 +16865,7 @@ mod stitch_tests { .expect("delivered") .unwrap() .unwrap(); - assert_eq!(group.sequence, SEQUENCE); + assert_eq!(group.start, KEY); let frame = group.read_frame().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), b"rewound"); assert!(h.track.closed().now_or_never().is_none()); @@ -16911,8 +16878,8 @@ mod stitch_tests { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - fill_stream(SEQUENCE, &[b"head-0", b"head-1"]), - tail_stream(SEQUENCE, 5, &[b"tail-5"]), + fill_stream(KEY, &[b"head-0", b"head-1"]), + tail_stream(KEY, 5, &[b"tail-5"]), ], ); let mut consumer = h.track.subscribe(None); @@ -16946,7 +16913,7 @@ mod stitch_tests { #[kio::test] async fn receive_drain_ends_a_fill_waiting_for_subscribe_ok() { - let h = Harness::new(Fill::Requested, vec![fill_stream(SEQUENCE, &[b"head"])]); + let h = Harness::new(Fill::Requested, vec![fill_stream(KEY, &[b"head"])]); let mut stream = h.stream().await; let mut subscriber = h.subscriber.clone(); let mut receiving = Box::pin(subscriber.recv_fill(&mut stream)); @@ -16962,7 +16929,7 @@ mod stitch_tests { async fn receive_drain_ends_a_tail_waiting_for_a_missing_fill() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![tail_stream(SEQUENCE, 1, &[b"tail"])], + vec![tail_stream(KEY, 1, &[b"tail"])], ); let mut stream = h.stream().await; let mut subscriber = h.subscriber.clone(); @@ -16976,7 +16943,7 @@ mod stitch_tests { async fn receive_drain_preserves_a_buffered_fill_and_tail() { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![tail_stream(SEQUENCE, 1, &[b"tail"]), fill_stream(SEQUENCE, &[b"head"])], + vec![tail_stream(KEY, 1, &[b"tail"]), fill_stream(KEY, &[b"head"])], ); let mut consumer = h.track.subscribe(None); let mut tail = h.stream().await; @@ -17001,7 +16968,7 @@ mod stitch_tests { #[kio::test] async fn receive_drain_ends_a_group_fetch_waiting_for_fetch_ok() { - let h = Harness::new(Fill::Done, vec![fill_stream(SEQUENCE, &[b"head"])]); + let h = Harness::new(Fill::Done, vec![fill_stream(KEY, &[b"head"])]); h.subscriber .state .lock() @@ -17022,7 +16989,7 @@ mod stitch_tests { /// for that response instead of looking up a producer that does not exist yet. #[kio::test] async fn an_early_fill_waits_for_subscribe_ok() { - let h = Harness::new(Fill::Requested, vec![fill_stream(SEQUENCE, &[b"head-0"])]); + let h = Harness::new(Fill::Requested, vec![fill_stream(KEY, &[b"head-0"])]); { let mut state = h.subscriber.state.lock(); state.subscribes.get_mut(&REQUEST).unwrap().producer = None; @@ -17044,7 +17011,7 @@ mod stitch_tests { /// subscription itself is delivering, so it is refused rather than written. #[kio::test] async fn an_unsolicited_fill_is_refused() { - let h = Harness::new(Fill::Done, vec![fill_stream(SEQUENCE, &[b"head-0"])]); + let h = Harness::new(Fill::Done, vec![fill_stream(KEY, &[b"head-0"])]); let mut fill = h.stream().await; assert!(matches!( @@ -17062,8 +17029,8 @@ mod stitch_tests { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - fill_stream_for(FETCH, &[(SEQUENCE, &[b"head-0", b"head-1"])]), - tail_stream(SEQUENCE, 2, &[b"tail-2"]), + fill_stream_for(FETCH, &[(KEY, &[b"head-0", b"head-1"])]), + tail_stream(KEY, 2, &[b"tail-2"]), ], ) .with_joining(FETCH); @@ -17078,8 +17045,8 @@ mod stitch_tests { head.expect("joining fetch"); tail.expect("tail"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 3); assert_eq!(frames[0].1, b"head-0"); assert_eq!(frames[1].1, b"head-1"); @@ -17093,8 +17060,8 @@ mod stitch_tests { let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), vec![ - tail_stream(SEQUENCE, 0, &[b"whole-0"]), - fill_stream_for(FETCH, &[(SEQUENCE, &[b"head-0", b"head-1"])]), + tail_stream(KEY, 0, &[b"whole-0"]), + fill_stream_for(FETCH, &[(KEY, &[b"head-0", b"head-1"])]), ], ) .with_joining(FETCH); @@ -17110,11 +17077,11 @@ mod stitch_tests { .expect("the whole group"); assert!( h.subscriber.clone().recv_fill(&mut fill).await.is_err(), - "the fetch cannot create a second producer for a live sequence" + "the fetch cannot create a second producer for a live key" ); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY); assert_eq!(frames.len(), 1); assert_eq!(frames[0].1, b"whole-0"); } @@ -17158,9 +17125,9 @@ mod stitch_tests { let headers = [ // Both IDs omitted: "the requested group, from its start" is exactly the bug. header(None, None, Zero, Some(0)), - header(Some(SEQUENCE), None, Zero, Some(0)), - header(Some(SEQUENCE), Some(0), Prior, Some(0)), - header(Some(SEQUENCE), Some(0), Zero, None), + header(Some(KEY), None, Zero, Some(0)), + header(Some(KEY), Some(0), Prior, Some(0)), + header(Some(KEY), Some(0), Zero, None), ]; for version in [VERSION] { @@ -17188,7 +17155,7 @@ mod stitch_tests { /// malformed rather than stamped with its arrival time. #[kio::test] async fn an_unstamped_object_on_a_timed_track_is_malformed() { - let mut run = GroupFetchRun::new(VERSION, group_fetch_objects(SEQUENCE, 0, &[b"a"])).await; + let mut run = GroupFetchRun::new(VERSION, group_fetch_objects(KEY, 0, &[b"a"])).await; let track = track::Producer::new( std::sync::Arc::new(crate::broadcast::Info::default()), "timed", @@ -17215,7 +17182,7 @@ mod stitch_tests { let short = Filter::is_draft20(version); for (count, complete) in [(2, short), (3, true), (4, false)] { let payloads: Vec<&[u8]> = [b"a", b"b", b"c", b"d"][..count].iter().map(|p| &p[..]).collect(); - let mut run = GroupFetchRun::new(version, group_fetch_objects(SEQUENCE, 0, &payloads)).await; + let mut run = GroupFetchRun::new(version, group_fetch_objects(KEY, 0, &payloads)).await; let (slot, fetch) = run.slot(None, Some(END)).await; let res = run.subscriber.recv_group_fetch(&mut run.stream, slot).await; @@ -17246,19 +17213,21 @@ mod stitch_tests { /// for now, since objects past the bound may still be written. #[kio::test] async fn a_range_fetch_must_reach_its_bounded_group() { - let last = SEQUENCE + 2; + let last = KEY + 2; for version in [Version::Draft18, VERSION] { - for objects in [Vec::new(), group_fetch_objects(SEQUENCE, 0, &[b"a"])] { + for objects in [Vec::new(), group_fetch_objects(KEY, 0, &[b"a"])] { let empty = objects.is_empty(); let mut run = GroupFetchRun::new(version, objects).await; let dynamic = run.track.requests(); let consumer = run.track.consume(); run.track.start_at(last + 1).unwrap(); - run.track.create_group((last + 1).into()).unwrap().finish().unwrap(); + run.track.create_group((last + 1).into()).unwrap().close().unwrap(); + // From group 0, which a time start names whatever is held, so the history below + // the feed is asked as a miss rather than looked up. let mut reading = consumer .subscribe(Some( track::Subscription::default() - .with_range(SEQUENCE..last + 1) + .with_range(0..last + 1) .with_order(crate::track::Order::Ascending) .with_delay(track::Delay::default().with_max(None)), )) @@ -17270,7 +17239,7 @@ mod stitch_tests { let slot = kio::Producer::new(GroupFetch::Ready(RangeFetch { fill: std::sync::Arc::new(fill), timescale: None, - first: SEQUENCE, + first: 0, last, start: 0, end: Some(2), @@ -17298,14 +17267,14 @@ mod stitch_tests { } /// One Object published as a datagram, as a fetch stream may carry it. - fn datagram_object(sequence: u64) -> Vec { + fn datagram_object(key: u64) -> Vec { let mut properties = Vec::new(); let w = &mut Encoder::new(&mut properties, VERSION.into()); ietf::encode_object_time(w, timestamp(0), Timescale::MICRO).unwrap(); let mut buf = Vec::new(); ietf::FetchObject::Object { subgroup: ietf::FetchSubgroup::Datagram, - group: Some(sequence), + group: Some(key), object: Some(0), priority: Some(0), properties: Some(properties), @@ -17321,13 +17290,13 @@ mod stitch_tests { /// drops its group, caching nothing, and reads on past its payload. #[kio::test] async fn a_group_fetch_drops_a_datagram_group() { - let mut run = GroupFetchRun::new(VERSION, datagram_object(SEQUENCE)).await; + let mut run = GroupFetchRun::new(VERSION, datagram_object(KEY)).await; let (slot, fetch) = run.slot(None, None).await; let res = run.subscriber.recv_group_fetch(&mut run.stream, slot).await; assert!(res.is_ok(), "{res:?}"); assert!(matches!(fetch.await, Err(Error::NotFound))); - assert!(run.track.consume().peek_group(SEQUENCE).is_none(), "nothing is cached"); + assert!(run.track.consume().peek_group(KEY).is_none(), "nothing is cached"); } /// A refused multi-group FETCH names every owed group dropped, without retrying @@ -17351,11 +17320,12 @@ mod stitch_tests { run.track.broadcast().pool.poll(now, &kio::Waiter::noop()); run.track.received(); run.track.start_at(3).unwrap(); - run.track.create_group(3u64.into()).unwrap().finish().unwrap(); + run.track.create_group(3u64.into()).unwrap().close().unwrap(); let requests = run.track.requests(); + // From group 0, which a time start names whatever is held. let mut reader = run.track.subscribe( track::Subscription::default() - .with_range(1..3) + .with_range(0..3) .with_delay(track::Delay::default().with_max(None)), ); let waiter = kio::Waiter::noop(); @@ -17364,7 +17334,7 @@ mod stitch_tests { let Poll::Ready(Ok(fill)) = requests.poll_requested_fill(&waiter) else { panic!("{version}: missing history was not requested"); }; - assert_eq!((fill.start().group, fill.end()), (1, 3)); + assert_eq!((fill.start().group, fill.end()), (0, 3)); run.subscriber .clone() .run_group_fetch(Path::from_static("broadcast"), "video".into(), fill, None) @@ -17376,7 +17346,7 @@ mod stitch_tests { .poll(now.saturating_add(crate::ledger::GRACE), &waiter); assert_eq!( reader.poll_drop(&waiter), - Poll::Ready((1..3, crate::track::Hole::drop(crate::StreamError::NotFound))), + Poll::Ready((0..3, crate::track::Hole::drop(crate::StreamError::NotFound))), "{version}" ); assert!(reader.poll_drop(&waiter).is_pending(), "each drop is reported once"); @@ -17399,11 +17369,13 @@ mod stitch_tests { let dynamic = run.track.requests(); let consumer = run.track.consume(); run.track.start_at(7).unwrap(); - run.track.create_group(7u64.into()).unwrap().finish().unwrap(); + run.track.create_group(7u64.into()).unwrap().close().unwrap(); + // From group 0, which a time start names whatever is held, so the history below + // the feed is asked as a miss rather than looked up. let mut reading = consumer .subscribe(Some( track::Subscription::default() - .with_range(3..7) + .with_range(0..7) .with_order(crate::track::Order::Ascending) .with_delay(track::Delay::default().with_max(None)), )) @@ -17413,15 +17385,15 @@ mod stitch_tests { let fill = dynamic.requested_fill().await.unwrap(); assert_eq!( (fill.start().group, fill.end()), - (3, 7), + (0, 7), "{version}: one fill for the run" ); // Readers of the missing groups join that fill. - let missing = [3, 5].map(|sequence| consumer.fetch_group(sequence, None)); + let missing = [3, 5].map(|key| consumer.fetch_group(key, None)); let slot = kio::Producer::new(GroupFetch::Ready(RangeFetch { fill: std::sync::Arc::new(fill), timescale: None, - first: 3, + first: 0, last: 6, start: 0, end: None, @@ -17434,13 +17406,13 @@ mod stitch_tests { .expect("recv"); assert!(res.is_ok(), "{version}: {res:?}"); assert_eq!(consumer.cached_groups(0..7), [4, 6], "{version}"); - for sequence in [4, 6] { - let mut group = consumer.fetch_group(sequence, None).await.unwrap(); + for key in [4, 6] { + let mut group = consumer.fetch_group(key, None).await.unwrap(); for object in 0..2 { let frame = group.read_frame().await.unwrap().unwrap(); assert_eq!( frame.payload.as_ref(), - format!("{sequence}-{object}").as_bytes(), + format!("{key}-{object}").as_bytes(), "{version}" ); } @@ -17470,11 +17442,11 @@ mod stitch_tests { ietf::FetchHeader { request_id: REQUEST } .encode(&mut crate::coding::Encoder::new(&mut fill, VERSION.into()), VERSION) .unwrap(); - fill.extend(datagram_object(SEQUENCE)); + fill.extend(datagram_object(KEY)); let h = Harness::new( Fill::Serving(Some(Timescale::MICRO)), - vec![fill, tail_stream(SEQUENCE + 1, 0, &[b"next"])], + vec![fill, tail_stream(KEY + 1, 0, &[b"next"])], ); let mut consumer = h.track.subscribe(None); let mut fill = h.stream().await; @@ -17490,21 +17462,21 @@ mod stitch_tests { .await .expect("the subscription carries on"); - let (sequence, frames) = read_group(&mut consumer).await; - assert_eq!(sequence, SEQUENCE + 1); + let (key, frames) = read_group(&mut consumer).await; + assert_eq!(key, KEY + 1); assert_eq!(frames.len(), 1); assert_eq!(frames[0].1, b"next"); } /// A group fetch's objects after the FETCH_HEADER: the first one names the group and /// `start`, and every later one is the next object. - fn group_fetch_objects(sequence: u64, start: u64, payloads: &[&[u8]]) -> Vec { + fn group_fetch_objects(key: u64, start: u64, payloads: &[&[u8]]) -> Vec { let mut buf = Vec::new(); for (index, payload) in payloads.iter().enumerate() { let first = index == 0; ietf::FetchObject::Object { subgroup: ietf::FetchSubgroup::Zero, - group: first.then_some(sequence), + group: first.then_some(key), object: first.then_some(start), priority: first.then_some(0), properties: None, @@ -17523,7 +17495,7 @@ mod stitch_tests { /// error instead of leaving the run's readers waiting on the stream. #[kio::test] async fn a_group_aborted_under_a_stalled_fetch_ends_the_run() { - let mut run = GroupFetchRun::stalled(VERSION, group_fetch_objects(SEQUENCE, 0, &[b"0"])).await; + let mut run = GroupFetchRun::stalled(VERSION, group_fetch_objects(KEY, 0, &[b"0"])).await; let (slot, _fetch) = run.slot(None, None).await; let mut receiving = run.subscriber.clone(); let mut receive = std::pin::pin!(receiving.recv_group_fetch(&mut run.stream, slot.clone())); @@ -17597,7 +17569,7 @@ mod stitch_tests { } /// Queue the cache miss through the same request state as a wire FETCH, answered - /// through group `SEQUENCE`'s object `end` when set. The fetch resolves once the stream + /// through group `KEY`'s object `end` when set. The fetch resolves once the stream /// writes the group. async fn slot( &self, @@ -17608,27 +17580,27 @@ mod stitch_tests { std::pin::Pin>>, ) { let dynamic = self.track.requests(); - let mut fetch = Box::pin(self.track.consume().fetch_group(SEQUENCE, None)); + let mut fetch = Box::pin(self.track.consume().fetch_group(KEY, None)); assert!(futures::poll!(fetch.as_mut()).is_pending()); let fill = dynamic.requested_fill().await.expect("group request"); let slot = kio::Producer::new(GroupFetch::Ready(RangeFetch { fill: std::sync::Arc::new(fill), timescale, - first: SEQUENCE, - last: SEQUENCE, + first: KEY, + last: KEY, start: 0, end, })); (slot, fetch) } - /// Decode the stream as the answer for group `SEQUENCE`. + /// Decode the stream as the answer for group `KEY`. async fn recv_objects(&mut self) -> Result<(), Error> { let (slot, _fetch) = self.slot(None, None).await; self.subscriber.recv_group_fetch(&mut self.stream, slot).await } - /// Decode the stream as the answer for group `SEQUENCE` through the exclusive object + /// Decode the stream as the answer for group `KEY` through the exclusive object /// `end`, in `timescale`, and read back the group it cached. async fn recv_cached( &mut self, @@ -17804,14 +17776,14 @@ mod joining_fetch_tests { track.fill.read().outstanding() } - fn fill_headless(&self, sequence: u64) -> bool { + fn fill_headless(&self, key: u64) -> bool { let state = self.subscriber.state.lock(); let track = state .subscribes .values() .next() .expect("the subscription is registered"); - track.fill.read().headless(sequence) + track.fill.read().headless(key) } } @@ -18249,7 +18221,7 @@ mod joining_fetch_tests { group = consumer.recv_group().fuse() => group.expect("track aborted").expect("track finished"), res = ingest.fuse() => panic!("the next group's stream ended: {res:?}"), }; - assert_eq!(group.sequence, 5, "the cut group was skipped"); + assert_eq!(group.start, 5, "the cut group was skipped"); } /// A publisher that serves no FETCH refuses the joining FETCH. That costs the head of @@ -18340,9 +18312,9 @@ mod joining_fetch_tests { }; let mut head = None; open_fill_group(&mut track, &mut head, 4).unwrap(); - let (sequence, next, producer) = head.unwrap(); + let (key, next, producer) = head.unwrap(); *fill.write().ok().unwrap() = Fill::Ready { - sequence, + key, next, producer: producer.into_inner(), }; @@ -18351,7 +18323,7 @@ mod joining_fetch_tests { let Poll::Ready(Ok(group)) = futures::poll!(fetch.as_mut()) else { panic!("the miss must resolve from the cache"); }; - assert_eq!(group.sequence, 4); + assert_eq!(group.start, 4); } /// Only the joined group waits for the subscription's fill. A head that never arrives @@ -18554,7 +18526,7 @@ mod joining_fetch_tests { let mut subscriber = (&mut run._hold.2).await.expect("subscribed"); let mut group = subscriber.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 4, "{version}"); + assert_eq!(group.start, 4, "{version}"); assert_eq!(group.read_frame().await.unwrap().unwrap().payload.as_ref(), b"x"); // The grace runs out with the head still being written, ending the subscription. @@ -18641,7 +18613,7 @@ mod joining_fetch_tests { let mut subscriber = (&mut run._hold.2).await.expect("subscribed"); let mut group = subscriber.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 4, "{version}/{status_code}"); + assert_eq!(group.start, 4, "{version}/{status_code}"); assert_eq!(group.read_frame().await.unwrap().unwrap().payload.as_ref(), b"x"); while !run.subscriber.state.lock().subscribes.is_empty() { @@ -19059,11 +19031,7 @@ mod joining_fetch_tests { .run_group_fetch(Path::from_static("broadcast"), "video".into(), request, None) .await; - assert_eq!( - consumer.final_sequence(), - None, - "{version}: rejected end changed the track" - ); + assert_eq!(consumer.final_key(), None, "{version}: rejected end changed the track"); assert!(matches!(fetch.await, Err(Error::ProtocolViolation))); let violation = SessionError::from(&Error::ProtocolViolation).to_code(); assert!( @@ -19202,7 +19170,7 @@ mod joining_fetch_tests { .unwrap_or_else(|err| panic!("{version}: {err:?}")); serving.await.expect("run_group_fetch"); - assert_eq!(consumer.final_sequence(), ended.then_some(GROUP + 1)); + assert_eq!(consumer.final_key(), ended.then_some(GROUP + 1)); let mut group = fetch.await.expect("fetched"); assert_eq!(group.index(), START, "the group starts where the reader wanted"); let mut payloads = Vec::new(); @@ -19337,15 +19305,19 @@ mod new_group_tests { (stream, msg.new_group) } - /// Group `sequence` lands in the copy: a first frame, and a delta when `delta`. - fn land(&mut self, sequence: u64, delta: bool) { + /// Group `key` lands in the copy: a first frame, and a delta when `delta`. + fn land(&mut self, key: u64, delta: bool) { let state = self.probe.state.lock(); let copy = state.subscribes.values().next().unwrap().producer.clone().unwrap(); drop(state); - let mut group = copy.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, b"key".as_slice()).unwrap(); + let mut group = copy.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), b"key".as_slice()) + .unwrap(); if delta { - group.write_frame(None, b"delta".as_slice()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), b"delta".as_slice()) + .unwrap(); } self.landed.push(group); } @@ -19739,7 +19711,7 @@ mod new_group_tests { let mut current = header(newest.request_id).await; relay.probe.clone().recv_fill(&mut current).await.unwrap(); assert!( - matches!(*fill.read(), Fill::Failed { sequence: 5 }), + matches!(*fill.read(), Fill::Failed { key: 5 }), "only the current header claimed the slot, and it carried no head" ); relay.serving.abort(); @@ -19803,14 +19775,14 @@ mod new_group_tests { let mut empty = header(widening.request_id).await; relay.probe.clone().recv_fill(&mut empty).await.unwrap(); let fill = relay.probe.state.lock().subscribes[&subscription].fill.clone(); - assert!(matches!(*fill.read(), Fill::Failed { sequence } if sequence == largest)); + assert!(matches!(*fill.read(), Fill::Failed { key } if key == largest)); failed.push((largest, fill)); } - for (sequence, fill) in &failed { + for (key, fill) in &failed { assert!( - fill.read().headless(*sequence), - "a stream already opening on the old fill still finds group {sequence} headless" + fill.read().headless(*key), + "a stream already opening on the old fill still finds group {key} headless" ); } assert_eq!( diff --git a/rs/moq-net/src/ietf/subscriber/tracks.rs b/rs/moq-net/src/ietf/subscriber/tracks.rs index fc10bd33974..ff282d90eb2 100644 --- a/rs/moq-net/src/ietf/subscriber/tracks.rs +++ b/rs/moq-net/src/ietf/subscriber/tracks.rs @@ -2387,9 +2387,13 @@ mod tests { .producer .clone() .unwrap(); - let mut group = copy.create_group(crate::model::group::Info { sequence: 5 }).unwrap(); - group.write_frame(None, b"key".as_slice()).unwrap(); - group.write_frame(None, b"delta".as_slice()).unwrap(); + let mut group = copy.create_group(crate::model::group::Info { start: 5 }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), b"key".as_slice()) + .unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), b"delta".as_slice()) + .unwrap(); // A join past the ratio asks for group 6, which stays outstanding. let join = track.subscribe(None).await.unwrap(); diff --git a/rs/moq-net/src/instance.rs b/rs/moq-net/src/instance.rs index 5d3b0343033..b53e553e820 100644 --- a/rs/moq-net/src/instance.rs +++ b/rs/moq-net/src/instance.rs @@ -4,7 +4,7 @@ use std::sync::Arc; /// Names one continuous stream at a path. /// -/// Within an instance, group sequences never restart and never name two different groups, +/// Within an instance, group keys never restart and never name two different groups, /// so a reader continues across every route that shares it. Different instances are /// independent streams, never stitched together. /// diff --git a/rs/moq-net/src/ledger.rs b/rs/moq-net/src/ledger.rs index 8dce93e7470..af60e72bb5d 100644 --- a/rs/moq-net/src/ledger.rs +++ b/rs/moq-net/src/ledger.rs @@ -1,4 +1,4 @@ -//! The sequences a feed has accounted for, and how long each gap between them is waited for. +//! The keys a feed has accounted for, and how long each gap between them is waited for. //! //! A subscription's tail (the session's `tail::Tail`) and a track's record of live arrivals //! both keep one, so they give up on a missing group by the same rule. @@ -34,7 +34,7 @@ pub(crate) const RUN_OVERHEAD: u64 = 2 * (size_of::<(u64, Run)>() + size_of::<(Instant, u64)>() + size_of::<(u64, (u64, u64))>() + size_of::<(u64, u64)>()) as u64; -/// A run of accounted sequences, keyed by its inclusive start. +/// A run of accounted keys, keyed by its inclusive start. #[derive(Debug, Clone, PartialEq, Eq)] struct Run { end: u64, @@ -42,15 +42,15 @@ struct Run { since: Instant, } -/// The sequences accounted for, and how late each gap between them is. +/// The keys accounted for, and how late each gap between them is. #[derive(Debug)] pub(crate) struct Ledger { - // Disjoint, sorted, non-adjacent runs of accounted sequences. A gap older than the grace + // Disjoint, sorted, non-adjacent runs of accounted keys. A gap older than the grace // can no longer be waited for, so it folds into the runs around it, which bounds this by // the gaps opened within the grace rather than every gap ever seen. A fold is final: a // later, longer grace or a lowered floor cannot reopen it. runs: BTreeMap, - // Demand can restart a gap below an older one, so sequence order is not age order. + // Demand can restart a gap below an older one, so key order is not age order. // Index both: one due gap must not make every arrival scan all the younger ones. ages: BTreeSet<(Instant, u64)>, grace: Duration, @@ -208,14 +208,14 @@ impl Ledger { self.demand(groups, now); } - /// Account a live arrival, refreshing a cached incarnation even if its sequence was + /// Account a live arrival, refreshing a cached incarnation even if its key was /// accounted before. Its gap ages stay unchanged, but readers must revisit a refusal. - pub fn arrive(&mut self, sequence: u64, now: Instant) { + pub fn arrive(&mut self, key: u64, now: Instant) { let revision = self.changes.revision(); - self.account(sequence..sequence.saturating_add(1), now); + self.account(key..key.saturating_add(1), now); if self.changes.revision() == revision - && let Some((&start, run)) = self.runs.range(..=sequence).next_back() - && sequence < run.end + && let Some((&start, run)) = self.runs.range(..=key).next_back() + && key < run.end { self.changes.insert(start..run.end); } @@ -240,7 +240,7 @@ impl Ledger { } /// Fold the oldest gap into the runs around it however young, giving up on it early, - /// and return the sequences it held; `None` when no gap is left to fold. + /// and return the keys it held; `None` when no gap is left to fold. pub fn fold_oldest(&mut self) -> Option> { let first = self.first()?; let &(_, start) = self.ages.iter().find(|(_, start)| *start != first)?; @@ -262,23 +262,23 @@ impl Ledger { self.runs.len() } - /// The lowest sequence accounted for. + /// The lowest key accounted for. pub fn first(&self) -> Option { self.runs.first_key_value().map(|(start, _)| *start) } - /// The exclusive end of the accounted run holding `sequence`, `None` while it is not. - pub fn run_end(&self, sequence: u64) -> Option { + /// The exclusive end of the accounted run holding `key`, `None` while it is not. + pub fn run_end(&self, key: u64) -> Option { self.runs - .range(..=sequence) + .range(..=key) .next_back() - .filter(|(_, run)| sequence < run.end) + .filter(|(_, run)| key < run.end) .map(|(_, run)| run.end) } - /// The exclusive end of the highest accounted run starting at or below `sequence`. - pub fn end_below(&self, sequence: u64) -> Option { - self.runs.range(..=sequence).next_back().map(|(_, run)| run.end) + /// The exclusive end of the highest accounted run starting at or below `key`. + pub fn end_below(&self, key: u64) -> Option { + self.runs.range(..=key).next_back().map(|(_, run)| run.end) } /// The accounted runs overlapping `groups`, clipped to it, lowest first. diff --git a/rs/moq-net/src/lite/group.rs b/rs/moq-net/src/lite/group.rs index abb78e3a368..f30ff2c2da6 100644 --- a/rs/moq-net/src/lite/group.rs +++ b/rs/moq-net/src/lite/group.rs @@ -7,8 +7,8 @@ pub struct Group { // The subscribe ID. pub subscribe: u64, - // The group sequence number - pub sequence: u64, + // The group key + pub key: u64, // The index of the first frame on this stream. 0 (the common case) means the stream // carries the group from its beginning; a higher value means the leading frames are @@ -30,7 +30,7 @@ impl<'a> Message<'a> for Group { fn decode_msg(r: &mut Decoder<'a>, version: Version) -> Result { let subscribe = r.varint()?; - let sequence = r.varint()?; + let key = r.varint()?; let frame_start = match version.has_frame_bounds() { true => r.varint()?, false => 0, @@ -38,14 +38,14 @@ impl<'a> Message<'a> for Group { Ok(Self { subscribe, - sequence, + key, frame_start, }) } fn encode_msg(&self, w: &mut Encoder<'_>, version: Version) -> Result<(), EncodeError> { w.varint(self.subscribe)?; - w.varint(self.sequence)?; + w.varint(self.key)?; if version.has_frame_bounds() { w.varint(self.frame_start)?; @@ -67,7 +67,7 @@ mod test { fn frame_start_roundtrips() { let msg = Group { subscribe: 1, - sequence: 7, + key: 7, frame_start: 4, }; let mut buf = Vec::new(); @@ -75,7 +75,7 @@ mod test { .unwrap(); let got = crate::coding::decode_buf(&mut buf.as_slice(), Version::Lite06, |r, v| Group::decode_msg(r, v)).unwrap(); - assert_eq!((got.sequence, got.frame_start), (7, 4)); + assert_eq!((got.key, got.frame_start), (7, 4)); } /// An older peer would number the frames from 0 and silently misalign the group. @@ -83,7 +83,7 @@ mod test { fn frame_start_rejected_before_lite06() { let msg = Group { subscribe: 1, - sequence: 7, + key: 7, frame_start: 4, }; let mut buf = Vec::new(); @@ -124,7 +124,7 @@ mod test { let fields = if version.has_frame_bounds() { 3 } else { 2 }; let msg = Group { subscribe: max, - sequence: max, + key: max, frame_start: if version.has_frame_bounds() { max } else { 0 }, }; @@ -140,8 +140,8 @@ mod test { let (got, used) = Group::decode_slice(&wire, version).unwrap(); assert_eq!(used, wire.len(), "{version:?}"); assert_eq!( - (got.subscribe, got.sequence, got.frame_start), - (msg.subscribe, msg.sequence, msg.frame_start), + (got.subscribe, got.key, got.frame_start), + (msg.subscribe, msg.key, msg.frame_start), "{version:?}" ); } diff --git a/rs/moq-net/src/lite/ietf.rs b/rs/moq-net/src/lite/ietf.rs index fae797bec88..85a1d4ad8ba 100644 --- a/rs/moq-net/src/lite/ietf.rs +++ b/rs/moq-net/src/lite/ietf.rs @@ -297,7 +297,7 @@ impl Records { if fields .properties .sealed_time - .is_some_and(|sealed| self.timescale != Some(sealed.scale())) + .is_some_and(|sealed| self.timescale != sealed.scale()) { return Err(Error::Unsupported); } @@ -827,7 +827,7 @@ mod hop_tests { .create_object(frame::Header { info: frame::Info { size: payload.len() as u64, - timestamp: Some(ts(millis)), + timestamp: ts(millis), }, fields: fields.map(Arc::new), inflated: None, @@ -891,7 +891,7 @@ mod hop_tests { info.properties = track_properties.clone(); let track = broadcast.broadcast().create_track("video", info).unwrap(); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); group .set_wire(group::Wire { sparse: false, @@ -920,7 +920,7 @@ mod hop_tests { // The upstream group is still open. let frame = next(&mut received).await.unwrap(); assert_eq!(&frame.payload.contiguous(0)[..], b"a"); - assert_eq!(frame.timestamp, Some(ts(0))); + assert_eq!(frame.timestamp, ts(0)); match negotiated { true => { assert_eq!(fields(&frame), Some((first.id, first.properties.clone())), "{offer:?}"); @@ -951,7 +951,7 @@ mod hop_tests { properties: Properties::default(), }) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let frame = next(&mut received).await.unwrap(); assert_eq!(&frame.payload.contiguous(0)[..], b"b"); @@ -1108,21 +1108,21 @@ mod hop_tests { let info = kio::clock::timeout(TIMEOUT, copy.info()).await.unwrap().unwrap(); assert!(info.properties.is_none()); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(5), b"key".to_vec()).unwrap(); let mut received = recv(&mut subscriber).await; let frame = next(&mut received).await.unwrap(); assert_eq!( (frame.timestamp, &frame.payload.contiguous(0)[..]), - (Some(ts(5)), &b"key"[..]) + (ts(5), &b"key"[..]) ); assert!(frame.fields.is_none() && received.wire().is_none()); group.write_frame(ts(6), b"delta".to_vec()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let frame = next(&mut received).await.unwrap(); assert_eq!( (frame.timestamp, &frame.payload.contiguous(0)[..]), - (Some(ts(6)), &b"delta"[..]) + (ts(6), &b"delta"[..]) ); assert!(frame.fields.is_none()); assert!(next(&mut received).await.is_none()); @@ -1146,7 +1146,7 @@ mod hop_tests { let track = broadcast.broadcast().create_track("video", info).unwrap(); let mut head = track.create_group(0u64.into()).unwrap(); head.write_frame(ts(100), b"frame".to_vec()).unwrap(); - head.finish().unwrap(); + head.close().unwrap(); let (downstream, sessions) = test_hop(&upstream, "room", offer).await; let broadcast = downstream.consume().request("room", None).await.unwrap(); // A listing carries no provenance, so it lists the track as native lite. @@ -1177,7 +1177,7 @@ mod hop_tests { .await .unwrap() .unwrap(); - assert_eq!(next(&mut fetched).await.unwrap().timestamp, Some(ts(100))); + assert_eq!(next(&mut fetched).await.unwrap().timestamp, ts(100)); assert!(next(&mut fetched).await.is_none()); let mut subscriber = kio::clock::timeout(TIMEOUT, copy.subscribe(track::Subscription::default().with_start(1))) @@ -1190,11 +1190,11 @@ mod hop_tests { let start = if negotiated { ts(50) } else { ts(150) }; let mut group = track.create_group(1u64.into()).unwrap(); group.write_frame(start, b"frame".to_vec()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut received = recv(&mut subscriber).await; - assert_eq!(received.sequence, 1); + assert_eq!(received.start, 1); let frame = next(&mut received).await.unwrap(); - assert_eq!(frame.timestamp, Some(start), "{offer:?}"); + assert_eq!(frame.timestamp, start, "{offer:?}"); assert_eq!(&frame.payload.contiguous(0)[..], b"frame"); assert!(next(&mut received).await.is_none()); drop(sessions); @@ -1216,16 +1216,16 @@ mod hop_tests { assert_eq!(subscriber.info().compression, crate::track::Compression::Deflate); let payload = |n: u8| vec![n; 300]; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(5), payload(1)).unwrap(); group.write_frame(ts(6), payload(2)).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut received = recv(&mut subscriber).await; for (millis, n) in [(5, 1), (6, 2)] { let frame = next(&mut received).await.unwrap(); assert_eq!( (frame.timestamp, &frame.payload.contiguous(0)[..]), - (Some(ts(millis)), &payload(n)[..]) + (ts(millis), &payload(n)[..]) ); assert!(frame.fields.is_none()); } diff --git a/rs/moq-net/src/lite/publisher.rs b/rs/moq-net/src/lite/publisher.rs index 04e5aa9b8f2..0c130a500c7 100644 --- a/rs/moq-net/src/lite/publisher.rs +++ b/rs/moq-net/src/lite/publisher.rs @@ -2322,18 +2322,6 @@ impl Request for TracksServe { /// Answers a SUBSCRIBE stream: streams the track's groups /// until the track ends. enum SubscribeServe { - /// A time anchor waits for the track's timescale, which places it, before the model - /// subscription opens. - Query { - /// Boxed: the request is far larger than the other states' handles. - msg: Box>, - track: track::Consumer, - querying: track::Resolving, - /// The newest group when the SUBSCRIBE arrived; see [`cursor_floor`]. - latest: Option, - update: Option, - legacy: Option>, - }, /// Waiting for the model subscription to be confirmed. Confirm { /// Boxed: the request is far larger than the other states' handles. @@ -2356,7 +2344,7 @@ enum SubscribeServe { priority: i8, /// Set once the track's info is known. send: Option, - /// One past the highest sequence served, which SUBSCRIBE_FINISH declares as the end. + /// One past the highest key served, which SUBSCRIBE_FINISH declares as the end. end: u64, }, /// The track finished: draining the in-flight group streams before the FIN. @@ -2411,25 +2399,14 @@ impl Request for SubscribeServe { Ok(track) => track, Err(err) => return Ok(Self::datagrams(Legacy::datagrams(legacy, err)?, &msg, None)), }; - let latest = track.latest_sequence(); - if Bounds::has_time(&msg.ranges) { - let querying = track.info().into_inner(); - return Ok(Self::Query { - msg: Box::new(msg), - track, - querying, - latest, - update: None, - legacy, - }); - } - let bounds = Bounds::new(&msg.ranges, None)?; + let latest = track.latest_key(); + let bounds = Bounds::new(&msg.ranges, !shared.version.has_ranges()); Ok(Self::subscribe(shared, msg, bounds, &track, latest, None, legacy)) } fn update(&mut self, update: LiveUpdate) { match self { - Self::Query { update: pending, .. } | Self::Confirm { update: pending, .. } => *pending = Some(update), + Self::Confirm { update: pending, .. } => *pending = Some(update), Self::Run(run) => run.update(update), Self::Datagrams { consumer, @@ -2456,51 +2433,6 @@ impl Request for SubscribeServe { ) -> Poll, Error>> { loop { match self { - Self::Query { querying, legacy, .. } => { - let info = match querying.poll_ok(waiter) { - Poll::Ready(info) => info, - Poll::Pending => { - let consumer = ready!(Legacy::poll_datagrams(legacy, waiter)); - let Self::Query { msg, update, .. } = std::mem::replace( - self, - Self::Drain { - children: Default::default(), - _track: None, - }, - ) else { - unreachable!() - }; - *self = Self::datagrams(consumer, &msg, update); - continue; - } - }; - let Self::Query { - msg, - track, - latest, - update, - legacy, - .. - } = std::mem::replace( - self, - Self::Drain { - children: Default::default(), - _track: None, - }, - ) - else { - unreachable!() - }; - let info = match info { - Ok(info) => info, - Err(err) => { - *self = Self::datagrams(Legacy::datagrams(legacy, err)?, &msg, update); - continue; - } - }; - let bounds = Bounds::new(&msg.ranges, info.timescale)?; - *self = Self::subscribe(shared, *msg, bounds, &track, latest, update, legacy); - } Self::Confirm { subscribing, legacy, .. } => { @@ -2592,7 +2524,6 @@ impl Request for SubscribeServe { steer_seen: steer, version: shared.version, stamp, - runtime: shared.runtime.clone(), outcomes: Default::default(), ietf, }; @@ -2600,10 +2531,13 @@ impl Request for SubscribeServe { let latest = match shared.version { Version::Lite01 | Version::Lite02 => latest, _ => latest - .max(track.latest_sequence()) + .max(track.latest_key()) .max(track.peek_largest().map(|largest| largest.group)), }; - let floor = cursor_floor(shared.version, bounds.groups().start(), latest); + // A legacy start serves from the first group at or after it, so the cursor + // never hands out the one before it: dropped here, it would stay consumed + // when an update lowers the floor to reach it. + let floor = cursor_floor(shared.version, bounds.groups().start(), latest).max(bounds.frames.floor); let mut run = TrackRun::new(sub, track, bounds, floor, steer_tx); run.update = msg.update; run.datagrams = LegacyDatagrams::join(legacy, msg.id, msg.priority, timescale); @@ -2945,8 +2879,6 @@ struct DatagramSend { version: Version, /// How datagrams are stamped, or `None` when they carry no timestamp. stamp: Option, - /// The clock an untimed track's Lite05/06 send times are read from. - runtime: crate::time::Deadlines, /// The publisher's priority for the track, which breaks subscriber priority ties. publisher: i8, /// The subscription's send group, or `None` on a transport without groups. @@ -2964,7 +2896,6 @@ impl DatagramSend { id, version: shared.version, stamp: Stamp::new(shared.version, info.timescale), - runtime: shared.runtime.clone(), publisher: info.priority, group, stats, @@ -2982,15 +2913,11 @@ impl DatagramSend { /// dropped, as a datagram may be anywhere, if its body doesn't fit the transport's /// datagram size or the transport has no room for it now. /// - /// Fails on a sequence the version's varints cannot carry (2^62 and up before lite-07): + /// Fails on a key the version's varints cannot carry (2^62 and up before lite-07): /// every datagram its source produces later is at least as far, so the subscription can /// deliver no more. One the network reordered below it is lost, as any datagram may be. fn send(&self, datagram: crate::datagram::Datagram) -> Result<(), Error> { - let Ok(timestamp) = self - .stamp - .map(|stamp| stamp.wire(datagram.timestamp, &self.runtime)) - .transpose() - else { + let Ok(timestamp) = self.stamp.map(|stamp| stamp.wire(datagram.timestamp)).transpose() else { return Ok(()); }; let sequence = datagram.sequence; @@ -3338,7 +3265,6 @@ impl Request for FetchServe { next.size, *stamp, prev_ts, - &shared.runtime, next.size - byte as u64, )?; *frame = Some(next); @@ -3368,7 +3294,6 @@ impl Request for FetchServe { batched.frame.payload.len() as u64, *stamp, prev_ts, - &shared.runtime, batched.frame.payload.len() as u64, )?; let payload = std::mem::take(&mut batched.frame.payload); @@ -3405,7 +3330,6 @@ impl Request for FetchServe { next.size, *stamp, prev_ts, - &shared.runtime, next.size, )?; *frame = Some(next); @@ -3444,11 +3368,11 @@ mod test { let producer = track_producer("test"); for second in 0..3 { - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); group .write_frame(Timestamp::from_millis(second * 1000).unwrap(), b"x".to_vec()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } // What run_subscribe hands the model for a peer that sent no budget at all. @@ -3459,11 +3383,11 @@ mod test { )) }; let drain = |subscriber: &mut track::Subscriber| { - let mut sequences = Vec::new(); + let mut keys = Vec::new(); while let Some(Ok(Some(group))) = subscriber.recv_group().now_or_never() { - sequences.push(group.sequence); + keys.push(group.start); } - sequences + keys }; let mut unpinned = served(Version::Lite01); @@ -3474,7 +3398,7 @@ mod test { ); let legacy = served(Version::Lite01); - let mut legacy = legacy.with_range(cursor_floor(Version::Lite01, None, producer.latest_sequence())..); + let mut legacy = legacy.with_range(cursor_floor(Version::Lite01, None, producer.latest_key())..); assert_eq!(drain(&mut legacy), vec![2]); // Lite03-05 declare a budget, but their drafts define it as a staleness tolerance @@ -3483,7 +3407,7 @@ mod test { track::Subscription::default() .with_delay(track::Delay::default().with_max(std::time::Duration::from_secs(5))), ); - let mut tolerant = tolerant.with_range(cursor_floor(Version::Lite05, None, producer.latest_sequence())..); + let mut tolerant = tolerant.with_range(cursor_floor(Version::Lite05, None, producer.latest_key())..); assert_eq!(drain(&mut tolerant), vec![2]); // On lite-06 the declared budget is what resolves the start, so it stands. @@ -3491,7 +3415,7 @@ mod test { track::Subscription::default() .with_delay(track::Delay::default().with_max(std::time::Duration::from_secs(5))), ); - let mut declared = declared.with_range(cursor_floor(Version::Lite06, None, producer.latest_sequence())..); + let mut declared = declared.with_range(cursor_floor(Version::Lite06, None, producer.latest_key())..); assert_eq!(drain(&mut declared), vec![0, 1, 2]); } @@ -3505,12 +3429,10 @@ mod test { let mut producer = track_producer("test"); let mut subscriber = producer.subscribe(None); - let write = |producer: &mut track::Producer, sequence: u64| { - let mut group = producer.create_group(crate::group::Info { sequence }).unwrap(); - group - .write_frame(Timestamp::from_millis(1).unwrap(), b"x".to_vec()) - .unwrap(); - group.finish().unwrap(); + let write = |producer: &mut track::Producer, key: u64| { + let mut group = producer.create_group(crate::group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), b"x".to_vec()).unwrap(); + group.close().unwrap(); }; // Group 7 arrives first: run_track resolves the start there and raises @@ -3518,8 +3440,8 @@ mod test { write(&mut producer, 7); match recv_next(&mut subscriber, false).await.unwrap() { Recv::Group(group) => { - assert_eq!(group.sequence, 7); - subscriber.set_range(group.sequence..); + assert_eq!(group.start, 7); + subscriber.set_range(group.start..); } _ => panic!("expected the first group"), } @@ -3544,23 +3466,21 @@ mod test { .with_delay(track::Delay::default().with_max(std::time::Duration::from_secs(60))), ); - let write = |producer: &mut track::Producer, sequence: u64| { - let mut group = producer.create_group(crate::group::Info { sequence }).unwrap(); - group - .write_frame(Timestamp::from_millis(1).unwrap(), b"x".to_vec()) - .unwrap(); - group.finish().unwrap(); + let write = |producer: &mut track::Producer, key: u64| { + let mut group = producer.create_group(crate::group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), b"x".to_vec()).unwrap(); + group.close().unwrap(); }; write(&mut producer, 7); match recv_next(&mut subscriber, false).await.unwrap() { - Recv::Group(group) => assert_eq!(group.sequence, 7), + Recv::Group(group) => assert_eq!(group.start, 7), _ => panic!("expected the first group"), } write(&mut producer, 5); match recv_next(&mut subscriber, false).await.unwrap() { - Recv::Group(group) => assert_eq!(group.sequence, 5), + Recv::Group(group) => assert_eq!(group.start, 5), _ => panic!("expected the late group above the floor"), } } @@ -3572,15 +3492,15 @@ mod test { // The last group is 6 (exclusive 7), but only group 5 has been produced so far. producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); producer.finish_at(track::Position::group(7), None).unwrap(); // Group 5 is delivered first. match recv_next(&mut subscriber, true).await.unwrap() { - Recv::Group(group) => assert_eq!(group.sequence, 5), + Recv::Group(group) => assert_eq!(group.start, 5), _ => panic!("expected group 5"), } @@ -3594,12 +3514,12 @@ mod test { // The caller stops requesting the boundary once sent. The trailing group arrives, // then the track finishes. producer - .create_group(group::Info { sequence: 6 }) + .create_group(group::Info { start: 6 }) .unwrap() - .finish() + .close() .unwrap(); match recv_next(&mut subscriber, false).await.unwrap() { - Recv::Group(group) => assert_eq!(group.sequence, 6), + Recv::Group(group) => assert_eq!(group.start, 6), _ => panic!("expected group 6"), } match recv_next(&mut subscriber, false).await.unwrap() { @@ -3641,7 +3561,7 @@ mod test { /// A relay can ingest back-to-back groups micro-reordered (the upstream leg /// sends newest-first). The older group is cached and in demand, so serving - /// must still deliver it; a sequence cursor would skip it permanently. + /// must still deliver it; a key cursor would skip it permanently. #[kio::test] async fn recv_next_serves_late_arrival_after_newer_group() { use futures::FutureExt; @@ -3652,23 +3572,23 @@ mod test { ); producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); match recv_next(&mut subscriber, false).await.unwrap() { - Recv::Group(group) => assert_eq!(group.sequence, 2), + Recv::Group(group) => assert_eq!(group.start, 2), _ => panic!("expected group 2"), } // Group 1 lands after group 2 was already served. producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); match recv_next(&mut subscriber, false).now_or_never() { - Some(Ok(Recv::Group(group))) => assert_eq!(group.sequence, 1), + Some(Ok(Recv::Group(group))) => assert_eq!(group.start, 1), Some(_) => panic!("expected the late-arriving group"), None => panic!("the late-arriving group was skipped"), } @@ -4663,12 +4583,11 @@ mod announce_test { fn buffer_frame_info( writer: &mut Writer, version: Version, - timestamp: Option, + timestamp: crate::Timestamp, inflated: Option, size: u64, stamp: Option, prev_ts: &mut u64, - runtime: &crate::time::Deadlines, // Payload bytes that follow: fewer than `size` only for a frame served from a byte. sent: u64, ) -> Result<(), Error> { @@ -4681,7 +4600,7 @@ fn buffer_frame_info( writer.buffer_varint(length)?; } if let Some(stamp) = stamp { - buffer_zigzag_delta(writer, stamp.wire(timestamp, runtime)?, prev_ts)?; + buffer_zigzag_delta(writer, stamp.wire(timestamp)?, prev_ts)?; } // Only a compressed group read in its stored form has one, and only on lite-07. if let Some(inflated) = inflated { @@ -4699,9 +4618,6 @@ fn buffer_frame_info( struct Stamp { /// The wire timescale TRACK_INFO declared (see [`Version::wire_timescale`]). timescale: crate::Timescale, - /// Whether each frame's own timestamp goes out. Lite05/06 cannot encode an untimed - /// track, so an untimed one carries its send time instead. - timed: bool, } impl Stamp { @@ -4709,16 +4625,17 @@ impl Stamp { fn new(version: Version, timescale: Option) -> Option { Some(Self { timescale: version.wire_timescale(timescale)?, - timed: timescale.is_some(), }) } - /// A frame or datagram's `timestamp` as its raw wire value, or its send time on - /// `runtime` when the track goes out untimed. - fn wire(self, timestamp: Option, runtime: &crate::time::Deadlines) -> Result { + /// A frame or datagram's `timestamp` as its raw wire value. A scale-less track's is its + /// group's key, or its datagram's sequence, which goes out verbatim: nothing is stamped + /// with a clock. + fn wire(self, timestamp: crate::Timestamp) -> Result { + if timestamp.scale().is_none() { + return Ok(timestamp.value()); + } timestamp - .filter(|_| self.timed) - .unwrap_or_else(|| crate::Timestamp::from(runtime.now())) .convert(self.timescale) .map(|time| time.value()) .map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded)) @@ -4741,7 +4658,7 @@ fn buffer_zigzag_delta( } /// What [`recv_next`] pulled from the one subscriber: the next group to serve, the track -/// declaring its exclusive final sequence, or the track finishing (the live edge having +/// declaring its exclusive final key, or the track finishing (the live edge having /// reached that boundary). #[allow( clippy::large_enum_variant, @@ -4755,25 +4672,25 @@ enum Recv { /// The group boundary SUBSCRIBE_FINISH carries. Lite cannot name a frame, so a mid-group end /// rounds up past its group, whose own stream FIN says where it stops. -fn final_sequence(end: track::Position) -> u64 { - end.final_sequence().unwrap_or(u64::MAX) +fn final_key(end: track::Position) -> u64 { + end.final_key().unwrap_or(u64::MAX) } -/// Every group sequence, for [`poll_recv_next`] to hand out any. +/// Every group key, for [`poll_recv_next`] to hand out any. const ALL: (Bound, Bound) = (Bound::Unbounded, Bound::Unbounded); /// Poll a single [`track::Subscriber`] for the next group (cap-aware, in arrival order). /// -/// Groups are served in arrival order (`poll_recv_group`), not sequence order: on a relay, a -/// burst can be ingested micro-reordered by the upstream leg, and a sequence cursor would then +/// Groups are served in arrival order (`poll_recv_group`), not key order: on a relay, a +/// burst can be ingested micro-reordered by the upstream leg, and a key cursor would then /// permanently skip the older group even though it is cached and in demand. Staleness is /// governed by max age and the subscriber's max delay, not arrival raciness. /// -/// The cursor hands out the next group only if its sequence lies in `groups`, leaving it in +/// The cursor hands out the next group only if its key lies in `groups`, leaving it in /// the cache otherwise, and without `groups` none at all. Either way it then never reports /// [`Recv::Finished`]. /// -/// When `emit_boundary` is set, a declared-but-not-yet-reached final sequence surfaces as +/// When `emit_boundary` is set, a declared-but-not-yet-reached final key surfaces as /// [`Recv::Boundary`] in an idle moment (after groups), so the caller can send /// SUBSCRIBE_FINISH as soon as the ending is known rather than waiting for the live edge to reach /// it. The caller clears `emit_boundary` after the first boundary so it fires once. @@ -4810,12 +4727,12 @@ fn poll_recv_next( let withheld = !end_error && !groups_finished && error.is_some() - && res.as_ref().is_ok_and(|end| track.reaches(final_sequence(*end))); + && res.as_ref().is_ok_and(|end| track.reaches(final_key(*end))); if !withheld { let error = error.filter(|_| end_error); return Poll::Ready(res.map(|end| { Recv::Boundary(lite::SubscribeFinish { - group: final_sequence(end), + group: final_key(end), error: error.as_ref().map(StreamError::from), }) })); @@ -4842,8 +4759,11 @@ async fn recv_next(track: &mut track::Subscriber, emit_boundary: bool) -> Result /// partial group, so it must be skipped: a group is the unit of decodability, and only /// the subscriber knows whether a partial one is any use to it. fn position_group(group: group::Consumer, frames: &Frames) -> Option { - let expected = frames.start(group.sequence); - let end = frames.end(group.sequence).map_or(Bound::Unbounded, Bound::Included); + if group.start < frames.floor { + return None; + } + let expected = frames.start(group.start); + let end = frames.end(group.start).map_or(Bound::Unbounded, Bound::Included); // The start clamps up to the first frame the group still holds, so landing higher // than asked means the frames below it are gone. @@ -4862,6 +4782,9 @@ struct Frames { first: Option<(u64, u64)>, /// The group the range ends partway through, and the last frame it serves of it, inclusive. last: Option<(u64, u64)>, + /// The lowest group a legacy wire's start serves: its start names the first group at or + /// after it, where the model's time start includes the group containing it. + floor: u64, } impl Frames { @@ -4888,26 +4811,19 @@ struct Bounds { } impl Bounds { - /// Read `ranges` as they arrived, with time anchors in `timescale`, the track's as this - /// peer was told it. + /// Read `ranges` as they arrived. Group IDs are group starts, so an absolute anchor and a + /// time anchor both name a start. /// - /// A time anchor on a track the peer was told is untimed is refused, since it names no - /// group. - fn new(ranges: &[lite::Range], timescale: Option) -> Result { + /// A `legacy` wire (one without ranges) means its start as the first group starting at + /// or after it, never one before it that contains it, so the frames narrow to that + /// floor (see [`Frames::floor`]). + fn new(ranges: &[lite::Range], legacy: bool) -> Self { use crate::track::{Anchor, Range}; - let anchor = |anchor: lite::Anchor| -> Result { - Ok(match anchor { - lite::Anchor::Absolute(position) => Anchor::Absolute(position.group), - lite::Anchor::Relative(back) => Anchor::Relative(back), - lite::Anchor::Time(pts) => { - let scale = timescale.ok_or(Error::TimestampMismatch)?; - Anchor::Time( - crate::Timestamp::new(pts, scale) - .map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded))?, - ) - } - }) + let anchor = |anchor: lite::Anchor| match anchor { + lite::Anchor::Absolute(position) => Anchor::Time(position.group), + lite::Anchor::Relative(back) => Anchor::Relative(back), + lite::Anchor::Time(time) => Anchor::Time(time), }; let mut bounds = Self { @@ -4921,13 +4837,16 @@ impl Bounds { if let (Some(lite::Anchor::Absolute(start)), Some(lite::Anchor::Absolute(end))) = (range.start, range.end) && end <= start { - bounds.ranges.push(Range::groups(start.group..start.group)); + bounds.ranges.push(Range::EMPTY); continue; } let start = match range.start { None => Range::oldest().start, - Some(start) => anchor(start)?, + Some(start) => anchor(start), }; + if legacy && let Some(lite::Anchor::Absolute(start)) = range.start { + bounds.frames.floor = bounds.frames.floor.max(start.group); + } if let Some(lite::Anchor::Absolute(start)) = range.start && start.frame != 0 { @@ -4936,18 +4855,25 @@ impl Bounds { let end = match range.end { None => None, // A mid-group end includes its group, up to the frame before it. - Some(lite::Anchor::Absolute(end)) => Some(Anchor::Absolute(match end.frame.checked_sub(1) { + Some(lite::Anchor::Absolute(end)) => match end.frame.checked_sub(1) { Some(frame) => { bounds.frames.last = Some((end.group, frame)); - end.group.checked_add(1).ok_or(Error::ProtocolViolation)? + Some(Anchor::Time(end.group)) } - None => end.group, - })), - Some(end) => Some(anchor(end)?), + // An exclusive end at group 0 holds nothing, whatever the start. + None => match end.group.checked_sub(1) { + Some(group) => Some(Anchor::Time(group)), + None => { + bounds.ranges.push(Range::EMPTY); + continue; + } + }, + }, + Some(end) => Some(anchor(end)), }; bounds.ranges.push(Range { start, end }); } - Ok(bounds) + bounds } /// Whether a subscription to these ranges over `version` is answered once they are @@ -4956,13 +4882,6 @@ impl Bounds { version.has_ranges() && self.ranges.iter().all(|range| range.end.is_some()) } - /// Whether any end is a presentation time, which needs the track's timescale. - fn has_time(ranges: &[lite::Range]) -> bool { - ranges.iter().any(|range| { - matches!(range.start, Some(lite::Anchor::Time(_))) || matches!(range.end, Some(lite::Anchor::Time(_))) - }) - } - /// The model's preferences for these ranges. fn groups(&self) -> crate::track::Subscription { crate::track::Subscription::default().with_ranges(self.ranges.iter().copied()) @@ -4993,8 +4912,6 @@ struct Subscription { version: Version, /// How frames are stamped, or `None` when they carry no timestamp. stamp: Option, - /// The clock an untimed track's Lite05/06 send times are read from. - runtime: crate::time::Deadlines, /// Outcomes shared by the subscription and every group it serves. outcomes: Arc, /// Whether the session negotiated the `ietf` extension, so streams carry relay state. @@ -5010,8 +4927,6 @@ struct GroupContext { version: Version, /// How frames are stamped, or `None` when they carry no timestamp. stamp: Option, - /// The clock an untimed track's Lite05/06 send times are read from. - runtime: crate::time::Deadlines, /// Outcomes shared by the subscription and every group it serves. outcomes: Arc, /// Whether the session negotiated the `ietf` extension, so streams carry relay state. @@ -5082,7 +4997,6 @@ impl Subscription { steer_seen: self.steer_seen, version: self.version, stamp: self.stamp, - runtime: self.runtime.clone(), outcomes: self.outcomes.clone(), ietf: self.ietf, } @@ -5104,14 +5018,14 @@ impl Subscription { #[cfg(test)] async fn serve_group( mut self, - sequence: u64, + key: u64, frame_start: u64, priority: PriorityHandle, group: group::Consumer, ) -> Result<(), Error> { let mut serve = Box::new(GroupServe::new( self.group_context(), - sequence, + key, frame_start, false, Rank::Queue(priority), @@ -5196,7 +5110,7 @@ struct TrackRun { // The expired groups an update re-offers, still cached and in range, each served again // on its own stream as stream credit allows. reoffered: std::collections::VecDeque, - // Lite-07: the paused groups by sequence, each continued on its own stream ahead of + // Lite-07: the paused groups by start, each continued on its own stream ahead of // re-offers and new groups, the highest-ranked first. Credit returns with each pause, so // sustained congestion can pause every group in turn: one the budget or the cache gives // up while it waits is dropped then, which bounds them by what the cache holds, and a @@ -5207,6 +5121,10 @@ struct TrackRun { // Lite-07: the group `opening` holds continues a paused one. continuing: bool, end_sent: bool, + // Lite-05/06: the track's end, declared while the start was still unresolved. Their + // SUBSCRIBE_START is the first message, and a SUBSCRIBE_FINISH ahead of it says no group + // was served, so the end waits for the start, or goes out alone once none can arrive. + held_end: Option, // Lite-07 with every range bounded: once every group in its ranges is given up, it is // answered without a group to resolve its start, rather than left unanswered. bounded: bool, @@ -5278,6 +5196,7 @@ impl TrackRun { paused_tasks: kio::Tasks::new(), continuing: false, end_sent: false, + held_end: None, opening: None, children: kio::Tasks::new(), idle: None, @@ -5335,13 +5254,8 @@ impl TrackRun { let floored = requested.is_some_and(|group| group > 0); let (ids, ranges): (Vec, Vec) = upd.ranges.iter().copied().unzip(); let ranges = self.pin(&ids, ranges); - let bounds = match Bounds::new(&ranges, self.ctx.stamp.map(|stamp| stamp.timescale)) { - Ok(bounds) => bounds, - Err(err) => { - self.refused = Some(err); - return; - } - }; + let bounds = Bounds::new(&ranges, !self.ctx.version.has_ranges()); + let groups = bounds.groups(); let (start, end) = (groups.start(), groups.end()); // A refusal leaves the model's ranges as they were, so this side keeps them too. @@ -5373,10 +5287,10 @@ impl TrackRun { }; // `position_cursor` pins an unfloored SUBSCRIBE to the latest group, so a pre-06 // update that drops a named floor re-pins there too. On an empty track the next - // group is the latest, whatever its sequence. + // group is the latest, whatever its key. let floor = match lowered { None if requested.is_some() && !self.ctx.version.resolves_start() => { - Some(self.track.latest_sequence().unwrap_or(0)) + Some(self.track.latest_key().unwrap_or(0)) } floor => floor, }; @@ -5411,12 +5325,12 @@ impl TrackRun { // one re-offered that has not opened yet, since they may no longer admit it. let mut expired = self.track.record_mut().expired(); for group in std::mem::take(&mut self.reoffered) { - expired.insert(group.sequence..group.sequence + 1); + expired.insert(group.start..group.start + 1); } expired.extend(std::mem::take(&mut self.redrop).iter().cloned()); for run in expired.iter() { let (groups, gone) = self.track.reoffer_groups(run.clone()); - self.owe(groups.iter().map(|group| group.sequence..group.sequence + 1).collect()); + self.owe(groups.iter().map(|group| group.start..group.start + 1).collect()); self.reoffered.extend(groups); self.redrop.extend(gone.iter().cloned()); } @@ -5590,23 +5504,23 @@ impl TrackRun { self.drop_given_up_paused(waiter); let dropped = std::mem::take(&mut *self.ctx.outcomes.dropped.lock().unwrap()); if !dropped.is_empty() { - for (sequence, err) in dropped { + for (key, err) in dropped { // A stream expired under this subscription's budget is a stale verdict. let hole = match err { // The pool folded the group while its stream was open, which is final for this // reader: no update would re-offer a stale drop on it. - StreamError::Old if self.ctx.version.has_skip() && self.track.record_mut().folded(sequence) => { + StreamError::Old if self.ctx.version.has_skip() && self.track.record_mut().folded(key) => { Hole::drop(StreamError::Evicted) } StreamError::Old => { if self.ctx.version.has_skip() { - self.sent_stale(sequence..sequence + 1); + self.sent_stale(key..key + 1); } Hole::stale(self.track.budget()) } err => Hole::drop(err), }; - writer.buffer(&self.hole(sequence..sequence + 1, &hole))?; + writer.buffer(&self.hole(key..key + 1, &hole))?; } return Poll::Ready(Ok(ControlFlow::Continue(()))); } @@ -5685,7 +5599,7 @@ impl TrackRun { { let PausedKey { key, .. } = self.paused.remove(&best).expect("best is paused"); let Paused { group, frame_start } = self.paused_tasks.remove(key).expect("a paused group waits"); - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = group.sequence, frame_start, "continuing paused group"); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, start = group.start, frame_start, "continuing paused group"); self.give_way(); self.open(group, frame_start); self.continuing = true; @@ -5697,20 +5611,20 @@ impl TrackRun { if self.opening.is_none() && let Some(group) = self.reoffered.pop_front() { - let sequence = group.sequence; + let key = group.start; // The pool folded it while it waited, which is final for this reader: tell the wire // it is gone, since an update re-offered it. - if self.start.emits_range() && self.track.record_mut().folded(sequence) { - writer.buffer(&self.hole(sequence..sequence + 1, &Hole::drop(StreamError::Evicted)))?; + if self.start.emits_range() && self.track.record_mut().folded(key) { + writer.buffer(&self.hole(key..key + 1, &Hole::drop(StreamError::Evicted)))?; return Poll::Ready(Ok(ControlFlow::Continue(()))); } - if self.start.emits_range() && self.track.record_mut().told(sequence) { + if self.start.emits_range() && self.track.record_mut().told(key) { return Poll::Ready(Ok(ControlFlow::Continue(()))); } match position_group(group, &self.frames) { Some(group) => self.serve(group), None => { - self.drop_range(sequence..sequence + 1, &Hole::drop(StreamError::NotFound), writer)?; + self.drop_range(key..key + 1, &Hole::drop(StreamError::NotFound), writer)?; } } return Poll::Ready(Ok(ControlFlow::Continue(()))); @@ -5728,49 +5642,48 @@ impl TrackRun { None => Some(ALL), Some(waiting) => stuck.map(|_| match order { Order::Descending => ALL, - Order::Ascending => (Bound::Unbounded, Bound::Excluded(waiting.sequence)), + Order::Ascending => (Bound::Unbounded, Bound::Excluded(waiting.key)), }), }; let end_error = self.ctx.version.has_end_error(); if let Poll::Ready(res) = poll_recv_next(&mut self.track, admit, emit_boundary, end_error, waiter) { match res? { Recv::Group(group) => { - // Never a second stream for a sequence this subscription already accounted + // Never a second stream for a key this subscription already accounted // for (served or dropped): a group re-created after its // DROP stays dropped, since the subscriber may settle before it arrives. A // stale drop a lite-07 update reopened is owed again (see `Self::owe`). A // lite-07 stale drop is no final verdict, and a Group Stream wins over it, so a // group handed out after one is served, as one a route the front switched to // delivers: the update that reopens the drop would never re-offer it. - let sequence = group.sequence; + let key = group.start; let record = self.track.record_mut(); - if self.start.emits_range() && record.told_stale(sequence) { - self.replace_stale(sequence..sequence + 1); - } else if self.start.emits_range() && record.told(sequence) { + if self.start.emits_range() && record.told_stale(key) { + self.replace_stale(key..key + 1); + } else if self.start.emits_range() && record.told(key) { return Poll::Ready(Ok(ControlFlow::Continue(()))); } let Some(group) = position_group(group, &self.frames) else { // Its head is gone, and this subscriber didn't ask for a // partial group. Skip it rather than open a stream that can // only be reset; the next servable group resolves the start. - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "skipping group with a missing head"); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, key, "skipping group with a missing head"); let hole = Hole::drop(StreamError::NotFound); match &mut self.start { - Start::Unresolved { skipped, .. } => skipped.insert(sequence..sequence + 1, hole), + Start::Unresolved { skipped, .. } => skipped.insert(key..key + 1, hole), Start::Resolved(_) => { - self.drop_range(sequence..sequence + 1, &hole, writer)?; + self.drop_range(key..key + 1, &hole, writer)?; } Start::Legacy => {} } return Poll::Ready(Ok(ControlFlow::Continue(()))); }; if let Some(waiting) = &self.opening - && crate::priority::in_group(sequence, 0, order) - < crate::priority::in_group(waiting.sequence, 0, order) + && crate::priority::in_group(key, 0, order) < crate::priority::in_group(waiting.key, 0, order) { - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "shedding a group ranked below one waiting for credit"); - self.mark_served(sequence); - self.shed(sequence); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, key, "shedding a group ranked below one waiting for credit"); + self.mark_served(key); + self.shed(key); return Poll::Ready(Ok(ControlFlow::Continue(()))); } match &mut self.start { @@ -5782,11 +5695,15 @@ impl TrackRun { } } Recv::Boundary(end) => { - // The track declared its exclusive final sequence. Forward it now, + // The track declared its exclusive final key. Forward it now, // even if trailing groups (below it) are still in flight, then // keep serving them until the live edge reaches the boundary. self.end_sent = true; - writer.buffer(&lite::SubscribeResponse::Finish(end))?; + if self.start.is_unresolved() && !self.ctx.version.has_ranges() { + self.held_end = Some(end); + } else { + writer.buffer(&lite::SubscribeResponse::Finish(end))?; + } } Recv::Finished => { // A continuation waiting for credit is still in flight too. @@ -5800,15 +5717,20 @@ impl TrackRun { if self.ctx.version.keeps_subscribe_open() && self.start.is_unresolved() && let Poll::Ready(Ok(fin)) = self.track.poll_finished(&kio::Waiter::noop()) - && final_sequence(fin) > 0 + && final_key(fin) > 0 { - self.resolve_held(final_sequence(fin), writer)?; + self.resolve_held(final_key(fin), writer)?; answered = true; } // Nothing arrives to resolve a range still waiting on the live feed. let started = self.report_started(true, writer)?; let dropped = self.drop_unserved(writer)? || answered || started; if !self.ctx.version.keeps_subscribe_open() { + // No group arrived to resolve the start, so the end goes out alone: nothing + // was served. + if let Some(end) = self.held_end.take() { + writer.buffer(&lite::SubscribeResponse::Finish(end))?; + } return Poll::Ready(Ok(ControlFlow::Break(()))); } // Lite-07 leaves the stream open past SUBSCRIBE_FINISH: an update may still @@ -5846,7 +5768,7 @@ impl TrackRun { self.bounded && self.opening.is_none() && self.track.covered(self.floor, skipped) } - /// Lite-05+ names a sequence as soon as this cursor gives it up; whether one was taken. + /// Lite-05+ names a key as soon as this cursor gives it up; whether one was taken. /// Before SUBSCRIBE_START there is no owed range yet, so the eventual start accounts /// for it instead, including ranges skipped while finding the first servable group. /// Without `probe`, only collects runs a read already gave up; see @@ -5893,10 +5815,10 @@ impl TrackRun { } } - /// Record `sequence` as served, for [`Self::drop_unserved`]. - fn mark_served(&mut self, sequence: u64) { + /// Record `key` as served, for [`Self::drop_unserved`]. + fn mark_served(&mut self, key: u64) { if self.start.emits_range() { - self.track.record_mut().tell(sequence..sequence + 1, Report::Settled); + self.track.record_mut().tell(key..key + 1, Report::Settled); } } @@ -5994,7 +5916,7 @@ impl TrackRun { } } - /// Name every sequence from the start to the end that this subscription never got. The + /// Name every key from the start to the end that this subscription never got. The /// track has ended, so none of them will be served now: groups the cursor passed over /// (stale, or missing their head), and gaps the publisher never declared skipped. /// @@ -6009,16 +5931,16 @@ impl TrackRun { return Ok(false); }; let Poll::Ready(Ok(fin)) = self.track.poll_finished(&kio::Waiter::noop()) else { - // A track closed without a final sequence sent no SUBSCRIBE_FINISH, so the subscriber + // A track closed without a final key sent no SUBSCRIBE_FINISH, so the subscriber // owes itself no range either. return Ok(false); }; // A cap below the end bounds what the subscriber is owed, as on its side. - let fin = final_sequence(fin); + let fin = final_key(fin); // Only the groups the ranges name: one between disjoint ranges is not owed, and // naming it would keep a later update widening over it from delivering it. let end = self.track.subscription().end().map_or(fin, |end| end.min(fin)); - // A sequence the publisher declared skipped went out as a skip already; one it never + // A key the publisher declared skipped went out as a skip already; one it never // declared might have existed, so it is not found. let hole = Hole::drop(StreamError::NotFound); let mut dropped = false; @@ -6058,15 +5980,15 @@ impl TrackRun { source: Option, writer: &mut Writer, ) -> Result<(), Error> { - let mut start = source.map_or(group.sequence, |source| source.min(group.sequence)); + let mut start = source.map_or(group.start, |source| source.min(group.start)); // A group given up cannot be where the feed starts. This stops at the held group // at the latest, since it was not skipped. // Adjacent runs given up with different errors stay separate, so walk them all. if let Start::Unresolved { skipped, .. } = &self.start { - while start < group.sequence + while start < group.start && let Some((given_up, _)) = skipped.get_key_value(&start) { - start = given_up.end.min(group.sequence); + start = given_up.end.min(group.start); } } self.resolve(start, None, writer)?; @@ -6142,16 +6064,20 @@ impl TrackRun { } self.forward_range(range.start.max(start)..range.end, hole, writer)?; } + // An end declared while this start was held follows it. + if let Some(end) = self.held_end.take() { + writer.buffer(&lite::SubscribeResponse::Finish(end))?; + } Ok(()) } /// Open a group machine for `group`, which waits for stream credit. The cursor hands it /// out past a group already waiting only if it outranks that one, which gives way. fn serve(&mut self, group: group::Consumer) { - let sequence = group.sequence; + let key = group.start; let frame_start = group.index(); - self.mark_served(sequence); - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence, "serving group"); + self.mark_served(key); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, key, "serving group"); self.give_way(); self.open(group, frame_start); } @@ -6163,7 +6089,7 @@ impl TrackRun { fn stuck(&mut self) -> Option { let waiting = self.opening.as_ref()?; let order = self.ctx.steer_current().order; - let rank = |serve: &GroupServe| crate::priority::in_group(serve.sequence, serve.frame_start, order); + let rank = |serve: &GroupServe| crate::priority::in_group(serve.key, serve.frame_start, order); let waiting = rank(waiting); let mut held = self .children @@ -6182,7 +6108,7 @@ impl TrackRun { let Some(mut waiting) = self.opening.take() else { return; }; - tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, sequence = waiting.sequence, "giving way to a higher-ranked group"); + tracing::debug!(subscribe = self.ctx.id, track = %self.ctx.track_name, key = waiting.key, "giving way to a higher-ranked group"); match std::mem::take(&mut self.continuing) { true => { self.pause(Paused { @@ -6196,35 +6122,30 @@ impl TrackRun { /// Give up a group the cursor handed out without a stream, as one expired under the /// subscription's budget is. - fn shed(&self, sequence: u64) { + fn shed(&self, key: u64) { if self.ctx.version.has_track_stream() { - self.ctx - .outcomes - .dropped - .lock() - .unwrap() - .push((sequence, StreamError::Old)); + self.ctx.outcomes.dropped.lock().unwrap().push((key, StreamError::Old)); } } /// Hold a paused group until its continuation, replacing the entry of an earlier pause. fn pause(&mut self, paused: Paused) { - let sequence = paused.group.sequence; + let start = paused.group.start; let frame_start = paused.frame_start; let key = self.paused_tasks.insert(paused); - if let Some(old) = self.paused.insert(sequence, PausedKey { key, frame_start }) { + if let Some(old) = self.paused.insert(start, PausedKey { key, frame_start }) { self.paused_tasks.remove(old.key); } } - /// The sequence and rank of the highest-ranked paused group in `order`, the next to - /// continue. The rank orders groups by sequence first, so it is an end of the map. + /// The start and rank of the highest-ranked paused group in `order`, the next to + /// continue. The rank orders groups by start first, so it is an end of the map. fn best_paused(&self, order: crate::track::Order) -> Option<(u64, i64)> { - let (&sequence, paused) = match order { + let (&start, paused) = match order { crate::track::Order::Descending => self.paused.last_key_value(), crate::track::Order::Ascending => self.paused.first_key_value(), }?; - Some((sequence, crate::priority::in_group(sequence, paused.frame_start, order))) + Some((start, crate::priority::in_group(start, paused.frame_start, order))) } /// Drop each paused group the subscription's budget or the cache gave up while it waited @@ -6235,13 +6156,13 @@ impl TrackRun { } let mut dropped = Vec::new(); let _ = self.paused_tasks.poll_each(waiter, |paused, err| { - dropped.push((paused.group.sequence, StreamError::from(&err))); + dropped.push((paused.group.start, StreamError::from(&err))); }); if dropped.is_empty() { return; } - for (sequence, _) in &dropped { - self.paused.remove(sequence); + for (start, _) in &dropped { + self.paused.remove(start); } if self.ctx.version.has_track_stream() { self.ctx.outcomes.dropped.lock().unwrap().extend(dropped); @@ -6251,9 +6172,9 @@ impl TrackRun { /// Open a group machine streaming `group` from `frame_start`, which waits for stream /// credit before the cursor hands out another. fn open(&mut self, group: group::Consumer, frame_start: u64) { - let sequence = group.sequence; + let key = group.start; // The stream stops at the end frame, before the group may end. - let capped = self.frames.end(sequence).is_some(); + let capped = self.frames.end(key).is_some(); // Use the latest steer for new groups so SUBSCRIBE_UPDATE applies to them too. let steer = self.ctx.steer_current(); @@ -6261,10 +6182,10 @@ impl TrackRun { Some(_) => Rank::Group, // The subscribe id scopes the group tie-break: one queue serves every // subscription on the session, and only groups of the same one may be - // ranked against each other by sequence, in that subscription's order. + // ranked against each other by key, in that subscription's order. None => Rank::Queue( self.ctx.priority.insert( - Priority::new(steer.priority, self.ctx.id, sequence) + Priority::new(steer.priority, self.ctx.id, key) .with_publisher(self.ctx.publisher) .with_order(steer.order), ), @@ -6273,7 +6194,7 @@ impl TrackRun { debug_assert!(self.opening.is_none(), "one group waits for stream credit at a time"); self.opening = Some(GroupServe::new( self.ctx.group_context(), - sequence, + key, frame_start, capped, rank, @@ -6296,12 +6217,22 @@ pub struct CoverageBench { impl CoverageBench { /// Subscribe to `ranges` one-group ranges, then give up the first `skipped` of them, one /// wake each. A range is left open, so the start never resolves and nothing is covered. + /// + /// Each range starts at a time no group held places, so it may reach back to the group + /// before it: giving a range up gives that group up too. pub fn new(ranges: u64, skipped: u64) -> Self { assert!(skipped < ranges, "a range must stay open"); // Every other group, so no two ranges merge into one run. let wire = (0..ranges).map(|range| 2 * range..2 * range + 1); + // Alternating errors keep adjacent drops separate entries in the skipped record. let drops = (0..skipped) - .map(|range| (2 * range..2 * range + 1, StreamError::NotFound)) + .map(|range| { + let err = match range % 2 { + 0 => StreamError::NotFound, + _ => StreamError::Cancel, + }; + ((2 * range).saturating_sub(1)..2 * range + 1, err) + }) .collect(); Self::build(wire, drops) } @@ -6337,7 +6268,7 @@ impl CoverageBench { end: group(range.end), }) .collect(); - let bounds = Bounds::new(&wire, None).unwrap(); + let bounds = Bounds::new(&wire, false); let track = track::Producer::new( Arc::new(crate::broadcast::Info::default()), "bench", @@ -6356,7 +6287,6 @@ impl CoverageBench { steer_seen: Steer::default(), version, stamp: None, - runtime: crate::time::Deadlines::new(kio::clock::now()), outcomes: Default::default(), ietf: false, }; @@ -6411,9 +6341,11 @@ impl PausedBench { let mut coverage = CoverageBench::new(2, 0); let mut groups = Vec::new(); // Past the subscribed ranges, so the run itself never serves them. - for sequence in 100..100 + paused { - let mut group = coverage.track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, b"x".to_vec()).unwrap(); + for start in 100..100 + paused { + let mut group = coverage.track.create_group(group::Info { start }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(start), b"x".to_vec()) + .unwrap(); coverage.run.pause(Paused { group: group.consume(), frame_start: 0, @@ -6433,7 +6365,8 @@ impl PausedBench { pub fn wake(&mut self, frames: usize) { for _ in 0..frames { let group = &mut self.groups[self.next]; - group.write_frame(None, b"x".to_vec()).unwrap(); + let key = crate::Timestamp::unscaled(group.start); + group.write_frame(key, b"x".to_vec()).unwrap(); self.next = (self.next + 1) % self.groups.len(); } let run = &mut self.coverage.run; @@ -6445,7 +6378,7 @@ impl PausedBench { /// Where a group stream is ranked. enum Rank { - /// In the subscription's send group, by its sequence in the subscription's order. + /// In the subscription's send group, by its key in the subscription's order. Group, /// In the session-wide queue, on a transport without send groups. Queue(PriorityHandle), @@ -6475,7 +6408,7 @@ struct GroupServe { /// Its place in line for the peer's stream credit while it waits to open. open: crate::transport::open::Open, group: group::Consumer, - sequence: u64, + key: u64, frame_start: u64, /// The stream stops at a subscription's end frame, so its FIN cannot end the group. capped: bool, @@ -6531,14 +6464,7 @@ impl GroupServe { matches!(self.state, GroupState::Serve { .. }) } - fn new( - ctx: GroupContext, - sequence: u64, - frame_start: u64, - capped: bool, - rank: Rank, - group: group::Consumer, - ) -> Self { + fn new(ctx: GroupContext, key: u64, frame_start: u64, capped: bool, rank: Rank, group: group::Consumer) -> Self { // A lite-07 peer receives a compressed group as stored, others inflated. let mut group = match ctx.version.has_compression() { true => group.compressed(), @@ -6553,7 +6479,7 @@ impl GroupServe { rank, open: Default::default(), group, - sequence, + key, frame_start, capped, prev_ts: 0, @@ -6591,7 +6517,7 @@ impl GroupServe { let options = match &mut self.rank { Rank::Group => SendOptions { group, - priority: crate::priority::in_group(self.sequence, self.frame_start, self.ctx.steer_seen.order), + priority: crate::priority::in_group(self.key, self.frame_start, self.ctx.steer_seen.order), }, Rank::Queue(handle) => SendOptions { group: None, @@ -6620,7 +6546,7 @@ impl GroupServe { let msg = lite::Group { subscribe: self.ctx.id, - sequence: self.sequence, + key: self.key, frame_start: self.frame_start, }; let header = writer.buffer(&lite::DataType::Group).and_then(|()| writer.buffer(&msg)); @@ -6687,7 +6613,7 @@ impl GroupServe { self.ctx.steer_seen = value; // The run loop moves the send group itself; a stream only follows the order. let order = match &mut self.rank { - Rank::Group => crate::priority::in_group(self.sequence, self.frame_start, value.order), + Rank::Group => crate::priority::in_group(self.key, self.frame_start, value.order), Rank::Queue(handle) => { PriorityHandle::send_order_of(handle.steer(value.priority, value.order)).into() } @@ -6786,7 +6712,6 @@ impl GroupServe { batched.frame.payload.len() as u64, self.ctx.stamp, &mut self.prev_ts, - &self.ctx.runtime, batched.frame.payload.len() as u64, ) }); @@ -6842,7 +6767,6 @@ impl GroupServe { next.size, self.ctx.stamp, &mut self.prev_ts, - &self.ctx.runtime, next.size, ) }); @@ -6913,7 +6837,7 @@ impl GroupServe { }; self.state = GroupState::Done; return Poll::Ready(res.map(|()| { - tracing::debug!(sequence = self.sequence, "finished group"); + tracing::debug!(key = self.key, "finished group"); })); } GroupState::Done => return Poll::Ready(Ok(())), @@ -6967,7 +6891,7 @@ impl GroupServe { .dropped .lock() .unwrap() - .push((self.sequence, StreamError::from(err))); + .push((self.key, StreamError::from(err))); } } @@ -6986,7 +6910,7 @@ mod serve_group_test { #[test] fn bounds_convert_to_whole_groups() { let group = |group, frame| Some(lite::Anchor::Absolute(track::Position { group, frame })); - let bounds = |start, end| Bounds::new(&[lite::Range { start, end }], None).unwrap(); + let bounds = |start, end| Bounds::new(&[lite::Range { start, end }], false); let whole = bounds(None, group(6, 0)); assert_eq!(whole.groups().end(), Some(6)); @@ -7003,20 +6927,54 @@ mod serve_group_test { // Empty partway through group 5 covers no group, rather than all of it. let empty = bounds(group(5, 3), group(5, 3)); - assert_eq!(empty.ranges, [crate::track::Range::groups(5..5)]); + assert_eq!(empty.ranges, [crate::track::Range::EMPTY]); assert_eq!(empty.frames, Frames::default()); - // A time anchor names no group on a track the peer was told is untimed. + // An exclusive end at group 0 holds nothing, and one at group 1 holds group 0. + assert_eq!(bounds(None, group(0, 0)).ranges, [crate::track::Range::EMPTY]); + let first = bounds(group(0, 0), group(1, 0)); + assert_eq!( + first.ranges, + [crate::track::Range { + start: crate::track::Anchor::Time(0), + end: Some(crate::track::Anchor::Time(0)), + }] + ); + + // A group ID is a start, so a time anchor names one too, timed or not. let time = [lite::Range { start: Some(lite::Anchor::Time(1_000)), end: None, }]; - assert!(matches!(Bounds::new(&time, None), Err(Error::TimestampMismatch))); - let timed = Bounds::new(&time, Some(crate::Timescale::MILLI)).unwrap(); assert_eq!( - timed.ranges[0].start, - crate::track::Anchor::Time(Timestamp::from_millis(1_000).unwrap()) + Bounds::new(&time, false).ranges[0].start, + crate::track::Anchor::Time(1_000) + ); + } + + /// A wire without ranges means its start as the first group at or after it, so the + /// frames floor there; lite-07 names the group containing it, with no floor. + #[test] + fn a_legacy_start_floors_at_its_group() { + let start = [lite::Range { + start: Some(lite::Anchor::Absolute(track::Position { group: 5, frame: 0 })), + end: None, + }]; + let legacy = Bounds::new(&start, true); + assert_eq!(legacy.ranges[0].start, crate::track::Anchor::Time(5)); + assert_eq!(legacy.frames.floor, 5); + assert_eq!(Bounds::new(&start, false).frames.floor, 0); + + // Sparse starts 4 and 7 with a legacy start of 5 serve 7, never 4 which contains 5. + let track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "video", + crate::track::Info::timed(), ); + let four = track.append_group(4).unwrap(); + let seven = track.append_group(7).unwrap(); + assert!(position_group(four.consume(), &legacy.frames).is_none()); + assert!(position_group(seven.consume(), &legacy.frames).is_some()); } /// A range inside one group keeps both of its bounds there. @@ -7028,9 +6986,8 @@ mod serve_group_test { start: group(5, 2), end: group(5, 4), }], - None, - ) - .unwrap(); + false, + ); assert_eq!((inside.frames.start(5), inside.frames.end(5)), (2, Some(3))); assert_eq!((inside.frames.start(4), inside.frames.end(6)), (0, None)); } @@ -7044,12 +7001,12 @@ mod serve_group_test { "video", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 3 }).unwrap(); + let mut group = track.create_group(group::Info { start: 3 }).unwrap(); group.start_at(5).unwrap(); - group.write_frame(Timestamp::ZERO, b"tail".to_vec()).unwrap(); + group.write_frame(group.key_time(), b"tail".to_vec()).unwrap(); let starting = |frame| Frames { first: Some((3, frame)), - last: None, + ..Frames::default() }; // Asked for whole groups, so the missing frames 0..5 make this unservable. @@ -7063,8 +7020,8 @@ mod serve_group_test { assert!(position_group(group.consume(), &starting(2)).is_none()); // The offset belongs to the start group only; a later group is served whole. - let mut other = track.create_group(group::Info { sequence: 4 }).unwrap(); - other.write_frame(Timestamp::ZERO, b"whole".to_vec()).unwrap(); + let mut other = track.create_group(group::Info { start: 4 }).unwrap(); + other.write_frame(other.key_time(), b"whole".to_vec()).unwrap(); let consumer = position_group(other.consume(), &starting(5)).unwrap(); assert_eq!(consumer.index(), 0); } @@ -7077,14 +7034,14 @@ mod serve_group_test { "video", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 7 }).unwrap(); + let mut group = track.create_group(group::Info { start: 7 }).unwrap(); for i in 0..4u8 { - group.write_frame(Timestamp::ZERO, vec![i]).unwrap(); + group.write_frame(group.key_time(), vec![i]).unwrap(); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let ending = |group, frame| Frames { - first: None, last: Some((group, frame)), + ..Frames::default() }; let mut consumer = position_group(group.consume(), &ending(7, 1)).unwrap(); @@ -7111,24 +7068,18 @@ mod serve_group_test { } } - /// Lite-05/06 cannot encode an untimed track, so one carries its send time there and - /// nothing on lite-07, while a timed track carries its own timestamps on every version. - #[kio::test] - async fn only_an_untimed_track_goes_out_with_send_times() { - let runtime = crate::time::sim(); + /// Lite-05/06 cannot encode a scale-less track, so its frames carry their group's key + /// there, never a clock, and nothing on lite-07, while a timed track carries its own + /// timestamps on every version. + #[test] + fn a_scale_less_track_goes_out_with_its_keys() { let stamped = Timestamp::from_millis(36_000_000).unwrap(); - let sent = Timestamp::from(runtime.now()).value(); - for version in [Version::Lite06, Version::Lite07] { let stamp = Stamp::new(version, Some(crate::Timescale::MILLI)).unwrap(); - assert_eq!( - stamp.wire(Some(stamped), &runtime).unwrap(), - stamped.value(), - "{version}" - ); + assert_eq!(stamp.wire(stamped).unwrap(), stamped.value(), "{version}"); } let stamp = Stamp::new(Version::Lite06, None).unwrap(); - assert_eq!(stamp.wire(None, &runtime).unwrap(), sent); + assert_eq!(stamp.wire(Timestamp::unscaled(7)).unwrap(), 7); assert!(Stamp::new(Version::Lite07, None).is_none()); } @@ -7149,7 +7100,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7159,7 +7109,7 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); group .write_frame(Timestamp::from_millis(0).unwrap(), b"hello".as_slice()) .unwrap(); @@ -7196,7 +7146,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7207,9 +7156,9 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut old = track.append_group().unwrap(); - old.write_frame(Timestamp::ZERO, b"old".as_slice()).unwrap(); - old.finish().unwrap(); + let mut old = track.append_next().unwrap(); + old.write_frame(old.key_time(), b"old".as_slice()).unwrap(); + old.close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("old group"); let handle = subscription.priority.insert(Priority::new(0, 0, 0)); @@ -7220,10 +7169,10 @@ mod serve_group_test { ); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(serve.await, Err(Error::Old))); assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); @@ -7264,7 +7213,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7274,11 +7222,11 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); for _ in 0..FRAMES { group.write_frame(Timestamp::ZERO, vec![b'x'; FRAME]).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let before = billed(); let meter = registry.publisher_meter(crate::Path::from_static("demo"), &crate::stats::Tier::default()); let consumer = subscriber @@ -7339,7 +7287,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7349,7 +7296,7 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(Timestamp::ZERO, vec![b'x'; FRAME]).unwrap(); let meter = registry.publisher_meter(crate::Path::from_static("demo"), &crate::stats::Tier::default()); let consumer = subscriber @@ -7410,7 +7357,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7421,8 +7367,8 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut old = track.append_group().unwrap(); - old.write_frame(Timestamp::ZERO, b"old".as_slice()).unwrap(); + let mut old = track.append_next().unwrap(); + old.write_frame(old.key_time(), b"old".as_slice()).unwrap(); let group = subscriber.recv_group().await.unwrap().expect("old group"); let handle = subscription.priority.insert(Priority::new(0, 0, 0)); @@ -7433,10 +7379,10 @@ mod serve_group_test { ); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(serve.await, Err(Error::Old))); assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); @@ -7461,7 +7407,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7472,10 +7417,10 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let old = track.append_group().unwrap(); + let old = track.append_next().unwrap(); let mut frame = old .create_frame(frame::Info { - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, size: 2, }) .unwrap(); @@ -7495,17 +7440,17 @@ mod serve_group_test { *open = false; drop(open); frame.write(b"b".as_slice()).unwrap(); - frame.finish().unwrap().finish().unwrap(); + frame.finish().unwrap().close().unwrap(); assert!( futures::poll!(serve.as_mut()).is_pending(), "the final byte is transport-blocked" ); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(serve.await, Err(Error::Old))); assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); @@ -7529,7 +7474,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7540,9 +7484,9 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut old = track.append_group().unwrap(); - old.write_frame(Timestamp::ZERO, b"old".as_slice()).unwrap(); - old.finish().unwrap(); + let mut old = track.append_next().unwrap(); + old.write_frame(old.key_time(), b"old".as_slice()).unwrap(); + old.close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("old group"); let handle = subscription.priority.insert(Priority::new(0, 0, 0)); @@ -7553,10 +7497,10 @@ mod serve_group_test { ); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(serve.await, Err(Error::Old))); } @@ -7594,13 +7538,13 @@ mod serve_group_test { }) .collect(); - for sequence in 0..groups { + for key in 0..groups { for (track, run, writer) in &mut runs { - write_group(track, sequence, sequence); + write_group(track, key, key); settle_run(run, writer).await; } let held: usize = runs.iter().map(|(_, run, _)| run.machines()).sum(); - assert_eq!(held, tracks, "{tracks} tracks after {} groups each", sequence + 1); + assert_eq!(held, tracks, "{tracks} tracks after {} groups each", key + 1); } *gate.write().ok().unwrap() = true; @@ -7629,8 +7573,8 @@ mod serve_group_test { ); let (mut run, mut writer, log) = lite_run(version, SinkSession::gated_open_uni(gate.consume()), subscriber); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert_eq!(run.machines(), 1, "group 0 waits for credit"); @@ -7667,8 +7611,8 @@ mod serve_group_test { track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(1))), ); let (mut run, mut writer, log) = lite_run(version, SinkSession::gated_open_uni(gate.consume()), subscriber); - for sequence in 0..10 { - write_group(&mut track, sequence, sequence * 1000); + for key in 0..10 { + write_group(&mut track, key, key * 1000); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert!(run.machines() <= 1); kio::clock::sim::advance(Duration::from_secs(1)).await; @@ -7709,7 +7653,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7720,7 +7663,7 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut producer = track.append_group().unwrap(); + let mut producer = track.append_next().unwrap(); producer.write_frame(Timestamp::ZERO, b"head".as_slice()).unwrap(); let group = subscriber.recv_group().await.unwrap().expect("group"); @@ -7782,7 +7725,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7792,12 +7734,12 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); group .write_frame(Timestamp::from_millis(0).unwrap(), b"hello".as_slice()) .unwrap(); let consumer = group.consume(); - group.finish().unwrap(); + group.close().unwrap(); let handle = subscription.priority.insert(Priority::new(0, 0, 0)); subscription.serve_group(0, 0, handle, consumer).await.unwrap(); @@ -7832,7 +7774,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7843,9 +7784,9 @@ mod serve_group_test { crate::track::Info::timed(), ); let mut subscriber = track.subscribe(None); - let mut old = track.append_group().unwrap(); - old.write_frame(Timestamp::ZERO, b"old".as_slice()).unwrap(); - old.finish().unwrap(); + let mut old = track.append_next().unwrap(); + old.write_frame(old.key_time(), b"old".as_slice()).unwrap(); + old.close().unwrap(); let group = subscriber.recv_group().await.unwrap().expect("old group"); let handle = subscription.priority.insert(Priority::new(0, 0, 0)); @@ -7854,10 +7795,10 @@ mod serve_group_test { assert!(log.resets().is_empty()); kio::clock::sim::advance(Duration::from_secs(1)).await; - let mut edge = track.append_group().unwrap(); + let mut edge = track.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(1000).unwrap(), b"edge".as_slice()) .unwrap(); - edge.finish().unwrap(); + edge.close().unwrap(); assert!(matches!(futures::poll!(serve.as_mut()), Poll::Ready(Err(Error::Old)))); assert_eq!(log.resets(), vec![crate::StreamError::Old.to_code()]); @@ -7881,7 +7822,6 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite06, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; @@ -7891,10 +7831,10 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(Timestamp::ZERO, b"hello".as_slice()).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), b"hello".as_slice()).unwrap(); let consumer = group.consume(); - group.finish().unwrap(); + group.close().unwrap(); let handle = subscription.priority.insert(Priority::new(0, 0, 0)); let queue = subscription.priority.clone(); @@ -7958,11 +7898,10 @@ mod serve_group_test { steer_seen: Steer::default(), version, stamp: Stamp::new(Version::Lite06, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: Default::default(), ietf: false, }; - let bounds = Bounds::new(&[lite::Range::span(Some(0), 0, None, None)], None).unwrap(); + let bounds = Bounds::new(&[lite::Range::span(Some(0), 0, None, None)], false); (TrackRun::new(ctx, track, bounds, 0, steer), writer, log) } @@ -7973,7 +7912,7 @@ mod serve_group_test { start: u64, ) -> (TrackRun, Writer, Log) { let (run, writer, log) = lite_run(version, SinkSession::new(Log::default()), track); - let bounds = Bounds::new(&[lite::Range::span(Some(start), 0, None, None)], None).unwrap(); + let bounds = Bounds::new(&[lite::Range::span(Some(start), 0, None, None)], false); ( TrackRun::new(run.ctx, run.track, bounds, start, run.steer_tx), writer, @@ -7990,10 +7929,14 @@ mod serve_group_test { ); let subscriber = track.subscribe(None); let (mut run, _, _) = lite_run(Version::Lite07, SinkSession::new(Log::default()), subscriber); - let mut group = track.append_group().unwrap(); - group.write_frame(None, &b"key"[..]).unwrap(); - group.write_frame(None, &b"more"[..]).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"key"[..]) + .unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"more"[..]) + .unwrap(); + group.close().unwrap(); let update = |wanted_group| lite::SubscribeUpdate { wanted_group, priority: 0, @@ -8007,21 +7950,23 @@ mod serve_group_test { assert_eq!(track.demand().wanted(), Some(0), "stale request is not a join"); run.replace(update(Some(1))); assert_eq!(track.demand().wanted(), Some(1)); - let mut group = track.append_group().unwrap(); - group.write_frame(None, &b"key"[..]).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"key"[..]) + .unwrap(); + group.close().unwrap(); run.replace(update(Some(1))); assert_eq!(track.demand().wanted(), Some(1), "answered request stays answered"); run.replace(update(Some(2))); assert_eq!(track.demand().wanted(), Some(1), "new hop judges its own ratio"); } - fn write_group(track: &mut track::Producer, sequence: u64, millis: u64) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); + fn write_group(track: &mut track::Producer, key: u64, millis: u64) { + let mut group = track.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } /// A datagram sender for subscription 0 on `session`, stamping in milliseconds. @@ -8033,7 +7978,6 @@ mod serve_group_test { id: 0, version, stamp: Stamp::new(version, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), publisher: 0, group, stats: crate::stats::Scope::default(), @@ -8053,7 +7997,7 @@ mod serve_group_test { datagram_send(version, session) .send(crate::datagram::Datagram { sequence: 1, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, payload: bytes::Bytes::from(vec![0; size - header]), }) .unwrap(); @@ -8067,7 +8011,7 @@ mod serve_group_test { } } - /// A sequence past QUIC's varints fails the send before lite-07, which carries any. + /// A key past QUIC's varints fails the send before lite-07, which carries any. #[kio::test] async fn a_datagram_past_the_wire_s_sequences_fails() { for version in [Version::Lite05, Version::Lite06, Version::Lite07] { @@ -8075,7 +8019,7 @@ mod serve_group_test { let log = session.log.clone(); let sent = datagram_send(version, session).send(crate::datagram::Datagram { sequence: 1 << 62, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, payload: bytes::Bytes::from_static(b"d"), }); let carried = version == Version::Lite07; @@ -8090,17 +8034,17 @@ mod serve_group_test { let session = SinkSession::new(Log::default()).with_groups().with_datagrams(1200); let log = session.log.clone(); let send = datagram_send(Version::Lite07, session); - for sequence in [4, 9] { + for key in [4, 9] { send.send(crate::datagram::Datagram { - sequence, - timestamp: Some(Timestamp::ZERO), + sequence: key, + timestamp: Timestamp::ZERO, payload: bytes::Bytes::from_static(b"d"), }) .unwrap(); } assert_eq!( log.datagram_ranks(), - [4, 9].map(|sequence| (Some(0), crate::priority::in_group(sequence, 0, Order::Descending))) + [4, 9].map(|key| (Some(0), crate::priority::in_group(key, 0, Order::Descending))) ); // Without send groups, the rank goes alone. @@ -8109,7 +8053,7 @@ mod serve_group_test { datagram_send(Version::Lite07, session) .send(crate::datagram::Datagram { sequence: 4, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, payload: bytes::Bytes::from_static(b"d"), }) .unwrap(); @@ -8120,7 +8064,7 @@ mod serve_group_test { } /// On a transport with send groups, a subscription opens one at its priority and its - /// group streams join it, ranked newest first by sequence alone: the session-wide queue + /// group streams join it, ranked newest first by key alone: the session-wide queue /// takes no part. A SUBSCRIBE_UPDATE moves the group once, and an order flip re-stamps /// the open streams. #[kio::test] @@ -8199,13 +8143,13 @@ mod serve_group_test { let log = Log::default(); let (mut run, mut writer, _) = lite07_run(SinkSession::new(log.clone()).with_unacked_fin(), subscriber); - let mut group = track.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = track.create_group(group::Info { start: 0 }).unwrap(); for millis in 0..FRAMES as u64 { group .write_frame(Timestamp::from_millis(millis).unwrap(), vec![0u8; PAYLOAD]) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let mut tasks: kio::Tasks = kio::Tasks::new(); let turns = Arc::new(AtomicU64::new(0)); @@ -8250,10 +8194,9 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); - for sequence in 0..5 { - write_group(&mut track, sequence, sequence * 1000); + for key in 0..5 { + write_group(&mut track, key * 1000, key * 1000); } - let at = |millis| Anchor::Time(Timestamp::from_millis(millis).unwrap()); let subscriber = track.subscribe( track::Subscription::default() .with_delay(track::Delay::default().with_max(Duration::from_secs(3600))) @@ -8263,15 +8206,15 @@ mod serve_group_test { end: None, }, Range { - start: at(1500), - end: Some(Anchor::Absolute(2)), + start: Anchor::Time(1500), + end: Some(Anchor::Time(1999)), }, // Empty, so it starts nowhere. - Range::groups(3..3), - // Empty only once its start resolves to group 4. + Range::EMPTY, + // Empty only once its start resolves to group 4000. Range { start: Anchor::Relative(0), - end: Some(Anchor::Absolute(2)), + end: Some(Anchor::Time(1999)), }, ]), ); @@ -8287,7 +8230,7 @@ mod serve_group_test { .unwrap() .0 { - lite::SubscribeResponse::Start(start) => assert_eq!(start.starts, [Some(3), Some(1), None, None]), + lite::SubscribeResponse::Start(start) => assert_eq!(start.starts, [Some(3000), Some(1000), None, None]), other => panic!("expected SUBSCRIBE_START, got {other:?}"), } } @@ -8334,8 +8277,8 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); - for sequence in 0..5 { - write_group(&mut track, sequence, sequence * 1000); + for key in 0..5 { + write_group(&mut track, key, key * 1000); } // Bounded below the groups the added range reaches, so its start is its own. let first = lite::Range::span(Some(0), 0, Some(0), None); @@ -8368,8 +8311,8 @@ mod serve_group_test { assert_eq!(started, [lite::SubscribeStarted { id: 1, start: Some(2) }]); // Newer groups arrive, and another update leaves the range live: it still starts at 2. - for sequence in 5..8 { - write_group(&mut track, sequence, sequence * 1000); + for key in 5..8 { + write_group(&mut track, key, key * 1000); } let live = [(0, first), (1, back)]; run.update(lite07_add(&live, 2, &[lite::Range::span(Some(9), 0, None, None)])); @@ -8377,9 +8320,9 @@ mod serve_group_test { assert_eq!(resolved, Some((2, None)), "the update kept where it started"); } - /// An update the model refuses, here a time start that a group-only copy's cache cannot - /// settle, ends the subscription with the refusal and keeps the ranges it had, rather - /// than serving under terms the model never took. + /// An update the model refuses, here a relative start that a group-only copy's cache + /// cannot settle, ends the subscription with the refusal once the run reads under it, + /// rather than serving under terms the model never took. #[kio::test] async fn lite07_a_refused_update_ends_the_subscription() { let mut track = track::Producer::new( @@ -8387,9 +8330,10 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); - track.anchored_cached(); - for sequence in 0..5 { - write_group(&mut track, sequence, sequence * 1000); + // A received copy whose wire can't say what a relative anchor counts back from. + track.received(); + for key in 0..5 { + write_group(&mut track, key, key * 1000); } let subscriber = track.subscribe( track::Subscription::default() @@ -8401,38 +8345,32 @@ mod serve_group_test { assert!(run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready()); } - // Swap range 0 for one ending at a time later groups may still start before. + // Swap range 0 for one counting back from a latest group the copy can't place. let later = lite::Range { - start: Some(lite::Anchor::Time(1_000)), + start: Some(lite::Anchor::Relative(2)), end: Some(lite::Anchor::Time(100_000)), }; run.update(lite07_add(&[], 1, &[later])); - assert_eq!(run.ids, [0], "kept the ranges the model holds"); - assert!(run.unstarted.is_empty(), "owes no start for a refused range"); - let result = loop { - if let Poll::Ready(Err(err)) = run.poll_step(&mut writer, &kio::Waiter::noop()) { - break err; - } - }; + let result = (0..100) + .find_map(|_| match run.poll_step(&mut writer, &kio::Waiter::noop()) { + Poll::Ready(Err(err)) => Some(err), + _ => None, + }) + .expect("the refusal ends the subscription"); assert!(matches!(result, Error::Unsupported), "ended with {result:?}"); } - /// Where an added time start `time` resolves among `groups` (sequence and start, in - /// milliseconds), before and after `newer` arrives and another update leaves it live, with - /// whether it names that group. The run serves group `first` first. - async fn time_start_across_an_update( - groups: &[(u64, u64)], - first: u64, - time: u64, - newer: (u64, u64), - ) -> [(u64, bool); 2] { + /// Where an added time start `time` resolves among `groups` (starts, in milliseconds), + /// before and after a group starting at `newer` arrives and another update leaves it live, + /// with whether it names that group. The run serves group `first` first. + async fn time_start_across_an_update(groups: &[u64], first: u64, time: u64, newer: u64) -> [(u64, bool); 2] { let mut track = track::Producer::new( Arc::new(broadcast::Info::default()), "test", crate::track::Info::timed(), ); - for (sequence, millis) in groups { - write_group(&mut track, *sequence, *millis); + for &key in groups { + write_group(&mut track, key, key); } let served = lite::Range::span(Some(first), 0, Some(first), None); let subscriber = track.subscribe( @@ -8452,7 +8390,7 @@ mod serve_group_test { while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} let before = run.track.resolved()[1].expect("resolved"); - write_group(&mut track, newer.0, newer.1); + write_group(&mut track, newer, newer); let live = [(0, served), (1, time)]; run.update(lite07_add(&live, 2, &[lite::Range::span(Some(99), 0, None, None)])); let after = run.track.resolved()[1].expect("resolved"); @@ -8464,21 +8402,21 @@ mod serve_group_test { /// still reaches back to the oldest, rather than becoming a floor at group 0. #[kio::test] async fn lite07_an_added_time_start_keeps_its_start() { - let five: Vec<_> = (0..5).map(|sequence| (sequence, sequence * 1000)).collect(); - // 4.5 s falls in group 4, the newest starting at or before it, until group 5 starts at - // 4.2 s. + let five = [0, 1000, 2000, 3000, 4000]; + // 4.5 s falls in the group at 4 s, the newest starting at or before it, until a group + // starts at 4.2 s. assert_eq!( - time_start_across_an_update(&five, 0, 4500, (5, 4200)).await, - [(4, true), (4, true)] + time_start_across_an_update(&five, 0, 4500, 4200).await, + [(4000, true), (4000, true)] ); - // 1 s falls in group 0 until group 1 starts at 0.5 s. + // 1 s falls in the group at 0 until a group starts at 0.5 s. assert_eq!( - time_start_across_an_update(&[(0, 0)], 0, 1000, (1, 500)).await, + time_start_across_an_update(&[0], 0, 1000, 500).await, [(0, true), (0, true)] ); - // 0.1 s is older than group 5, the oldest held. + // 0.1 s is older than the group at 5 s, the oldest held. assert_eq!( - time_start_across_an_update(&[(5, 5000)], 5, 100, (6, 6000)).await, + time_start_across_an_update(&[5000], 5000, 100, 6000).await, [(0, false), (0, false)] ); } @@ -8542,7 +8480,7 @@ mod serve_group_test { ); } - /// A sequence the publisher declared it will never produce is accounted for by SKIP + /// A key the publisher declared it will never produce is accounted for by SKIP /// rather than a stream count. #[kio::test] async fn lite07_skips_a_declared_skip() { @@ -8854,8 +8792,8 @@ mod serve_group_test { write_group(&mut track, 0, 0); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} // Groups 1 and 2 reach 2s and 3s, at least 1.5s behind the 6s edge. - for (sequence, millis) in [(1, 1000), (2, 2000), (3, 3000), (4, 6000)] { - write_group(&mut track, sequence, millis); + for (key, millis) in [(1, 1000), (2, 2000), (3, 3000), (4, 6000)] { + write_group(&mut track, key, millis); } while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} let case = format!("{version:?}, widen={widen}"); @@ -8900,8 +8838,8 @@ mod serve_group_test { let (mut run, mut writer, log) = lite07_run(SinkSession::new(Log::default()), subscriber); write_group(&mut track, 0, 0); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} - for (sequence, millis) in [(1, 1000), (2, 2000), (3, 3000), (4, 6000)] { - write_group(&mut track, sequence, millis); + for (key, millis) in [(1, 1000), (2, 2000), (3, 3000), (4, 6000)] { + write_group(&mut track, key, millis); } while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert_eq!(drop_updates(&mut writer, &log, Version::Lite07).await, [(1, 2, 0)]); @@ -9264,10 +9202,10 @@ mod serve_group_test { assert_eq!(drop_updates(&mut writer, &log, Version::Lite07).await, [(0, 0, 0)]); } - /// Lite-05 and later name every sequence below the end that the subscription never + /// Lite-05 and later name every key below the end that the subscription never /// got once the track ends: one skipped between groups, and one past the last. #[kio::test] - async fn the_end_drops_the_sequences_never_served() { + async fn the_end_drops_the_keys_never_served() { for version in [Version::Lite05, Version::Lite06] { let mut track = track::Producer::new( Arc::new(broadcast::Info::default()), @@ -9337,7 +9275,7 @@ mod serve_group_test { while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert!(run.start.resolved().is_some()); - let group = track.create_group(group::Info { sequence: 1 }).unwrap(); + let group = track.create_group(group::Info { start: 1 }).unwrap(); group.abort(Error::App(1)).unwrap(); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert_eq!(drops(&mut writer, &log, version).await, [(1, 1)]); @@ -9387,7 +9325,7 @@ mod serve_group_test { } } - /// A wide upstream DROP is held and forwarded as a range, never one entry per sequence, + /// A wide upstream DROP is held and forwarded as a range, never one entry per key, /// and a wider one later forwards only what is new. #[kio::test] async fn a_wide_drop_is_forwarded_as_a_range() { @@ -9495,7 +9433,7 @@ mod serve_group_test { while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert!(run.start.resolved().is_some()); - let group = track.create_group(group::Info { sequence: 1 }).unwrap(); + let group = track.create_group(group::Info { start: 1 }).unwrap(); group.abort(Error::App(1)).unwrap(); while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert_eq!(drops(&mut writer, &log, version).await, [(1, 1)]); @@ -9565,7 +9503,7 @@ mod serve_group_test { while run.poll_step(&mut writer, &kio::Waiter::noop()).is_ready() {} assert!(run.start.resolved().is_some()); - let mut headless = track.create_group(group::Info { sequence: 1 }).unwrap(); + let mut headless = track.create_group(group::Info { start: 1 }).unwrap(); headless.start_at(1).unwrap(); headless .write_frame(Timestamp::from_millis(1).unwrap(), b"x".as_slice()) @@ -9668,10 +9606,10 @@ mod serve_group_test { ), false => None, }; - for sequence in 0..3 { - write_group(&mut track, sequence, sequence * 10); + for key in 0..3 { + write_group(&mut track, key, key * 10); if let Some(keeper) = &mut keeper { - assert_eq!(keeper.recv_group().await.unwrap().unwrap().sequence, sequence); + assert_eq!(keeper.recv_group().await.unwrap().unwrap().start, key); } } kio::clock::sim::advance(Duration::from_secs(1)).await; @@ -9691,7 +9629,7 @@ mod serve_group_test { .subscribe(track::Subscription::default().with_range(0..)) .await .unwrap(); - assert_eq!(live.recv_group().await.unwrap().unwrap().sequence, 2); + assert_eq!(live.recv_group().await.unwrap().unwrap().start, 2); starts.push(kio::wait(|waiter| live.poll_start(waiter)).await); } assert_eq!( @@ -9714,8 +9652,8 @@ mod serve_group_test { crate::track::Info::timed(), ); track.start_at(if warm { 0 } else { 2 }).unwrap(); - for sequence in (if warm { 0 } else { 2 })..3 { - write_group(&mut track, sequence, sequence * 10); + for key in (if warm { 0 } else { 2 })..3 { + write_group(&mut track, key, key * 10); } kio::clock::sim::advance(Duration::from_secs(1)).await; let subscriber = track.subscribe(track::Subscription::default()); @@ -9823,12 +9761,13 @@ mod serve_group_test { /// stays open for an update. #[kio::test] async fn giving_up_every_range_before_the_start_answers_it() { - // Disjoint ranges, then one range given up a group at a time with alternating errors. - for (mut bench, last) in [(CoverageBench::new(3, 2), 4), (CoverageBench::adjacent(4), 4)] { + // Disjoint ranges, the last given up with the group before it, which its time start may + // reach back to; then one range given up a group at a time with alternating errors. + for (mut bench, last) in [(CoverageBench::new(3, 2), 3..5), (CoverageBench::adjacent(4), 4..5)] { bench.wake(); bench.wake(); assert!(bench.run.start.resolved().is_none(), "a range is still open"); - bench.track.drop_groups(last..last + 1, StreamError::NotFound); + bench.track.drop_groups(last, StreamError::NotFound); loop { match bench.run.poll_step(&mut bench.writer, &kio::Waiter::noop()) { Poll::Ready(Ok(ControlFlow::Continue(()))) => continue, @@ -9901,7 +9840,7 @@ mod serve_group_test { .with_delay(track::Delay::default().with_max(Duration::from_secs(1))); let subscriber = track.subscribe(subscription); let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); - write_group(&mut track, 5, 0); + write_group(&mut track, 5, 5); while run.start.resolved().is_none() { let step = run.poll_step(&mut writer, &kio::Waiter::noop()); assert!(step.is_ready(), "the start never resolved"); @@ -9922,8 +9861,8 @@ mod serve_group_test { // Group 2's successor arrives with it: group 5 arrived a full budget before // the update, so group 2 bounded by it would be stale on the wall clock. Group 4 // is, and goes unserved. - write_group(&mut track, 4, 0); - write_group(&mut track, 2, 0); + write_group(&mut track, 4, 4); + write_group(&mut track, 2, 2); assert!(futures::poll!(run.as_mut()).is_pending()); track.finish_at(track::Position::group(6), None).unwrap(); drop(track); @@ -9967,7 +9906,7 @@ mod serve_group_test { run.replace(update(5)); { let mut run = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); - write_group(&mut track, 5, 0); + write_group(&mut track, 5, 5); assert!(futures::poll!(run.as_mut()).is_pending()); } @@ -10016,8 +9955,8 @@ mod serve_group_test { assert_eq!(drops(&mut writer, &log, version).await, []); run.replace(update(0)); - for sequence in 1..5 { - write_group(&mut track, sequence, 0); + for key in 1..5 { + write_group(&mut track, key, key); } track.finish_at(track::Position::group(5), None).unwrap(); drop(track); @@ -10065,6 +10004,68 @@ mod serve_group_test { assert_eq!(settle(&mut run), 2); } + /// Lite-05/06 make SUBSCRIBE_START the first message, and a SUBSCRIBE_FINISH ahead of it + /// says no group was served. A copy whose end lands before its first group, as a relay's can + /// when its source answered both, holds the end until that group resolves the start. + #[kio::test] + async fn a_pre07_end_declared_before_the_first_group_follows_the_start() { + for version in [Version::Lite05, Version::Lite06] { + let mut track = track::Producer::new( + Arc::new(broadcast::Info::default()), + "test", + crate::track::Info::timed(), + ); + let subscriber = track.subscribe( + track::Subscription::default() + .with_start(0) + .with_delay(track::Delay::default().with_max(Duration::from_secs(60))), + ); + let (mut run, mut writer, log) = lite_run(version, SinkSession::new(Log::default()), subscriber); + let step = |run: &mut TrackRun, writer: &mut Writer<_, _>| { + while let Poll::Ready(flow) = run.poll_step(writer, &kio::Waiter::noop()) { + if flow.unwrap().is_break() { + break; + } + } + }; + let sent = |log: &Log| { + let writes = log.writes.lock().unwrap().clone(); + let mut buf = writes.as_slice(); + let mut sent = Vec::new(); + while !buf.is_empty() { + let (msg, size) = lite::SubscribeResponse::decode_slice(buf, version).unwrap(); + buf = &buf[size..]; + sent.push(match msg { + lite::SubscribeResponse::Start(start) => format!("start {:?}", start.starts), + lite::SubscribeResponse::Finish(finish) => format!("finish {}", finish.group), + other => format!("{other:?}"), + }); + } + sent + }; + + track.finish_at(track::Position::group(273), None).unwrap(); + step(&mut run, &mut writer); + kio::wait(|waiter| writer.poll_flush(waiter)).await.unwrap(); + assert_eq!( + sent(&log), + Vec::::new(), + "{version:?}: the end waits for the start" + ); + + write_group(&mut track, 272, 272); + step(&mut run, &mut writer); + kio::wait(|waiter| writer.poll_flush(waiter)).await.unwrap(); + // Drops for the starts skipped below the group may go between. + let sent = sent(&log); + assert_eq!( + (sent.first().map(String::as_str), sent.last().map(String::as_str)), + (Some("start [Some(272)]"), Some("finish 273")), + "{version:?}: {sent:?}" + ); + } + } + /// On an empty track the re-pin has no latest group to name, so the first group the /// track produces is served, even below the dropped floor. #[kio::test] @@ -10118,11 +10119,11 @@ mod serve_group_test { let mut run = std::pin::pin!(kio::wait(|waiter| run.poll(&mut writer, waiter))); write_group(&mut track, 0, 0); assert!(futures::poll!(run.as_mut()).is_pending()); - write_group(&mut track, 2, 0); + write_group(&mut track, 2, 2); assert!(futures::poll!(run.as_mut()).is_pending()); kio::clock::sim::advance(Duration::from_secs(2)).await; - write_group(&mut track, 4, 0); + write_group(&mut track, 4, 4); assert!(futures::poll!(run.as_mut()).is_pending()); track.finish_at(track::Position::group(5), None).unwrap(); drop(track); @@ -10167,11 +10168,10 @@ mod serve_group_test { steer_seen: Steer::default(), version: Version::Lite07, stamp: Stamp::new(Version::Lite07, Some(crate::Timescale::MILLI)), - runtime: crate::time::sim(), outcomes: outcomes.clone(), ietf: false, }; - let bounds = Bounds::new(&[lite::Range::span(Some(start_group), 0, None, None)], None).unwrap(); + let bounds = Bounds::new(&[lite::Range::span(Some(start_group), 0, None, None)], false); Self { run: TrackRun::new(ctx, subscriber, bounds, 0, steer), writer, @@ -10240,8 +10240,8 @@ mod serve_group_test { "test", crate::track::Info::timed(), ); - for sequence in 0..3 { - write_group(&mut track, sequence, sequence); + for key in 0..3 { + write_group(&mut track, key, key); } let mut relay = RelayRun::new(&mut track, 1); relay.settle(); @@ -10299,8 +10299,9 @@ mod serve_group_test { assert_eq!(relay.opened(), 1, "served a group below the floor"); } - /// A source whose feed starts below the subscriber's floor serves the floor's group, so - /// a newer group arriving first must not resolve the start past it. + /// A source whose feed starts below the subscriber's time start serves the group + /// containing it, so a newer group arriving first must not resolve the start past it: + /// with no group held placing the time yet, the start is where the source's feed starts. #[kio::test] async fn held_first_group_resolves_to_the_floor_under_the_source() { let mut track = track::Producer::new( @@ -10312,14 +10313,14 @@ mod serve_group_test { write_group(&mut track, 6, 6); relay.settle(); - track.start_at(0).unwrap(); + track.start_at(2).unwrap(); relay.settle(); assert_eq!(relay.opened(), 1, "the held group is served"); write_group(&mut track, 5, 5); relay.settle(); - assert_eq!(relay.opened(), 2, "dropped the floor's group"); - assert!(relay.started_at(5)); + assert_eq!(relay.opened(), 2, "dropped the group containing the start"); + assert!(relay.started_at(2)); } /// A group skipped for a missing head before the start resolves is never served, so @@ -10334,12 +10335,12 @@ mod serve_group_test { let mut relay = RelayRun::new(&mut track, 5); // Group 5's first frame is gone. - let mut headless = track.create_group(group::Info { sequence: 5 }).unwrap(); + let mut headless = track.create_group(group::Info { start: 5 }).unwrap(); headless.start_at(1).unwrap(); headless .write_frame(Timestamp::from_millis(5).unwrap(), b"x".as_slice()) .unwrap(); - headless.finish().unwrap(); + headless.close().unwrap(); write_group(&mut track, 6, 6); relay.settle(); @@ -10362,12 +10363,12 @@ mod serve_group_test { track.drop_groups(4..5, StreamError::Old); // Group 5's first frame is gone. - let mut headless = track.create_group(group::Info { sequence: 5 }).unwrap(); + let mut headless = track.create_group(group::Info { start: 5 }).unwrap(); headless.start_at(1).unwrap(); headless .write_frame(Timestamp::from_millis(5).unwrap(), b"x".as_slice()) .unwrap(); - headless.finish().unwrap(); + headless.close().unwrap(); write_group(&mut track, 6, 6); relay.settle(); @@ -11401,7 +11402,9 @@ mod tests { let mut track_info = legacy_request::(&publisher, version, &legacy_track()).await; assert!(!drive(&mut track_info).await, "{case}: the TRACK stream is held"); - datagrams.append(None, b"between".as_slice()).unwrap(); + datagrams + .append(crate::Timestamp::unscaled(0), b"between".as_slice()) + .unwrap(); let mut subscribe = legacy_request::(&publisher, version, &legacy_subscribe(-5)).await; assert!(!drive(&mut subscribe).await, "{case}: the subscription is live"); @@ -11442,7 +11445,9 @@ mod tests { let _track = reserved.accept(crate::track::Info::timed()); assert!(!drive(&mut track_info).await, "{version:?}: the TRACK stream is held"); - datagrams.append(None, b"between".as_slice()).unwrap(); + datagrams + .append(crate::Timestamp::unscaled(0), b"between".as_slice()) + .unwrap(); assert!(!drive(&mut subscribe).await, "{version:?}: the subscription is live"); assert_eq!(log.lock().unwrap().len(), 1, "{version:?}"); } @@ -11464,7 +11469,9 @@ mod tests { let mut older = legacy_request::(&publisher, version, &legacy_track()).await; assert!(!drive(&mut older).await, "{version:?}: the older TRACK stream is held"); - datagrams.append(None, b"between".as_slice()).unwrap(); + datagrams + .append(crate::Timestamp::unscaled(0), b"between".as_slice()) + .unwrap(); let mut newer = legacy_request::(&publisher, version, &legacy_track()).await; assert!(!drive(&mut newer).await, "{version:?}: the newer TRACK stream is held"); @@ -11490,7 +11497,9 @@ mod tests { let mut older = legacy_request::(&publisher, version, &legacy_track()).await; assert!(!drive(&mut older).await, "{version:?}: the older TRACK stream is held"); - first.append(None, b"first".as_slice()).unwrap(); + first + .append(crate::Timestamp::unscaled(0), b"first".as_slice()) + .unwrap(); first.finish(); let mut current = broadcast @@ -11499,7 +11508,9 @@ mod tests { .unwrap(); let mut track_info = legacy_request::(&publisher, version, &legacy_track()).await; assert!(!drive(&mut track_info).await, "{version:?}: the TRACK stream is held"); - current.append(None, b"current".as_slice()).unwrap(); + current + .append(crate::Timestamp::unscaled(0), b"current".as_slice()) + .unwrap(); let mut subscribe = legacy_request::(&publisher, version, &legacy_subscribe(0)).await; assert!(!drive(&mut subscribe).await, "{version:?}: the subscription is live"); @@ -11631,11 +11642,11 @@ mod tests { Empty, } - fn write(track: &mut track::Producer, sequences: &[u64]) { - for &sequence in sequences { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"x".as_slice()).unwrap(); - group.finish().unwrap(); + fn write(track: &mut track::Producer, keys: &[u64]) { + for &key in keys { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); + group.close().unwrap(); } } @@ -11760,17 +11771,17 @@ mod tests { streams.entry(id).or_default().extend(bytes); } } - let mut sequences = streams + let mut keys = streams .values() .map(|bytes| { crate::coding::decode_buf(&mut bytes.as_slice(), version, |r, v| { assert_eq!(lite::DataType::decode(r, v)?, lite::DataType::Group); - Ok(lite::Group::decode(r, v)?.sequence) + Ok(lite::Group::decode(r, v)?.key) }) .unwrap() }) .collect::>(); - sequences.sort(); + keys.sort(); let all: &[u64] = match answer { Answer::Reported => &[3, 7, 9], Answer::Warm => &[0, 3, 7, 9], @@ -11789,7 +11800,7 @@ mod tests { Answer::Empty => &[0, 3, 7], } }; - assert_eq!(sequences, expected, "{version}, start={start:?}"); + assert_eq!(keys, expected, "{version}, start={start:?}"); track.abort(Error::Cancel).unwrap(); } } @@ -11881,11 +11892,11 @@ mod tests { .expect("track requested") .unwrap(); let track = request.accept(crate::track::Info::timed().with_refresh(1)); - let mut group = track.create_group(group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(group::Info { start: 1 }).unwrap(); for frame in [&b"key"[..], b"abc", b"d"] { - group.write_frame(crate::Timestamp::ZERO, frame).unwrap(); + group.write_frame(group.key_time(), frame).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); assert!(!drive(&mut serve).await, "the subscription ended"); let RequestState::Serve(SubscribeServe::Run(run)) = &serve.state else { panic!("the subscription is not running"); @@ -11895,11 +11906,11 @@ mod tests { // The asked-for group arrives, over the ratio too. A restated wanted group is now // stale, so it is not another join. - let mut group = track.create_group(group::Info { sequence: 2 }).unwrap(); + let mut group = track.create_group(group::Info { start: 2 }).unwrap(); for frame in [&b"key"[..], b"abc", b"d"] { - group.write_frame(crate::Timestamp::ZERO, frame).unwrap(); + group.write_frame(group.key_time(), frame).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); session.push(&update(Some(2), 5).encode_bytes(version).unwrap()); assert!(!drive(&mut serve).await, "the subscription ended"); let RequestState::Serve(SubscribeServe::Run(run)) = &serve.state else { @@ -12048,11 +12059,11 @@ mod tests { let mut appended = 0; tasks.push(Box::new(move |waiter: &kio::Waiter| { for _ in 0..BATCH.min(GROUPS - appended) { - let mut group = producer.create_group(group::Info { sequence: appended }).unwrap(); + let mut group = producer.create_group(group::Info { start: appended }).unwrap(); group .write_frame(crate::Timestamp::from_millis(appended).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); appended += 1; } if appended == GROUPS { @@ -12260,11 +12271,11 @@ mod tests { .create_track("video", track::Info::timed()) .unwrap(); broadcast.announce().unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); for _ in 0..FRAMES { group.write_frame(crate::Timestamp::ZERO, b"x".to_vec()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let peer_setup = crate::lite::PeerSetup::new(false, crate::transport::Address::Setup); peer_setup.set(crate::lite::Setup::default()).unwrap(); @@ -12343,8 +12354,8 @@ mod tests { .create_track("video", track::Info::timed()) .unwrap(); broadcast.announce().unwrap(); - let mut open = track.append_group().unwrap(); - open.write_frame(crate::Timestamp::ZERO, b"a".to_vec()).unwrap(); + let mut open = track.append_next().unwrap(); + open.write_frame(open.key_time(), b"a".to_vec()).unwrap(); let peer_setup = crate::lite::PeerSetup::new(false, crate::transport::Address::Setup); peer_setup.set(crate::lite::Setup::default()).unwrap(); @@ -12393,11 +12404,11 @@ mod tests { // The track runs a full second past the group's successor. for ms in [1000, 2000] { - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group .write_frame(crate::Timestamp::from_millis(ms).unwrap(), b"x".to_vec()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } assert!(drive(&mut serve).await, "the budget ends the response"); assert_eq!(session.log.resets(), vec![crate::StreamError::Old.to_code()]); @@ -12421,9 +12432,9 @@ mod tests { } let track = broadcast.broadcast().create_track("video", info).unwrap(); broadcast.announce().unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(crate::Timestamp::ZERO, b"abcd".to_vec()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let peer_setup = crate::lite::PeerSetup::new(false, crate::transport::Address::Setup).offer_ietf(true); peer_setup @@ -12907,7 +12918,7 @@ mod tests { run.drop_given_up_paused(&kio::Waiter::noop()); assert!(run.paused.is_empty() && run.paused_tasks.is_empty()); let dropped = run.ctx.outcomes.dropped.lock().unwrap(); - assert_eq!(dropped.iter().map(|(sequence, _)| *sequence).collect::>(), [100]); + assert_eq!(dropped.iter().map(|(start, _)| *start).collect::>(), [100]); } /// The next paused group to continue is the newest when descending, the oldest when @@ -12915,7 +12926,7 @@ mod tests { #[test] fn the_best_paused_group_follows_the_order() { let mut bench = PausedBench::new(3); - let best = |run: &TrackRun, order| run.best_paused(order).map(|(sequence, _)| sequence); + let best = |run: &TrackRun, order| run.best_paused(order).map(|(start, _)| start); assert_eq!(best(&bench.coverage.run, track::Order::Descending), Some(102)); assert_eq!(best(&bench.coverage.run, track::Order::Ascending), Some(100)); @@ -12931,12 +12942,8 @@ mod tests { fn a_released_paused_group_with_nothing_unread_is_dropped() { let mut bench = PausedBench::new(1); let run = &mut bench.coverage.run; - let empty = bench - .coverage - .track - .create_group(group::Info { sequence: 200 }) - .unwrap(); - let empty = empty.finish().unwrap(); + let empty = bench.coverage.track.create_group(group::Info { start: 200 }).unwrap(); + let empty = empty.close().unwrap(); run.pause(Paused { group: empty.consume(), frame_start: 0, @@ -12949,6 +12956,6 @@ mod tests { assert!(!run.paused.contains_key(&200)); assert_eq!(run.paused_tasks.len(), 1, "only the released group leaves"); let dropped = run.ctx.outcomes.dropped.lock().unwrap(); - assert_eq!(dropped.iter().map(|(sequence, _)| *sequence).collect::>(), [200]); + assert_eq!(dropped.iter().map(|(start, _)| *start).collect::>(), [200]); } } diff --git a/rs/moq-net/src/lite/subscribe.rs b/rs/moq-net/src/lite/subscribe.rs index b8989127c9e..12f67ff9ee3 100644 --- a/rs/moq-net/src/lite/subscribe.rs +++ b/rs/moq-net/src/lite/subscribe.rs @@ -445,7 +445,7 @@ fn encode_order(w: &mut Encoder<'_>, version: Version, order: Order) -> Result<( /// Decode the `Group Start` field shared by SUBSCRIBE and SUBSCRIBE_UPDATE. /// /// Lite-06 carries the raw floor, so every value names a concrete group and 0 decodes as -/// `Some(0)`. Pre-06 wires encode the sequence + 1, with 0 meaning the latest group +/// `Some(0)`. Pre-06 wires encode the key + 1, with 0 meaning the latest group /// (`None`). Callers canonicalize with [`canonical_start_group`] once the frame bounds /// are known. fn decode_start_group(r: &mut Decoder<'_>, version: Version) -> Result, DecodeError> { @@ -469,7 +469,7 @@ fn canonical_start_group(version: Version, start_group: Option, start_frame /// Encode the `Group Start` field shared by SUBSCRIBE and SUBSCRIBE_UPDATE. /// /// The inverse of [`decode_start_group`]. Lite-06 writes the raw floor, so `None` and -/// `Some(0)` are the same group 0. A pre-06 wire encodes the sequence + 1: `None` is 0 +/// `Some(0)` are the same group 0. A pre-06 wire encodes the key + 1: `None` is 0 /// (the latest group, where the publisher starts) and `Some(0)` is 1, replay from the /// beginning. fn encode_start_group(w: &mut Encoder<'_>, version: Version, start_group: Option) -> Result<(), EncodeError> { @@ -1059,11 +1059,11 @@ impl<'a> Message<'a> for SubscribeStarted { /// Names groups that existed but will not be delivered: aborted or reset, evicted or /// expired with no copy left, or stale. /// -/// Lite03+; publishers send it on Lite05 and later. Before lite-07 it also names sequences +/// Lite03+; publishers send it on Lite05 and later. Before lite-07 it also names keys /// declared skipped, which lite-07 names in [`SubscribeSkip`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SubscribeDrop { - /// The dropped group sequences. + /// The dropped group keys. pub groups: std::ops::Range, /// Why the groups were given up: a stream error code on lite-07 and later, or an @@ -1107,7 +1107,7 @@ impl SubscribeDrop { /// A SUBSCRIBE_DROP or SUBSCRIBE_SKIP range: `Group Start` and `Group Count` on lite-07, an /// inclusive `Group Start` and `Group End` before. Either way it ends by the reserved -/// sequence `u64::MAX`, so it never names that group. +/// key `u64::MAX`, so it never names that group. fn decode_groups(r: &mut Decoder<'_>, version: Version) -> Result, DecodeError> { let start = r.varint()?; let end = match version.has_group_count() { @@ -1161,13 +1161,13 @@ impl<'a> Message<'a> for SubscribeDrop { } } -/// Names sequences the publisher declared it will never produce (see +/// Names keys the publisher declared it will never produce (see /// `track::Producer::skip_groups`): final, with no error, ending no reader. /// /// Lite07+ only; earlier versions name them in a [`SubscribeDrop`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct SubscribeSkip { - /// The skipped group sequences. + /// The skipped group keys. pub groups: std::ops::Range, } @@ -1672,8 +1672,8 @@ mod test { assert_eq!(got.ranges, msg.ranges); } - /// Lite06 carries the raw floor; pre-06 wires encode the sequence + 1 with 0 meaning - /// the latest group. An explicit group 0 is that sequence, so it round-trips as group 0 + /// Lite06 carries the raw floor; pre-06 wires encode the key + 1 with 0 meaning + /// the latest group. An explicit group 0 is that key, so it round-trips as group 0 /// rather than as the latest group. #[test] fn group_start_is_absolute_on_lite06() { @@ -1703,7 +1703,7 @@ mod test { .unwrap(); assert_eq!(got.ranges, [Range::OLDEST]); - // On the pre-06 wire an explicit group 0 is sequence + 1, distinct from an + // On the pre-06 wire an explicit group 0 is key + 1, distinct from an // absent start (the latest group). let explicit = encode(&msg, Version::Lite05).unwrap(); assert_eq!(roundtrip(&msg, Version::Lite05).ranges, msg.ranges); diff --git a/rs/moq-net/src/lite/subscriber.rs b/rs/moq-net/src/lite/subscriber.rs index 186b23c2ecf..209edd416e1 100644 --- a/rs/moq-net/src/lite/subscriber.rs +++ b/rs/moq-net/src/lite/subscriber.rs @@ -238,7 +238,7 @@ struct Parked { } /// Lite-07: the groups whose stream the publisher reset with YIELDED, each held open for a -/// later stream to continue, by sequence. +/// later stream to continue, by key. #[derive(Default)] struct Paused { groups: BTreeMap, @@ -492,29 +492,29 @@ impl TrackEntry { /// Open the group of a stream whose header is `hdr` in the copy. fn open(&mut self, hdr: &lite::Group, version: Version, now: crate::time::Instant) -> Result { - if self.is_paused(hdr.sequence) { - let reading = Reading::open(&self.tail, Some(hdr.sequence), now); + if self.is_paused(hdr.key) { + let reading = Reading::open(&self.tail, Some(hdr.key), now); return self.resume(hdr, reading).expect("checked paused above"); } let Some(Accepted { producer, .. }) = &mut self.copy else { return Err(Error::Cancel); }; - let group_info = group::Info { sequence: hdr.sequence }; + let group_info = group::Info { start: hdr.key }; // A stale drop that overtook this stream, which the group replaces: the next update // supersedes that verdict, so a YIELDED reset after it pauses the group. - let stale = producer.holds_stale(hdr.sequence); + let stale = producer.holds_stale(hdr.key); // Stats (groups/frames/bytes) are counted in the model as the group // is written, through the tagged `track::Producer`. The track counts // it as content once its first frame lands. let received = producer.create_group_at(group_info, hdr.frame_start); // Asked before this stream accounts for the group: a drop that overtook it named it. - let owed = !self.tail.read().accounts(hdr.sequence); + let owed = !self.tail.read().accounts(hdr.key); // The subscription's end waits until this stream is read. Counted // whatever the cache said, since the publisher counts every stream it // opens (a resubscribe resends a cached group), and only after the // cache holds the group, so a fetch held for the feed that sees it // accounted also finds it. - let reading = Reading::open(&self.tail, Some(hdr.sequence), now); + let reading = Reading::open(&self.tail, Some(hdr.key), now); // A route's first group says where its live feed is, when its answer does // not: even one the cache already holds, as an idle copy asked from its // head gets back. The copy goes live once the cache shows that group. @@ -522,7 +522,7 @@ impl TrackEntry { // track the first to land is the newest, not where the feed starts. if !producer.is_live() { let first = Position { - group: hdr.sequence, + group: hdr.key, frame: hdr.frame_start, }; if !version.has_track_stream() { @@ -547,21 +547,21 @@ impl TrackEntry { self.place(hdr, version, received, reading, verdict) } - /// Whether a YIELDED reset paused the group `sequence`, which the cache still holds. One the + /// Whether a YIELDED reset paused the group `key`, which the cache still holds. One the /// cache gave up on is let go here, releasing the subscription's end its stream held, since /// the stream asking opens the group afresh. - fn is_paused(&mut self, sequence: u64) -> bool { + fn is_paused(&mut self, key: u64) -> bool { let Some(registry) = &self.paused else { return false; }; let Ok(mut state) = registry.write() else { return false; }; - match state.groups.get(&sequence) { + match state.groups.get(&key) { None => false, Some(paused) if !paused.group.handle().is_aborted() => true, Some(_) => { - let given_up = state.groups.remove(&sequence); + let given_up = state.groups.remove(&key); drop(state); drop(given_up); false @@ -570,9 +570,9 @@ impl TrackEntry { } /// Continue the group a YIELDED reset paused with the stream whose header is `hdr`, - /// counted by `reading`, or `None` if no group with its sequence is paused. + /// counted by `reading`, or `None` if no group with its key is paused. fn resume(&mut self, hdr: &lite::Group, reading: Reading) -> Option> { - let paused = self.paused.as_ref()?.write().ok()?.groups.remove(&hdr.sequence)?; + let paused = self.paused.as_ref()?.write().ok()?.groups.remove(&hdr.key)?; let Some(Accepted { producer, framing }) = &self.copy else { return Some(Err(Error::Cancel)); }; @@ -586,7 +586,7 @@ impl TrackEntry { // A continuation starting past what the group holds would leave a hole in it. if hdr.frame_start > PausedGroup::resumes_by_parts(&group, partial.as_ref()) { tracing::warn!( - group = hdr.sequence, + group = hdr.key, frame_start = hdr.frame_start, "continuation past the frames held" ); @@ -605,7 +605,7 @@ impl TrackEntry { /// Pause `group`, whose stream the publisher reset with YIELDED, for a later stream to /// continue. Groups the cache gave up on meanwhile are swept out as the held ones double. - fn pause(&mut self, sequence: u64, paused: PausedGroup) { + fn pause(&mut self, key: u64, paused: PausedGroup) { let registry = self.paused.get_or_insert_with(Default::default); let Ok(mut state) = registry.write() else { return; @@ -614,24 +614,24 @@ impl TrackEntry { state.groups.retain(|_, paused| !paused.group.handle().is_aborted()); state.swept = state.groups.len(); } - state.groups.insert(sequence, paused); + state.groups.insert(key, paused); } - /// Track a stream reading `sequence`, returning its token for [`Self::end_yielding`]. - fn read_yielding(&mut self, sequence: u64, verdict: YieldVerdict) -> u64 { + /// Track a stream reading `key`, returning its token for [`Self::end_yielding`]. + fn read_yielding(&mut self, key: u64, verdict: YieldVerdict) -> u64 { self.yield_streams += 1; let stream = self.yield_streams; - self.yielding.insert(sequence, Yielding { stream, verdict }); + self.yielding.insert(key, Yielding { stream, verdict }); stream } - /// Stop tracking the stream reading `sequence`, returning whether a YIELDED reset pauses it. + /// Stop tracking the stream reading `key`, returning whether a YIELDED reset pauses it. /// A stream that a later one for the same group replaced finds nothing, leaving the /// later stream's record in place. A stale drop an update sent since superseded no /// longer ends the group. - fn end_yielding(&mut self, sequence: u64, stream: u64) -> Option { - let mut yielding = match self.yielding.get(&sequence) { - Some(yielding) if yielding.stream == stream => self.yielding.remove(&sequence)?, + fn end_yielding(&mut self, key: u64, stream: u64) -> Option { + let mut yielding = match self.yielding.get(&key) { + Some(yielding) if yielding.stream == stream => self.yielding.remove(&key)?, _ => return None, }; yielding.verdict = yielding.verdict.updated(self.tail.read().updates()); @@ -652,7 +652,7 @@ impl TrackEntry { }; let ended: Vec<_> = match registry.write() { Ok(mut state) => { - let keys: Vec = state.groups.range(groups).map(|(sequence, _)| *sequence).collect(); + let keys: Vec = state.groups.range(groups).map(|(key, _)| *key).collect(); keys.into_iter().filter_map(|key| state.groups.remove(&key)).collect() } Err(_) => return, @@ -672,7 +672,7 @@ impl TrackEntry { let Some(Accepted { producer, .. }) = &self.copy else { return Err(Error::Cancel); }; - let received = producer.create_group_at(group::Info { sequence: hdr.sequence }, hdr.frame_start); + let received = producer.create_group_at(group::Info { start: hdr.key }, hdr.frame_start); // Served again, so owed again. self.place(hdr, version, received, reading, YieldVerdict::Owed) } @@ -709,7 +709,7 @@ impl TrackEntry { Version::Lite01 | Version::Lite02 | Version::Lite03 | Version::Lite04 | Version::Lite05 ) => { - tracing::warn!(group = hdr.sequence, "group past the declared end of track"); + tracing::warn!(group = hdr.key, "group past the declared end of track"); let _ = producer.clone().abort(Error::ProtocolViolation); return Err(Error::ProtocolViolation); } @@ -1185,12 +1185,13 @@ impl Subscriber { let Some((mut producer, timescale)) = target else { return Ok(Some(dg.subscribe)); }; - let timestamp = dg - .timestamp - .zip(timescale) - .map(|(value, scale)| Timestamp::new(value, scale)) - .transpose() - .map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded))?; + // A scale-less datagram's time is its sequence. + let timestamp = match (dg.timestamp, timescale) { + (Some(value), Some(scale)) => { + Timestamp::new(value, scale).map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded))? + } + _ => Timestamp::unscaled(dg.sequence), + }; producer.insert(crate::datagram::Datagram { sequence: dg.sequence, timestamp, @@ -1776,7 +1777,7 @@ impl GroupRecv { }; let paused = duplicate.paused.get_or_insert_with(|| entry.watch_paused()); let _ = paused.poll(waiter, |_| Poll::<()>::Pending); - if entry.is_paused(hdr.sequence) { + if entry.is_paused(hdr.key) { let resumed = entry.resume(hdr, duplicate.reading).expect("checked paused above"); match resumed { Ok(resumed) => { @@ -1825,14 +1826,14 @@ impl GroupRecv { .subscribes .lock() .get_mut(&hdr.subscribe) - .map(|entry| entry.read_yielding(hdr.sequence, verdict)), + .map(|entry| entry.read_yielding(hdr.key, verdict)), false => None, }; let (priority, order) = track .subscription() .map_or((0, Default::default()), |sub| (sub.priority, sub.order)); self.reader - .set_priority(crate::priority::recv_group(priority, hdr.sequence, order)); + .set_priority(crate::priority::recv_group(priority, hdr.key, order)); // The timescale came from TRACK_INFO, so frames decode immediately. No // SUBSCRIBE_OK to wait on. @@ -1860,7 +1861,7 @@ impl GroupRecv { .subscribes .lock() .get_mut(&hdr.subscribe) - .and_then(|entry| entry.end_yielding(hdr.sequence, stream)) + .and_then(|entry| entry.end_yielding(hdr.key, stream)) }); let res = match res { Ok(()) => { @@ -1894,15 +1895,15 @@ impl GroupRecv { group, reading, }; - tracing::debug!(group = hdr.sequence, "group paused"); + tracing::debug!(group = hdr.key, "group paused"); // A subscription gone meanwhile ends it as cancelled. if let Some(entry) = self.subscriber.subscribes.lock().get_mut(&hdr.subscribe) { - entry.pause(hdr.sequence, paused); + entry.pause(hdr.key, paused); } return Poll::Ready(Ok(())); } Err(err) => { - tracing::debug!(%err, group = %group.sequence, "group error"); + tracing::debug!(%err, group = %group.key, "group error"); let _ = group.abort(err.clone()); Err(err) } @@ -2107,6 +2108,7 @@ impl FrameIngest { } None => None, }; + let timestamp = Timestamp::received(timestamp, group.key); Ok(frame::Header { info: frame::Info { size, timestamp }, fields, @@ -3728,14 +3730,14 @@ mod tests { const VERSION: Version = Version::Lite05; - /// A pre-lite-05 relay's track, untimed, with a fetch of `sequence` queued on it. - fn feed_fetch(sequence: u64) -> (track::Producer, kio::Pending, track::Fill) { + /// A pre-lite-05 relay's track, untimed, with a fetch of `key` queued on it. + fn feed_fetch(key: u64) -> (track::Producer, kio::Pending, track::Fill) { let info = track::Info::default().with_timescale(None).with_max_age(None); let track = track::Producer::new(std::sync::Arc::new(crate::broadcast::Info::default()), "video", info); // The relay's copy of the upstream feed, whose groups may land out of order. track.received(); let dynamic = track.requests(); - let fetching = track.consume().fetch_group(sequence, None); + let fetching = track.consume().fetch_group(key, None); let req = dynamic.requested_fill().now_or_never().expect("queued").unwrap(); (track, fetching, req) } @@ -3774,14 +3776,16 @@ mod tests { } /// A group arriving at the relay as its header lands: cached, then accounted for. - fn arrive(track: &track::Producer, tail: &kio::Producer, sequence: u64, now: crate::time::Instant) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + fn arrive(track: &track::Producer, tail: &kio::Producer, key: u64, now: crate::time::Instant) { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); let Ok(mut ledger) = tail.write() else { panic!("ledger closed") }; - ledger.account(sequence..sequence + 1, now); + ledger.account(key..key + 1, now); } /// A fetch held for a pre-lite-05 feed is released once its caller gives up, rather @@ -3862,7 +3866,7 @@ mod tests { .now_or_never() .expect("answered") .expect("served from the feed"); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); } /// A group still missing a grace after a later one arrived is a miss, not a hold. @@ -3891,10 +3895,12 @@ mod tests { for update in [true, false] { let h = Harness::new(Version::Lite04); let (mut track, fetching, req) = feed_fetch(3); - for sequence in [5, 6] { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + for key in [5, 6] { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); } kio::clock::sleep(crate::tail::GRACE * 2).await; @@ -3952,21 +3958,23 @@ mod tests { "group 3 was asked for just now (update: {update})" ); - let mut group = track.create_group(group::Info { sequence: 3 }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(group::Info { start: 3 }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); fetches.poll(&sub, now, &kio::Waiter::noop()); let group = fetching .now_or_never() .expect("answered") .expect("served from the feed"); - assert_eq!(group.sequence, 3); + assert_eq!(group.start, 3); } } /// Many group fetches on one track share model state, but never share readiness. fn feed_batch( - sequences: impl IntoIterator, + keys: impl IntoIterator, ) -> (track::Producer, Vec>, Vec) { let info = track::Info::default().with_timescale(None).with_max_age(None); let track = track::Producer::new(Arc::new(crate::broadcast::Info::default()), "video", info); @@ -3975,8 +3983,8 @@ mod tests { let dynamic = track.requests(); let mut callers = Vec::new(); let mut requests = Vec::new(); - for sequence in sequences { - callers.push(track.consume().fetch_group(sequence, None)); + for key in keys { + callers.push(track.consume().fetch_group(key, None)); requests.push(dynamic.requested_fill().now_or_never().expect("queued").unwrap()); } (track, callers, requests) @@ -4192,9 +4200,7 @@ mod tests { assert_eq!(fetches.held.len(), 3, "one entry per fill"); assert!(callers.iter_mut().all(|caller| caller.now_or_never().is_none())); - let mut joined: Vec<_> = (1..=3) - .map(|sequence| track.consume().fetch_group(sequence, None)) - .collect(); + let mut joined: Vec<_> = (1..=3).map(|key| track.consume().fetch_group(key, None)).collect(); assert!( dynamic.requested_fill().now_or_never().is_none(), "joins existing fills" @@ -4277,14 +4283,14 @@ mod tests { if req.poll_unwanted(waiter).is_ready() { continue; } - let sequence = req.sequence(); + let key = req.key(); // The ledger before the cache: a group it accounts for by arrival is cached first. - let missing = !tail.covers(sequence..sequence.saturating_add(1)); - let folds = tail.folds_at(sequence); - // Every held sequence is ahead of the measured arrivals, so the + let missing = !tail.covers(key..key.saturating_add(1)); + let folds = tail.folds_at(key); + // Every held key is ahead of the measured arrivals, so the // newer-group check is inert. Keep the scanner on the model's global // wake channel rather than giving it the new per-group index. - match feed.poll_group(sequence, waiter) { + match feed.poll_group(key, waiter) { Poll::Ready(Ok(Some(_))) => drop(req), Poll::Ready(Ok(None)) => req.reject(Error::NotFound), Poll::Ready(Err(err)) => req.reject(err), @@ -4320,8 +4326,8 @@ mod tests { let (track, callers, requests) = feed_batch(1_000_000..1_000_000 + held); let tail = kio::Producer::new(Tail::default()); let sub = feed_sub(&tail).await; - for sequence in 0..runs { - arrive(&track, &tail, sequence * 2, start); + for key in 0..runs { + arrive(&track, &tail, key * 2, start); } let mut scan = ScanFetches::new(&clock); let mut fetches = FeedFetches::new(&clock); @@ -4347,8 +4353,8 @@ mod tests { reason = "the scan cost is real elapsed time" )] let began = std::time::Instant::now(); - for sequence in runs * 2..runs * 2 + EVENTS { - arrive(track, tail, sequence, start); + for key in runs * 2..runs * 2 + EVENTS { + arrive(track, tail, key, start); if indexed { fetches.poll(sub, start, &waiter); } else { @@ -4829,10 +4835,10 @@ mod tests { timescale: None, }, ); - for sequence in 0..QUEUED { + for key in 0..QUEUED { let body = lite::Datagram { subscribe: 1, - sequence, + sequence: key, timestamp: None, payload: bytes::Bytes::from_static(b"x"), }; @@ -4924,7 +4930,7 @@ mod tests { .unwrap(); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut crate::coding::Encoder::new(&mut bytes, version.into()), version) @@ -4994,7 +5000,7 @@ mod tests { assert!(accept.poll(&kio::Waiter::noop()).is_pending()); assert!(log.stops().is_empty()); let mut group = consumer.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 3); + assert_eq!(group.start, 3); let frame = group.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), &[42]); assert!(group.read_frame().await.unwrap().is_none()); @@ -5002,14 +5008,14 @@ mod tests { } } - /// Serve one Group Stream for `sequence` on subscription 7, accepted into `track`. - async fn serve_group(version: Version, sequence: u64, track: &track::Producer) -> Result<(), Error> { + /// Serve one Group Stream for `key` on subscription 7, accepted into `track`. + async fn serve_group(version: Version, key: u64, track: &track::Producer) -> Result<(), Error> { use crate::transport::poll::Session as _; let mut script = Vec::new(); lite::Group { subscribe: 7, - sequence, + key, frame_start: 0, } .encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) @@ -5083,7 +5089,7 @@ mod tests { /// Group `u64::MAX` is reserved, so a stream naming it is refused, and the track lives on. #[kio::test] - async fn a_group_at_the_reserved_sequence_is_refused() { + async fn a_group_at_the_reserved_key_is_refused() { let track = track::Producer::new( std::sync::Arc::new(crate::broadcast::Info::default()), "video", @@ -5092,7 +5098,7 @@ mod tests { let res = serve_group(Version::Lite07, u64::MAX, &track).await; assert!(matches!(res, Err(Error::BoundsExceeded(_))), "{res:?}"); assert!(track.closed().now_or_never().is_none()); - assert_eq!(track.latest_sequence(), None); + assert_eq!(track.latest_key(), None); } /// A track in `timescale` that a draft-18 moq-transport publisher published. @@ -5119,7 +5125,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 0, + key: 0, frame_start: 0, } .encode(&mut encoder, version) @@ -5266,7 +5272,7 @@ mod tests { if ok { let mut group = track.consume().peek_group(0).unwrap(); let frame = group.read_frame().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(frame.timestamp, Some(Timestamp::from_millis(500).unwrap()), "{reason}"); + assert_eq!(frame.timestamp, Timestamp::from_millis(500).unwrap(), "{reason}"); } } } @@ -5306,7 +5312,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 5, + key: 5, frame_start: 0, } .encode(&mut encoder, version) @@ -5340,7 +5346,7 @@ mod tests { before .write_frame(Timestamp::from_millis(1000).unwrap(), &b"before"[..]) .unwrap(); - before.finish().unwrap(); + before.close().unwrap(); subscriber.subscribes.lock().insert( 7, TrackEntry::accepted( @@ -5375,10 +5381,10 @@ mod tests { assert!(track.closed().now_or_never().is_none(), "{version}"); let mut consumer = track.subscribe(None); let mut group = consumer.recv_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(group.sequence, 5, "{version}"); + assert_eq!(group.start, 5, "{version}"); let frame = group.read_frame().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), b"frame", "{version}"); - assert_eq!(frame.timestamp, Timestamp::from_millis(500).ok(), "{version}"); + assert_eq!(frame.timestamp, Timestamp::from_millis(500).unwrap(), "{version}"); } } @@ -5401,7 +5407,7 @@ mod tests { res.unwrap_or_else(|err| panic!("{version}: {err:?}")); let mut group = track.consume().peek_group(5).unwrap(); let frame = group.read_frame().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(frame.timestamp.map(Timestamp::value), Some(max as u64), "{version}"); + assert_eq!(frame.timestamp.value(), max as u64, "{version}"); let (res, _) = receive_group(version, &[max, 1]).await; assert!(matches!(res, Err(Error::BoundsExceeded(_))), "{version}: {res:?}"); @@ -5421,9 +5427,9 @@ mod tests { "video", crate::track::Info::timed(), ); - let mut cached = track.create_group(group::Info { sequence: 0 }).unwrap(); - cached.write_frame(Timestamp::ZERO, &b"0"[..]).unwrap(); - cached.finish().unwrap(); + let mut cached = track.create_group(group::Info { start: 0 }).unwrap(); + cached.write_frame(cached.key_time(), &b"0"[..]).unwrap(); + cached.close().unwrap(); track.set_idle(); let mut entry = TrackEntry::accepted(track.clone(), Default::default(), Default::default()); @@ -5437,23 +5443,23 @@ mod tests { assert!(!track.is_live(), "the start waits for a group"); entry.answered(None, Some(Position::group(3))); } - let mut newest = track.create_group(group::Info { sequence: 3 }).unwrap(); + let mut newest = track.create_group(group::Info { start: 3 }).unwrap(); newest .write_frame(Timestamp::from_millis(300).unwrap(), &b"3"[..]) .unwrap(); - newest.finish().unwrap(); + newest.close().unwrap(); assert!(track.is_live(), "header first: {header_first}"); let subscription = crate::track::Subscription::default() .with_delay(crate::track::Delay::default().with_max(std::time::Duration::from_secs(10))) .with_range(0..); let mut reader = track.subscribe(subscription); - let mut sequences = Vec::new(); + let mut keys = Vec::new(); while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&kio::Waiter::noop()) { - sequences.push(group.sequence); + keys.push(group.start); } - sequences.sort(); - assert_eq!(sequences, [0, 3], "header first: {header_first}"); + keys.sort(); + assert_eq!(keys, [0, 3], "header first: {header_first}"); } } @@ -5468,7 +5474,7 @@ mod tests { let mut script = Vec::new(); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) @@ -5493,9 +5499,9 @@ mod tests { "video", crate::track::Info::timed(), ); - let mut cached = track.create_group(group::Info { sequence: 3 }).unwrap(); - cached.write_frame(Timestamp::ZERO, &b"x"[..]).unwrap(); - cached.finish().unwrap(); + let mut cached = track.create_group(group::Info { start: 3 }).unwrap(); + cached.write_frame(cached.key_time(), &b"x"[..]).unwrap(); + cached.close().unwrap(); let tail = kio::Producer::new(Tail::default()); subscriber.subscribes.lock().insert( 7, @@ -5538,7 +5544,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut encoder, version) @@ -5572,7 +5578,7 @@ mod tests { .with_range(0..); let mut reader = track.subscribe(subscription.clone()); // The original copy, still arriving: one frame in, no end yet. - let Ok(track::Received::Group(mut first)) = track.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(track::Received::Group(mut first)) = track.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first @@ -5608,7 +5614,7 @@ mod tests { let frame = original.read_frame().await.unwrap().unwrap(); assert_eq!(&frame.payload[..], b"first"); if !first_fails { - first.finish().unwrap(); + first.close().unwrap(); let res = kio::wait(|waiter| duplicate.poll_serve(waiter)).await; assert!(matches!(res, Err(Error::Duplicate)), "{res:?}"); assert!(original.read_frame().await.unwrap().is_none()); @@ -5626,7 +5632,7 @@ mod tests { // One that had not reached it reads the re-offer whole. let mut later = track.subscribe(subscription); let mut group = later.assert_group(); - assert_eq!(group.sequence, 3); + assert_eq!(group.start, 3); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"first"); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"again"); assert!(group.read_frame().await.unwrap().is_none()); @@ -5652,7 +5658,7 @@ mod tests { let mut script = Vec::new(); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) @@ -5734,7 +5740,7 @@ mod tests { assert!(res.is_ok(), "overtakes: {overtakes}: {res:?}"); let mut group = reader.assert_group(); - assert_eq!(group.sequence, 3); + assert_eq!(group.start, 3); for (_, payload) in frames { let read = group.read_frame().await.unwrap().unwrap(); assert_eq!(&read.payload[..], payload, "overtakes: {overtakes}"); @@ -5784,7 +5790,7 @@ mod tests { let mut script = Vec::new(); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut crate::coding::Encoder::new(&mut script, version.into()), version) @@ -5873,7 +5879,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut encoder, version) @@ -5954,7 +5960,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut encoder, version) @@ -6037,7 +6043,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 3, + key: 3, frame_start: 0, } .encode(&mut encoder, version) @@ -6152,9 +6158,9 @@ mod tests { let broadcast = crate::broadcast::Info::new().produce(); let request = broadcast.reserve_track("video").unwrap(); let serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); - let mut group = serving.serving.create_group(group::Info { sequence: 2 }).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"2".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = serving.serving.create_group(group::Info { start: 2 }).unwrap(); + group.write_frame(group.key_time(), b"2".as_slice()).unwrap(); + group.close().unwrap(); let mut reader = broadcast .consume() .track("video") @@ -6174,7 +6180,7 @@ mod tests { let end = loop { match reader.recv_group().await { Ok(Some(_)) => continue, - end => break end.map(|group| group.map(|group| group.sequence)), + end => break end.map(|group| group.map(|group| group.start)), } }; match version { @@ -6251,7 +6257,7 @@ mod tests { reader .recv_group() .now_or_never() - .map(|res| res.map(|group| group.map(|group| group.sequence))) + .map(|res| res.map(|group| group.map(|group| group.start))) } /// An answer read only after another instance took the path may be that instance's, @@ -6426,38 +6432,40 @@ mod tests { .with_max_age(None), ); let requests = copy.requests(); - for sequence in 0..6 { - let mut group = copy.create_group(group::Info { sequence }).unwrap(); + for key in 0..6 { + let mut group = copy.create_group(group::Info { start: key }).unwrap(); group - .write_frame(crate::Timestamp::from_millis(sequence * 1000).unwrap(), b"x".as_slice()) + .write_frame(crate::Timestamp::from_millis(key * 1000).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - for sequence in 1..5 { - copy.evict(sequence); + // From group 0, which a time start names whatever is held, so the reader asks the + // missing groups straight away rather than looking up where its start falls. + for key in 0..4 { + copy.evict(key); } let mut reader = copy .subscribe( crate::track::Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) - .with_range(1..5), + .with_range(0..4), ) .ordered(); - assert!(reader.poll_next_group(&waiter).is_pending(), "groups 1..5 are owed"); + assert!(reader.poll_next_group(&waiter).is_pending(), "groups 0..4 are owed"); let Poll::Ready(Ok(fill)) = requests.poll_requested_fill(&waiter) else { - panic!("groups 1..5 are a miss"); + panic!("groups 0..4 are a miss"); }; let mut fills = fills::RangeFills::default(); fills.push(fill); assert!(fills.poll(&serve, ©, None, true, &waiter).unwrap()); - assert_eq!(fills.ask.groups.iter().cloned().collect::>(), [1..5]); + assert_eq!(fills.ask.groups.iter().cloned().collect::>(), [0..4]); - fills.refuse(1..3, crate::StreamError::Old); + fills.refuse(0..2, crate::StreamError::Old); assert!(fills.poll(&serve, ©, None, true, &waiter).unwrap()); - assert!(fills.recovering(), "1..3 recovered on its own subscription"); + assert!(fills.recovering(), "0..2 recovered on its own subscription"); assert_eq!( fills.ask.groups.iter().cloned().collect::>(), - [3..5], + [2..4], "the recovered run is no longer asked" ); } @@ -6571,20 +6579,20 @@ mod tests { &version, &crate::StreamError::Old, )); - let old = |sequence: u64| { + let old = |key: u64| { lite::SubscribeResponse::Drop(lite::SubscribeDrop { - groups: sequence..sequence + 1, + groups: key..key + 1, error: code, update: 0, }) }; let frame = |mut group: crate::group::Producer| { - group.write_frame(crate::Timestamp::from_millis(1).unwrap(), &b"x"[..])?; - group.finish().map(drop) + group.write_frame(group.key_time(), &b"x"[..])?; + group.close().map(drop) }; - let arrive = |serving: &mut ServeLoop, sequence| { - let group = serving.serving.create_group(crate::group::Info { sequence }).unwrap(); - (group, Reading::open(&tail, Some(sequence), now())) + let arrive = |serving: &mut ServeLoop, key| { + let group = serving.serving.create_group(crate::group::Info { start: key }).unwrap(); + (group, Reading::open(&tail, Some(key), now())) }; // OLD, then the group, which finishes. @@ -6627,9 +6635,9 @@ mod tests { &version, &crate::StreamError::Old, )); - let old = |sequence: u64, update| { + let old = |key: u64, update| { lite::SubscribeResponse::Drop(lite::SubscribeDrop { - groups: sequence..sequence + 1, + groups: key..key + 1, error: code, update, }) @@ -6648,19 +6656,20 @@ mod tests { ), ); // A group stream that arrives whole, through the path that records it, then is evicted. - let arrive = async |serving: &mut ServeLoop, sequence| { + let arrive = async |serving: &mut ServeLoop, key| { use crate::transport::poll::Session as _; let mut script = Vec::new(); let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 0, - sequence, + key, frame_start: 0, } .encode(&mut encoder, version) .unwrap(); - encoder.varint(crate::coding::varint::zigzag(1)).unwrap(); + // A frame at the group's key, since starts strictly increase. + encoder.varint(crate::coding::varint::zigzag(key as i64)).unwrap(); encoder.bytes(b"x").unwrap(); let session = crate::lite::test_transport::ScriptedSession::eof(script); let (_, recv) = session.open_bi(Default::default()).await.unwrap(); @@ -6669,7 +6678,7 @@ mod tests { Reader::new(crate::transport::boxed::RecvStream::new(recv), version), ); kio::wait(|waiter| group.poll_serve(waiter)).await.unwrap(); - serving.serving.evict(sequence); + serving.serving.evict(key); }; let update = |serving: &mut ServeLoop| { let Sub::Active(active) = &mut serving.sub else { @@ -6698,8 +6707,8 @@ mod tests { // Under the pool's debt the tail folds what finished, gaps and all. A group in the span // whose stream arrived before its stale drop still wins over it; a gap there the drop // accounted first is owed again. - for sequence in [3, 5, 7] { - arrive(&mut serving, sequence).await; + for key in [3, 5, 7] { + arrive(&mut serving, key).await; } if let Ok(mut state) = tail.write() { assert!(state.fold()); @@ -6767,7 +6776,7 @@ mod tests { assert!(covers(4..5), "the drop is final"); assert!( matches!( - serving.serving.create_group(crate::group::Info { sequence: 1 }), + serving.serving.create_group(crate::group::Info { start: 1 }), Err(Error::Skipped) ), "still skipped after the update" @@ -6789,14 +6798,11 @@ mod tests { groups: first..last + 1, }) }; - let mut group = serving - .serving - .create_group(crate::group::Info { sequence: 5 }) - .unwrap(); + let mut group = serving.serving.create_group(crate::group::Info { start: 5 }).unwrap(); group .write_frame(crate::Timestamp::from_millis(5).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); serving .apply_response(&serve, drop(1, 2, &crate::StreamError::App(5))) @@ -6804,7 +6810,7 @@ mod tests { serving.apply_response(&serve, skip(1, 2)).unwrap(); assert!( matches!( - serving.serving.create_group(crate::group::Info { sequence: 2 }), + serving.serving.create_group(crate::group::Info { start: 2 }), Err(Error::Skipped) ), "the skip replaced the final drop" @@ -6814,7 +6820,7 @@ mod tests { .unwrap(); assert!( matches!( - serving.serving.create_group(crate::group::Info { sequence: 2 }), + serving.serving.create_group(crate::group::Info { start: 2 }), Err(Error::Skipped) ), "a later drop leaves the skip" @@ -6832,7 +6838,7 @@ mod tests { assert_eq!(reader.poll_drop(&waiter), Poll::Ready((6..7, track::Hole::Skip))); assert!(reader.poll_drop(&waiter).is_pending()); let group = reader.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 5, "the group that arrived is kept"); + assert_eq!(group.start, 5, "the group that arrived is kept"); } /// A SUBSCRIBE_DROP ends a group still arriving now, with its Error Code on lite-07. @@ -6888,8 +6894,8 @@ mod tests { let request = broadcast.reserve_track("video").unwrap(); let serving = ServeLoop::new(&serve, request, crate::track::Info::timed()); let producer = serving.serving.clone(); - let mut arriving = serving.serving.create_group(group::Info { sequence: 2 }).unwrap(); - arriving.write_frame(crate::Timestamp::ZERO, b"2".as_slice()).unwrap(); + let mut arriving = serving.serving.create_group(group::Info { start: 2 }).unwrap(); + arriving.write_frame(arriving.key_time(), b"2".as_slice()).unwrap(); let mut reader = broadcast .consume() .track("video") @@ -6920,15 +6926,15 @@ mod tests { ); // The track goes on: the next group still reaches the reader. - let mut following = producer.create_group(group::Info { sequence: 3 }).unwrap(); - following.write_frame(crate::Timestamp::ZERO, b"3".as_slice()).unwrap(); + let mut following = producer.create_group(group::Info { start: 3 }).unwrap(); + following.write_frame(following.key_time(), b"3".as_slice()).unwrap(); let mut next = reader .recv_group() .now_or_never() .expect("the next group") .unwrap() .unwrap(); - assert_eq!(next.sequence, 3, "{version:?}"); + assert_eq!(next.start, 3, "{version:?}"); assert_eq!(next.read_frame().await.unwrap().unwrap().payload.as_ref(), b"3"); } } @@ -7254,11 +7260,11 @@ mod tests { } } - let payload = |sequence, size| { + let payload = |key, size| { lite::Datagram { subscribe: 1, - sequence, - timestamp: Some(sequence), + sequence: key, + timestamp: Some(key), payload: bytes::Bytes::from(vec![0; size]), } .encode_bytes(version) @@ -7269,7 +7275,7 @@ mod tests { panic!("{version}: the datagram was not delivered"); }; assert_eq!(datagram.sequence, 1); - assert_eq!(datagram.timestamp, Some(Timestamp::new(1, Timescale::MILLI).unwrap())); + assert_eq!(datagram.timestamp, Timestamp::new(1, Timescale::MILLI).unwrap()); let max_payload = 1200 - payload(2, 0).len(); let oversized = payload(2, max_payload + 1); @@ -7706,20 +7712,21 @@ mod tests { #[kio::test] async fn a_group_that_beats_its_track_info_waits_for_it() { let version = Version::Lite07; - let group = |sequence: u64, payload: u8| { + let group = |key: u64, payload: u8| { let mut bytes = Vec::new(); let mut w = crate::coding::Encoder::new(&mut bytes, version.into()); lite::DataType::Group.encode(&mut w, version).unwrap(); lite::Group { subscribe: 0, - sequence, + key, frame_start: 0, } .encode(&mut w, version) .unwrap(); - // A one-byte payload's length plus one, then a zero timestamp delta. + // A one-byte payload's length plus one, then the frame at the group's key, since + // starts strictly increase. w.varint(2).unwrap(); - w.varint(0).unwrap(); + w.varint(crate::coding::varint::zigzag(key as i64)).unwrap(); w.u8(payload); bytes }; @@ -7758,14 +7765,14 @@ mod tests { subscriber.runtime.now(), ); // Both opened at the accept, the first to arrive first. - assert_eq!(track.latest_sequence(), Some(3)); + assert_eq!(track.latest_key(), Some(3)); assert!(accept.poll(&kio::Waiter::noop()).is_pending()); assert_eq!(subscriber.park_waits.load(atomic::Ordering::Relaxed), 0); let mut read = Vec::new(); for _ in 0..2 { let mut group = consumer.recv_group().await.unwrap().unwrap(); - read.push((group.sequence, group.read_frame().await.unwrap().unwrap().payload[0])); + read.push((group.start, group.read_frame().await.unwrap().unwrap().payload[0])); } read.sort_unstable(); assert_eq!(read, [(2, 20), (3, 30)]); @@ -7780,9 +7787,9 @@ mod tests { #[kio::test] async fn a_failed_parked_stream_withdraws_its_group() { let version = Version::Lite07; - let header = |sequence| lite::Group { + let header = |key| lite::Group { subscribe: 0, - sequence, + key, frame_start: 0, }; let mut entry = TrackEntry::new(None, Default::default()); @@ -7830,11 +7837,11 @@ mod tests { let _track = track.finish().unwrap(); // Only the kept group reaches the subscriber: the withdrawn one ended, so nothing // waits on it. - let mut sequences = Vec::new(); + let mut keys = Vec::new(); while let Some(group) = consumer.recv_group().await.unwrap() { - sequences.push(group.sequence); + keys.push(group.start); } - assert_eq!(sequences, [3]); + assert_eq!(keys, [3]); assert!(entry.parked.is_none(), "released once the last stream took its group"); } @@ -8620,10 +8627,10 @@ mod tests { assert!(serving.poll(&serve, &kio::Waiter::noop()).is_pending()); // Held for the whole round: a dropped tail settles at once. let tail = kio::Producer::new(Tail::default()); - let mut group = Some(serving.serving.append_group().unwrap()); + let mut group = Some(serving.serving.append_next().unwrap()); if let Some(finished) = ended { if finished { - group.take().unwrap().finish().unwrap(); + group.take().unwrap().close().unwrap(); } if version.has_track_stream() { serving.serving.finish_at(Position::group(1), None).unwrap(); @@ -8861,7 +8868,7 @@ mod tests { #[test] fn wire_ranges_name_whole_groups() { let group = |group| Some(lite::Anchor::Absolute(Position::group(group))); - let ranges = wire_ranges(&Subscription::default().with_range(5..6), None).unwrap(); + let ranges = wire_ranges(&Subscription::default().with_range(5..6)); assert_eq!( ranges, Some(vec![lite::Range { @@ -8869,22 +8876,32 @@ mod tests { end: group(6), }]) ); - assert_eq!(wire_ranges(&Subscription::default().with_end(0), None).unwrap(), None); - - // A time anchor needs the track's timescale. - let timed = Subscription { - ranges: vec![track::Range { - start: Anchor::Time(Timestamp::from_millis(5).unwrap()), - end: None, - }], - ..Default::default() - }; - assert!(matches!(wire_ranges(&timed, None), Err(Error::TimestampMismatch))); + assert_eq!(wire_ranges(&Subscription::default().with_end(0)), None); + + // A time anchor names a group start, which goes out as its group ID: an end as the + // exclusive group past it, on any track. + let timed = Subscription::default().with_ranges([track::Range { + start: Anchor::Time(5), + end: Some(Anchor::Time(9)), + }]); + assert_eq!( + wire_ranges(&timed), + Some(vec![lite::Range { + start: group(5), + end: group(10), + }]) + ); + // An end before its start asks for nothing. + let empty = Subscription::default().with_ranges([track::Range { + start: Anchor::Time(5), + end: Some(Anchor::Time(4)), + }]); + assert_eq!(wire_ranges(&empty), None); // Only the oldest start goes out without an anchor; any count back stays one. let start = |start| { let subscription = Subscription::default().with_ranges([track::Range { start, end: None }]); - wire_ranges(&subscription, None).unwrap().unwrap()[0].start + wire_ranges(&subscription).unwrap()[0].start }; assert_eq!(start(Anchor::Oldest), None); assert_eq!( @@ -9022,12 +9039,14 @@ mod tests { assert!(matches!(serving.sub, Sub::Active(_)), "the reader opens the upstream"); let established = h.wire().len(); - let arrive = |serving: &ServeLoop, sequence| { - let mut group = serving.serving.create_group(group::Info { sequence }).unwrap(); + let arrive = |serving: &ServeLoop, key| { + let mut group = serving.serving.create_group(group::Info { start: key }).unwrap(); for frame in [&b"key"[..], b"abc", b"d"] { - group.write_frame(None, frame).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), frame) + .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); }; let wanted = |h: &Harness| { let wire = h.wire(); @@ -9127,21 +9146,21 @@ mod tests { .handle_subscription(&mut h.producer, &mut sub, demand(5), true) .await .unwrap(); - assert_eq!(h.producer.start_sequence(), Some(5)); + assert_eq!(h.producer.start_key(), Some(5)); // Forward: a reader waiting in [5, 8) must fail over, not stall. h.serve .handle_subscription(&mut h.producer, &mut sub, demand(8), true) .await .unwrap(); - assert_eq!(h.producer.start_sequence(), Some(8)); + assert_eq!(h.producer.start_key(), Some(8)); // Backward: the reopened range must stop being a permanent miss. h.serve .handle_subscription(&mut h.producer, &mut sub, demand(3), true) .await .unwrap(); - assert_eq!(h.producer.start_sequence(), Some(3)); + assert_eq!(h.producer.start_key(), Some(3)); // Live edge: the floor is unknown until the next declaration, so a // group below the stale one must not be a permanent miss. @@ -9149,7 +9168,7 @@ mod tests { .handle_subscription(&mut h.producer, &mut sub, Some(Subscription::default()), true) .await .unwrap(); - assert_eq!(h.producer.start_sequence(), None); + assert_eq!(h.producer.start_key(), None); } /// A second announce for a live path is a protocol error whatever its hops say. It @@ -9603,9 +9622,9 @@ mod tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"local".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"local".as_ref()).unwrap(); + group.close().unwrap(); // The peer's own subscription, excluding the hop the server minted for // it, is served from the local front before anything is announced back. @@ -9664,9 +9683,9 @@ mod tests { "the reflected announce must not replace the local front" ); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"still".as_ref()).unwrap(); + group.close().unwrap(); let mut group = sub.recv_group().await.expect("recv group").expect("track ended early"); assert_eq!( &group.read_frame().await.expect("read frame").expect("frame").payload[..], @@ -10816,7 +10835,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 0, + key: 0, frame_start: u64::MAX - 1, } .encode(&mut encoder, version) @@ -10878,7 +10897,7 @@ mod tests { let mut encoder = crate::coding::Encoder::new(&mut script, version.into()); lite::Group { subscribe: 7, - sequence: 0, + key: 0, frame_start: start, } .encode(&mut encoder, version) @@ -10916,47 +10935,41 @@ mod tests { } } -/// The ranges a SUBSCRIBE carries for `subscription`, without the empty ones, with time -/// anchors in `timescale`, the track's as the publisher declared it. +/// The ranges a SUBSCRIBE carries for `subscription`, without the empty ones. /// -/// A subscription always starts and ends at a group boundary, so every absolute end names -/// a group's first frame. `None` when nothing is left to ask for, which no wire can say. -/// The caller narrows a subscription for a wire without ranges to the one span it carries -/// first (see [`wire_span`]). -fn wire_ranges(subscription: &Subscription, timescale: Option) -> Result>, Error> { - let time = |pts: crate::Timestamp| -> Result { - let scale = timescale.ok_or(Error::TimestampMismatch)?; - let pts = pts - .convert(scale) - .map_err(|_| Error::BoundsExceeded(crate::coding::BoundsExceeded))?; - Ok(lite::Anchor::Time(pts.value())) - }; +/// A group ID is the group's start, so a time anchor goes out as the group ID it names, +/// on every version: a time start as the start, and a time end as the exclusive group past +/// it. `None` when nothing is left to ask for, which no wire can say. The caller narrows a +/// subscription for a wire without ranges to the one span it carries first (see +/// [`wire_span`]). +/// +/// A wire without ranges reads a start as the first group at or after it, so a time start +/// between two groups misses the one containing it there. +fn wire_ranges(subscription: &Subscription) -> Option> { let mut ranges = Vec::with_capacity(subscription.ranges.len()); for range in &subscription.ranges { - // Nothing sorts below group 0, whatever the start. - if let Some(Anchor::Absolute(end)) = range.end - && match range.start { - Anchor::Absolute(start) => end <= start, - _ => end == 0, - } { + if let (Anchor::Time(start), Some(Anchor::Time(end))) = (range.start, range.end) + && end < start + { continue; } let start = match range.start { Anchor::Oldest => None, Anchor::Relative(back) => Some(lite::Anchor::Relative(back)), - Anchor::Absolute(group) => Some(lite::Anchor::Absolute(Position::group(group))), - Anchor::Time(pts) => Some(time(pts)?), + Anchor::Time(time) => Some(lite::Anchor::Absolute(Position::group(time))), }; - // The copy resolves a relative or time end itself, against the groups it receives past - // it, so it asks through the live edge. SUBSCRIBE_START names a resolved start, but - // nothing names a resolved end, so one resolved upstream would leave the copy short. + // The copy resolves a relative end itself, against the groups it receives past it, so + // it asks through the live edge. SUBSCRIBE_START names a resolved start, but nothing + // names a resolved end, so one resolved upstream would leave the copy short. let end = match range.end { - Some(Anchor::Absolute(group)) => Some(lite::Anchor::Absolute(Position::group(group))), - None | Some(Anchor::Oldest | Anchor::Relative(_) | Anchor::Time(_)) => None, + Some(Anchor::Time(time)) => time + .checked_add(1) + .map(|end| lite::Anchor::Absolute(Position::group(end))), + None | Some(Anchor::Oldest | Anchor::Relative(_)) => None, }; ranges.push(lite::Range { start, end }); } - Ok((!ranges.is_empty()).then_some(ranges)) + (!ranges.is_empty()).then_some(ranges) } /// `subscription` narrowed to the one span a wire without ranges carries, which the @@ -10972,7 +10985,7 @@ fn wire_span(subscription: Subscription) -> Subscription { None if subscription .ranges .iter() - .any(|range| matches!(range.start, Anchor::Absolute(_))) => + .any(|range| matches!(range.start, Anchor::Time(_))) => { Some(0) } @@ -11075,8 +11088,27 @@ impl SubStream { Some((start, self.started.is_none())) } + /// Tell the copy where the publisher started each live range asked from a group's start, + /// which a reader's time start there resolves to when the copy holds no group at it. + /// Lite-07 only, where each range's start is reported. + fn report_time_starts(&self, serving: &track::Producer) { + if !self.live.reports_starts() { + return; + } + let starts = self + .live + .iter() + .filter_map(|(id, range)| match (range.start, self.starts.get(&id)) { + (Some(lite::Anchor::Absolute(time)), Some(Some(start))) if time.frame == 0 => { + Some((time.group, *start)) + } + _ => None, + }); + serving.time_starts(starts); + } + /// Declare on the copy where the live feed starts, by [`Self::feed_start`], so a reader - /// waiting on each group in sequence starts there rather than at whatever arrives first. + /// waiting on each group in key starts there rather than at whatever arrives first. fn declare(&mut self, serving: &mut track::Producer) { let Some(next) = self.feed_start() else { return; @@ -11606,7 +11638,7 @@ impl TrackServe { (None, Sub::Active(active)) if !fills.is_empty() => active.demand.clone(), (pref, _) => pref, }; - let request = self.wire_request(pref, opening, producer.idle_newest(), producer.info().timescale)?; + let request = self.wire_request(pref, opening, producer.idle_newest()); // What goes on the wire: the demand's ranges, then the fills', under the demand's // parameters. The demand alone sets the floor. let request = match request { @@ -11729,24 +11761,20 @@ impl TrackServe { /// `None` when it asks for nothing. /// /// `opening` when this opens a subscription for a copy whose newest cached group, while - /// idle, is `newest`. `timescale` is the track's, which only a time anchor needs. + /// idle, is `newest`. fn wire_request( &self, pref: Option, opening: bool, newest: Option, - timescale: Option, - ) -> Result)>, Error> { + ) -> Option<(Subscription, Vec)> { // An empty half-open range asks for nothing, and the wire cannot say that: its // bounds are inclusive, so the nearest encoding either hands back the position // the caller excluded or inverts the range outright once the two bounds meet. No // demand at all is the faithful translation. An absent start is the live edge, wherever that lands, so the only end that is // certainly empty is the very first position, which is what `Position::default()` // stands in for. - let Some(subscription) = pref.filter(|sub| sub.end().is_none_or(|end| end > sub.start().unwrap_or_default())) - else { - return Ok(None); - }; + let subscription = pref.filter(|sub| sub.end().is_none_or(|end| end > sub.start().unwrap_or_default()))?; let ranged = self.subscriber.version.has_ranges(); // Lite-07 carries every range asked for; an older wire one span covering them. let mut subscription = match ranged { @@ -11767,18 +11795,15 @@ impl TrackServe { if opening && idle { subscription = subscription.with_start(newest); } - Ok(wire_ranges(&subscription, timescale)?.map(|ranges| (subscription, ranges))) + wire_ranges(&subscription).map(|ranges| (subscription, ranges)) } /// Open the SUBSCRIBE for `pref` alongside the TRACK still waiting on its TRACK_INFO, - /// registered with its copy pending. `None` when it asks for nothing, or names a time - /// that needs the track's timescale, so it waits for the info. + /// registered with its copy pending. `None` when it asks for nothing. fn pipeline(&self, pref: Subscription) -> Result, Error> { // A copy not accepted yet caches nothing. - let (subscription, ranges) = match self.wire_request(Some(pref.clone()), true, None, None) { - Ok(Some(request)) => request, - Ok(None) | Err(Error::TimestampMismatch) => return Ok(None), - Err(err) => return Err(err), + let Some((subscription, ranges)) = self.wire_request(Some(pref.clone()), true, None) else { + return Ok(None); }; let establish = self.register(subscription, ranges, None); Ok(Some(Pipelined { @@ -11908,7 +11933,7 @@ impl TrackServe { true => subscription, false => wire_span(subscription), }; - let ranges = wire_ranges(&subscription, producer.info().timescale)?.expect("a non-empty subscription"); + let ranges = wire_ranges(&subscription).expect("a non-empty subscription"); let wire = Wire { subscription: subscription.clone(), demand: Some(subscription), @@ -12885,7 +12910,7 @@ enum ServeMode { Establish(Establish), /// The publisher FINned once the track ended, as before lite-07, but QUIC does not order /// streams: keep the subscription routable until every owed group is accounted for. The - /// grace bounds an unaccounted sequence. Lite-07 keeps serving + /// grace bounds an unaccounted key. Lite-07 keeps serving /// instead, settling its tail in [`SubStream::finishing`]. Tail { settle: Settle, @@ -12933,7 +12958,7 @@ struct HeldFetch { } struct FetchWake { - sequence: u64, + key: u64, ready: kio::Shared>, } @@ -12942,7 +12967,7 @@ impl std::task::Wake for FetchWake { self.wake_by_ref(); } fn wake_by_ref(self: &Arc) { - self.ready.lock().insert(self.sequence); + self.ready.lock().insert(self.key); } } @@ -12976,22 +13001,22 @@ impl FeedFetches { fn hold(&mut self, fill: track::Fill, fresh: bool) { let mut next = Some((fill, fresh)); while let Some((fill, fresh)) = next.take() { - let Some(sequence) = fill.wanted() else { + let Some(key) = fill.wanted() else { continue; }; // A replacement attempt may arrive before the poll that would see the held one's // readers leave this group: that one moves on to the next group it owes, if any. - if let Some(previous) = self.held.remove(&sequence) + if let Some(previous) = self.held.remove(&key) && previous.fill.wanted().is_some() { next = Some((previous.fill, false)); } let waiter = kio::Waiter::new(std::task::Waker::from(Arc::new(FetchWake { - sequence, + key, ready: self.ready.clone(), }))); - self.held.insert(sequence, HeldFetch { fill, waiter, fresh }); - self.ready.lock().insert(sequence); + self.held.insert(key, HeldFetch { fill, waiter, fresh }); + self.ready.lock().insert(key); } } @@ -13020,7 +13045,7 @@ impl FeedFetches { } for (gap, at) in arrivals.folds() { if at <= now { - let mut groups = self.held.range(gap).map(|(sequence, _)| *sequence); + let mut groups = self.held.range(gap).map(|(key, _)| *key); if let Some(first) = groups.next() { let mut ready = self.ready.lock(); ready.insert(first); @@ -13041,8 +13066,8 @@ impl FeedFetches { let _ = self.ready.poll(waiter, |_| Poll::<()>::Pending); let mut unused = Vec::new(); let mut settled = Vec::new(); - for sequence in ready { - let Some(mut held) = self.held.remove(&sequence) else { + for key in ready { + let Some(mut held) = self.held.remove(&key) else { continue; }; #[cfg(test)] @@ -13053,20 +13078,20 @@ impl FeedFetches { unused.push(held.fill); continue; } - let verdict = arrivals.poll_group(sequence, now, &held.waiter); + let verdict = arrivals.poll_group(key, now, &held.waiter); if verdict.is_pending() && !held.fresh { - self.held.insert(sequence, held); + self.held.insert(key, held); continue; } let gone = arrivals.given_up(held.fill.start().group..held.fill.end(), now); if verdict.is_pending() && gone.is_empty() { held.fresh = false; - self.held.insert(sequence, held); + self.held.insert(key, held); continue; } // Never coming is a miss; a group given up keeps the error it was given up with. let decided = match verdict { - Poll::Ready(res) => Some((sequence, res.err().unwrap_or(Error::NotFound))), + Poll::Ready(res) => Some((key, res.err().unwrap_or(Error::NotFound))), Poll::Pending => None, }; settled.push((held.fill, decided, gone)); @@ -13083,10 +13108,10 @@ impl FeedFetches { // the frames the fill starts from is no answer, and the cache serves what it has. // Answered before the bulk give-up, which knows no reason, so a drop's own error // reaches the fetch. - if let Some((sequence, err)) = decided - && fill.wanted() == Some(sequence) + if let Some((key, err)) = decided + && fill.wanted() == Some(key) { - fill.group(sequence).reject(err); + fill.group(key).reject(err); } for groups in gone.iter() { fill.drop_groups(groups.clone()); @@ -13128,6 +13153,11 @@ impl ServeLoop { if serve.subscriber.version.resolves_anchors() { serving.anchored(); } + // A wire without ranges starts a time anchor at the first group at or after it, so + // its readers do too, rather than at an earlier group the copy happens to hold. + if !serve.subscriber.version.has_ranges() { + serving.starts_at_keys(); + } if serve.subscriber.version.fills_from_start() { serving.fills_from_start(); } @@ -13234,7 +13264,7 @@ impl ServeLoop { }; match &msg { // SUBSCRIBE_FINISH declares the track's exclusive final - // sequence, which may arrive while trailing groups are + // key, which may arrive while trailing groups are // still in flight. Record it on this segment's producer so // consumers learn the boundary early; the tail then waits for // the groups below it. @@ -13349,6 +13379,7 @@ impl ServeLoop { // Lite-07 says where every live range starts, so the declaration follows them // whatever an update changed since. if active.live.reports_starts() { + active.report_time_starts(&self.serving); active.declare(&mut self.serving); } else if active.start == active.requested && active.demand.is_some() { let _ = match (serve.subscriber.version.resolves_start(), served) { @@ -13393,6 +13424,7 @@ impl ServeLoop { if let Ok(mut tail) = active.tail.write() { tail.expire(now); } + active.report_time_starts(&self.serving); active.declare(&mut self.serving); } } @@ -13433,7 +13465,7 @@ impl ServeLoop { } match stale { // Judged under the budget this subscriber sent with that update. A - // sequence already skipped stays skipped, and one whose group arrived + // key already skipped stays skipped, and one whose group arrived // keeps it, so no update reopens either. A group a fill asked for is // recovered on its own, so the copy does not record it given up meanwhile. true => { @@ -13853,7 +13885,7 @@ impl ServeLoop { } tracing::debug!(broadcast = %serve.subscriber.log_path(&serve.path), track = %serve.name, "subscribe complete"); // Upstream FIN'd the subscription: the publisher only FINs - // once the track's final sequence is known and every group + // once the track's final key is known and every group // stream finished, so the logical track is over for good // (bounded demand alone never FINs; the publisher parks, since // a cap can be raised). Those streams can still be in flight, @@ -13941,9 +13973,9 @@ impl kio::Task for FillServeRun { fn poll(&mut self, waiter: &kio::Waiter) -> Poll<()> { loop { - if let Some((sequence, run)) = &mut self.run { + if let Some((key, run)) = &mut self.run { ready!(run.poll(waiter)); - self.last = Some(*sequence); + self.last = Some(*key); self.run = None; } // Withdrawn: nobody waits on what is left, so it records nothing. @@ -13953,17 +13985,17 @@ impl kio::Task for FillServeRun { // The next fetch waits for a reader to pick up the group the last one brought, so a // fill wider than the cache never evicts its own groups before they are read. ready!(self.fill.poll_paced(waiter)); - let Some(sequence) = self.fill.wanted() else { + let Some(key) = self.fill.wanted() else { return Poll::Ready(()); }; // Every run answers its group, so the same one coming back is a bug; ending the // fill names it dropped rather than fetching it forever. - if self.last == Some(sequence) { - debug_assert!(false, "fill group {sequence} left unanswered"); + if self.last == Some(key) { + debug_assert!(false, "fill group {key} left unanswered"); return Poll::Ready(()); } - let run = FetchServeRun::new(self.serve.clone(), self.fill.group(sequence), self.info.clone()); - self.run = Some((sequence, run)); + let run = FetchServeRun::new(self.serve.clone(), self.fill.group(key), self.info.clone()); + self.run = Some((key, run)); } } } @@ -13971,7 +14003,7 @@ impl kio::Task for FillServeRun { /// Serves one downstream fetch end-to-end on its own bidi stream: send FETCH, /// then fill the group from the bare FRAME messages that follow. The track's /// info comes from its TRACK_INFO, which the response waits for unread if it -/// is still in flight, and the group sequence is implicit from the request. +/// is still in flight, and the group key is implicit from the request. struct FetchServeRun { /// Its place in line for the peer's stream credit while its stream opens. open: crate::transport::open::Open, @@ -14029,7 +14061,7 @@ impl FetchRunState { impl FetchServeRun { fn new(serve: TrackServe, request: track::GroupFill, info: FetchInfo) -> Self { let session = serve.subscriber.session.clone(); - let group = request.sequence(); + let group = request.key(); Self { open: Default::default(), validity: serve.validity.start(), diff --git a/rs/moq-net/src/lite/subscriber/fills.rs b/rs/moq-net/src/lite/subscriber/fills.rs index 885b74c4d84..3b1d481e8a5 100644 --- a/rs/moq-net/src/lite/subscriber/fills.rs +++ b/rs/moq-net/src/lite/subscriber/fills.rs @@ -401,11 +401,11 @@ impl RangeFills { } // Only the groups the cache holds, so a wide run costs what it holds. for gap in self.draining.gaps(run) { - for sequence in edge.cached_groups(gap) { - if let Some(mut group) = edge.peek_group(sequence) + for key in edge.cached_groups(gap) { + if let Some(mut group) = edge.peek_group(key) && group.poll_finished(waiter).is_pending() { - self.reading.insert(sequence, group); + self.reading.insert(key, group); } } } @@ -414,7 +414,7 @@ impl RangeFills { draining.insert(run.clone()); } } - self.reading.retain(|sequence, _| draining.contains(sequence)); + self.reading.retain(|key, _| draining.contains(key)); self.draining = draining; // A run being recovered leaves the subscription, even out of a run kept whole: any later // update reopens its stale drop there, and the publisher would serve it on both. @@ -492,7 +492,7 @@ impl Refusal { /// publisher gives it up, then FINs and waits for the publisher's FIN in reply. struct Recovery { serve: TrackServe, - /// The copy its groups land in. + /// The copy the recovered groups land in, which records what the publisher skipped. serving: track::Producer, id: u64, groups: Range, @@ -521,7 +521,7 @@ enum RecoveryState { impl Recovery { fn new(serve: &TrackServe, serving: &track::Producer, groups: Range, priority: i8) -> Self { let subscription = Subscription::default() - .with_ranges([track::Range::groups(groups.clone())]) + .with_ranges(track::Range::span(Some(groups.start), Some(groups.end))) .with_priority(priority) .with_delay(Delay::default().with_max(None)) .with_order(Order::Ascending); @@ -655,13 +655,14 @@ impl Recovery { Ok(Some(lite::SubscribeResponse::Drop(dropped))) => { given_up(dropped.groups.clone(), Refusal::dropped(dropped.error(version))); } - // Never produced: final, recorded on the copy as the subscription feeding it - // records it, which answers every fill owing the groups. + // Never produced, such as the keys between two sparse group starts: the copy + // records the skip, as the subscription itself does, and a fill owing one + // is answered rather than asked again. Ok(Some(lite::SubscribeResponse::Skip(skipped))) => { - if let Ok(mut tail) = active.tail.write() { - tail.account(skipped.groups.clone(), now); + if !skipped.groups.is_empty() { + self.serving.skip_received(skipped.groups.clone()); + given_up(skipped.groups, Refusal::dropped(StreamError::NotFound)); } - self.serving.skip_received(skipped.groups); } // Nothing at or past the track's end will be produced. Ok(Some(lite::SubscribeResponse::Finish(finish))) if active.end.is_none() => { @@ -764,12 +765,12 @@ mod tests { let (_producer, _requests, _reader, fill) = missed(); let mut fills = RangeFills::default(); fills.held.push(fill); - fills.refused.insert(1..3, Refusal::passing(StreamError::Cancel)); - fills.refused.insert(3..6, Refusal::dropped(StreamError::NotFound)); + fills.refused.insert(0..2, Refusal::passing(StreamError::Cancel)); + fills.refused.insert(2..5, Refusal::dropped(StreamError::NotFound)); fills.forget_answered(); - assert!(fills.refused.get(&1).is_some(), "the held fill still owes it"); - assert!(fills.refused.get(&3).is_some(), "the held fill still owes it"); - assert_eq!(fills.refused.get(&4), None, "trimmed to the groups a held fill covers"); + assert!(fills.refused.get(&0).is_some(), "the held fill still owes it"); + assert!(fills.refused.get(&2).is_some(), "the held fill still owes it"); + assert_eq!(fills.refused.get(&3), None, "trimmed to the groups a held fill covers"); fills.held.clear(); fills.forget_answered(); @@ -798,7 +799,7 @@ mod tests { assert!(fills.refused.is_empty(), "not read back"); } - /// A track whose cache evicted groups 1..4, with a reader of them whose fill is the next + /// A track whose cache evicted groups 0..3, with a reader of them whose fill is the next /// request. fn missed() -> (track::Producer, track::Requests, track::Ordered, track::Fill) { let producer = track::Producer::new( @@ -809,33 +810,35 @@ mod tests { .with_max_age(None), ); let requests = producer.requests(); - for sequence in 0..6 { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + for key in 0..6 { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group - .write_frame(crate::Timestamp::from_millis(sequence * 1000).unwrap(), b"x".as_slice()) + .write_frame(crate::Timestamp::from_millis(key * 1000).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - for sequence in 1..4 { - producer.evict(sequence); + // From group 0, which a time start names whatever is held, so the reader asks the + // missing groups straight away rather than looking up where its start falls. + for key in 0..3 { + producer.evict(key); } let (reader, fill) = ask(&producer, &requests); (producer, requests, reader, fill) } - /// A reader of groups 1..4, and the fill it asks for. + /// A reader of groups 0..3, and the fill it asks for. fn ask(producer: &track::Producer, requests: &track::Requests) -> (track::Ordered, track::Fill) { let waiter = kio::Waiter::noop(); let mut reader = producer .subscribe( Subscription::default() .with_delay(Delay::default().with_max(None)) - .with_range(1..4), + .with_range(0..3), ) .ordered(); - assert!(reader.poll_next_group(&waiter).is_pending(), "groups 1..4 are owed"); + assert!(reader.poll_next_group(&waiter).is_pending(), "groups 0..3 are owed"); let Poll::Ready(Ok(fill)) = requests.poll_requested_fill(&waiter) else { - panic!("groups 1..4 are a miss"); + panic!("groups 0..3 are a miss"); }; (reader, fill) } @@ -847,7 +850,7 @@ mod tests { fn a_skipped_start_is_never_a_loss() { let waiter = kio::Waiter::noop(); let (producer, requests, mut reader, fill) = missed(); - Refusal::passing(StreamError::NotFound).answer(&fill, 1..4); + Refusal::passing(StreamError::NotFound).answer(&fill, 0..3); assert!( reader.poll_next_group(&waiter).is_ready(), "the reader stops waiting for the skipped groups" @@ -856,29 +859,29 @@ mod tests { // The lowered start asks for them again, and they arrive. let (mut reader, fill) = ask(&producer, &requests); - for sequence in 1..4 { - let mut group = fill.accept(sequence, None).unwrap(); + for key in 0..3 { + let mut group = fill.accept(key, None).unwrap(); group - .write_frame(crate::Timestamp::from_millis(sequence * 1000).unwrap(), b"x".as_slice()) + .write_frame(crate::Timestamp::from_millis(key * 1000).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - for sequence in 1..4 { + for key in 0..3 { let Poll::Ready(Ok(Some(group))) = reader.poll_next_group(&waiter) else { - panic!("group {sequence} was delivered"); + panic!("group {key} was delivered"); }; - assert_eq!(group.sequence, sequence); + assert_eq!(group.start, key); } // The publisher naming them not found is a loss: nobody asks again. let (producer, requests, reader, fill) = missed(); - Refusal::dropped(StreamError::NotFound).answer(&fill, 1..4); + Refusal::dropped(StreamError::NotFound).answer(&fill, 0..3); drop((reader, fill)); let mut reader = producer .subscribe( Subscription::default() .with_delay(Delay::default().with_max(None)) - .with_range(1..4), + .with_range(0..3), ) .ordered(); let _ = reader.poll_next_group(&waiter); diff --git a/rs/moq-net/src/lite/version.rs b/rs/moq-net/src/lite/version.rs index 287e9a38cf5..dbb86a17e6d 100644 --- a/rs/moq-net/src/lite/version.rs +++ b/rs/moq-net/src/lite/version.rs @@ -161,7 +161,7 @@ impl Version { } /// Whether SUBSCRIBE, SUBSCRIBE_UPDATE, FETCH, and GROUP carry frame indices - /// alongside their group sequences, so a subscription or fetch can start and end + /// alongside their group keys, so a subscription or fetch can start and end /// partway through a group. Added in lite-06. /// /// Older versions only address whole groups, so a route change has to wait for the @@ -233,10 +233,10 @@ impl Version { } /// Whether SUBSCRIBE's `Group Start` is an absolute floor the publisher resolves a - /// start from: the raw minimum group sequence (default 0), with `Subscriber Max Age` + /// start from: the raw minimum group key (default 0), with `Subscriber Max Age` /// as the only gate on how far back delivery begins. Changed in lite-06. /// - /// Older versions encode `Group Start` as the sequence + 1, with 0 meaning the + /// Older versions encode `Group Start` as the key + 1, with 0 meaning the /// latest group, so an absent start there pins the cursor to the live edge instead /// of resolving it from the budget. pub(crate) fn resolves_start(self) -> bool { @@ -336,7 +336,7 @@ impl Version { ) } - /// Whether holes split into SUBSCRIBE_SKIP, for sequences declared never produced, and + /// Whether holes split into SUBSCRIBE_SKIP, for keys declared never produced, and /// SUBSCRIBE_DROP, for groups that existed, which names the `Update` in effect when it was /// given up; SUBSCRIBE and SUBSCRIBE_UPDATE carry that `Update` number. Added in lite-07; /// earlier versions name every hole in SUBSCRIBE_DROP. @@ -458,8 +458,10 @@ impl Version { /// The timescale the wire carries for a track with `timescale`, or `None` when its /// frames or datagrams carry no timestamp. /// - /// Lite05 and Lite06 always carry one, so an untimed track declares milliseconds there - /// and its payloads carry their send times instead. + /// Lite05 and Lite06 always carry one, and those published drafts treat a TRACK_INFO + /// `Timescale` of 0 as invalid, so their wire forces a scale-less track to declare + /// milliseconds. Its frames then carry their group's key, and its datagrams their + /// sequence, never a clock. pub(crate) fn wire_timescale(self, timescale: Option) -> Option { match self { Self::Lite01 | Self::Lite02 | Self::Lite03 | Self::Lite04 => None, diff --git a/rs/moq-net/src/model/aborts.rs b/rs/moq-net/src/model/aborts.rs index aa9995f7e2f..9abe1fdab5e 100644 --- a/rs/moq-net/src/model/aborts.rs +++ b/rs/moq-net/src/model/aborts.rs @@ -3,7 +3,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; -/// One registration per wire publisher, with pending sequences bounded by the cuts +/// One registration per wire publisher, with pending keys bounded by the cuts /// between two polls. #[derive(Default)] pub(super) struct Aborts { @@ -32,14 +32,14 @@ impl Aborts { } } - /// Tell every watcher that the publisher cut group `sequence`. - pub(super) fn notify(&self, sequence: u64) { + /// Tell every watcher that the publisher cut group `key`. + pub(super) fn notify(&self, key: u64) { let mut wakes = Vec::new(); { let readers = self.readers.lock(); for reader in readers.values() { let mut state = reader.lock(); - if state.pending.insert(sequence) { + if state.pending.insert(key) { wakes.push(state.waiters.take()); } } diff --git a/rs/moq-net/src/model/broadcast.rs b/rs/moq-net/src/model/broadcast.rs index 0f2530481b5..5d5b5f0565b 100644 --- a/rs/moq-net/src/model/broadcast.rs +++ b/rs/moq-net/src/model/broadcast.rs @@ -1570,11 +1570,11 @@ mod test { let producer = Info::new().produce(); let consumer = producer.consume(); let track = producer.create_track("a", crate::track::Info::timed()).unwrap(); - let _open = track.append_group().unwrap(); + let _open = track.append_next().unwrap(); let handle = consumer.track("a").unwrap(); - assert_eq!(handle.latest_sequence(), Some(0)); + assert_eq!(handle.latest_key(), Some(0)); drop(track); - assert_eq!(handle.latest_sequence(), None); + assert_eq!(handle.latest_key(), None); } /// A handle's largest reads the instance its open broadcast serves now, without asking @@ -1584,8 +1584,10 @@ mod test { let producer = Info::new().produce(); let consumer = producer.consume(); let track = producer.create_track("a", crate::track::Info::default()).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(None, &b"a"[..]).unwrap(); + let mut group = track.append_next().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"a"[..]) + .unwrap(); let mut handle = consumer.track("a").unwrap(); let waiter = kio::Waiter::noop(); assert_eq!( @@ -1604,11 +1606,11 @@ mod test { let producer = Info::new().produce(); let consumer = producer.consume(); let track = producer.create_track("a", crate::track::Info::timed()).unwrap(); - let _open = track.append_group().unwrap(); + let _open = track.append_next().unwrap(); let handle = consumer.track("a").unwrap(); producer.close(); drop(track); - assert_eq!(handle.latest_sequence(), Some(0)); + assert_eq!(handle.latest_key(), Some(0)); } #[test] @@ -1798,7 +1800,7 @@ mod test { let mut demand = producer.demand(); let waiter = kio::Waiter::noop(); let mut subscriber = video.subscribe(None); - let mut writing = video.append_group().unwrap(); + let mut writing = video.append_next().unwrap(); writing.write_frame(crate::Timestamp::ZERO, &b"key"[..]).unwrap(); assert_eq!(wanted_changed(&mut demand, &waiter), Some(true)); @@ -1840,7 +1842,7 @@ mod test { // Create the track before any consumer exists. let track1 = producer.assert_create_track("track1", crate::track::Info::timed()); - track1.append_group().unwrap().finish().unwrap(); + track1.append_next().unwrap().close().unwrap(); let consumer = producer.consume(); @@ -1854,7 +1856,7 @@ mod test { let mut track2_consumer = consumer2.track("track2").unwrap().subscribe(None).await.unwrap(); track2_consumer.assert_no_group(); - track2.append_group().unwrap().finish().unwrap(); + track2.append_next().unwrap().close().unwrap(); track2_consumer.assert_group(); } @@ -1942,7 +1944,7 @@ mod test { track3.subscribe(None).assert_is_clone(&track1); // Append a group and make sure they all get it. - track3.append_group().unwrap().finish().unwrap(); + track3.append_next().unwrap().close().unwrap(); track1.assert_group(); track2.assert_group(); @@ -1968,7 +1970,7 @@ mod test { let mut track1 = track1_fut.await.unwrap(); // Close the producer (simulating publisher disconnect). - producer1.append_group().unwrap().finish().unwrap(); + producer1.append_next().unwrap().close().unwrap(); let producer1 = producer1.finish().unwrap(); drop(producer1); @@ -1984,9 +1986,9 @@ mod test { // The new consumer should receive the new group, which continues past the first's. producer2 - .create_group(crate::group::Info { sequence: 1 }) + .create_group(crate::group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); track2.assert_group(); } @@ -2003,16 +2005,16 @@ mod test { let track = consumer.track("media").unwrap().subscribe(None); let first = dynamic.assert_request().accept(crate::track::Info::timed()); - let mut group = first.create_group(crate::group::Info { sequence: 8 }).unwrap(); + let mut group = first.create_group(crate::group::Info { start: 8 }).unwrap(); group.write_frame(at(1000), b"a".as_slice()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let first = first.finish().unwrap(); drop((first, track)); let track = consumer.track("media").unwrap().subscribe(None); let second = dynamic.assert_request().accept(crate::track::Info::timed()); - let mut group = second.append_group().unwrap(); - assert_eq!(group.sequence, 0); + let mut group = second.append_next().unwrap(); + assert_eq!(group.start, 0); group.write_frame(at(500), b"b".as_slice()).unwrap(); drop((group, second, track)); @@ -2030,7 +2032,7 @@ mod test { for i in 0..1000 { let track = consumer.track(&format!("requested/{i}")).unwrap().subscribe(None); let served = dynamic.assert_request().accept(crate::track::Info::timed()); - served.append_group().unwrap().finish().unwrap(); + served.append_next().unwrap().close().unwrap(); drop((served, track)); let track = consumer.track(&format!("refused/{i}")).unwrap().subscribe(None); @@ -2039,7 +2041,7 @@ mod test { let created = producer .create_track(format!("created/{i}"), crate::track::Info::timed()) .unwrap(); - created.append_group().unwrap().finish().unwrap(); + created.append_next().unwrap().close().unwrap(); } let state = producer.state.lock(); assert!(state.tracks.len() < 16, "{} names retained", state.tracks.len()); @@ -2114,7 +2116,7 @@ mod test { let mut sub = subscribing.await.expect("fulfilled by create_track"); // The fulfilled subscription is live against this very producer. - track.append_group().unwrap().finish().unwrap(); + track.append_next().unwrap().close().unwrap(); sub.recv_group().await.expect("recv").expect("group"); // The handler never sees the request; a fresh subscribe reuses the track. @@ -2239,7 +2241,7 @@ mod test { assert!(matches!(queued.await, Err(Error::Unroutable))); assert!(matches!(consumer.track("later"), Err(Error::Unroutable))); - served.append_group().unwrap().finish().unwrap(); + served.append_next().unwrap().close().unwrap(); accepted.assert_group(); } @@ -2307,7 +2309,7 @@ mod test { let pending_fetch = track.fetch_group(0, None); let fetch = dynamic.requested_fill_group().await.unwrap(); let group = fetch.accept(crate::track::Info::timed()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pending_fetch.await.unwrap(); let mut subscriber = track.subscribe(None).await.unwrap(); @@ -2315,7 +2317,7 @@ mod test { assert!(matches!(subscriber.recv_group().await, Err(Error::Unroutable))); let stale = request.accept(crate::track::Info::timed()); - assert!(stale.append_group().is_err()); + assert!(stale.append_next().is_err()); } /// A staged track resolves no subscriber until it commits, and then with what was @@ -2336,7 +2338,7 @@ mod test { let _track = staged.commit(); let mut subscriber = subscribing.await.unwrap(); - assert_eq!(subscriber.recv_group().await.unwrap().unwrap().sequence, 0); + assert_eq!(subscriber.recv_group().await.unwrap().unwrap().start, 0); } /// Dropping a staged track uncommitted rejects the request, so its subscribers fail @@ -2363,7 +2365,7 @@ mod test { let producer = Info::new().produce(); let request = producer.reserve_track("track1").unwrap(); let staged = request.stage(crate::track::Info::timed()); - let mut group = staged.append_group().unwrap(); + let mut group = staged.append_next().unwrap(); drop(staged); let write = group.write_frame(crate::Timestamp::ZERO, b"x".as_slice()); assert!(matches!(write, Err(Error::Dropped)), "{write:?}"); @@ -2383,7 +2385,7 @@ mod test { producer.close(); assert!(matches!(consumer.track("track1"), Err(Error::Unroutable))); - track.append_group().unwrap().finish().unwrap(); + track.append_next().unwrap().close().unwrap(); subscriber.assert_group(); let _track = track.finish().unwrap(); } @@ -2404,7 +2406,7 @@ mod test { assert!(matches!(pending.await, Err(Error::Unroutable))); let track = request.accept(crate::track::Info::timed()); - assert!(track.append_group().is_err()); + assert!(track.append_next().is_err()); let mut subscriber = track.subscribe(None); assert!(matches!(subscriber.recv_group().await, Err(Error::Unroutable))); assert!(consumer.track("track1").is_err()); @@ -2468,7 +2470,7 @@ mod test { assert!(!track.is_closed()); let mut subscriber = viewer.await.unwrap(); subscriber.assert_no_group(); - track.append_group().unwrap().finish().unwrap(); + track.append_next().unwrap().close().unwrap(); assert!(subscriber.recv_group().await.unwrap().is_some()); // Once the viewer really leaves, the same teardown commits, and the lookup @@ -2517,7 +2519,7 @@ mod test { )); let mut reader = consumer.datagrams("audio").unwrap(); - datagrams.append(None, "x").unwrap(); + datagrams.append(crate::Timestamp::unscaled(0), "x").unwrap(); assert_eq!(received(&mut reader), 0); // Once it ends, the name is free again, and a reader no longer joins it. @@ -2549,7 +2551,7 @@ mod test { let info = datagram::Info::default().with_priority(2); let mut served = request.accept(info); assert!(matches!(second.poll_info(&kio::Waiter::noop()), Poll::Ready(Ok(got)) if got == info)); - served.append(None, "x").unwrap(); + served.append(crate::Timestamp::unscaled(0), "x").unwrap(); assert_eq!(received(&mut first), 0); assert_eq!(received(&mut second), 0); @@ -2572,7 +2574,7 @@ mod test { let info = datagram::Info::default().with_priority(2); let mut datagrams = producer.create_datagrams("audio", info).unwrap(); assert!(matches!(reader.poll_info(&kio::Waiter::noop()), Poll::Ready(Ok(got)) if got == info)); - datagrams.append(None, "x").unwrap(); + datagrams.append(crate::Timestamp::unscaled(0), "x").unwrap(); assert_eq!(received(&mut reader), 0); assert!(handler.poll_requested(&kio::Waiter::noop()).is_pending()); } diff --git a/rs/moq-net/src/model/cache.rs b/rs/moq-net/src/model/cache.rs index 8527ee463e3..c4eb29abb92 100644 --- a/rs/moq-net/src/model/cache.rs +++ b/rs/moq-net/src/model/cache.rs @@ -760,11 +760,11 @@ impl Track { } } - /// Group `sequence` holds every frame it will have: a whole copy finished. A group name + /// Group `key` holds every frame it will have: a whole copy finished. A group name /// fixes its content, so no legal write raises the largest past it. - pub(super) fn finish_largest(&self, sequence: u64) { + pub(super) fn finish_largest(&self, key: u64) { let mut largest = self.largest.lock(); - if largest.position.is_some_and(|position| position.group == sequence) { + if largest.position.is_some_and(|position| position.group == key) { largest.open = false; } } @@ -881,15 +881,15 @@ impl Track { self.declared.load(Ordering::Acquire) } - /// Watch every group the publisher cuts, rather than chosen sequences. + /// Watch every group the publisher cuts, rather than chosen keys. pub(super) fn watch_cuts(&self) -> aborts::Watch { self.aborts.get_or_init(Arc::default).watch_cuts() } - /// Notify watchers that group `sequence` aborted, if the publisher cut it. - pub(super) fn aborted(&self, sequence: u64, cut: bool) { + /// Notify watchers that group `key` aborted, if the publisher cut it. + pub(super) fn aborted(&self, key: u64, cut: bool) { if cut && let Some(aborts) = self.aborts.get() { - aborts.notify(sequence); + aborts.notify(key); } } diff --git a/rs/moq-net/src/model/datagram.rs b/rs/moq-net/src/model/datagram.rs index 0a8370e534c..c901bd30841 100644 --- a/rs/moq-net/src/model/datagram.rs +++ b/rs/moq-net/src/model/datagram.rs @@ -69,8 +69,9 @@ const DEFAULT_PRIORITY: i8 = 0; pub struct Datagram { /// The datagram track's own sequence number. pub sequence: u64, - /// Presentation timestamp in the track's timescale, or `None` on a track without one. - pub timestamp: Option, + /// Presentation timestamp in the track's timescale, or on a scale-less track + /// [`Timestamp::unscaled`] at its sequence. + pub timestamp: Timestamp, /// The datagram payload. pub payload: Bytes, } @@ -104,16 +105,14 @@ impl Info { /// `timestamp` at this track's [`timescale`](Self::timescale). /// - /// A track is all timed or all untimed: a timestamp that doesn't match it, or one the + /// A track is all timed or all scale-less: a timestamp that doesn't match it, or one the /// track's scale can't hold, is [`Error::TimestampMismatch`]. A writer that changes /// state before its datagram reaches the track, such as encrypting it, rescales first so /// a refused timestamp changes nothing. - pub fn rescale(&self, timestamp: Option) -> Result> { - match (timestamp, self.timescale) { - (Some(timestamp), Some(timescale)) => Ok(Some( - timestamp.convert(timescale).map_err(|_| Error::TimestampMismatch)?, - )), - (None, None) => Ok(None), + pub fn rescale(&self, timestamp: Timestamp) -> Result { + match (timestamp.scale(), self.timescale) { + (Some(_), Some(timescale)) => timestamp.convert(timescale).map_err(|_| Error::TimestampMismatch), + (None, None) => Ok(timestamp), _ => Err(Error::TimestampMismatch), } } @@ -284,17 +283,21 @@ impl Producer { /// Append a datagram with the next sequence number, returning that sequence. /// - /// Pass `None` on an untimed track; a timestamp that doesn't match the track is - /// [`Error::TimestampMismatch`]. Keep payloads well under the 1200-byte minimum path + /// A scale-less track has no time, so its datagram's [`Timestamp::unscaled`] names the + /// sequence instead, as its key names a group. A timestamp that doesn't match the track + /// is [`Error::TimestampMismatch`]. Keep payloads well under the 1200-byte minimum path /// MTU: a hop drops a datagram that does not fit. - pub fn append(&mut self, timestamp: impl Into>, payload: B) -> Result { + pub fn append(&mut self, timestamp: Timestamp, payload: B) -> Result { let mut state = self.state.lock(); - let sequence = state.next.ok_or(crate::coding::BoundsExceeded)?; + let sequence = match timestamp.scale() { + Some(_) => state.next.ok_or(crate::coding::BoundsExceeded)?, + None => timestamp.value(), + }; self.push( &mut state, Datagram { sequence, - timestamp: timestamp.into(), + timestamp, payload: payload.into_bytes(), }, )?; @@ -316,6 +319,10 @@ impl Producer { } state.closed()?; datagram.timestamp = self.info.rescale(datagram.timestamp)?; + // A scale-less datagram's timestamp is its sequence. + if datagram.timestamp.scale().is_none() && datagram.timestamp.value() != datagram.sequence { + return Err(Error::TimestampMismatch); + } self.stats.meter().datagram(datagram.payload.len() as u64); state.push(datagram); Ok(()) @@ -684,6 +691,15 @@ mod tests { consumer.poll_recv(&kio::Waiter::noop()) } + /// A scale-less datagram's timestamp: its sequence. + fn seq(sequence: u64) -> Timestamp { + Timestamp::unscaled(sequence) + } + + fn ms(millis: u64) -> Timestamp { + Timestamp::from_millis(millis).unwrap() + } + fn sequence(consumer: &mut Consumer) -> u64 { match recv(consumer) { Poll::Ready(Ok(Some(datagram))) => datagram.sequence, @@ -702,7 +718,7 @@ mod tests { assert_eq!(producer.priority(), Some(-7)); drop(reader); assert_eq!(producer.priority(), None); - producer.append(None, &b"tail"[..]).unwrap(); + producer.append(seq(0), &b"tail"[..]).unwrap(); producer.finish(); for consumer in [&mut observer, &mut copy] { let Poll::Ready(Ok(Some(datagram))) = consumer.poll_recv(&kio::Waiter::noop()) else { @@ -718,13 +734,13 @@ mod tests { #[test] fn append_counts_its_own_sequence_past_inserts() { - let mut producer = Info::default().produce("dg"); + let mut producer = Info::default().with_timescale(Timescale::MILLI).produce("dg"); let mut consumer = producer.consume(); - assert_eq!(producer.append(None, "a").unwrap(), 0); + assert_eq!(producer.append(ms(0), "a").unwrap(), 0); producer .insert(Datagram { sequence: 10, - timestamp: None, + timestamp: ms(10), payload: Bytes::from_static(b"b"), }) .unwrap(); @@ -732,11 +748,11 @@ mod tests { producer .insert(Datagram { sequence: 3, - timestamp: None, + timestamp: ms(3), payload: Bytes::from_static(b"c"), }) .unwrap(); - assert_eq!(producer.append(None, "d").unwrap(), 11); + assert_eq!(producer.append(ms(11), "d").unwrap(), 11); let got: Vec = (0..4).map(|_| sequence(&mut consumer)).collect(); assert_eq!(got, [0, 10, 3, 11]); assert!(recv(&mut consumer).is_pending()); @@ -745,24 +761,24 @@ mod tests { /// Past `u64::MAX` no sequence remains, so `append` refuses instead of repeating it. #[test] fn append_refuses_once_the_sequence_is_spent() { - let mut producer = Info::default().produce("dg"); + let mut producer = Info::default().with_timescale(Timescale::MILLI).produce("dg"); producer .insert(Datagram { sequence: u64::MAX, - timestamp: None, + timestamp: ms(0), payload: Bytes::from_static(b"last"), }) .unwrap(); - assert!(matches!(producer.append(None, "x"), Err(Error::BoundsExceeded(_)))); + assert!(matches!(producer.append(ms(1), "x"), Err(Error::BoundsExceeded(_)))); } #[test] fn a_consumer_starts_live() { let mut producer = Info::default().produce("dg"); - producer.append(None, "old").unwrap(); + producer.append(seq(0), "old").unwrap(); let mut consumer = producer.consume(); assert!(recv(&mut consumer).is_pending()); - producer.append(None, "new").unwrap(); + producer.append(seq(1), "new").unwrap(); assert_eq!(sequence(&mut consumer), 1); } @@ -770,8 +786,8 @@ mod tests { fn a_slow_consumer_skips_to_the_oldest_held() { let mut producer = Info::default().produce("dg"); let mut consumer = producer.consume(); - for _ in 0..MAX_BUFFERED + 5 { - producer.append(None, "x").unwrap(); + for sequence in 0..MAX_BUFFERED as u64 + 5 { + producer.append(seq(sequence), "x").unwrap(); } assert_eq!(sequence(&mut consumer), 5); } @@ -781,16 +797,16 @@ mod tests { let mut producer = Info::default().produce("dg"); let mut consumer = producer.consume(); let payload = Bytes::from(vec![0; MAX_BUFFERED_BYTES / 4]); - for _ in 0..8 { - producer.append(None, payload.clone()).unwrap(); + for sequence in 0..8 { + producer.append(seq(sequence), payload.clone()).unwrap(); } // Four fit, so the oldest four went. assert_eq!(sequence(&mut consumer), 4); // The newest is always held, even alone over the budget. let mut producer = Info::default().produce("dg"); let mut consumer = producer.consume(); - for _ in 0..2 { - producer.append(None, vec![0; MAX_PAYLOAD]).unwrap(); + for sequence in 0..2 { + producer.append(seq(sequence), vec![0; MAX_PAYLOAD]).unwrap(); } assert_eq!(sequence(&mut consumer), 1); } @@ -799,18 +815,26 @@ mod tests { fn timestamps_must_match_the_track() { let mut timed = Info::default().with_timescale(Timescale::MILLI).produce("dg"); let mut consumer = timed.consume(); - assert!(matches!(timed.append(None, "x"), Err(Error::TimestampMismatch))); + assert!(matches!(timed.append(seq(0), "x"), Err(Error::TimestampMismatch))); timed.append(Timestamp::from_secs(1).unwrap(), "x").unwrap(); let Poll::Ready(Ok(Some(datagram))) = recv(&mut consumer) else { panic!("expected a datagram"); }; // Converted into the track's scale. - assert_eq!(datagram.timestamp, Some(Timestamp::from_millis(1000).unwrap())); + assert_eq!(datagram.timestamp, Timestamp::from_millis(1000).unwrap()); let mut untimed = Info::default().produce("dg"); assert!(matches!( untimed.append(Timestamp::from_secs(1).unwrap(), "x"), Err(Error::TimestampMismatch) )); + // A scale-less datagram's timestamp names its sequence, so another is a mismatch. + let wrong = Datagram { + sequence: 4, + timestamp: seq(5), + payload: Bytes::from_static(b"x"), + }; + assert!(matches!(untimed.insert(wrong), Err(Error::TimestampMismatch))); + assert_eq!(untimed.append(seq(4), "x").unwrap(), 4); } #[test] @@ -819,7 +843,7 @@ mod tests { let mut finished = producer.consume(); producer.clone().finish(); assert!(matches!(recv(&mut finished), Poll::Ready(Ok(None)))); - assert!(matches!(producer.append(None, "x"), Err(Error::Closed))); + assert!(matches!(producer.append(seq(0), "x"), Err(Error::Closed))); let producer = Info::default().produce("dg"); let mut dropped = producer.consume(); @@ -831,8 +855,8 @@ mod tests { fn a_finish_delivers_what_is_buffered() { let mut producer = Info::default().produce("dg"); let mut consumer = producer.consume(); - producer.append(None, "a").unwrap(); - producer.append(None, "b").unwrap(); + producer.append(seq(0), "a").unwrap(); + producer.append(seq(1), "b").unwrap(); producer.finish(); assert_eq!(sequence(&mut consumer), 0); assert_eq!(sequence(&mut consumer), 1); @@ -841,7 +865,7 @@ mod tests { // A consumer arriving after the finish never reads them. let mut producer = Info::default().produce("dg"); let _reading = producer.consume(); - producer.append(None, "a").unwrap(); + producer.append(seq(0), "a").unwrap(); producer.clone().finish(); let mut late = producer.consume(); assert!(matches!(recv(&mut late), Poll::Ready(Ok(None)))); @@ -849,12 +873,12 @@ mod tests { // Nobody left to read them, they go: on a finish with no consumer, or with the last // consumer of a finished track. let mut producer = Info::default().produce("dg"); - producer.append(None, "a").unwrap(); + producer.append(seq(0), "a").unwrap(); producer.clone().finish(); assert!(producer.state.read().buffer.is_empty()); let mut producer = Info::default().produce("dg"); let reading = producer.consume(); - producer.append(None, "a").unwrap(); + producer.append(seq(0), "a").unwrap(); producer.clone().finish(); assert_eq!(producer.state.read().buffer.len(), 1); drop(reading); @@ -864,7 +888,7 @@ mod tests { // An abort drops them. let mut producer = Info::default().produce("dg"); let mut consumer = producer.consume(); - producer.append(None, "a").unwrap(); + producer.append(seq(0), "a").unwrap(); producer.abort(Error::Cancel); assert!(matches!(recv(&mut consumer), Poll::Ready(Err(Error::Cancel)))); } diff --git a/rs/moq-net/src/model/delivered.rs b/rs/moq-net/src/model/delivered.rs index fd39fcb714b..c1b2894e0bb 100644 --- a/rs/moq-net/src/model/delivered.rs +++ b/rs/moq-net/src/model/delivered.rs @@ -1,34 +1,34 @@ -//! The group sequences a reader handed out, so it never hands one out twice. +//! The group keys a reader handed out, so it never hands one out twice. use std::collections::BTreeMap; -/// The group sequences a reader handed out, as runs of consecutive sequences. +/// The group keys a reader handed out, as runs of consecutive keys. /// /// A reader stops expecting a group once it is past its budget, and counts everything /// below it as handed out (see [`Self::forget_below`]). The run starting at 0 is that /// floor, so only the gaps above it cost memory. It never holds the reserved group -/// sequence `u64::MAX`, so one past any run fits. +/// key `u64::MAX`, so one past any run fits. #[derive(Default)] pub(super) struct Delivered { - /// First to last sequence, inclusive, never adjacent to each other. + /// First to last key, inclusive, never adjacent to each other. runs: BTreeMap, } impl Delivered { - /// Whether `sequence` was handed out, or is below the floor. - pub fn contains(&self, sequence: u64) -> bool { + /// Whether `key` was handed out, or is below the floor. + pub fn contains(&self, key: u64) -> bool { self.runs - .range(..=sequence) + .range(..=key) .next_back() - .is_some_and(|(_, last)| sequence <= *last) + .is_some_and(|(_, last)| key <= *last) } - /// The first sequence above the handed-out prefix. + /// The first key above the handed-out prefix. pub fn floor(&self) -> u64 { self.runs.get(&0).map_or(0, |last| last + 1) } - /// Whether every sequence in `groups` was handed out, or is below the floor. + /// Whether every key in `groups` was handed out, or is below the floor. pub fn covers(&self, groups: std::ops::Range) -> bool { groups.is_empty() || self @@ -38,28 +38,28 @@ impl Delivered { .is_some_and(|(_, last)| groups.end - 1 <= *last) } - /// Record `sequence` as handed out; `false` when it already was. - pub fn insert(&mut self, sequence: u64) -> bool { - if self.contains(sequence) { + /// Record `key` as handed out; `false` when it already was. + pub fn insert(&mut self, key: u64) -> bool { + if self.contains(key) { return false; } - let first = sequence + let first = key .checked_sub(1) .and_then(|below| self.runs.range(..=below).next_back()) - .filter(|(_, last)| **last + 1 == sequence) - .map_or(sequence, |(first, _)| *first); - let last = self.runs.remove(&(sequence + 1)).unwrap_or(sequence); + .filter(|(_, last)| **last + 1 == key) + .map_or(key, |(first, _)| *first); + let last = self.runs.remove(&(key + 1)).unwrap_or(key); self.runs.insert(first, last); true } - /// Record every sequence in `groups` as handed out, joining the runs it touches. + /// Record every key in `groups` as handed out, joining the runs it touches. pub fn insert_run(&mut self, groups: std::ops::Range) { let Some(mut last) = groups.end.checked_sub(1).filter(|_| !groups.is_empty()) else { return; }; let mut first = groups.start; - // A run ending at or just below the first sequence joins it. + // A run ending at or just below the first key joins it. if let Some((&start, &end)) = self.runs.range(..=first).next_back() && end + 1 >= first { @@ -109,7 +109,7 @@ impl Delivered { gaps } - /// Count every sequence below `floor` as handed out. + /// Count every key below `floor` as handed out. pub fn forget_below(&mut self, floor: u64) { let Some(below) = floor.checked_sub(1) else { return; @@ -122,7 +122,7 @@ impl Delivered { self.runs.insert(0, last); } - /// The lowest sequence handed out above a gap: once it is past the reader's budget, + /// The lowest key handed out above a gap: once it is past the reader's budget, /// so is the gap, since a timed track refuses a group whose start rewinds. pub fn above_gap(&self) -> Option { let mut runs = self.runs.keys(); @@ -132,17 +132,17 @@ impl Delivered { } } - /// The last run starting below `sequence`. - pub fn last_before(&self, sequence: u64) -> Option { - self.runs.range(..sequence).next_back().map(|(start, _)| *start) + /// The last run starting below `key`. + pub fn last_before(&self, key: u64) -> Option { + self.runs.range(..key).next_back().map(|(start, _)| *start) } - /// One past the last sequence of the run holding `sequence`, if it was handed out. - pub fn run_end(&self, sequence: u64) -> Option { + /// One past the last key of the run holding `key`, if it was handed out. + pub fn run_end(&self, key: u64) -> Option { self.runs - .range(..=sequence) + .range(..=key) .next_back() - .filter(|(_, last)| sequence <= **last) + .filter(|(_, last)| key <= **last) .map(|(_, last)| last + 1) } @@ -163,8 +163,8 @@ mod test { #[test] fn gaps_are_what_no_run_covers() { let mut delivered = Delivered::default(); - for sequence in [2, 4, 5, 9] { - delivered.insert(sequence); + for key in [2, 4, 5, 9] { + delivered.insert(key); } assert_eq!(delivered.gaps(0..12), [0..2, 3..4, 6..9, 10..12]); assert_eq!(delivered.gaps(4..6), []); @@ -196,8 +196,8 @@ mod test { #[test] fn insert_run_joins_every_run_it_touches() { let mut delivered = Delivered::default(); - for sequence in [2, 5, 6, 9, 14] { - delivered.insert(sequence); + for key in [2, 5, 6, 9, 14] { + delivered.insert(key); } delivered.insert_run(3..5); assert_eq!(runs(&delivered), [(2, 6), (9, 9), (14, 14)], "adjacent on both sides"); @@ -212,7 +212,7 @@ mod test { } #[test] - fn the_extremes_are_sequences_too() { + fn the_extremes_are_keys_too() { let mut delivered = Delivered::default(); assert!(delivered.insert(0)); assert!(delivered.insert(u64::MAX - 1)); @@ -226,8 +226,8 @@ mod test { #[test] fn forget_below_closes_the_gaps_under_the_floor() { let mut delivered = Delivered::default(); - for sequence in [3, 5, 6, 9, 12] { - delivered.insert(sequence); + for key in [3, 5, 6, 9, 12] { + delivered.insert(key); } assert_eq!(delivered.above_gap(), Some(3)); assert_eq!(delivered.last_before(0), None); diff --git a/rs/moq-net/src/model/expiry.rs b/rs/moq-net/src/model/expiry.rs index 42259592cd5..7a51842b007 100644 --- a/rs/moq-net/src/model/expiry.rs +++ b/rs/moq-net/src/model/expiry.rs @@ -16,7 +16,7 @@ use super::Timestamp; /// successor starts, plus its budget. The edge only ever takes a timestamp some frame /// write presented, so the write that crosses a deadline wakes the reads behind it and /// nobody else. The successor itself is watched per group (its first frame, its abort), -/// except for a group landing between the read and it, which reads park on by sequence. +/// except for a group landing between the read and it, which reads park on by key. /// /// A parked read also goes stale once the wall clock reaches its successor's arrival plus /// its budget. Nothing written moves that clock, so those reads park by instant and the @@ -34,10 +34,6 @@ pub(super) struct Wakes { /// Relaxed: a read registers before resolving the edge, under the group locks a /// write takes before it presents, so a write the read missed sees its deadline. earliest: AtomicU64, - /// The lowest sequence a parked time end watches for a start, or `u64::MAX`, so a first - /// frame below every one skips the lock. Ordered by the starts index's lock, which a - /// read takes after registering and a first frame before checking this. - starts_from: AtomicU64, /// Whether either landing index has entries. Insertion checks this after raising /// `newest`, and a landing read sets it before reading `newest`, each across a SeqCst /// fence: whichever goes second sees the other, so a read registering without the @@ -45,7 +41,7 @@ pub(super) struct Wakes { /// read lock, which orders them against insertion too. First-frame reveal holds only /// a track read lock and must always take the parked lock instead. landing_waiters: AtomicBool, - /// The highest sequence a servable group ever landed at, or 0, so a read with nothing + /// The highest key a servable group ever landed at, or 0, so a read with nothing /// above it skips the track lock. Nothing landing and group 0 landing put no group /// above any read, so 0 serves both. Never lowered: a group that later fails or leaves /// only sends the read the slow way. @@ -61,7 +57,7 @@ struct Parked { /// The `deadlines` size that sweeps out the lists every reader has left. A read that /// returns or is dropped leaves its deadline behind, and distinct budgets never share one. deadlines_sweep: usize, - /// Reads by their group's sequence, woken when a group lands above them. + /// Reads by their group's key, woken when a group lands above them. landings: BTreeMap, /// `landings` for groups held from another copy, also woken by their own group /// landing here. Kept apart because this track may never have held them, so falling @@ -72,9 +68,6 @@ struct Parked { /// Exact cache arrivals, independent of the live edge and neighbouring groups. arrivals: BTreeMap, arrival_sweep: usize, - /// Reads parked on a time end, by the lowest sequence whose start could move it. - starts: BTreeMap, - starts_sweep: usize, /// Reads by wall-clock deadline, released by the pool's driver. wall: BTreeMap, wall_sweep: usize, @@ -103,7 +96,6 @@ impl Default for Wakes { fn default() -> Self { Self { earliest: AtomicU64::new(u64::MAX), - starts_from: AtomicU64::new(u64::MAX), landing_waiters: AtomicBool::new(false), newest: AtomicU64::new(0), parked: kio::Lock::default(), @@ -114,7 +106,11 @@ impl Default for Wakes { impl Wakes { /// A frame write left its group presenting up to `latest`: wake the reads it expires. pub(super) fn presented(&self, latest: Timestamp) { - let at = micros(latest.as_micros()); + // A group key presents nothing in time. + let Ok(at) = latest.as_micros() else { + return; + }; + let at = micros(at); if at < self.earliest.load(Ordering::Relaxed) { return; } @@ -135,7 +131,10 @@ impl Wakes { /// Park `waiter` until the edge reaches `reach` plus `budget`. pub(super) fn watch_deadline(&self, reach: Timestamp, budget: Duration, waiter: &kio::Waiter) { - let deadline = micros(reach.as_micros()).saturating_add(micros(budget.as_micros())); + let Ok(reach) = reach.as_micros() else { + return; + }; + let deadline = micros(reach).saturating_add(micros(budget.as_micros())); // No timestamp reaches it. if deadline == u64::MAX { return; @@ -196,34 +195,34 @@ impl Wakes { next } - /// A servable group was inserted at `sequence`: returns whether a landing read may be + /// A servable group was inserted at `key`: returns whether a landing read may be /// parked for it to wake. - pub(super) fn land(&self, sequence: u64) -> bool { - self.newest.fetch_max(sequence, Ordering::Relaxed); + pub(super) fn land(&self, key: u64) -> bool { + self.newest.fetch_max(key, Ordering::Relaxed); // Pairs with the fence in `watch_landing`. fence(Ordering::SeqCst); self.landing_waiters.load(Ordering::Relaxed) } - /// Park `waiter` until a group lands above `sequence`, returning whether one already + /// Park `waiter` until a group lands above `key`, returning whether one already /// has. Needs no track lock: a group landing after it either wakes `waiter` or is seen. /// - /// `false` means no servable group ever landed above `sequence`, so it has no successor + /// `false` means no servable group ever landed above `key`, so it has no successor /// and neither of the budget's clocks can convict it yet. - pub(super) fn watch_landing(&self, sequence: u64, waiter: &kio::Waiter) -> bool { + pub(super) fn watch_landing(&self, key: u64, waiter: &kio::Waiter) -> bool { { let mut parked = self.parked.lock(); - parked.landings.entry(sequence).or_default().register(waiter); + parked.landings.entry(key).or_default().register(waiter); self.landing_waiters.store(true, Ordering::Relaxed); } // Pairs with the fence in `land`. fence(Ordering::SeqCst); - self.newest.load(Ordering::Relaxed) > sequence + self.newest.load(Ordering::Relaxed) > key } - /// Park `waiter`, holding group `sequence` from another copy, until it or a group + /// Park `waiter`, holding group `key` from another copy, until it or a group /// above it lands. - pub(super) fn watch_held(&self, sequence: u64, waiter: &kio::Waiter) { + pub(super) fn watch_held(&self, key: u64, waiter: &kio::Waiter) { let mut parked = self.parked.lock(); let parked = &mut *parked; // Sweep only once the map has doubled, so each sweep is paid for by the keys @@ -233,49 +232,18 @@ impl Wakes { parked.held.retain(|_, list| !list.is_empty()); parked.held_sweep = sweep_at(parked.held.len()); } - parked.held.entry(sequence).or_default().register(waiter); + parked.held.entry(key).or_default().register(waiter); self.landing_waiters.store(true, Ordering::Relaxed); } /// Park until this exact group is cached, or a declared boundary excludes it. - pub(super) fn watch_arrival(&self, sequence: u64, waiter: &kio::Waiter) { + pub(super) fn watch_arrival(&self, key: u64, waiter: &kio::Waiter) { let mut parked = self.parked.lock(); if parked.arrivals.len() >= parked.arrival_sweep { parked.arrivals.retain(|_, list| !list.is_empty()); parked.arrival_sweep = sweep_at(parked.arrivals.len()); } - parked.arrivals.entry(sequence).or_default().register(waiter); - } - - /// Park `waiter` until a group at or above `from` starts. - pub(super) fn watch_start(&self, from: u64, waiter: &kio::Waiter) { - let mut parked = self.parked.lock(); - if parked.starts.len() >= parked.starts_sweep { - parked.starts.retain(|_, list| !list.is_empty()); - parked.starts_sweep = sweep_at(parked.starts.len()); - } - parked.starts.entry(from).or_default().register(waiter); - self.starts_from.fetch_min(from, Ordering::Relaxed); - } - - /// Group `sequence` started: wake the time ends it could move. - pub(super) fn started(&self, sequence: u64) { - if sequence < self.starts_from.load(Ordering::Relaxed) { - return; - } - let mut woken = Vec::new(); - { - let mut parked = self.parked.lock(); - while let Some(entry) = parked.starts.first_entry() - && *entry.key() <= sequence - { - woken.push(entry.remove()); - } - let from = parked.starts.first_key_value().map_or(u64::MAX, |(from, _)| *from); - self.starts_from.store(from, Ordering::Relaxed); - parked.starts_sweep = lower(parked.starts_sweep, parked.starts.len()); - } - woken.iter_mut().for_each(kio::WaiterList::wake); + parked.arrivals.entry(key).or_default().register(waiter); } /// Wake only cache arrivals whose outcome this insertion or boundary changes. @@ -292,11 +260,11 @@ impl Wakes { woken.iter_mut().for_each(kio::WaiterList::wake); } - /// A servable group landed at `sequence`, so it succeeds every read from `below`, the + /// A servable group landed at `key`, so it succeeds every read from `below`, the /// nearest servable group beneath it, up, and continues a held group of its own - /// sequence. Reads below `oldest`, the first group still cached, can no longer expire + /// key. Reads below `oldest`, the first group still cached, can no longer expire /// and are forgotten; held groups are not. - pub(super) fn landed(&self, below: Option, sequence: u64, oldest: u64) { + pub(super) fn landed(&self, below: Option, key: u64, oldest: u64) { let mut woken = Vec::new(); { let mut parked = self.parked.lock(); @@ -307,10 +275,10 @@ impl Wakes { entry.remove(); } let from = below.unwrap_or(0); - while let Some(at) = parked.landings.range(from..sequence).next().map(|(at, _)| *at) { + while let Some(at) = parked.landings.range(from..key).next().map(|(at, _)| *at) { woken.extend(parked.landings.remove(&at)); } - while let Some(at) = parked.held.range(from..=sequence).next().map(|(at, _)| *at) { + while let Some(at) = parked.held.range(from..=key).next().map(|(at, _)| *at) { woken.extend(parked.held.remove(&at)); } parked.held_sweep = lower(parked.held_sweep, parked.held.len()); @@ -327,7 +295,6 @@ impl Wakes { let parked = { let mut parked = self.parked.lock(); self.earliest.store(u64::MAX, Ordering::Relaxed); - self.starts_from.store(u64::MAX, Ordering::Relaxed); self.landing_waiters.store(false, Ordering::Relaxed); std::mem::take(&mut *parked) }; @@ -335,8 +302,7 @@ impl Wakes { let lists = lists .chain(parked.held.into_values()) .chain(parked.wall.into_values()) - .chain(parked.arrivals.into_values()) - .chain(parked.starts.into_values()); + .chain(parked.arrivals.into_values()); for mut list in lists { list.wake(); } @@ -385,10 +351,7 @@ mod test { assert!(wakes.watch_landing(12, &waiter), "a group landed above it"); assert!(!wakes.watch_landing(13, &waiter), "nothing landed above it"); assert!(wakes.land(u64::MAX)); - assert!( - wakes.watch_landing(u64::MAX - 1, &waiter), - "the last sequence is above it" - ); + assert!(wakes.watch_landing(u64::MAX - 1, &waiter), "the last key is above it"); wakes.wake_all(); assert!(!wakes.landing_waiters.load(Ordering::Relaxed)); } @@ -399,11 +362,11 @@ mod test { fn held_groups_do_not_accumulate_across_route_switches() { let wakes = Wakes::default(); let mut reader = None; - for sequence in 0..100 { + for key in 0..100 { // The last switch's reader gave its group up. drop(reader.take()); let waiter = parked(); - wakes.watch_held(sequence, &waiter); + wakes.watch_held(key, &waiter); reader = Some(waiter); } assert!( diff --git a/rs/moq-net/src/model/frame.rs b/rs/moq-net/src/model/frame.rs index 3f0c7f5399b..fb23bc2a4d2 100644 --- a/rs/moq-net/src/model/frame.rs +++ b/rs/moq-net/src/model/frame.rs @@ -29,15 +29,12 @@ use crate::{Error, IntoBytes, Result, Timestamp, stats}; pub struct Info { /// Total payload size in bytes. Declared up front so consumers can preallocate. pub size: u64, - /// Presentation timestamp, or `None` on an untimed track. It must match the track - /// ([`Error::TimestampMismatch`] otherwise). + /// Presentation timestamp, or on a scale-less track [`Timestamp::unscaled`] at its + /// group's key. It must match the track ([`Error::TimestampMismatch`] otherwise). /// /// [`group::Producer::create_frame`] converts it into the parent track's /// timescale, so the scale you build it with doesn't have to match the track. - /// No receiver fills one in, so an untimed track stays untimed across hops, except - /// over moq-lite 05 and 06, which can't mark a track untimed and carry the encoder's - /// send time instead. - pub timestamp: Option, + pub timestamp: Timestamp, } /// A completed frame: an optional timestamp, its full, contiguous payload, and when it @@ -49,8 +46,9 @@ pub struct Info { /// [`Consumer::poll_read_chunk`] hands out each segment without a copy. #[derive(Clone, Debug)] pub struct Frame { - /// Presentation timestamp, at the parent track's timescale, or `None` when untimed. - pub timestamp: Option, + /// Presentation timestamp, at the parent track's timescale, or on a scale-less track + /// [`Timestamp::unscaled`] at its group's key. + pub timestamp: Timestamp, /// The full frame payload. pub payload: Bytes, // Stamped as the frame enters a group's cache; see [`Self::arrived`]. @@ -58,10 +56,11 @@ pub struct Frame { } impl Frame { - /// A frame of `payload` presented at `timestamp`, or `None` on an untimed track. - pub fn new(timestamp: impl Into>, payload: impl IntoBytes) -> Self { + /// A frame of `payload` presented at `timestamp`, or on a scale-less track at + /// [`Timestamp::unscaled`] of its group's key. + pub fn new(timestamp: Timestamp, payload: impl IntoBytes) -> Self { Self { - timestamp: timestamp.into(), + timestamp, payload: payload.into_bytes(), // Stamped when written to a group, so building one never reads the clock. arrived: Instant::ZERO, @@ -246,7 +245,7 @@ impl Buf for Payload { /// A completed frame as its group stores it. #[derive(Clone, Debug)] pub(crate) struct Cached { - pub timestamp: Option, + pub timestamp: Timestamp, pub payload: Payload, // When the frame's last byte landed in this node's cache; see [`Frame::arrived`]. pub arrived: Instant, @@ -1458,10 +1457,7 @@ impl Drop for Producer { && let Some(group) = self.group.as_mut() { if !group.is_aborted() && !self.cancel_on_drop { - tracing::warn!( - group = group.sequence, - "frame::Producer dropped before writing all bytes" - ); + tracing::warn!(group = group.start, "frame::Producer dropped before writing all bytes"); } group.frame_abort(if self.cancel_on_drop { Error::Cancel diff --git a/rs/moq-net/src/model/front.rs b/rs/moq-net/src/model/front.rs index 37611b052a8..aa69378c2a8 100644 --- a/rs/moq-net/src/model/front.rs +++ b/rs/moq-net/src/model/front.rs @@ -11,7 +11,7 @@ //! //! The driver only offers a front routes that serve its broadcast: those of the //! instance it first resolved, at its newest epoch, whose tracks resume where they -//! stopped since the instance promises one continuous sequence space. An instance a +//! stopped since the instance promises one continuous key space. An instance a //! session minted, for a wire that carries none, promises nothing past its route: //! that route can answer a re-request with another stream, so a track whose copy //! fails there ends with the error instead of re-splicing. Another @@ -1798,10 +1798,10 @@ mod tests { } } - /// Every sequence of events up to a small depth, over a small alphabet, + /// Every key of events up to a small depth, over a small alphabet, /// checked against the invariants the driver relies on. #[test] - fn bounded_sequences_hold_the_invariants() { + fn bounded_keys_hold_the_invariants() { let t0 = Instant::now(); let alphabet: Vec = vec![ Event::Selected { @@ -1870,7 +1870,7 @@ mod tests { Event::Held, ]; - fn walk(front: &Front, alphabet: &[Event], depth: usize, sequences: &mut usize) { + fn walk(front: &Front, alphabet: &[Event], depth: usize, keys: &mut usize) { // Rendered only when an event needs it: formatting dominates the walk. let mut before = None; for event in alphabet { @@ -1883,7 +1883,7 @@ mod tests { } let mut next = front.clone(); let actions = next.step(event.clone()); - *sequences += 1; + *keys += 1; // Route-wide events skip the index update, so they must leave parked tracks alone. assert_index_matches_scan(&next); @@ -1944,14 +1944,14 @@ mod tests { assert!(next.idle() && !front.retiring, "retired a front in use: {next:?}"); } if depth > 1 { - walk(&next, alphabet, depth - 1, sequences); + walk(&next, alphabet, depth - 1, keys); } } } - let mut sequences = 0; - walk(&Front::new(LINGER), &alphabet, 5, &mut sequences); - assert!(sequences > 100_000); + let mut keys = 0; + walk(&Front::new(LINGER), &alphabet, 5, &mut keys); + assert!(keys > 100_000); } /// The newcomer refused the track, so it stays on the source the newcomer replaced. diff --git a/rs/moq-net/src/model/group.rs b/rs/moq-net/src/model/group.rs index ebc4cf30972..7331fdb0fda 100644 --- a/rs/moq-net/src/model/group.rs +++ b/rs/moq-net/src/model/group.rs @@ -11,7 +11,7 @@ //! back but never in the middle: [Producer::start_at] starts it later, so a handle can //! carry the tail of a group whose leading frames came from somewhere else, and //! [Producer::finish] ends it wherever writing stopped. [Consumer::with_range] bounds a reader to a sub-range the same way [`track::Subscriber`] -//! bounds group sequences. +//! bounds group keys. //! //! A group has no size cap. Each copy holds at most its newest 32 MiB, less //! under the cache pool's memory pressure, and sheds whole frames from its front to stay @@ -72,14 +72,17 @@ pub(super) const CACHE_OVERHEAD: u64 = (kio::Producer::::HEAP + size_of::() + kio::log::Writer::::HEAP) as u64; -/// A group contains a sequence number because they can arrive out of order. +/// A group's identity: where it starts, in its track's timescale. /// -/// You can use [track::Producer::append_group] if you just want to +1 the sequence number. +/// A group's start is its key on the track: starts strictly increase along a track, so no +/// two groups share one, and a later group never starts earlier. A group a local producer +/// writes starts at its first frame's timestamp; one received over a wire that numbers +/// groups starts at that wire's group ID, verbatim. On a scale-less track (no +/// [`track::Info::timescale`]) a start is a bare number in no known unit. #[derive(Clone, Copy, Debug, Hash, Eq, PartialEq, Ord, PartialOrd)] pub struct Info { - /// Per-track sequence number used to detect ordering and gaps. Higher numbers - /// supersede lower ones; consumers may skip late arrivals. - pub sequence: u64, + /// Where the group starts, in its track's timescale. + pub start: u64, } impl Info { @@ -104,39 +107,33 @@ impl Info { } impl From for Info { - fn from(sequence: usize) -> Self { - Self { - sequence: sequence as u64, - } + fn from(start: usize) -> Self { + Self { start: start as u64 } } } impl From for Info { - fn from(sequence: u64) -> Self { - Self { sequence } + fn from(start: u64) -> Self { + Self { start } } } impl From for Info { - fn from(sequence: u32) -> Self { - Self { - sequence: sequence as u64, - } + fn from(start: u32) -> Self { + Self { start: start as u64 } } } impl From for Info { - fn from(sequence: u16) -> Self { - Self { - sequence: sequence as u64, - } + fn from(start: u16) -> Self { + Self { start: start as u64 } } } /// The in-flight (tail) frame being written. At most one exists at a time, since a /// group is a single ordered stream. pub(super) struct Partial { - timestamp: Option, + timestamp: Timestamp, fields: Option>, buf: FrameBuf, // How much of `buf` has been charged to the cache so far. @@ -314,8 +311,9 @@ pub(super) enum Outcome { /// Frames may still be written. #[default] Open, - /// Finished with this many frames. - Finished(usize), + /// Finished with this many frames, and where it ends in its track's timescale when its + /// producer said it is whole: `None` when it only closed (see [`Producer::close`]). + Finished(usize, Option), /// Finished with this many frames, then aborted to free its cache: a reader that took /// every frame got the whole group, and one short of it gets the error. Released(usize, Error), @@ -327,7 +325,7 @@ impl Outcome { /// The total number of frames, once finished. pub(super) fn frames(&self) -> Option { match self { - Self::Finished(frames) | Self::Released(frames, _) => Some(*frames), + Self::Finished(frames, _) | Self::Released(frames, _) => Some(*frames), Self::Open | Self::Aborted(_) => None, } } @@ -336,7 +334,7 @@ impl Outcome { pub(super) fn error(&self) -> Option<&Error> { match self { Self::Released(_, err) | Self::Aborted(err) => Some(err), - Self::Open | Self::Finished(_) => None, + Self::Open | Self::Finished(..) => None, } } } @@ -800,7 +798,7 @@ enum Source { Inflate { job: Inflate, local: usize, - timestamp: Option, + timestamp: Timestamp, fields: Option>, }, } @@ -1267,7 +1265,7 @@ impl GroupState { } match &self.outcome { Outcome::Released(total, err) if index < *total => Poll::Ready(Err(err.clone())), - Outcome::Finished(_) | Outcome::Released(..) => Poll::Ready(Ok(())), + Outcome::Finished(..) | Outcome::Released(..) => Poll::Ready(Ok(())), Outcome::Aborted(err) => Poll::Ready(Err(err.clone())), Outcome::Open => { // Waiting on frames the writer has yet to commit, wherever the reader skipped @@ -1309,7 +1307,7 @@ impl GroupState { /// the cache of, or the abort error. fn poll_outcome(&self) -> Poll> { match &self.outcome { - Outcome::Finished(_) | Outcome::Released(..) => Poll::Ready(Ok(())), + Outcome::Finished(..) | Outcome::Released(..) => Poll::Ready(Ok(())), Outcome::Aborted(err) => Poll::Ready(Err(err.clone())), Outcome::Open => Poll::Pending, } @@ -1330,8 +1328,8 @@ impl GroupState { /// Check frame 0's `timestamp` against the track's `neighbors`, returning how to record /// it. Anything but a timed frame 0 checked under an open local track is unchecked. - fn check_start(&self, neighbors: Option<&track::Neighbors>, timestamp: Option) -> Result { - match (neighbors, timestamp) { + fn check_start(&self, neighbors: Option<&track::Neighbors>, timestamp: Timestamp) -> Result { + match (neighbors, timestamp.timed()) { (Some(neighbors), Some(start)) if self.next_index == 0 => neighbors.admit(start), _ => Ok(self.origin), } @@ -1350,11 +1348,11 @@ impl GroupState { /// Record a frame's place in presentation time: the first frame starts the group, /// and later frames can extend a timed group's end. Every frame matches its track /// ([`track::Info::rescale`]), so a group never mixes the two. - fn stamp(&mut self, timestamp: Option) { - self.timeline = match (self.timeline, timestamp) { + fn stamp(&mut self, timestamp: Timestamp) { + self.timeline = match (self.timeline, timestamp.timed()) { (Timeline::Timed { start, latest }, Some(timestamp)) => Timeline::Timed { start, - latest: latest.max(timestamp), + latest: latest.later(timestamp), }, (_, Some(start)) => Timeline::Timed { start, latest: start }, (_, None) => Timeline::Untimed, @@ -1656,10 +1654,7 @@ impl Drop for Producer { if !matches!(state.outcome, Outcome::Open) { return; } - tracing::warn!( - sequence = self.sequence, - "group::Producer dropped without finish() or abort()" - ); + tracing::warn!(key = self.start, "group::Producer dropped without finish() or abort()"); self.handle.commit_abort(state, Error::Dropped); } } @@ -1894,10 +1889,11 @@ impl Producer { /// If you want to write multiple chunks, use [Self::create_frame] to get a frame producer. /// But an upfront size is required. /// - /// `timestamp` is converted into the parent track's timescale. Pass `None` on an - /// untimed track; a frame that doesn't match its track is [`Error::TimestampMismatch`]. - pub fn write_frame(&mut self, timestamp: impl Into>, data: B) -> Result<()> { - let timestamp = self.handle.track.rescale(timestamp.into())?; + /// `timestamp` is converted into the parent track's timescale. On a scale-less track it + /// is [`Timestamp::unscaled`] at this group's start; a frame that doesn't match its track + /// is [`Error::TimestampMismatch`]. + pub fn write_frame(&mut self, timestamp: Timestamp, data: B) -> Result<()> { + let timestamp = self.handle.track.rescale_in(timestamp, self.handle.info.start)?; let payload = data.into_bytes(); if payload.len() as u64 > MAX_CACHE_BYTES { return Err(Error::FrameTooLarge); @@ -1905,9 +1901,7 @@ impl Producer { let producer = self.first_frame_track(); let track = producer.as_ref().map(|track| track.read()); - let neighbors = track - .as_ref() - .and_then(|track| track.neighbors(self.handle.info.sequence)); + let neighbors = track.as_ref().and_then(|track| track.neighbors(self.handle.info.start)); let mut state = modify(&self.handle.state)?; if state.outcome.frames().is_some() { return Err(Error::Closed); @@ -1951,7 +1945,7 @@ impl Producer { drop(state); // The group's lock is released first: indexing reads it back through its slot. if let Some(track) = &track { - track.first_frame(self.handle.info.sequence, bound); + track.first_frame(self.handle.info.start, bound); } drop(track); self.handle.start(producer); @@ -1960,7 +1954,7 @@ impl Producer { // eviction debt if enough has been written since the track last paid. self.handle.cache.settle(now); // An untimed frame presents nothing, so it can expire no read. - if let Some(timestamp) = timestamp { + if let Some(timestamp) = timestamp.timed() { self.handle.cache.wakes().presented(timestamp); } @@ -1982,7 +1976,7 @@ impl Producer { #[cfg(any(test, feature = "fuzz"))] pub(crate) fn write_frames(&mut self, frames: &mut frame::Buffer) -> Result<()> { for frame in frames.filled() { - self.handle.track.rescale(frame.timestamp)?; + self.handle.track.rescale_in(frame.timestamp, self.handle.info.start)?; if frame.payload.len() as u64 > MAX_CACHE_BYTES { return Err(Error::FrameTooLarge); } @@ -1992,9 +1986,7 @@ impl Producer { let bytes: u64 = frames.filled().iter().map(|frame| frame.payload.len() as u64).sum(); let producer = self.first_frame_track(); let track = producer.as_ref().map(|track| track.read()); - let neighbors = track - .as_ref() - .and_then(|track| track.neighbors(self.handle.info.sequence)); + let neighbors = track.as_ref().and_then(|track| track.neighbors(self.handle.info.start)); let mut state = modify(&self.handle.state)?; if state.outcome.frames().is_some() { return Err(Error::Closed); @@ -2011,7 +2003,7 @@ impl Producer { neighbors.as_ref(), self.handle .track - .rescale(timestamp) + .rescale_in(timestamp, self.handle.info.start) .expect("timestamp scale checked above"), )?, None => state.origin, @@ -2032,7 +2024,7 @@ impl Producer { // The last frame's tick, reused below so settling does not re-read the clock. let mut now = None; - let mut latest = None; + let mut latest: Option = None; let mut deflated = deflated.into_iter(); let arrived = super::clock::now(); // Validation is done, so the batch is taken by value: a conflict below fails the group, @@ -2045,7 +2037,7 @@ impl Producer { let timestamp = self .handle .track - .rescale(frame.timestamp) + .rescale_in(frame.timestamp, self.handle.info.start) .expect("timestamp scale checked above"); let (payload, inflated) = match state.flate.is_some() { true => ( @@ -2062,7 +2054,10 @@ impl Producer { }, inflated, ); - latest = latest.max(timestamp); + latest = match (latest, timestamp.timed()) { + (Some(latest), Some(timestamp)) => Some(latest.later(timestamp)), + (latest, timestamp) => latest.or(timestamp), + }; if !canonical { now = state.commit(entry); continue; @@ -2087,7 +2082,7 @@ impl Producer { drop(state); // The group's lock is released first: indexing reads it back through its slot. if let Some(track) = &track { - track.first_frame(self.handle.info.sequence, bound); + track.first_frame(self.handle.info.start, bound); } drop(track); if count > 0 { @@ -2211,7 +2206,7 @@ impl Producer { /// size is only a promise until they arrive. fn open_frame(&mut self, header: frame::Header) -> Result<(frame::Info, FrameBuf, Option)> { let frame = header.info; - let timestamp = self.handle.track.rescale(frame.timestamp)?; + let timestamp = self.handle.track.rescale_in(frame.timestamp, self.handle.info.start)?; if frame.size > MAX_CACHE_BYTES || header.inflated.is_some_and(|inflated| inflated > MAX_CACHE_BYTES) { return Err(Error::FrameTooLarge); } @@ -2224,9 +2219,7 @@ impl Producer { let producer = self.first_frame_track(); let track = producer.as_ref().map(|track| track.read()); - let neighbors = track - .as_ref() - .and_then(|track| track.neighbors(self.handle.info.sequence)); + let neighbors = track.as_ref().and_then(|track| track.neighbors(self.handle.info.start)); let mut state = modify(&self.handle.state)?; if state.outcome.frames().is_some() { return Err(Error::Closed); @@ -2296,7 +2289,7 @@ impl Producer { drop(state); // The group's lock is released first: indexing reads it back through its slot. if let Some(track) = &track { - track.first_frame(self.handle.info.sequence, bound); + track.first_frame(self.handle.info.start, bound); } drop(track); self.handle.start(producer); @@ -2305,7 +2298,7 @@ impl Producer { // eviction debt if enough has been written since the track last paid. self.handle.cache.settle(now); // An untimed frame presents nothing, so it can expire no read. - if let Some(timestamp) = timestamp { + if let Some(timestamp) = timestamp.timed() { self.handle.cache.wakes().presented(timestamp); } @@ -2406,7 +2399,7 @@ impl Producer { /// Raise the track's largest to the last committed frame. fn advance_largest(&self, state: &GroupState) { self.handle.cache.advance_largest(track::Position { - group: self.handle.info.sequence, + group: self.handle.info.start, frame: (state.committed() - 1) as u64, }); } @@ -2417,15 +2410,37 @@ impl Producer { let _ = self.handle.close_aborted(err); } - /// Mark the group as complete, returning the [`Finished`] group: no more frames can be - /// written, but it can still be read or aborted. + /// End the group whole at `end`, the exclusive end of its presentation in its track's + /// timescale, returning the [`Finished`] group: no more frames can be written, but it can + /// still be read or aborted. + /// + /// A finished group's own end bounds how long it stays fresh, so the last group of a + /// paused or finished track ages out without a successor (see + /// [`track::Info::max_age`]). An `end` at or before the group's start is + /// [`Error::TimestampMismatch`]. Fails with the abort error if the group was already + /// aborted. + pub fn finish(self, end: u64) -> Result { + if end <= self.start { + return Err(Error::TimestampMismatch); + } + self.end(Some(end)) + } + + /// End the group without saying where it ends, returning the [`Finished`] group: every + /// frame written still reaches its readers, but they can't tell the group is whole. /// - /// Fails with the abort error if the group was already aborted. - pub fn finish(self) -> Result { + /// What a producer that doesn't know its group's end calls, and how a relay ends a + /// group that expired or arrived partial. Fails with the abort error if the group was + /// already aborted. + pub fn close(self) -> Result { + self.end(None) + } + + fn end(self, end: Option) -> Result { let mut state = modify(&self.handle.state)?; debug_assert!(state.partial.is_none(), "a frame is already open"); let fin = state.next_index; - state.outcome = Outcome::Finished(fin); + state.outcome = Outcome::Finished(fin, end); // No frame follows to compress. let original_fin = state.range_end.is_none().then_some(fin); if let Some(flate) = &mut state.flate { @@ -2437,7 +2452,7 @@ impl Producer { // A finished group can no longer raise the track's largest, which may end its stream. // A copy a transport bound cut short finished only its own range. if original_fin.is_some() { - self.handle.cache.finish_largest(self.sequence); + self.handle.cache.finish_largest(self.start); } Ok(Finished { handle: self.handle.clone(), @@ -2729,7 +2744,7 @@ impl Handle { pub(super) fn register_content(&self, contents: &Contents) { let state = self.state.read(); if let Some(content) = &state.content { - contents.insert(self.sequence, content, &state.charge); + contents.insert(self.start, content, &state.charge); } } @@ -2839,12 +2854,15 @@ impl Handle { } state.offset = head.start; state.cache += payload; - let start = entries.iter().find_map(|entry| entry.frame.timestamp); - let head_latest = entries.iter().filter_map(|entry| entry.frame.timestamp).max(); + let start = entries.iter().find_map(|entry| entry.frame.timestamp.timed()); + let head_latest = entries + .iter() + .filter_map(|entry| entry.frame.timestamp.timed()) + .reduce(Timestamp::later); state.timeline = match (state.timeline, start) { (Timeline::Timed { latest, .. }, Some(start)) => Timeline::Timed { start, - latest: latest.max(head_latest.unwrap_or(start)), + latest: latest.later(head_latest.unwrap_or(start)), }, (Timeline::Empty, Some(start)) => Timeline::Timed { start, @@ -2938,7 +2956,7 @@ impl Handle { let fall = match guard.outcome { // A finished group's outcome is settled: an abort only releases its cache, so it // neither failed nor was cut. - Outcome::Finished(_) => Fall::Released, + Outcome::Finished(..) => Fall::Released, _ if policy => Fall::Released, // A reset carrying a policy code is the upstream's cache policy, a cancel is a // reader no longer wanting the group, and a fill refused as unfetchable abandons @@ -2965,7 +2983,7 @@ impl Handle { } let finished = guard.outcome.frames().is_some(); guard.outcome = match guard.outcome { - Outcome::Finished(frames) => Outcome::Released(frames, err), + Outcome::Finished(frames, _) => Outcome::Released(frames, err), _ => Outcome::Aborted(err), }; // After the outcome exists, so a scan reading the mirror never hides a live group. @@ -2982,7 +3000,7 @@ impl Handle { std::sync::atomic::fence(Ordering::SeqCst); self.alive.carries_drop.load(Ordering::Relaxed) }; - self.cache.aborted(self.sequence, cut || carried); + self.cache.aborted(self.start, cut || carried); } /// Mark that a final drop of this group waits on this copy, so a failure before it @@ -3066,6 +3084,15 @@ impl Handle { self.state.read().outcome.frames().is_some() } + /// Where the group ends in its track's timescale, once its producer finished it whole + /// (see [`Producer::finish`]). + pub(crate) fn end(&self) -> Option { + match self.state.read().outcome { + Outcome::Finished(_, end) => end, + _ => None, + } + } + /// Whether the group was finished: it holds every frame it will ever have. pub(crate) fn is_finished(&self) -> bool { { @@ -3137,17 +3164,10 @@ impl Handle { }; let answers = track.read().reveal(self); if answers && let Ok(mut state) = track.write() { - state.shown(self.sequence); + state.shown(self.start); } } - /// Whether this copy may yet present the group's frame 0: it holds it, and nothing - /// opened or ended it. - pub(super) fn may_start(&self) -> bool { - let state = self.state.read(); - state.offset == 0 && !state.started() - } - /// Where the group starts in presentation time: its first frame's timestamp, or /// `None` while no frame has been opened or when that frame is untimed. /// @@ -3166,16 +3186,6 @@ impl Handle { } } - /// Where the group starts in presentation time, accepted unverified or not: what a time - /// range counts on a track whose history a local handler serves, where no remote - /// publisher wrote it (see [`Self::timestamp`]). - pub(super) fn first_timestamp(&self) -> Option { - match self.state.read().timeline { - Timeline::Timed { start, .. } => Some(start), - _ => None, - } - } - /// The start this group lends its neighbors' rewind check: its first frame's timestamp, /// once frame 0 was checked to fit between theirs. pub(super) fn bound(&self) -> Option { @@ -3345,7 +3355,7 @@ impl Handle { // Sessions observe a group while its sole writer belongs to a frame. pub(crate) fn demand(&self) -> Demand { Demand { - sequence: self.info.sequence, + key: self.info.start, state: self.state.weak(), prev: None, } @@ -3386,7 +3396,7 @@ fn settle(poll: Poll, kio::Ref<'_, GroupState>>>) /// its own, so its first [`poll_wanted_changed`](Self::poll_wanted_changed) reports the /// current value. pub struct Demand { - sequence: u64, + key: u64, state: kio::ProducerWeak, // The value `poll_wanted_changed` last returned, or `None` before it returned any. prev: Option, @@ -3395,7 +3405,7 @@ pub struct Demand { impl Clone for Demand { fn clone(&self) -> Self { Self { - sequence: self.sequence, + key: self.key, state: self.state.clone(), prev: None, } @@ -3403,9 +3413,9 @@ impl Clone for Demand { } impl Demand { - /// The group sequence this handle watches. - pub fn sequence(&self) -> u64 { - self.sequence + /// The start of the group this handle watches. + pub fn start(&self) -> u64 { + self.key } /// Whether the group is open and has readers right now. @@ -4312,7 +4322,7 @@ impl Consumer { /// end removes the cap. Raising the cap re-offers frames that are still cached. pub fn with_range(mut self, frames: impl RangeBounds) -> Self { Idle::disarm(&mut self.idle); - let (start, end) = super::subscription::sequence_bounds(frames); + let (start, end) = super::subscription::key_bounds(frames); self.cursor.start_at(start); self.cursor.end = end.map(|end| usize::try_from(end).unwrap_or(usize::MAX)); self @@ -4441,6 +4451,15 @@ impl Consumer { state.outcome.frames().unwrap_or(state.next_index) } + /// Where the group ends in its track's timescale, once its producer finished it whole + /// with [`Producer::finish`]: `None` while it is open, or when it only closed. + pub fn end(&self) -> Option { + match self.cursor.state.read().outcome { + Outcome::Finished(_, end) => end, + _ => None, + } + } + /// The next frame, as a consumer for chunked reading. /// /// Returns None if the group is finished and the index is out of range, or the cursor @@ -4475,7 +4494,7 @@ impl Consumer { Poll::Ready(Ok(None)) => self.release_if_finished(), Poll::Ready(Err(Error::Evicted)) => { if let Some(track) = self.cursor.cache.track() { - track::TrackState::shed_read(track.read(), self.info.sequence); + track::TrackState::shed_read(track.read(), self.info.start); } } _ => {} @@ -5292,6 +5311,15 @@ impl Fetch { } } +#[cfg(test)] +impl Producer { + /// This group's key read as milliseconds: a first frame there starts the group where its + /// key says, so test groups start in key order on a millisecond track. + pub(crate) fn key_time(&self) -> Timestamp { + Timestamp::from_millis(self.start).expect("a test key fits a timestamp") + } +} + #[cfg(test)] mod test { use super::*; @@ -5304,7 +5332,7 @@ mod test { #[test] fn reordered_frames_keep_the_highest_group_pts() { for mode in 0..3 { - let mut group = Info { sequence: 0 }.produce(); + let mut group = Info { start: 0 }.produce(); let timestamps = [0, 80, 40].map(|ms| Timestamp::from_millis(ms).unwrap()); if mode == 1 { let mut frames = frame::Buffer::<3>::new(); @@ -5317,19 +5345,14 @@ mod test { if mode == 0 { group.write_frame(timestamp, Bytes::new()).unwrap(); } else { - let frame = group - .create_frame(frame::Info { - size: 0, - timestamp: Some(timestamp), - }) - .unwrap(); + let frame = group.create_frame(frame::Info { size: 0, timestamp }).unwrap(); group = frame.finish().unwrap(); } } } assert_eq!(group.handle.timestamp(), Some(timestamps[0]), "mode {mode}"); assert_eq!(group.handle.latest(), Some(timestamps[1]), "mode {mode}"); - group.finish().unwrap(); + group.close().unwrap(); } } @@ -5346,7 +5369,7 @@ mod test { compressed: false, }; for tail in [None, Some(20), Some(120)] { - let mut group = Info { sequence: 0 }.produce(); + let mut group = Info { start: 0 }.produce(); group.start_at(3).unwrap(); if let Some(ms) = tail { group @@ -5363,14 +5386,14 @@ mod test { Timestamp::from_millis(tail.unwrap_or(0).max(80)).unwrap(), "tail {tail:?}" ); - group.finish().unwrap(); + group.close().unwrap(); } } #[test] fn a_zero_group_pts_keeps_its_track_scale_and_untimed_stays_unstamped() { let mut timed = Producer::new( - Info { sequence: 0 }, + Info { start: 0 }, track::Info::default().with_timescale(Timescale::MICRO), Default::default(), ); @@ -5378,12 +5401,14 @@ mod test { .write_frame(Timestamp::from_micros(0).unwrap(), Bytes::new()) .unwrap(); assert_eq!(timed.handle.latest(), Some(Timestamp::from_micros(0).unwrap())); - timed.finish().unwrap(); - let mut untimed = Info { sequence: 0 }.produce_untimed(); + timed.close().unwrap(); + let mut untimed = Info { start: 0 }.produce_untimed(); assert_eq!(untimed.handle.latest(), None); - untimed.write_frame(None, Bytes::new()).unwrap(); + untimed + .write_frame(crate::Timestamp::unscaled(untimed.start), Bytes::new()) + .unwrap(); assert_eq!(untimed.handle.latest(), None); - untimed.finish().unwrap(); + untimed.close().unwrap(); } /// Fields sit beside the frames only from the first fielded one, so every read path @@ -5394,14 +5419,16 @@ mod test { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let mut group = Producer::new(Info::from(0u64), track::Info::default(), cache); - group.write_frame(None, Bytes::from_static(b"p")).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from_static(b"p")) + .unwrap(); let plain = pool.used(); // Ten fielded frames span two prefetch batches, then plain frames follow. for id in 1..=10u64 { let header = frame::Header { info: frame::Info { size: 1, - timestamp: None, + timestamp: Timestamp::unscaled(0), }, fields: Some(Arc::new(super::super::object::Fields { id: id * 10, @@ -5415,9 +5442,11 @@ mod test { } assert!(pool.used() > plain + 10, "the metadata table is charged"); for _ in 0..Prefetch::CAP { - group.write_frame(None, Bytes::from_static(b"p")).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from_static(b"p")) + .unwrap(); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let expected = |index: u64| (1..=10).contains(&index).then_some(index * 10); let mut objects = group.consume(); @@ -5463,7 +5492,9 @@ mod test { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let mut group = Producer::new(Info::from(0u64), track::Info::default(), cache); - group.write_frame(None, Bytes::from_static(b"payload")).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from_static(b"payload")) + .unwrap(); let fields = super::super::object::Fields { id: 4, properties: Default::default(), @@ -5471,7 +5502,7 @@ mod test { let charged = size_of::() as u64 + fields.heap(); group.set_end_object(fields.clone()).unwrap(); assert!(matches!(group.set_end_object(fields), Err(Error::MalformedTrack))); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let consumer = group.consume(); group.abort(Error::Cancel).unwrap(); assert_eq!(consumer.end_object().unwrap().id, 4); @@ -5484,7 +5515,7 @@ mod test { #[cfg(feature = "async")] fn a_leading_gap_may_reach_below_where_the_copy_starts() { for (gap, fetchable) in [(Some(6), true), (Some(4), true), (Some(3), false), (None, false)] { - let mut group = Info { sequence: 0 }.produce(); + let mut group = Info { start: 0 }.produce(); group.start_at(2).unwrap(); group .set_wire(Wire { @@ -5505,9 +5536,9 @@ mod test { #[test] fn demand_watches_readers_without_keeping_the_group_alive() { let waiter = kio::Waiter::noop(); - let producer = Info { sequence: 7 }.produce(); + let producer = Info { start: 7 }.produce(); let mut demand = producer.demand(); - assert_eq!(demand.sequence(), 7); + assert_eq!(demand.start(), 7); assert!(!demand.wanted()); assert!(matches!(demand.poll_wanted_changed(&waiter), Poll::Ready(Ok(false)))); assert!(demand.poll_wanted_changed(&waiter).is_pending()); @@ -5555,7 +5586,7 @@ mod test { .max(kio::log::Writer::::HEAP); let before = pool.used(); match mode { - 0 => group.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(), + 0 => group.write_frame(group.key_time(), bytes::Bytes::new()).unwrap(), 1 => { let mut buffer = frame::Buffer::<1>::new(); buffer.push(Frame::new(Timestamp::ZERO, bytes::Bytes::new())).unwrap(); @@ -5565,7 +5596,7 @@ mod test { let frame = group .create_frame(frame::Info { size: 0, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); group = frame.finish().unwrap(); @@ -5590,14 +5621,14 @@ mod test { #[test] #[cfg(feature = "async")] fn basic_frame_reading() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .write_frame(Timestamp::ZERO, Bytes::from_static(b"frame0")) .unwrap(); producer .write_frame(Timestamp::ZERO, Bytes::from_static(b"frame1")) .unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); let f0 = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap(); @@ -5611,14 +5642,14 @@ mod test { #[test] #[cfg(feature = "async")] fn read_frame_preserves_timestamp() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); let timestamp = Timestamp::from_micros(20_000).unwrap(); producer.write_frame(timestamp, Bytes::from_static(b"hello")).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); let frame = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(frame.timestamp.unwrap().as_micros(), 20_000); + assert_eq!(frame.timestamp.as_micros().unwrap(), 20_000); assert_eq!(frame.payload, Bytes::from_static(b"hello")); } @@ -5627,15 +5658,17 @@ mod test { #[test] #[cfg(feature = "async")] fn an_untimed_frame_still_starts_the_group() { - let mut producer = Info { sequence: 0 }.produce_untimed(); + let mut producer = Info { start: 0 }.produce_untimed(); let waiter = kio::Waiter::noop(); assert!( producer.handle.poll_started(&waiter).is_pending(), "nothing presented yet" ); - producer.write_frame(None, Bytes::from_static(b"hello")).unwrap(); - let producer = producer.finish().unwrap(); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), Bytes::from_static(b"hello")) + .unwrap(); + let producer = producer.close().unwrap(); assert!(producer.handle.poll_started(&waiter).is_ready()); assert_eq!(producer.handle.timestamp(), None); @@ -5643,51 +5676,53 @@ mod test { let mut consumer = producer.consume(); let frame = consumer.read_frame().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(frame.timestamp, None); + assert_eq!(frame.timestamp, Timestamp::unscaled(0)); assert_eq!(frame.payload, Bytes::from_static(b"hello")); } /// A track is all timed or all untimed, so a frame that doesn't match it is refused. #[test] fn a_mismatched_frame_is_refused() { + let time = |timed| match timed { + true => Timestamp::ZERO, + false => Timestamp::unscaled(0), + }; for timed in [false, true] { let mut group = if timed { - Info { sequence: 0 }.produce() + Info { start: 0 }.produce() } else { - Info { sequence: 0 }.produce_untimed() + Info { start: 0 }.produce_untimed() }; let mut frames = frame::Buffer::<2>::new(); - frames - .push(Frame::new(timed.then_some(Timestamp::ZERO), Bytes::from_static(b"a"))) - .unwrap(); - frames - .push(Frame::new( - (!timed).then_some(Timestamp::ZERO), - Bytes::from_static(b"b"), - )) - .unwrap(); + frames.push(Frame::new(time(timed), Bytes::from_static(b"a"))).unwrap(); + frames.push(Frame::new(time(!timed), Bytes::from_static(b"b"))).unwrap(); assert!(matches!(group.write_frames(&mut frames), Err(Error::TimestampMismatch))); assert_eq!(frames.len(), 2, "a refused tail leaves the whole batch intact"); assert_eq!(group.frame_count(), 0, "a refused tail publishes no prefix"); assert!(!group.is_aborted(), "a bad timestamp does not fail the group"); - group.finish().unwrap(); + group.close().unwrap(); } - let mut untimed = Info { sequence: 0 }.produce_untimed(); + let mut untimed = Info { start: 0 }.produce_untimed(); assert!(matches!( - untimed.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")), + untimed.write_frame(untimed.key_time(), Bytes::from_static(b"x")), Err(Error::TimestampMismatch) )); - let mut timed = Info { sequence: 0 }.produce(); + let mut timed = Info { start: 0 }.produce(); assert!(matches!( - timed.write_frame(None, Bytes::from_static(b"x")), + timed.write_frame(crate::Timestamp::unscaled(timed.start), Bytes::from_static(b"x")), + Err(Error::TimestampMismatch) + )); + // A scale-less frame carries its group's key, so another key is a mismatch. + assert!(matches!( + untimed.write_frame(Timestamp::unscaled(1), Bytes::from_static(b"x")), Err(Error::TimestampMismatch) )); assert!(matches!( timed.create_frame(frame::Info { size: 1, - timestamp: None + timestamp: Timestamp::unscaled(0) }), Err(Error::TimestampMismatch) )); @@ -5696,17 +5731,17 @@ mod test { #[test] #[cfg(feature = "async")] fn chunked_frame_reads_whole() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let mut frame = producer .create_frame(frame::Info { size: 10, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); frame.write(Bytes::from_static(b"hello")).unwrap(); frame.write(Bytes::from_static(b"world")).unwrap(); let producer = frame.finish().unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); // Frame data is held in a single per-frame buffer; a whole-frame read returns // the full contents in one slice. @@ -5718,13 +5753,13 @@ mod test { #[test] #[cfg(feature = "async")] fn chunked_frame_streams_partial() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let mut consumer = producer.consume(); let mut frame = producer .create_frame(frame::Info { size: 6, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); frame.write(Bytes::from_static(b"foo")).unwrap(); @@ -5747,8 +5782,8 @@ mod test { #[test] #[cfg(feature = "async")] fn group_finish_returns_none() { - let producer = Info { sequence: 0 }.produce(); - let producer = producer.finish().unwrap(); + let producer = Info { start: 0 }.produce(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); let end = consumer.next_frame().now_or_never().unwrap().unwrap(); @@ -5766,10 +5801,10 @@ mod test { .with_delay(track::Delay::default().with_max(std::time::Duration::from_secs(3600))), ); let groups: Vec<_> = (0..3) - .map(|sequence| { - let mut group = track.create_group(Info { sequence }).unwrap(); - group.write_frame(Timestamp::ZERO, b"f".as_ref()).unwrap(); - group.finish().unwrap() + .map(|key| { + let mut group = track.create_group(Info { start: key }).unwrap(); + group.write_frame(group.key_time(), b"f".as_ref()).unwrap(); + group.close().unwrap() }) .collect(); subscriber.set_range(..0); @@ -5780,13 +5815,13 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("group zero is in range") }; - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); } #[test] #[cfg(feature = "async")] fn abort_propagates() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let mut consumer = producer.consume(); producer.abort(crate::Error::Cancel).unwrap(); @@ -5796,7 +5831,7 @@ mod test { #[test] fn abort_unwanted_pairs_with_try_consume() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let consumer = producer.handle.try_consume().expect("open"); assert!( @@ -5814,7 +5849,7 @@ mod test { #[test] fn abort_clears_cached_frames() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .write_frame(Timestamp::ZERO, Bytes::from_static(b"data")) .unwrap(); @@ -5843,8 +5878,12 @@ mod test { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); - producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); - producer.write_frame(None, Bytes::from_static(b"b")).unwrap(); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), Bytes::from_static(b"a")) + .unwrap(); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), Bytes::from_static(b"b")) + .unwrap(); let handoff = producer.consume().handoff(); assert_eq!(handoff.head().map(|head| head.entries.len()), Some(2), "read live"); @@ -5870,7 +5909,9 @@ mod test { let pool = cache::Pool::new(cache::Config::default().with_expiry(std::time::Duration::from_secs(1))); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); - producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), Bytes::from_static(b"a")) + .unwrap(); let handoff = producer.consume().handoff(); producer.handle().abort(Error::Cancel).unwrap(); drop(producer); @@ -5889,17 +5930,19 @@ mod test { const BURST: usize = 64; let pool = cache::Pool::new(cache::Config::default().with_expiry(std::time::Duration::from_secs(1))); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); - let fail = |sequence: u64| { - let mut producer = Producer::new(Info::from(sequence), track::Info::default(), cache.clone()); - producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + let fail = |key: u64| { + let mut producer = Producer::new(Info::from(key), track::Info::default(), cache.clone()); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), Bytes::from_static(b"a")) + .unwrap(); let handoff = producer.consume().handoff(); producer.abort(Error::Cancel).unwrap(); handoff }; // Recovery adopts each head as soon as its group fails. - for sequence in 0..BURST as u64 { - drop(fail(sequence)); + for key in 0..BURST as u64 { + drop(fail(key)); } assert!(pool.held_capacity() < BURST, "adopted hand-offs pinned the registry"); @@ -5918,7 +5961,9 @@ mod test { let pool = cache::Pool::new(cache::Config::default().with_capacity(1)); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let mut producer = Producer::new(Info::from(0u64), track::Info::default(), cache); - producer.write_frame(None, Bytes::from_static(b"a")).unwrap(); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), Bytes::from_static(b"a")) + .unwrap(); let handoff = producer.consume().handoff(); producer.handle().abort(Error::Cancel).unwrap(); assert!(handoff.head().is_none()); @@ -5937,14 +5982,17 @@ mod test { let _frame = tail .create_frame(frame::Info { size: MAX_CACHE_BYTES - 1, - timestamp: None, + timestamp: Timestamp::unscaled(0), }) .unwrap(); let head = Head { content: None, start: 0, first_object: true, - entries: vec![Entry::plain(Frame::new(None, Bytes::from_static(b"ab")).into(), 0)], + entries: vec![Entry::plain( + Frame::new(Timestamp::unscaled(0), Bytes::from_static(b"ab")).into(), + 0, + )], compressed: false, }; adopter.adopt_head(&head); @@ -5959,7 +6007,7 @@ mod test { /// that frame whole beside its rest, outside the live tail's admission. #[test] fn a_copy_with_a_lead_adopts_no_head() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .bound(Bounds { start: 1, @@ -5972,7 +6020,7 @@ mod test { content: None, start: 0, first_object: true, - entries: vec![Entry::plain(Frame::new(None, Bytes::from_static(b"ab")).into(), 0); 2], + entries: vec![Entry::plain(Frame::new(Timestamp::ZERO, Bytes::from_static(b"ab")).into(), 0); 2], compressed: false, }; adopter.adopt_head(&head); @@ -5990,7 +6038,7 @@ mod test { let count = MAX_CACHE_BYTES as usize / size_of::() + 1; tail.start_at(count as u64).unwrap(); let adopter = tail.handle(); - let entry = Entry::plain(Frame::new(None, Bytes::new()).into(), 0); + let entry = Entry::plain(Frame::new(Timestamp::unscaled(0), Bytes::new()).into(), 0); let head = Head { content: None, start: 0, @@ -6018,10 +6066,10 @@ mod test { start: 0, first_object: true, entries: vec![ - Entry::plain(Frame::new(None, Bytes::from_static(b"a")).into(), 0), + Entry::plain(Frame::new(Timestamp::unscaled(0), Bytes::from_static(b"a")).into(), 0), Entry { owner: Mutex::new(None), - frame: Frame::new(None, Bytes::from_static(b"b")).into(), + frame: Frame::new(Timestamp::unscaled(0), Bytes::from_static(b"b")).into(), fields: fields(1), inflated: 0, end: 0, @@ -6063,8 +6111,9 @@ mod test { tail.handle.state.read().wire().unwrap().fetchable, "no gap below the stream" ); - tail.write_frame(None, Bytes::from_static(b"c")).unwrap(); - let tail = tail.finish().unwrap(); + tail.write_frame(crate::Timestamp::unscaled(tail.start), Bytes::from_static(b"c")) + .unwrap(); + let tail = tail.close().unwrap(); let waiter = kio::Waiter::noop(); let mut whole = tail.consume(); @@ -6095,10 +6144,10 @@ mod test { #[test] #[cfg(feature = "async")] fn drop_unfinished_clears_cached_frames() { - let mut writer = Info { sequence: 0 }.produce(); + let mut writer = Info { start: 0 }.produce(); let producer = writer.handle(); writer - .write_frame(Timestamp::ZERO, Bytes::from_static(b"data")) + .write_frame(writer.key_time(), Bytes::from_static(b"data")) .unwrap(); // A stale consumer keeps the channel (and thus the cache) alive. @@ -6116,11 +6165,11 @@ mod test { #[test] fn drop_after_abort_does_not_warn() { let warns = count_drop_warnings("group::Producer dropped without finish", || { - let mut writer = Info { sequence: 0 }.produce(); + let mut writer = Info { start: 0 }.produce(); let keep = writer.handle(); let producer = writer.handle(); writer - .write_frame(Timestamp::ZERO, Bytes::from_static(b"data")) + .write_frame(writer.key_time(), Bytes::from_static(b"data")) .unwrap(); let _consumer = producer.consume(); writer.abort(crate::Error::Cancel).unwrap(); @@ -6132,10 +6181,10 @@ mod test { #[test] fn drop_unfinished_warns() { let warns = count_drop_warnings("group::Producer dropped without finish", || { - let mut writer = Info { sequence: 0 }.produce(); + let mut writer = Info { start: 0 }.produce(); let producer = writer.handle(); writer - .write_frame(Timestamp::ZERO, Bytes::from_static(b"data")) + .write_frame(writer.key_time(), Bytes::from_static(b"data")) .unwrap(); let _consumer = producer.consume(); drop(writer); @@ -6147,11 +6196,11 @@ mod test { #[test] #[cfg(feature = "async")] fn drop_finished_keeps_cached_frames() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .write_frame(Timestamp::ZERO, Bytes::from_static(b"data")) .unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); drop(producer); @@ -6164,7 +6213,7 @@ mod test { #[test] #[cfg(feature = "async")] fn pending_then_ready() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); let mut consumer = producer.consume(); // Consumer blocks because no frames yet. @@ -6173,7 +6222,7 @@ mod test { producer .write_frame(Timestamp::ZERO, Bytes::from_static(b"data")) .unwrap(); - producer.finish().unwrap(); + producer.close().unwrap(); let frame = consumer.next_frame().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(frame.size, 4); @@ -6211,7 +6260,9 @@ mod test { let frames = mebibytes(48); for (index, frame) in frames.iter().enumerate() { - producer.write_frame(None, frame.clone()).unwrap(); + producer + .write_frame(crate::Timestamp::unscaled(producer.start), frame.clone()) + .unwrap(); let read = live.read_frame().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(read.payload, *frame, "frame {index}"); assert!(pool.used() <= MAX_CACHE_BYTES + (1 << 20), "{} bytes held", pool.used()); @@ -6229,7 +6280,7 @@ mod test { Err(Error::Evicted) )); - producer.finish().unwrap(); + producer.close().unwrap(); assert!(live.read_frame().now_or_never().unwrap().unwrap().is_none()); } @@ -6237,7 +6288,7 @@ mod test { #[test] #[cfg_attr(miri, ignore = "fills the 32 MiB group tail, past Miri's 15-minute cap")] fn empty_frames_shed_by_their_slots() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); let slot = size_of::(); let count = 2 * MAX_CACHE_BYTES as usize / slot; for _ in 0..count { @@ -6254,7 +6305,7 @@ mod test { #[test] #[cfg_attr(miri, ignore = "fills the 32 MiB group tail, past Miri's 15-minute cap")] fn empty_frame_batches_shed_by_their_slots() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); let count = 2 * MAX_CACHE_BYTES as usize / size_of::(); for _ in 0..count / 64 { let mut batch = frame::Buffer::<64>::new(); @@ -6279,11 +6330,11 @@ mod test { #[test] #[cfg_attr(miri, ignore = "each write takes Miri ~30 ms, so 10,000 run past its SLOW mark")] fn a_group_has_no_frame_cap() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); for _ in 0..10_000 { producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap(); } - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let state = producer.handle.state.read(); assert_eq!(state.offset, 0); @@ -6295,7 +6346,7 @@ mod test { /// reader about to take it. #[test] fn debt_sheds_the_front_but_not_the_newest_frame() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); // Small frames keep Miri's per-byte cost down; shedding is per frame, not per byte. for index in 0..4u8 { producer @@ -6306,7 +6357,7 @@ mod test { assert!(freed >= 3 * 1024, "freed {freed}"); assert_eq!(producer.handle.state.read().held(), 3); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); producer.handle.shed(u64::MAX, u64::MAX); assert!(producer.handle.state.read().is_empty()); assert_eq!(producer.handle.state.read().cache, 0); @@ -6315,7 +6366,7 @@ mod test { #[test] #[cfg(feature = "async")] fn clone_consumer_independent() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); let mut c1 = producer.consume(); @@ -6326,7 +6377,7 @@ mod test { let mut c2 = c1.clone(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap(); - producer.finish().unwrap(); + producer.close().unwrap(); // c2 should get the second frame (inherited index) let f = c2.next_frame().now_or_never().unwrap().unwrap().unwrap(); @@ -6340,10 +6391,10 @@ mod test { fn prefetched_consumer(pool: &cache::Pool, max_age: std::time::Duration) -> (Finished, Consumer) { let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let track = track::Info::timed().with_max_age(max_age); - let mut producer = Producer::new(Info { sequence: 0 }, track, cache); + let mut producer = Producer::new(Info { start: 0 }, track, cache); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); consumer.read_frame().now_or_never().unwrap().unwrap().unwrap(); @@ -6429,13 +6480,13 @@ mod test { #[cfg(feature = "async")] fn read_frame_crosses_prefetch_batches() { let n = Prefetch::CAP * 3 + 5; - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); for i in 0..n { producer .write_frame(Timestamp::ZERO, Bytes::from(vec![i as u8; 4])) .unwrap(); } - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); for i in 0..n { @@ -6451,11 +6502,11 @@ mod test { #[test] #[cfg(feature = "async")] fn abort_after_finish_keeps_the_clean_end_for_a_drained_reader() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .write_frame(Timestamp::ZERO, Bytes::from_static(b"hello")) .unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut drained = producer.consume(); let mut behind = producer.consume(); @@ -6478,10 +6529,10 @@ mod test { #[test] #[cfg(feature = "async")] fn finished_answers_for_the_cursor() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut drained = producer.consume(); let mut behind = producer.consume(); @@ -6501,7 +6552,7 @@ mod test { #[cfg_attr(miri, ignore = "fills the 32 MiB group tail, past Miri's 15-minute cap")] #[cfg(feature = "async")] fn finished_counts_prefetched_frames_as_read() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); let frames = mebibytes(36); for frame in &frames[..8] { producer.write_frame(Timestamp::ZERO, frame.clone()).unwrap(); @@ -6524,11 +6575,11 @@ mod test { #[test] #[cfg(feature = "async")] fn interleave_read_and_next_frame() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); for i in 0..5u8 { producer.write_frame(Timestamp::ZERO, Bytes::from(vec![i; 1])).unwrap(); } - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); // The first whole-frame read prefetches all five frames into the batch. @@ -6549,7 +6600,7 @@ mod test { #[test] #[cfg(feature = "async")] fn read_frame_past_cleared_frames_does_not_panic() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"b")).unwrap(); @@ -6570,11 +6621,11 @@ mod test { #[test] #[cfg(feature = "async")] fn drop_with_partial_batch() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); for _ in 0..Prefetch::CAP { producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap(); } - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); // Take one frame so the batch is filled but only partially drained. @@ -6589,12 +6640,12 @@ mod test { #[kio::test] #[cfg(feature = "async")] async fn chunk_write_wakes_parked_reader() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let mut consumer = producer.consume(); let mut frame = producer .create_frame(frame::Info { size: 6, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); let mut f = consumer.next_frame().await.unwrap().unwrap(); @@ -6618,14 +6669,14 @@ mod test { use std::time::Duration; let start = kio::clock::now(); - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); kio::clock::sleep(Duration::from_millis(10)).await; let mut chunked = producer .create_frame(frame::Info { size: 2, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); chunked.write(Bytes::from_static(b"b")).unwrap(); @@ -6643,7 +6694,7 @@ mod test { .push(Frame::new(Timestamp::ZERO, Bytes::from_static(b"e"))) .unwrap(); producer.write_frames(&mut batch).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); kio::clock::sleep(Duration::from_secs(1)).await; let mut consumer = producer.consume(); @@ -6660,13 +6711,13 @@ mod test { #[cfg(feature = "async")] async fn groups_carry_when_they_landed() { let start = kio::clock::now(); - let alone = Info { sequence: 0 }.produce(); + let alone = Info { start: 0 }.produce(); let track = crate::broadcast::Info::new() .produce() .create_track("test", None) .unwrap(); let mut subscriber = track.subscribe(None); - let live = track.create_group(Info { sequence: 0 }).unwrap(); + let live = track.create_group(Info { start: 0 }).unwrap(); kio::clock::sleep(std::time::Duration::from_secs(1)).await; assert_eq!(alone.consume().arrived(), start); @@ -6674,8 +6725,8 @@ mod test { assert_eq!(group.arrived(), start); assert_eq!(group.clone().arrived(), start, "a clone keeps it"); - alone.finish().unwrap(); - live.finish().unwrap(); + alone.close().unwrap(); + live.close().unwrap(); } /// A frame whose timestamp is at a different scale is converted to the group's @@ -6685,38 +6736,17 @@ mod test { use crate::{Timescale, Timestamp}; let producer = Producer::new( - Info { sequence: 0 }, + Info { start: 0 }, track::Info::default().with_timescale(Timescale::MICRO), Default::default(), ); let frame = frame::Info { size: 3, - timestamp: Some(Timestamp::from_millis(1).unwrap()), // 1ms -> 1000µs + timestamp: Timestamp::from_millis(1).unwrap(), // 1ms -> 1000µs }; let writer = producer.create_frame(frame).unwrap(); - assert_eq!(writer.timestamp.unwrap().scale(), Timescale::MICRO); - assert_eq!(writer.timestamp.unwrap().value(), 1000); - } - - /// An explicit current timestamp is converted to the group's scale. - #[test] - #[cfg_attr(miri, ignore = "reads the wall clock, which Miri's isolation refuses")] - fn create_frame_converts_current_timestamp() { - use crate::Timescale; - - let producer = Producer::new( - Info { sequence: 0 }, - track::Info::default().with_timescale(Timescale::MICRO), - Default::default(), - ); - let writer = producer - .create_frame(frame::Info { - size: 3, - timestamp: Some(Timestamp::now()), - }) - .unwrap(); - assert_eq!(writer.timestamp.unwrap().scale(), Timescale::MICRO); - assert!(!writer.timestamp.unwrap().is_zero(), "local clock should be non-zero"); + assert_eq!(writer.timestamp.scale(), Some(Timescale::MICRO)); + assert_eq!(writer.timestamp.value(), 1000); } /// A group can start partway in, so a route can serve the tail of a group whose @@ -6724,10 +6754,10 @@ mod test { #[test] #[cfg(feature = "async")] fn start_at_starts_the_group_later() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.start_at(3).unwrap(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"d")).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); // The frame landed at index 3, so the group's length counts the missing head. assert_eq!(producer.frame_count(), 4); @@ -6752,10 +6782,10 @@ mod test { #[test] #[cfg(feature = "async")] fn start_at_clamps_up_to_the_first_frame() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.start_at(3).unwrap(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"d")).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume().with_range(1..); assert_eq!(consumer.index(), 3, "clamped up to the first frame that exists"); @@ -6770,11 +6800,11 @@ mod test { #[test] #[cfg(feature = "async")] fn end_at_caps_and_reopens() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); for i in 0..4u8 { producer.write_frame(Timestamp::ZERO, Bytes::from(vec![i])).unwrap(); } - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); consumer = consumer.with_range(..2); @@ -6801,11 +6831,11 @@ mod test { #[test] #[cfg(feature = "async")] fn frame_ranges_preserve_progress_and_make_inclusion_explicit() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); for i in 0..4u8 { producer.write_frame(Timestamp::ZERO, Bytes::from(vec![i])).unwrap(); } - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); consumer = consumer.with_range(1..=1); assert_eq!( @@ -6831,9 +6861,9 @@ mod test { #[test] #[cfg(feature = "async")] fn end_at_zero_is_empty() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"x")).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let mut consumer = producer.consume(); consumer = consumer.with_range(..0); @@ -6852,7 +6882,7 @@ mod test { /// Where the group begins is part of its shape, so it can't move once frames exist. #[test] fn start_at_rejected_after_a_frame() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); // Re-declaring before the first frame is fine; the shape isn't committed yet. producer.start_at(2).unwrap(); producer.start_at(3).unwrap(); @@ -6869,7 +6899,7 @@ mod test { /// The start must leave room for at least one frame index. #[test] fn start_at_rejects_the_largest_index() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); assert!(matches!( producer.start_at(usize::MAX as u64), Err(Error::BoundsExceeded(_)) @@ -6879,10 +6909,10 @@ mod test { /// The per-frame size cap (the group byte budget) is enforced before allocating. #[test] fn create_frame_rejects_oversized() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let result = producer.create_frame(frame::Info { size: MAX_CACHE_BYTES + 1, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }); assert!(matches!(result, Err(Error::FrameTooLarge))); } @@ -6890,7 +6920,7 @@ mod test { fn sized(size: u64) -> frame::Info { frame::Info { size, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, } } @@ -6901,7 +6931,7 @@ mod test { #[cfg(feature = "async")] fn a_copy_started_partway_into_a_frame_serves_only_its_rest() { let bounded = || { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .bound(Bounds { start: 1, @@ -6968,7 +6998,7 @@ mod test { ); let mut producer = lead.finish().unwrap(); producer.write_frame(Timestamp::ZERO, b"g".as_ref()).unwrap(); - producer.finish().unwrap(); + producer.close().unwrap(); assert_eq!(&frame.read_chunk().now_or_never().unwrap().unwrap().unwrap()[..], b"f"); assert!(frame.read_chunk().now_or_never().unwrap().unwrap().is_none()); assert_eq!( @@ -6989,7 +7019,7 @@ mod test { #[cfg(feature = "async")] fn a_lead_is_shed_first_past_the_live_tail() { let half = MAX_CACHE_BYTES / 2; - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .bound(Bounds { start: 0, @@ -7035,7 +7065,7 @@ mod test { fn frame_is_charged_as_written() { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); - let producer = Producer::new(Info { sequence: 0 }, track::Info::timed(), cache); + let producer = Producer::new(Info { start: 0 }, track::Info::timed(), cache); let before = pool.used(); let observing = producer.handle(); @@ -7061,20 +7091,20 @@ mod test { #[test] #[cfg(feature = "async")] fn an_open_frame_reads_its_group() { - let mut producer = Info { sequence: 3 }.produce(); + let mut producer = Info { start: 3 }.produce(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); let mut consumer = producer.consume(); let mut frame = producer.create_frame(sized(1)).unwrap(); let group = frame.group(); - assert_eq!(group.sequence, 3); + assert_eq!(group.start, 3); assert_eq!(group.frame_count(), 2, "the open frame counts"); assert!(group.demand().wanted()); assert!(group.poll_closed(&kio::Waiter::noop()).is_pending()); frame.write(Bytes::from_static(b"b")).unwrap(); let producer = frame.finish().unwrap(); - producer.finish().unwrap(); + producer.close().unwrap(); let mut read = || consumer.read_frame().now_or_never().unwrap().unwrap(); assert_eq!(read().unwrap().payload, Bytes::from_static(b"a")); assert_eq!(read().unwrap().payload, Bytes::from_static(b"b")); @@ -7083,7 +7113,7 @@ mod test { #[test] fn refused_frame_aborts_with_its_error() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let handle = producer.handle(); assert!(matches!( producer.create_frame(sized(MAX_CACHE_BYTES + 1)), @@ -7099,7 +7129,7 @@ mod test { /// outcome stays clean, a drained reader ends cleanly, and nothing records a cut. #[test] fn abort_after_finish_keeps_the_clean_outcome() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); let handle = producer.handle(); let demand = producer.demand(); let mut consumer = producer.consume(); @@ -7108,7 +7138,7 @@ mod test { assert!(demand.poll_closed(&waiter).is_pending()); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); assert!(matches!(handle.poll_closed(&waiter), Poll::Ready(Ok(())))); assert!(matches!(demand.poll_closed(&waiter), Poll::Ready(Ok(())))); assert!( @@ -7130,10 +7160,10 @@ mod test { #[cfg(feature = "async")] #[test] fn writers_dropped_after_a_finish_abort_nothing() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer.write_frame(Timestamp::ZERO, Bytes::from_static(b"a")).unwrap(); let mut consumer = producer.consume(); - drop(producer.finish().unwrap()); + drop(producer.close().unwrap()); let waiter = kio::Waiter::noop(); assert!(consumer.poll_aborted(&waiter).is_pending()); @@ -7144,7 +7174,7 @@ mod test { /// An abort before the finish is the outcome, on every observer. #[test] fn abort_before_finish_is_the_outcome() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let handle = producer.handle(); let demand = producer.demand(); producer.abort(Error::Timeout).unwrap(); @@ -7158,7 +7188,7 @@ mod test { fn frame_commit_charges_the_tail() { let pool = cache::Pool::unbounded(); let cache = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); - let producer = Producer::new(Info { sequence: 0 }, track::Info::timed(), cache); + let producer = Producer::new(Info { start: 0 }, track::Info::timed(), cache); let before = pool.used(); let observing = producer.handle(); @@ -7177,9 +7207,9 @@ mod test { fn frame_budget_bounds_upfront_allocation() { let budget = frame::Budget::new(1024); let pool = cache::Pool::unbounded(); - let group = |sequence| { + let group = |key| { let cache = cache::Track::new(pool.clone(), budget.clone(), kio::Weak::new()); - Producer::new(Info { sequence }, track::Info::timed(), cache) + Producer::new(Info { start: key }, track::Info::timed(), cache) }; let a = group(0); let b = group(1); @@ -7199,7 +7229,7 @@ mod test { first.write(Bytes::from(vec![0u8; 1023])).unwrap(); let a = first.finish().unwrap(); - a.finish().unwrap(); + a.close().unwrap(); let a = group(2); let mut third = a.create_frame(sized(1024)).unwrap(); @@ -7213,14 +7243,14 @@ mod test { fn a_large_frame_allocates_one_segment_ahead() { let budget = frame::Budget::new(2 * frame::SEGMENT); let pool = cache::Pool::unbounded(); - let group = |sequence| { + let group = |key| { let cache = cache::Track::new(pool.clone(), budget.clone(), kio::Weak::new()); - Producer::new(Info { sequence }, track::Info::timed(), cache) + Producer::new(Info { start: key }, track::Info::timed(), cache) }; let frames: Vec<_> = (0..2) - .map(|sequence| { - let mut frame = group(sequence).create_frame(sized(MAX_CACHE_BYTES)).unwrap(); + .map(|key| { + let mut frame = group(key).create_frame(sized(MAX_CACHE_BYTES)).unwrap(); frame.write(Bytes::from_static(b"x")).unwrap(); frame }) @@ -7235,7 +7265,7 @@ mod test { #[test] #[cfg(feature = "async")] fn a_batch_mixes_segmented_frames() { - let mut group = Info { sequence: 0 }.produce(); + let mut group = Info { start: 0 }.produce(); let payloads: Vec = (0..Prefetch::CAP) .map(|i| match i % 2 { 0 => Bytes::from(vec![i as u8; 3]), @@ -7246,7 +7276,7 @@ mod test { let mut frame = group .create_frame(frame::Info { size: payload.len() as u64, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); // Written in pieces, so the large ones chain. @@ -7255,7 +7285,7 @@ mod test { } group = frame.finish().unwrap(); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let mut whole = group.consume(); for (i, payload) in payloads.iter().enumerate() { @@ -7295,10 +7325,11 @@ mod test { Bytes::from(format!("{index}:{}", "moq".repeat(300))) } + /// A scale-less frame of `size` bytes in group 0. fn untimed(size: usize) -> frame::Info { frame::Info { size: size as u64, - timestamp: None, + timestamp: Timestamp::unscaled(0), } } @@ -7332,15 +7363,21 @@ mod test { let mut raw = group.consume().compressed(); let frames = mebibytes(80); - group.write_frame(None, frames[0].clone()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), frames[0].clone()) + .unwrap(); assert_eq!(read_plain(&mut plain).unwrap().unwrap(), frames[0]); // 34 frames behind, past the stored tail but within the inflated one: the slices it // needs inflate before they go. for frame in &frames[1..35] { - group.write_frame(None, frame.clone()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), frame.clone()) + .unwrap(); } for (index, frame) in frames.iter().enumerate().skip(35) { - group.write_frame(None, frame.clone()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), frame.clone()) + .unwrap(); let read = read_plain(&mut plain).unwrap().unwrap(); assert_eq!(read, frames[index - 34], "frame {}", index - 34); assert!(pool.used() <= 3 * MAX_CACHE_BYTES, "{} bytes held", pool.used()); @@ -7362,7 +7399,9 @@ mod test { let pool = cache::Pool::unbounded(); let mut group = compressed_group(&pool); for frame in &mebibytes(40) { - group.write_frame(None, frame.clone()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), frame.clone()) + .unwrap(); } let held = group.handle.state.read().held(); assert!(held > 1, "the slices were shed"); @@ -7385,7 +7424,9 @@ mod test { let mut group = compressed_group(&pool); let mut plain = group.consume(); for _ in 0..1_000 { - group.write_frame(None, Bytes::new()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::new()) + .unwrap(); assert!(read_plain(&mut plain).unwrap().unwrap().is_empty()); } let flate = group.handle.state.read().flate.as_ref().unwrap().plain.clone(); @@ -7402,9 +7443,11 @@ mod test { let mut group = compressed_group(&pool); let frame = Bytes::from(vec![7u8; 31 << 20]); for _ in 0..8 { - group.write_frame(None, frame.clone()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), frame.clone()) + .unwrap(); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let mut plain = group.consume(); for _ in 0..8 { assert_eq!(read_plain(&mut plain).unwrap().unwrap(), frame); @@ -7427,16 +7470,20 @@ mod test { let account = cache::Track::new(pool.clone(), Default::default(), kio::Weak::new()); let info = track::Info::default().with_compression(crate::track::Compression::Deflate); let mut concurrent = Producer::new(0u64.into(), info.clone(), account.clone()); - concurrent.write_frame(None, verbose(0)).unwrap(); - for sequence in 1..5u64 { - let mut group = Producer::new(sequence.into(), info.clone(), account.clone()); + concurrent + .write_frame(crate::Timestamp::unscaled(concurrent.start), verbose(0)) + .unwrap(); + for key in 1..5u64 { + let mut group = Producer::new(key.into(), info.clone(), account.clone()); let mut reference = flate::Encoder::new(); - let frames = [verbose(0), Bytes::new(), verbose(sequence as usize)]; + let frames = [verbose(0), Bytes::new(), verbose(key as usize)]; let slices: Vec<_> = frames.iter().map(|frame| reference.frame(frame)).collect(); for frame in &frames { - group.write_frame(None, frame.clone()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), frame.clone()) + .unwrap(); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let mut raw = group.consume().compressed(); let mut decoder = flate::Decoder::new(); for (frame, expected) in frames.iter().zip(&slices) { @@ -7448,7 +7495,9 @@ mod test { assert_eq!(decoder.frame(&slice, frame.len()).unwrap(), *frame); } } - concurrent.write_frame(None, verbose(1)).unwrap(); + concurrent + .write_frame(crate::Timestamp::unscaled(concurrent.start), verbose(1)) + .unwrap(); let mut reader = concurrent.consume(); assert_eq!(read_plain(&mut reader).unwrap().unwrap(), verbose(0)); assert_eq!(read_plain(&mut reader).unwrap().unwrap(), verbose(1)); @@ -7466,15 +7515,19 @@ mod test { fn compressed_group_stores_slices_and_inflates_once() { let pool = cache::Pool::unbounded(); let mut group = compressed_group(&pool); - group.write_frame(None, verbose(0)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(0)) + .unwrap(); // A chunked frame is invisible until its last chunk, then compressed whole. let mut frame = group.create_frame(untimed(verbose(1).len())).unwrap(); for chunk in verbose(1).chunks(100) { frame.write(Bytes::copy_from_slice(chunk)).unwrap(); } group = frame.finish().unwrap(); - group.write_frame(None, verbose(2)).unwrap(); - let group = group.finish().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(2)) + .unwrap(); + let group = group.close().unwrap(); let plain: usize = (0..3).map(|index| verbose(index).len()).sum(); let stored = pool.used(); @@ -7485,9 +7538,11 @@ mod test { cache::Track::new(uncompressed.clone(), Default::default(), kio::Weak::new()), ); for index in 0..3 { - reference.write_frame(None, verbose(index)).unwrap(); + reference + .write_frame(crate::Timestamp::unscaled(reference.start), verbose(index)) + .unwrap(); } - let _reference = reference.finish().unwrap(); + let _reference = reference.close().unwrap(); assert!( stored - flate::ENCODER_HEAP + (plain as u64) / 2 < uncompressed.used(), "slices are stored: {stored} against {}", @@ -7542,8 +7597,12 @@ mod test { fn a_compressed_group_adopts_a_compressed_head() { let pool = cache::Pool::unbounded(); let fail = |group: &mut Producer| { - group.write_frame(None, verbose(0)).unwrap(); - group.write_frame(None, verbose(1)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(0)) + .unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(1)) + .unwrap(); let handoff = group.consume().handoff(); group.handle().abort(Error::Cancel).unwrap(); handoff.head().expect("held for its resumers") @@ -7566,7 +7625,7 @@ mod test { let head = fail(&mut compressed_group(&pool)); let group = successor(); group.handle.adopt_head(&head); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let mut reader = group.consume(); for index in 0..2 { assert_eq!( @@ -7606,9 +7665,11 @@ mod test { let write = || { let mut group = compressed_group(&pool); for index in 0..3 { - group.write_frame(None, verbose(index)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(index)) + .unwrap(); } - group.finish().unwrap() + group.close().unwrap() }; let group = write(); @@ -7653,16 +7714,20 @@ mod test { }) .unwrap(); for index in 0..2 { - released.write_frame(None, verbose(index)).unwrap(); + released + .write_frame(crate::Timestamp::unscaled(released.start), verbose(index)) + .unwrap(); } - let released = released.finish().unwrap(); + let released = released.close().unwrap(); let content = released.handle.activate_content().unwrap().unwrap(); let mut survivor = Producer::new(Info::from(0u64), info, cache); survivor.handle.share_content(content); for index in 0..3 { - survivor.write_frame(None, verbose(index)).unwrap(); + survivor + .write_frame(crate::Timestamp::unscaled(survivor.start), verbose(index)) + .unwrap(); } - let survivor = survivor.finish().unwrap(); + let survivor = survivor.close().unwrap(); let late = take_slices(&released.handle, 1); released.handle().abort(Error::Evicted).unwrap(); let used = pool.used(); @@ -7700,10 +7765,12 @@ mod test { let meter = registry.publisher_meter(crate::Path::from_static("demo"), &stats::Tier::default()); let mut group = compressed_group(&pool).with_meter(meter); - group.write_frame(None, verbose(0)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(0)) + .unwrap(); let mut batch = frame::Buffer::<2>::new(); for index in 1..3 { - batch.push(Frame::new(None, verbose(index))).unwrap(); + batch.push(Frame::new(Timestamp::unscaled(0), verbose(index))).unwrap(); } group.write_frames(&mut batch).unwrap(); let mut frame = group.create_frame(untimed(verbose(3).len())).unwrap(); @@ -7711,7 +7778,7 @@ mod test { frame.write(Bytes::copy_from_slice(chunk)).unwrap(); } group = frame.finish().unwrap(); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let stored: u64 = (group.handle.state.read().entries(0, usize::MAX)) .map(|entry| entry.frame.payload.len() as u64) @@ -7728,10 +7795,14 @@ mod test { fn codec_state_is_billed_while_held() { let pool = cache::Pool::unbounded(); let mut group = compressed_group(&pool); - group.write_frame(None, verbose(0)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(0)) + .unwrap(); let stored = group.handle.state.read().charge.size() + Flate::RELEASE_HEAP; assert_eq!(group.handle.cache_size(), stored + flate::ENCODER_HEAP); - group.write_frame(None, verbose(1)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(1)) + .unwrap(); // The read inflates both frames ahead, but the open group may still get a third. let mut reader = group.consume(); @@ -7742,7 +7813,7 @@ mod test { let held = stored + flate::ENCODER_HEAP + flate::DECODER_HEAP + inflated; assert_eq!(group.handle.cache_size(), held, "an open group keeps both"); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); assert_eq!( group.handle.cache_size(), stored + inflated, @@ -7752,21 +7823,29 @@ mod test { // A finished group's decoder goes with the last frame it inflates. let mut group = compressed_group(&pool); - group.write_frame(None, verbose(0)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(0)) + .unwrap(); let mut reader = group.consume(); assert_eq!(read_plain(&mut reader).unwrap().unwrap(), verbose(0)); - group.write_frame(None, verbose(1)).unwrap(); - let group = group.finish().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(1)) + .unwrap(); + let group = group.close().unwrap(); let stored = group.handle.state.read().charge.size() + Flate::RELEASE_HEAP; assert_eq!(read_plain(&mut reader).unwrap().unwrap(), verbose(1)); assert_eq!(group.handle.cache_size(), stored + inflated); // So does one whose reader took the last slice before the group finished. let mut group = compressed_group(&pool); - group.write_frame(None, verbose(0)).unwrap(); - group.write_frame(None, verbose(1)).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(0)) + .unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(1)) + .unwrap(); let job = take_slices(&group.handle, 1); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); job.run(&stats::Meter::default()); let stored = group.handle.state.read().charge.size() + Flate::RELEASE_HEAP; assert_eq!(group.handle.cache_size(), stored + inflated, "the decoder is dropped"); @@ -7779,8 +7858,10 @@ mod test { let pool = cache::Pool::unbounded(); let mut group = compressed_group(&pool); group.start_at(1).unwrap(); - group.write_frame(None, verbose(1)).unwrap(); - let group = group.finish().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(1)) + .unwrap(); + let group = group.close().unwrap(); let mut consumer = group.consume().with_range(1..); assert!(matches!(read_plain(&mut consumer), Err(Error::Lagged))); @@ -7795,8 +7876,12 @@ mod test { let cache = cache::Track::new(cache::Pool::unbounded(), Default::default(), kio::Weak::new()); let info = track::Info::default().with_compression(compression); let mut first = Producer::new(Info::from(0u64), info.clone(), cache.clone()); - first.write_frame(None, verbose(0)).unwrap(); - first.write_frame(None, verbose(1)).unwrap(); + first + .write_frame(crate::Timestamp::unscaled(first.start), verbose(0)) + .unwrap(); + first + .write_frame(crate::Timestamp::unscaled(first.start), verbose(1)) + .unwrap(); let content = first.handle.activate_content().unwrap().unwrap(); let copy = || { let copy = Producer::new(Info::from(0u64), info.clone(), cache.clone()); @@ -7806,15 +7891,15 @@ mod test { let mut whole = copy(); assert!(matches!( - whole.write_frame(None, verbose(10)), + whole.write_frame(crate::Timestamp::unscaled(whole.start), verbose(10)), Err(Error::ProtocolViolation) )); assert!(whole.handle.is_aborted()); let mut batch = copy(); let mut frames = frame::Buffer::<2>::new(); - frames.push(Frame::new(None, verbose(0))).unwrap(); - frames.push(Frame::new(None, verbose(10))).unwrap(); + frames.push(Frame::new(Timestamp::unscaled(0), verbose(0))).unwrap(); + frames.push(Frame::new(Timestamp::unscaled(0), verbose(10))).unwrap(); assert!(matches!(batch.write_frames(&mut frames), Err(Error::ProtocolViolation))); assert!(batch.handle.is_aborted()); } @@ -7834,7 +7919,7 @@ mod test { let mut frame = group.create_object(header).unwrap(); frame.write(slice).unwrap(); let group = frame.finish().unwrap(); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); assert!(matches!(read_plain(&mut group.consume()), Err(Error::WrongSize))); } @@ -7852,7 +7937,7 @@ mod test { let res = compressed_group(&pool).create_object(header(MAX_CACHE_BYTES + 1)); assert!(matches!(res, Err(Error::FrameTooLarge))); - let res = Info::from(1u64).produce_untimed().create_object(header(1)); + let res = Info::from(0u64).produce_untimed().create_object(header(1)); assert!(matches!(res, Err(Error::ProtocolViolation))); } @@ -7884,7 +7969,7 @@ mod test { /// prefix off as the whole group. #[test] fn a_drained_unfinished_group_expires_as_old() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"a")) .unwrap(); @@ -7914,7 +7999,7 @@ mod test { #[cfg(feature = "async")] fn a_lead_read_expires_as_old() { let bounded = || { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .bound(Bounds { start: 0, @@ -7979,7 +8064,7 @@ mod test { /// parks anew for another task, which the next frame then wakes instead. #[test] fn an_idle_frame_read_parks_for_the_task_polling_it() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"a")) .unwrap(); @@ -8005,7 +8090,7 @@ mod test { /// A read that parks once and is then woken by its next frame never sets up a gate. #[test] fn a_read_woken_by_its_frame_takes_no_gate() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); let mut group = producer.consume(); let waiter = kio::Waiter::new(Arc::new(Woken::default()).into()); for payload in [b"a", b"b", b"c"] { @@ -8022,12 +8107,12 @@ mod test { /// that read looks again rather than waiting out a frame it no longer reads. #[test] fn a_chunked_read_moves_a_parked_frame_read_on() { - let producer = Info { sequence: 0 }.produce(); + let producer = Info { start: 0 }.produce(); let mut group = producer.consume().with_range(0..1); let mut frame = producer .create_frame(frame::Info { size: 6, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); frame.write(Bytes::from_static(b"foo")).unwrap(); @@ -8049,7 +8134,7 @@ mod test { /// it, so a publisher that served the range keeps its stream when the rest fails upstream. #[test] fn a_capped_reader_finishes_at_its_range_end() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); for payload in [b"a", b"b"] { producer .write_frame(crate::Timestamp::ZERO, Bytes::from_static(payload)) @@ -8072,7 +8157,7 @@ mod test { /// holds two frames, so a reader past them reads a clean end rather than a cut. #[test] fn a_bounded_copy_finished_short_of_its_bound_ends_cleanly() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .bound(Bounds { byte: 0, @@ -8086,7 +8171,7 @@ mod test { .write_frame(crate::Timestamp::ZERO, Bytes::from_static(payload)) .unwrap(); } - producer.finish().unwrap(); + producer.close().unwrap(); reader.skip_to(4); let waiter = kio::Waiter::noop(); let res = reader.poll_read_frame(&waiter); @@ -8099,11 +8184,11 @@ mod test { /// gives up on it, so it can abandon the stream it is still writing. #[test] fn a_fully_read_group_still_reports_its_expiry() { - let mut producer = Info { sequence: 0 }.produce(); + let mut producer = Info { start: 0 }.produce(); producer .write_frame(crate::Timestamp::ZERO, Bytes::from_static(b"a")) .unwrap(); - let producer = producer.finish().unwrap(); + let producer = producer.close().unwrap(); let expired = Switch::default(); let mut group = producer.consume().with_expiry(Arc::new(expired.clone())); @@ -8125,8 +8210,10 @@ mod test { fn inflater_release_guard_charge_follows_its_last_job() { let pool = cache::Pool::unbounded(); let mut group = compressed_group(&pool); - group.write_frame(None, verbose(0)).unwrap(); - let group = group.finish().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), verbose(0)) + .unwrap(); + let group = group.close().unwrap(); let job = take_slices(&group.handle, 0); let retained = job.release.clone(); group.abort(Error::Evicted).unwrap(); diff --git a/rs/moq-net/src/model/group/content.rs b/rs/moq-net/src/model/group/content.rs index 4e0353744a6..877a8cf3976 100644 --- a/rs/moq-net/src/model/group/content.rs +++ b/rs/moq-net/src/model/group/content.rs @@ -213,17 +213,10 @@ impl Drop for Content { { plain.release(Error::Dropped); } - if let Some((index, sequence)) = self.index.get_mut().unwrap_or_else(PoisonError::into_inner).take() { + if let Some((index, key)) = self.index.get_mut().unwrap_or_else(PoisonError::into_inner).take() { let mut state = index.state.lock().unwrap_or_else(PoisonError::into_inner); - if state - .get(&sequence) - .is_some_and(|owner| std::ptr::eq(owner.as_ptr(), self)) - { - remove( - &mut state, - &sequence, - index.bill.get().expect("registered index is billed"), - ); + if state.get(&key).is_some_and(|owner| std::ptr::eq(owner.as_ptr(), self)) { + remove(&mut state, &key, index.bill.get().expect("registered index is billed")); } } } @@ -241,25 +234,21 @@ struct Index { } impl Contents { - pub(in crate::model) fn get(&self, sequence: u64) -> Option> { + pub(in crate::model) fn get(&self, key: u64) -> Option> { let index = self.0.lock().unwrap_or_else(PoisonError::into_inner).upgrade()?; let state = index.state.lock().unwrap_or_else(PoisonError::into_inner); - state.get(&sequence).and_then(Weak::upgrade) + state.get(&key).and_then(Weak::upgrade) } - pub(in crate::model) fn remove(&self, sequence: u64) { + pub(in crate::model) fn remove(&self, key: u64) { let Some(index) = self.0.lock().unwrap_or_else(PoisonError::into_inner).upgrade() else { return; }; let mut state = index.state.lock().unwrap_or_else(PoisonError::into_inner); - remove( - &mut state, - &sequence, - index.bill.get().expect("registered index is billed"), - ); + remove(&mut state, &key, index.bill.get().expect("registered index is billed")); } - pub(in crate::model) fn insert(&self, sequence: u64, content: &Arc, charge: &cache::Charge) { + pub(in crate::model) fn insert(&self, key: u64, content: &Arc, charge: &cache::Charge) { let index = { let mut current = self.0.lock().unwrap_or_else(PoisonError::into_inner); match current.upgrade() { @@ -277,9 +266,9 @@ impl Contents { bill.add((2 * size_of::() + size_of::()) as u64); bill }); - insert(&mut state, sequence, Arc::downgrade(content), bill); + insert(&mut state, key, Arc::downgrade(content), bill); drop(state); - *content.index.lock().unwrap_or_else(PoisonError::into_inner) = Some((index, sequence)); + *content.index.lock().unwrap_or_else(PoisonError::into_inner) = Some((index, key)); } } diff --git a/rs/moq-net/src/model/idle.rs b/rs/moq-net/src/model/idle.rs index 441d8f6b773..b3d008ea6c8 100644 --- a/rs/moq-net/src/model/idle.rs +++ b/rs/moq-net/src/model/idle.rs @@ -16,19 +16,19 @@ pub(super) use crate::util::Idle; -/// Which group read parked: a track reader parks the arrival and sequence walks +/// Which group read parked: a track reader parks the arrival and key walks /// separately, since one finding nothing says nothing about the other. #[derive(Clone, Copy, PartialEq, Eq)] pub(super) enum Walk { Arrival, - Sequence, + Key, } impl Walk { - /// The walk a read in sequence order for `ordered`, or arrival order otherwise, takes. + /// The walk a read in key order for `ordered`, or arrival order otherwise, takes. pub(super) fn new(ordered: bool) -> Self { match ordered { - true => Self::Sequence, + true => Self::Key, false => Self::Arrival, } } diff --git a/rs/moq-net/src/model/mod.rs b/rs/moq-net/src/model/mod.rs index b8d7da65d05..028c6e0114d 100644 --- a/rs/moq-net/src/model/mod.rs +++ b/rs/moq-net/src/model/mod.rs @@ -41,7 +41,6 @@ mod resume; mod resume_model; mod subscription; mod time; -mod timed; mod weak_cache; use request_queue::RequestQueue; @@ -49,7 +48,6 @@ use weak_cache::{WeakCache, WeakEntry}; pub use bytes::*; pub use time::*; -pub use timed::Timed; /// Routing broadcasts into an origin, and consuming them through it. pub mod origin { diff --git a/rs/moq-net/src/model/origin.rs b/rs/moq-net/src/model/origin.rs index fdb6355dbff..6e567ec3e91 100644 --- a/rs/moq-net/src/model/origin.rs +++ b/rs/moq-net/src/model/origin.rs @@ -558,7 +558,7 @@ pub enum Source { impl RouteInfo { /// The stream the route serves: routes with one instance serve one continuous - /// sequence space, so a subscription resumes from one to another. Pass it to + /// key space, so a subscription resumes from one to another. Pass it to /// [`Consumer::request`] to pin this stream. pub fn instance(&self) -> &Instance { self.instance.as_ref().expect("a delivered route has an instance") @@ -723,10 +723,10 @@ impl RouteOptions { /// Serve a stable `instance` at `epoch`, in place of a freshly minted one. /// - /// Only for a publisher that continues the instance's sequence space across restarts, + /// Only for a publisher that continues the instance's key space across restarts, /// announcing a higher `epoch` each time, such as a counter persisted on disk. Readers /// stitch every route of one instance together, so a publisher that restarts its - /// group sequences under an old instance splices unrelated groups into their tracks. + /// group keys under an old instance splices unrelated groups into their tracks. /// The highest epoch wins over every older one of the instance, cheaper or not. /// Naming an instance grants nothing: publishing still needs the path's grant. pub fn with_instance(mut self, instance: Instance, epoch: u64) -> Self { @@ -935,7 +935,7 @@ enum Held { /// bounded by the number of distinct prefixes. A metadata change on a live route /// overwrites the pending `Announce` (or is delivered as another active update), /// a pending `Restart` stays one until delivered whatever metadata follows it, -/// and `UnannounceAnnounce` preserves a real retract-then-announce sequence. +/// and `UnannounceAnnounce` preserves a real retract-then-announce key. type AnnounceMeta = (RouteMeta, Option>); enum PendingUpdate { @@ -2432,8 +2432,8 @@ impl Producer { /// This is how local content enters an origin. Fill the broadcast through /// [`Route::broadcast`]. The route serves a fresh random instance, minted here and /// kept for its lifetime, across [`Route::withdraw`] and [`Route::announce`], since - /// its sequences continue: a new route is a new stream that never splices into an - /// old one. A publisher that continues one sequence space across restarts may name + /// its keys continue: a new route is a new stream that never splices into an + /// old one. A publisher that continues one key space across restarts may name /// a stable instance instead ([`RouteOptions::with_instance`]). /// /// Announced at once unless created [`RouteOptions::hidden`]. A hidden route exists @@ -2701,7 +2701,7 @@ impl Producer { /// routes answering its requests, so a reader that resolved a path through it /// continues on that path's answer rather than restarting. Once an exact route holding /// the lent instance stops serving it (withdrawn, closed, or renamed), this route moves - /// to a fresh instance, as a restarted job's sequences are another stream: see + /// to a fresh instance, as a restarted job's keys are another stream: see /// [`Claim::instance`]. A claim naming a stable instance lends nothing, and its /// answers each mint their own. /// @@ -3052,7 +3052,7 @@ struct Serving { /// /// A route mints its instance when it is created and keeps it for its lifetime, across /// [`withdraw`](Self::withdraw), [`announce`](Self::announce) and reconnects, since it -/// owns the same broadcast and its sequences continue. A new run is a new route. +/// owns the same broadcast and its keys continue. A new run is a new route. /// /// ``` /// # fn run(origin: moq_net::origin::Producer) -> Result<(), moq_net::Error> { @@ -4429,7 +4429,7 @@ impl kio::Task for FrontServeRun { } /// The route a front may serve from. `resolved` is the instance the front first -/// resolved, once it has. Its routes serve one continuous sequence space, so any of them +/// resolved, once it has. Its routes serve one continuous key space, so any of them /// at its newest epoch may take over. The front stays until those routes go, even once /// another instance wins the path: its subscriptions are sticky, and new requests /// resolve the winner on a fresh front. @@ -4774,7 +4774,7 @@ impl FrontServe { Err(Error::Cancel) } }; - let delivered = copy.latest_sequence().is_some(); + let delivered = copy.latest_key().is_some(); return Some(FrontStep::Ended(name.clone(), *source, result, delivered)); } // Watch the demand edge in whichever direction is unmet. Only `Pending` @@ -5269,7 +5269,7 @@ impl kio::Task for FrontServe { // nothing is a refusal, which leaves a draining incumbent serving. let result = match result { Ok(info) => match asked.copy.poll_complete(&kio::Waiter::noop()) { - Poll::Ready(Err(err)) if asked.copy.latest_sequence().is_none() => Err(err), + Poll::Ready(Err(err)) if asked.copy.latest_key().is_none() => Err(err), _ => Ok(info), }, Err(err) => Err(err), @@ -6626,9 +6626,9 @@ impl OriginState { /// A job's route carrying `spent`, the instance it took from the prefix route `id` at /// `prefix`, stopped serving it: that prefix route moves to a fresh instance. /// - /// One instance is one stream at each path, and the job's sequences end with it. Were the + /// One instance is one stream at each path, and the job's keys end with it. Were the /// prefix route to keep the instance, its next answer for the path, a restarted job, - /// would splice a new sequence space into everything that holds the old one, and nothing + /// would splice a new key space into everything that holds the old one, and nothing /// short of remembering every path it ever answered could tell. Under the table's lock, /// with the job's retraction, so no request resolves the path through the old instance /// in between. A prefix route already past `spent` has nothing to do. @@ -6890,7 +6890,7 @@ impl Request { /// continues on this answer; a prefix route that lends nothing has the route mint its /// own. Ending the route withdraws only this path, and moves a prefix route that lent /// its instance to a fresh one, so a job that ends and restarts beneath the same prefix - /// is a new stream for its restarted sequences. The prefix route keeps answering later + /// is a new stream for its restarted keys. The prefix route keeps answering later /// requests. Created visible, the route announces now and answers the requesters /// waiting on its path. Created [`RouteOptions::hidden`], they wait until /// [`Route::announce`], so the job can fill its catalog first; dropping the route @@ -6909,7 +6909,7 @@ impl Request { /// Serve a held broadcast under this prefix route without announcing an exact route. /// /// The answer belongs to the prefix route's instance at this path: every answer for - /// that path must continue the same sequence space and bytes. Start a new prefix + /// that path must continue the same key space and bytes. Start a new prefix /// route to serve different content there. For a job that may restart independently, /// use [`Self::route`]: its end moves the prefix route to another instance. pub fn serve(self, broadcast: impl Consume) { @@ -7944,7 +7944,7 @@ impl Consumer { /// /// When its serving source dies or a better route of the same instance appears, /// including a higher epoch, the front switches to it, invisibly to subscribers: the - /// instance promises one continuous sequence space. Its routes retracting with no + /// instance promises one continuous key space. Its routes retracting with no /// replacement ends the broadcast, and the next request re-serves the path; /// tracks already in flight carry on to their own end. The broadcast also ends /// once no handle holds it and none of its tracks has been read for a linger, @@ -8887,9 +8887,9 @@ mod tests { // The query is already in flight when metadata and the abort arrive together. let track = track.accept(track::Info::timed()); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"history".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"history".as_ref()).unwrap(); + group.close().unwrap(); track.abort(Error::App(7)).unwrap(); driver.poll(now, &waiter).unwrap(); let Poll::Ready(Ok(mut subscriber)) = subscribing.poll_ok(&waiter) else { @@ -8917,8 +8917,8 @@ mod tests { let broadcast = Arc::new(broadcast::Info::default()); let source = track::Producer::new(broadcast.clone(), "catalog.json", crate::track::Info::timed()); let mut group = source.create_group(7u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"snapshot".as_ref()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"snapshot".as_ref()).unwrap(); + group.close().unwrap(); let source = source.finish().unwrap(); let routes = crate::model::resume::Producer::new(); @@ -8944,7 +8944,7 @@ mod tests { io.end(Ok(()), None); let mut group = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(group.sequence, 7); + assert_eq!(group.start, 7); let frame = group.read_frame().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), b"snapshot"); assert!(subscriber.recv_group().now_or_never().unwrap().unwrap().is_none()); @@ -9681,9 +9681,9 @@ mod tests { .await .unwrap(); let mut metered = broadcast.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"frame".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"frame".as_ref()).unwrap(); + group.close().unwrap(); let mut read = unmetered.recv_group().await.unwrap().expect("a group"); assert_eq!( @@ -9730,10 +9730,10 @@ mod tests { let mut subscriber = broadcast.track("video").unwrap().subscribe(None).await.unwrap(); let bill = subscriber.bill(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(crate::Timestamp::ZERO, b"dropped".as_ref()).unwrap(); group.write_frame(crate::Timestamp::ZERO, b"sent".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut read = subscriber.recv_group().await.unwrap().expect("a group"); let mut chunked = read.clone(); @@ -9783,9 +9783,10 @@ mod tests { .sum() }; - let mut live = track.create_group(crate::group::Info { sequence: 1 }).unwrap(); - live.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); - live.finish().unwrap(); + let mut live = track.create_group(crate::group::Info { start: 1 }).unwrap(); + live.write_frame(crate::Timestamp::from_millis(1).unwrap(), b"live".as_ref()) + .unwrap(); + live.close().unwrap(); assert_eq!(ingress(), 4); let fetch = track.consume().fetch_group(0, None); @@ -9794,7 +9795,7 @@ mod tests { fetched .write_frame(crate::Timestamp::ZERO, b"fetched".as_ref()) .unwrap(); - fetched.finish().unwrap(); + fetched.close().unwrap(); fetch.await.unwrap(); assert_eq!(ingress(), 4 + 7, "the fetched group counts as well"); @@ -11289,9 +11290,9 @@ mod tests { .await; let source = request.accept(crate::track::Info::timed()); - let mut group = source.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"late".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = source.append_next().unwrap(); + group.write_frame(group.key_time(), b"late".as_ref()).unwrap(); + group.close().unwrap(); let source = source.finish().unwrap(); let mut subscription = subscribing.await.expect("subscribe survives the retraction"); @@ -11336,16 +11337,12 @@ mod tests { .expect("request"); let source = request.request_start(None).accept(crate::track::Info::timed()); let mut group = source.create_group(0u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"snapshot".as_ref()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"snapshot".as_ref()).unwrap(); + group.close().unwrap(); let source = source.finish().unwrap(); let mut subscription = subscribing.await.unwrap().expect("subscribe"); assert_eq!( - next_group(&mut subscription) - .await - .unwrap() - .expect("the catalog") - .sequence, + next_group(&mut subscription).await.unwrap().expect("the catalog").start, 0 ); assert!(next_group(&mut subscription).await.unwrap().is_none()); @@ -11355,11 +11352,7 @@ mod tests { // Within the linger, a returning reader gets the finished track from the front. let mut subscription = resolved.track("catalog").unwrap().subscribe(None).await.unwrap(); assert_eq!( - next_group(&mut subscription) - .await - .unwrap() - .expect("the catalog") - .sequence, + next_group(&mut subscription).await.unwrap().expect("the catalog").start, 0 ); assert!(next_group(&mut subscription).await.unwrap().is_none()); @@ -11395,8 +11388,8 @@ mod tests { .unwrap(); let upstream = broadcast::Info::new().produce(); let source = upstream.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = source.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); + let mut group = source.append_next().unwrap(); + group.write_frame(group.key_time(), b"live".as_ref()).unwrap(); let counts = || { let table = producer.shared.lock(); @@ -11474,8 +11467,8 @@ mod tests { .unwrap(); let upstream = broadcast::Info::new().produce(); let source = upstream.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = source.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); + let mut group = source.append_next().unwrap(); + group.write_frame(group.key_time(), b"live".as_ref()).unwrap(); let fronts = || producer.shared.lock().fronts.len(); let pending = consumer.request("room/alice", None); @@ -11607,8 +11600,8 @@ mod tests { .unwrap(); let upstream = broadcast::Info::new().produce(); let source = upstream.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = source.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); + let mut group = source.append_next().unwrap(); + group.write_frame(group.key_time(), b"live".as_ref()).unwrap(); let fronts = || producer.shared.lock().fronts.len(); let pending = producer.consume().request("room/alice", None); @@ -11718,8 +11711,8 @@ mod tests { .unwrap() .accept(crate::track::Info::timed()); let mut group = source.create_group(0u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"a".as_ref()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"b".as_ref()).unwrap(); + group.write_frame(group.key_time(), b"a".as_ref()).unwrap(); + group.write_frame(group.key_time(), b"b".as_ref()).unwrap(); let mut subscription = subscribing.await.unwrap().unwrap(); let mut reading = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(read(&mut reading).await, b"a"); @@ -11766,7 +11759,7 @@ mod tests { while let Ok(request) = fetches.requested_fill_group().await { let mut group = request.accept(crate::track::Info::timed()).unwrap(); for frame in [b"a", b"b", b"c"] { - group.write_frame(crate::Timestamp::ZERO, frame.as_ref()).unwrap(); + group.write_frame(group.key_time(), frame.as_ref()).unwrap(); } serving.lock().unwrap().push(group); } @@ -11775,14 +11768,14 @@ mod tests { next.start_at(0).unwrap(); let mut next_group = next.create_group(0u64.into()).unwrap(); next_group.start_at(2).unwrap(); - next_group.write_frame(crate::Timestamp::ZERO, b"c".as_ref()).unwrap(); + next_group.write_frame(next_group.key_time(), b"c".as_ref()).unwrap(); assert_eq!(read(&mut reading).await, b"c", "dies_first={dies_first}"); // A preempted route still owns its open group: its writer and other readers carry // on, and its copy of the frame the replacement already wrote is a duplicate. if let Some((group, ..)) = &mut first { let mut independent = group.consume(); - group.write_frame(crate::Timestamp::ZERO, b"c".as_ref()).unwrap(); + group.write_frame(group.key_time(), b"c".as_ref()).unwrap(); for expect in [b"a", b"b", b"c"] { assert_eq!(read(&mut independent).await, expect); } @@ -11799,7 +11792,7 @@ mod tests { .unwrap_or_else(|_| panic!("dies_first={dies_first}: the later reader never got the group")) .unwrap() .expect("track ended"); - assert_eq!(late.sequence, 0); + assert_eq!(late.start, 0); for expect in [b"a", b"b", b"c"] { assert_eq!(read(&mut late).await, expect, "dies_first={dies_first}"); } @@ -11809,7 +11802,7 @@ mod tests { .await .expect("the reader joining mid-outage never got the group") .unwrap(); - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); for expect in [b"a", b"b", b"c"] { assert_eq!(read(&mut group).await, expect); } @@ -11825,16 +11818,16 @@ mod tests { ); // The continuation still flows to both. - next_group.write_frame(crate::Timestamp::ZERO, b"d".as_ref()).unwrap(); + next_group.write_frame(next_group.key_time(), b"d".as_ref()).unwrap(); for group in fetched.lock().unwrap().iter_mut() { - group.write_frame(crate::Timestamp::ZERO, b"d".as_ref()).unwrap(); + group.write_frame(group.key_time(), b"d".as_ref()).unwrap(); } assert_eq!(read(&mut reading).await, b"d"); assert_eq!(read(&mut late).await, b"d"); if let Some((_, group)) = &mut joined { assert_eq!(read(group).await, b"d"); } - next_group.finish().unwrap(); + next_group.close().unwrap(); drop(( joined, reading, @@ -11866,9 +11859,9 @@ mod tests { .expect("the front asked the source") .expect("request"); let source = request.accept(crate::track::Info::timed()); - let mut group = source.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"cached".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = source.append_next().unwrap(); + group.write_frame(group.key_time(), b"cached".as_ref()).unwrap(); + group.close().unwrap(); let mut subscription = subscribing.await.unwrap().expect("subscribe"); subscription.recv_group().await.unwrap().expect("the cached group"); drop(subscription); @@ -11893,8 +11886,8 @@ mod tests { // A fresh source copy numbers groups past what the cache already delivered. let source = request.accept(crate::track::Info::timed()); let mut group = source.create_group(1u64.into()).unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"late".as_ref()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"late".as_ref()).unwrap(); + group.close().unwrap(); let _source = source.finish().unwrap(); let mut subscription = subscribing.await.unwrap().expect("subscribe survives the retraction"); @@ -13555,7 +13548,7 @@ mod tests { let front = producer.consume().request("room", None).await.unwrap(); let mut reader = front.datagrams("audio").unwrap(); - datagrams.append(None, "x").unwrap(); + datagrams.append(crate::Timestamp::unscaled(0), "x").unwrap(); let datagram = reader.recv().now_or_never().unwrap().unwrap().unwrap(); assert_eq!(datagram.sequence, 0); assert!(matches!(front.datagrams("video"), Err(Error::NotFound))); @@ -13573,9 +13566,9 @@ mod tests { let source = broadcast::Info::new().produce(); for name in ["a", "b"] { let track = source.create_track(name, crate::track::Info::timed()).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, name.as_bytes()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), name.as_bytes()).unwrap(); + group.close().unwrap(); // The producer stays alive: the track is open, like a live SI track. std::mem::forget(track); } @@ -13906,9 +13899,9 @@ mod tests { let request = queued(&server).await; let source = broadcast::Info::new().produce(); let track = source.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); request.serve(&source); let resolved = pending.await.expect("resolves"); @@ -13961,14 +13954,14 @@ mod tests { let track = replacement.create_track("video", crate::track::Info::timed()).unwrap(); // The same content: group 0 was already delivered through the old route, // so the track resumes at group 1. - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); request.serve(&replacement); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"resumed".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"resumed".as_ref()).unwrap(); + group.close().unwrap(); let mut group = rig .subscription @@ -13982,7 +13975,7 @@ mod tests { /// Only an instance's highest epoch is eligible: a costlier route at a higher epoch /// takes the front over from the cheaper older one still standing, and the reader - /// continues across the epoch change without a seam, since the instance's sequences + /// continues across the epoch change without a seam, since the instance's keys /// continue. A new request joins the same broadcast. #[kio::test] #[cfg(feature = "async")] @@ -14098,9 +14091,9 @@ mod tests { /// Write one single-frame group and read it back through `subscription`. #[cfg(feature = "async")] async fn deliver(track: &track::Producer, subscription: &mut track::Subscriber, payload: &'static [u8]) { - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, payload).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), payload).unwrap(); + group.close().unwrap(); let mut group = next_group(subscription).await.unwrap().expect("track ended early"); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], payload); } @@ -14492,9 +14485,9 @@ mod tests { let request = queued(&standby_server).await; let successor = broadcast::Info::new().produce(); let track = successor.create_track("video", replacement).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); request.serve(&successor); rig.incumbent_track.clone().abort(Error::Dropped).unwrap(); @@ -14540,11 +14533,11 @@ mod tests { let (mut rig, incumbent, source) = ResumeRig::new(&[10]).await; let mut group = rig .incumbent_track - .create_group(crate::group::Info { sequence: 5 }) + .create_group(crate::group::Info { start: 5 }) .unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"five".as_ref()).unwrap(); - group.finish().unwrap(); - assert_eq!(rig.subscription.recv_group().await.unwrap().unwrap().sequence, 5); + group.write_frame(group.key_time(), b"five".as_ref()).unwrap(); + group.close().unwrap(); + assert_eq!(rig.subscription.recv_group().await.unwrap().unwrap().start, 5); let standby = rig.standby(&[10, 20]); drop(incumbent); drop(source); @@ -14572,9 +14565,9 @@ mod tests { let standby = rig.standby(&[10, 20]); server.update(RouteInfo::new().with_hops(hops(&[11]))).unwrap(); - let mut group = rig.incumbent_track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"after".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = rig.incumbent_track.append_next().unwrap(); + group.write_frame(group.key_time(), b"after".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut rig.subscription) .await .expect("the subscription survives the update") @@ -14621,9 +14614,9 @@ mod tests { let request = queued(&incumbent).await; let source = broadcast::Info::new().produce(); let track = source.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); request.serve(&source); let resolved = pending.await.expect("resolves through the incumbent"); @@ -14649,13 +14642,13 @@ mod tests { let request = queued(&echo).await; let replacement = broadcast::Info::new().produce(); let track = replacement.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); request.serve(&replacement); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"resumed".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"resumed".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut subscription).await.unwrap().expect("resumed group"); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"resumed"); @@ -14687,9 +14680,9 @@ mod tests { let request = queued(&server_a).await; let source_a = broadcast::Info::new().produce(); let track_a = source_a.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = track_a.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"from-a".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track_a.append_next().unwrap(); + group.write_frame(group.key_time(), b"from-a".as_ref()).unwrap(); + group.close().unwrap(); request.serve(&source_a); let resolved_a = pending.await.expect("resolves"); @@ -14739,9 +14732,9 @@ mod tests { let request = queued(&server_b).await; let source_b = broadcast::Info::new().produce(); let track_b = source_b.create_track("video", crate::track::Info::timed()).unwrap(); - let mut group = track_b.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"from-b".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track_b.append_next().unwrap(); + group.write_frame(group.key_time(), b"from-b".as_ref()).unwrap(); + group.close().unwrap(); request.serve(&source_b); let resolved_b = pending.await.expect("B's front is served immediately"); @@ -14779,9 +14772,9 @@ mod tests { .unwrap(); let track = source.create_track("audio", crate::track::Info::timed()).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"steady".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"steady".as_ref()).unwrap(); + group.close().unwrap(); let mut audio = rig .resolved @@ -14930,9 +14923,9 @@ mod tests { .await .expect("subscribe"); // A textbook clean end, innermost first: the group, the track, the broadcast. - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"tail".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"tail".as_ref()).unwrap(); + group.close().unwrap(); let track = track.finish().unwrap(); drop(track); broadcast.broadcast().close(); @@ -14984,9 +14977,9 @@ mod tests { source.close(); drop(server); settle(|| resolved.is_closed()).await; - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"tail".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"tail".as_ref()).unwrap(); + group.close().unwrap(); let track = track.finish().unwrap(); drop(track); @@ -15040,9 +15033,9 @@ mod tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"cached".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"cached".as_ref()).unwrap(); + group.close().unwrap(); let resolved = consumer.request("room/alice", None).await.expect("resolves"); let mut subscription = resolved @@ -15083,9 +15076,9 @@ mod tests { .expect("returning reader re-splices the source") .expect("source closed"); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"live".as_ref()).unwrap(); + group.close().unwrap(); let mut group = kio::clock::timeout(Duration::from_secs(1), again.recv_group()) .await .expect("groups past the cached edge come from the re-splice") @@ -15111,15 +15104,15 @@ mod tests { let resolved = pending.await.unwrap(); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut writing = track.append_group().unwrap(); - writing.write_frame(crate::Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut writing = track.append_next().unwrap(); + writing.write_frame(writing.key_time(), b"head".as_ref()).unwrap(); let mut reading = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"head"); drop(subscription); kio::clock::sleep(Duration::from_millis(10)).await; - writing.write_frame(crate::Timestamp::ZERO, b"tail".as_ref()).unwrap(); - writing.finish().unwrap(); + writing.write_frame(writing.key_time(), b"tail".as_ref()).unwrap(); + writing.close().unwrap(); let tail = kio::clock::timeout(Duration::from_secs(1), reading.read_frame()) .await @@ -15150,8 +15143,8 @@ mod tests { let resolved = producer.consume().request("live", None).await.unwrap(); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut aborted = track.append_group().unwrap(); - aborted.write_frame(crate::Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut aborted = track.append_next().unwrap(); + aborted.write_frame(aborted.key_time(), b"head".as_ref()).unwrap(); let mut reading = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&reading.read_frame().await.unwrap().unwrap().payload[..], b"head"); aborted.abort(Error::Cancel).unwrap(); @@ -15160,9 +15153,9 @@ mod tests { assert!(matches!(end, Ok(Err(_))), "the group should fail: {end:?}"); // The track carries on. - let mut next = track.append_group().unwrap(); - next.write_frame(crate::Timestamp::ZERO, b"next".as_ref()).unwrap(); - next.finish().unwrap(); + let mut next = track.append_next().unwrap(); + next.write_frame(next.key_time(), b"next".as_ref()).unwrap(); + next.close().unwrap(); let mut group = kio::clock::timeout(Duration::from_secs(1), subscription.recv_group()) .await .expect("the next group") @@ -15185,9 +15178,9 @@ mod tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"cached".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"cached".as_ref()).unwrap(); + group.close().unwrap(); // The leaf's front view, which the next front serves from. let leaf_front = leaf_consumer.request("room/alice", None).await.expect("resolves"); @@ -15233,9 +15226,9 @@ mod tests { .await .expect("resubscribe should reach the leaf") .expect("source open"); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"live".as_ref()).unwrap(); + group.close().unwrap(); let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"cached"); drop(group); @@ -15251,9 +15244,9 @@ mod tests { // A group the leaf produced while every front was parked: the fetch re-splices // each hop to reach the leaf. - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"fetched".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"fetched".as_ref()).unwrap(); + group.close().unwrap(); let fetch = edge_resolved.track("video").unwrap().fetch_group(2, None); let mut fetch = std::pin::pin!(fetch); assert!(futures::poll!(fetch.as_mut()).is_pending(), "fetch should re-splice"); @@ -15291,9 +15284,9 @@ mod tests { .subscribe(None) .await .expect("subscribe"); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); @@ -15310,9 +15303,9 @@ mod tests { } // Still spliced to the incumbent, still delivering. - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"still".as_ref()).unwrap(); + group.close().unwrap(); let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"still"); @@ -15339,9 +15332,9 @@ mod tests { queued(&server).await.serve(&upstream); let resolved = pending.await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut subscription).await.unwrap().expect("first group"); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); (producer, server, upstream, old, subscription) @@ -15397,9 +15390,9 @@ mod tests { kio::clock::sim::yield_now().await; } - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"still".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut subscription) .await .expect("the incumbent keeps serving") @@ -15440,9 +15433,9 @@ mod tests { queued(&server).await.serve(&upstream); let resolved = pending.await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut subscription).await.unwrap().expect("first group"); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); @@ -15463,9 +15456,9 @@ mod tests { kio::clock::sim::yield_now().await; } - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"last".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut subscription) .await .expect("the incumbent keeps serving") @@ -15475,7 +15468,7 @@ mod tests { let _old = old.finish().unwrap(); let end = next_group(&mut subscription) .await - .map(|group| group.map(|group| group.sequence)); + .map(|group| group.map(|group| group.start)); assert!(matches!(end, Ok(None)), "the incumbent's clean end, not {end:?}"); } @@ -15505,9 +15498,9 @@ mod tests { kio::clock::sim::yield_now().await; } - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"still".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut subscription) .await .expect("the incumbent keeps serving") @@ -15539,9 +15532,9 @@ mod tests { for _ in 0..10 { kio::clock::sim::yield_now().await; } - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"last".as_ref()).unwrap(); + group.close().unwrap(); let _old = old.finish().unwrap(); replacement.abort(Error::App(7)).unwrap(); @@ -15554,26 +15547,27 @@ mod tests { } /// The incumbent finishes and the newcomer aborts before its first group: a reader whose - /// range starts past the end, and one joining while the newcomer is awaited, end cleanly. + /// range starts past the last group, which contains its start, and one joining while the + /// newcomer is awaited, each read that group and end cleanly. #[kio::test] #[cfg(feature = "async")] async fn readers_owed_nothing_below_the_incumbents_end_end_cleanly() { let (producer, _server, _upstream, old, _subscription) = remote_front().await; let resolved = producer.consume().request("room/alice", None).await.unwrap(); let video = resolved.track("video").unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"last".as_ref()).unwrap(); + group.close().unwrap(); let mut later = video .subscribe(crate::track::Subscription { ranges: vec![crate::track::Range { - start: crate::track::Anchor::Absolute(3), + start: crate::track::Anchor::Time(3), end: None, }], ..Default::default() }) .await .unwrap(); - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"last".as_ref()).unwrap(); - group.finish().unwrap(); let newcomer = producer .route("room/alice", RouteOptions::hidden().with_instance(instance(), 0)) @@ -15595,13 +15589,14 @@ mod tests { let _old = old.finish().unwrap(); replacement.abort(Error::App(7)).unwrap(); - assert!(matches!(next_group(&mut later).await, Ok(None))); - let mut group = next_group(&mut joiner) - .await - .expect("the incumbent's latest group") - .expect("track open"); - assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); - assert!(matches!(next_group(&mut joiner).await, Ok(None))); + for reader in [&mut later, &mut joiner] { + let mut group = next_group(reader) + .await + .expect("the incumbent's latest group") + .expect("track open"); + assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"last"); + assert!(matches!(next_group(reader).await, Ok(None))); + } } /// While the front waits for the newcomer's first group, a reader changing its @@ -15733,9 +15728,9 @@ mod tests { for _ in 0..10 { kio::clock::sim::yield_now().await; } - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"draining".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"draining".as_ref()).unwrap(); + group.close().unwrap(); let mut group = next_group(&mut subscription).await.unwrap().expect("draining group"); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"draining"); drop(group); @@ -15754,9 +15749,9 @@ mod tests { } let mut again = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = old.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"after".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = old.append_next().unwrap(); + group.write_frame(group.key_time(), b"after".as_ref()).unwrap(); + group.close().unwrap(); let outcome = kio::clock::timeout(Duration::from_secs(5), async { loop { match again.recv_group().await { @@ -15783,8 +15778,8 @@ mod tests { async fn a_parked_track_serves_whole_groups_from_its_copy() { let (producer, _server, _upstream, old, subscription) = remote_front().await; let resolved = producer.consume().request("room/alice", None).await.unwrap(); - let mut open = old.append_group().unwrap(); - open.write_frame(crate::Timestamp::ZERO, b"open".as_ref()).unwrap(); + let mut open = old.append_next().unwrap(); + open.write_frame(open.key_time(), b"open".as_ref()).unwrap(); drop(subscription); kio::clock::timeout(Duration::from_secs(1), old.demand().until(|wanted| wanted.is_none())) .await @@ -15804,7 +15799,7 @@ mod tests { (&mut fetching).now_or_never().is_none(), "an open group needs the route" ); - open.finish().unwrap(); + open.close().unwrap(); let mut group = fetching.await.unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"open"); } @@ -15857,9 +15852,9 @@ mod tests { .unwrap(); let resolved = consumer.request("room/alice", None).await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); @@ -15901,9 +15896,9 @@ mod tests { .unwrap(); let resolved = consumer.request("room/alice", None).await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); drop(group); @@ -15923,13 +15918,13 @@ mod tests { second.withdraw(); settle(|| resolved.is_closed()).await; - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"still".as_ref()).unwrap(); + group.close().unwrap(); for payload in [b"wrong0".as_ref(), b"wrong1".as_ref()] { - let mut group = incompatible.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, payload).unwrap(); - group.finish().unwrap(); + let mut group = incompatible.append_next().unwrap(); + group.write_frame(group.key_time(), payload).unwrap(); + group.close().unwrap(); } let mut group = next_group(&mut subscription) @@ -15961,9 +15956,9 @@ mod tests { .unwrap(); let resolved = consumer.request("room/alice", None).await.expect("resolves"); let mut subscription = resolved.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"before".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"before".as_ref()).unwrap(); + group.close().unwrap(); let mut group = subscription.recv_group().await.unwrap().unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"before"); drop(group); @@ -15982,9 +15977,9 @@ mod tests { second.withdraw(); settle(|| resolved.is_closed()).await; - let mut group = track.append_group().unwrap(); - group.write_frame(crate::Timestamp::ZERO, b"still".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.key_time(), b"still".as_ref()).unwrap(); + group.close().unwrap(); request.reject(Error::NotFound); let mut group = next_group(&mut subscription) @@ -16673,9 +16668,9 @@ mod tests { .broadcast() .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut written = track.append_group().unwrap(); - written.write_frame(crate::Timestamp::ZERO, b"frame".as_ref()).unwrap(); - written.finish().unwrap(); + let mut written = track.append_next().unwrap(); + written.write_frame(written.key_time(), b"frame".as_ref()).unwrap(); + written.close().unwrap(); let _track = track.finish().unwrap(); let mut reader = resolved.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = reader.recv_group().await.unwrap().unwrap(); @@ -16762,12 +16757,12 @@ mod tests { .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut stalled = track.append_group().unwrap(); - stalled.write_frame(crate::Timestamp::ZERO, b"x".as_slice()).unwrap(); + let mut stalled = track.append_next().unwrap(); + stalled.write_frame(stalled.key_time(), b"x".as_slice()).unwrap(); // A successor, so the stalled group is not the protected live edge. Its // timestamp is inside the max age, so the stalled group stays fresh and // subscription expiry never skips it: only reclamation can bound it. - let _successor = track.append_group().unwrap(); + let _successor = track.append_next().unwrap(); let mut reading = stalled.consume(); assert!(reading.read_frame().await.unwrap().is_some()); @@ -16798,9 +16793,9 @@ mod tests { .create_track("video", crate::track::Info::timed()) .unwrap(); - let mut stalled = track.append_group().unwrap(); - stalled.write_frame(crate::Timestamp::ZERO, b"x".as_slice()).unwrap(); - let _successor = track.append_group().unwrap(); + let mut stalled = track.append_next().unwrap(); + stalled.write_frame(stalled.key_time(), b"x".as_slice()).unwrap(); + let _successor = track.append_next().unwrap(); let mut reading = stalled.consume(); assert!(reading.read_frame().await.unwrap().is_some()); diff --git a/rs/moq-net/src/model/pace.rs b/rs/moq-net/src/model/pace.rs index 5301c75a16f..7029c0fe857 100644 --- a/rs/moq-net/src/model/pace.rs +++ b/rs/moq-net/src/model/pace.rs @@ -178,10 +178,12 @@ mod tests { /// An untimed group with `frames` frames of a megabyte each. fn written(frames: usize) -> group::Producer { - let mut group = group::Info { sequence: 0 }.produce_untimed(); + let mut group = group::Info { start: 0 }.produce_untimed(); let payload = Bytes::from(vec![0u8; MB]); for _ in 0..frames { - group.write_frame(None, payload.clone()).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), payload.clone()) + .unwrap(); } group } @@ -189,7 +191,7 @@ mod tests { fn info(size: usize) -> frame::Info { frame::Info { size: size as u64, - timestamp: None, + timestamp: crate::Timestamp::unscaled(0), } } @@ -213,7 +215,9 @@ mod tests { "within the window" ); - group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from(vec![0u8; MB])) + .unwrap(); assert_eq!(group.poll_paced(&waiter, 0, AHEAD), Poll::Pending, "past the window"); assert!(!flag.woken()); @@ -267,7 +271,7 @@ mod tests { frame::Header { info: frame::Info { size: 1, - timestamp: None, + timestamp: crate::Timestamp::unscaled(0), }, fields: Some(Arc::new(Fields { id, @@ -286,7 +290,7 @@ mod tests { #[cfg(feature = "async")] fn metadata_counts_toward_the_window() { let (_flag, waiter) = flag(); - let mut group = group::Info { sequence: 0 }.produce_untimed(); + let mut group = group::Info { start: 0 }.produce_untimed(); let _reader = group.consume(); for id in 0..15 { let mut frame = group.create_object(fielded(id, MB)).unwrap(); @@ -311,7 +315,7 @@ mod tests { let mut frame = group .create_frame(frame::Info { size: 2 * MB as u64, - timestamp: None, + timestamp: crate::Timestamp::unscaled(0), }) .unwrap(); assert_eq!(frame.poll_ready(&waiter, AHEAD), Poll::Ready(())); @@ -361,14 +365,16 @@ mod tests { fn a_local_writer_waits_for_a_reader_and_room() { let (flag, waiter) = flag(); let small = info(0); - let mut group = group::Info { sequence: 0 }.produce_untimed(); + let mut group = group::Info { start: 0 }.produce_untimed(); assert_eq!(group.poll_ready(&waiter, &small), Poll::Pending, "nobody reads yet"); let mut reader = group.consume(); assert!(flag.woken(), "a reader arriving wakes the writer"); assert_eq!(group.poll_ready(&waiter, &small), Poll::Ready(())); for _ in 0..17 { - group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from(vec![0u8; MB])) + .unwrap(); } assert_eq!(group.poll_ready(&waiter, &small), Poll::Pending, "past the window"); read(&mut reader); @@ -392,12 +398,16 @@ mod tests { #[test] fn a_local_writer_waits_for_room_for_its_frame() { let (_flag, waiter) = flag(); - let mut group = group::Info { sequence: 0 }.produce_untimed(); + let mut group = group::Info { start: 0 }.produce_untimed(); let mut reader = group.consume(); - group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from(vec![0u8; MB])) + .unwrap(); read(&mut reader); for _ in 0..8 { - group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from(vec![0u8; MB])) + .unwrap(); } assert_eq!( group.poll_ready(&waiter, &info(MB)), @@ -432,7 +442,7 @@ mod tests { let mut frame = group .create_frame(frame::Info { size: 20 * MB as u64, - timestamp: None, + timestamp: crate::Timestamp::unscaled(0), }) .unwrap(); frame.write(Bytes::from(vec![0u8; 18 * MB])).unwrap(); @@ -448,7 +458,9 @@ mod tests { let _reader = group.consume(); // A local writer never waits, so it sheds the front past the 32 MB tail. for _ in 0..40 { - group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from(vec![0u8; MB])) + .unwrap(); } assert_eq!(group.poll_paced(&waiter, 0, AHEAD), Poll::Ready(())); } @@ -458,11 +470,13 @@ mod tests { #[test] fn a_reader_ahead_of_the_writer_never_stalls_it() { let (_flag, waiter) = flag(); - let mut group = group::Info { sequence: 0 }.produce_untimed(); + let mut group = group::Info { start: 0 }.produce_untimed(); let mut reader = group.consume().with_range(20..); assert!(reader.poll_read_frame(&kio::Waiter::noop()).is_pending()); for _ in 0..17 { - group.write_frame(None, Bytes::from(vec![0u8; MB])).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), Bytes::from(vec![0u8; MB])) + .unwrap(); } // Each commit wakes the parked reader, which polls again and reports the edge. assert!(reader.poll_read_frame(&kio::Waiter::noop()).is_pending()); diff --git a/rs/moq-net/src/model/record.rs b/rs/moq-net/src/model/record.rs index 791b7f896b9..7b7382484e4 100644 --- a/rs/moq-net/src/model/record.rs +++ b/rs/moq-net/src/model/record.rs @@ -1,4 +1,4 @@ -//! What one reader did with each group sequence, as runs: handed out, skipped as stale, given +//! What one reader did with each group key, as runs: handed out, skipped as stale, given //! up, and what its wire was told. use std::ops::Range; @@ -17,10 +17,10 @@ use crate::Error; /// settles costs what the cache would hold, never more. pub(super) const RUN_OVERHEAD: u64 = 2 * size_of::<(Range, Pending)>() as u64; -/// One reader's record of every sequence it settled, as runs. +/// One reader's record of every key it settled, as runs. #[derive(Default)] pub(crate) struct Record { - /// The groups handed out, so none is handed out twice: not when its sequence is re-created, + /// The groups handed out, so none is handed out twice: not when its key is re-created, /// nor when another route delivers it too. pub(super) delivered: Delivered, /// Cached groups skipped as stale, which a wider budget owes again. @@ -29,10 +29,10 @@ pub(crate) struct Record { /// wider budget owes them again. Kept under an open window too, which a reader may narrow to /// a bounded one. pub(super) written_off: RangeSet, - /// Sequences given up and not reported yet, with why: stale skips, cuts, and upstream skips + /// Keys given up and not reported yet, with why: stale skips, cuts, and upstream skips /// and drops. pub(super) pending: RangeMap, - /// Every sequence already reported, with how. + /// Every key already reported, with how. pub(super) reported: RangeMap, /// Everything below was settled for good and folded away under pool pressure: handed /// out, or reported final, so none is handed out or reported again. @@ -172,13 +172,13 @@ impl Record { None } - /// Mark a stale report on `sequence` settled by its group being handed out. - pub fn retire(&mut self, sequence: u64) { - if let Some(end) = sequence.checked_add(1) - && let Some(Report::Stale { received, expired, .. }) = self.reported.get(&sequence).copied() + /// Mark a stale report on `key` settled by its group being handed out. + pub fn retire(&mut self, key: u64) { + if let Some(end) = key.checked_add(1) + && let Some(Report::Stale { received, expired, .. }) = self.reported.get(&key).copied() { self.reported.insert( - sequence..end, + key..end, Report::Stale { received, expired, @@ -189,22 +189,22 @@ impl Record { } } -/// What a publisher told the wire about a reader's sequences. +/// What a publisher told the wire about a reader's keys. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] impl Record { - /// Whether the wire was told of `sequence`: served, or given up. - pub fn told(&self, sequence: u64) -> bool { - sequence < self.floor || self.reported.contains_key(&sequence) + /// Whether the wire was told of `key`: served, or given up. + pub fn told(&self, key: u64) -> bool { + key < self.floor || self.reported.contains_key(&key) } - /// Whether the pool folded `sequence` into this record's floor, final for this reader. - pub fn folded(&self, sequence: u64) -> bool { - sequence < self.floor + /// Whether the pool folded `key` into this record's floor, final for this reader. + pub fn folded(&self, key: u64) -> bool { + key < self.floor } - /// Whether `sequence` went out as a stale drop, which a group still replaces. - pub fn told_stale(&self, sequence: u64) -> bool { - matches!(self.reported.get(&sequence), Some(Report::Stale { .. })) + /// Whether `key` went out as a stale drop, which a group still replaces. + pub fn told_stale(&self, key: u64) -> bool { + matches!(self.reported.get(&key), Some(Report::Stale { .. })) } /// The parts of `range` the wire was told nothing of. @@ -359,7 +359,7 @@ impl Record { /// not final. Final are the groups handed out, the reports nothing reopens, and this /// reader's own stale skips, which a wider budget would offer again only while the pool /// still held their groups. Not final are the reports an update reopens, the runs given - /// up and not reported yet, and the sequences nothing settled. + /// up and not reported yet, and the keys nothing settled. fn fold(&mut self, handed: Option) { // The runs copied from the track's drop set go first: the next scan folds in again // whatever the track still holds. @@ -458,13 +458,13 @@ mod test { fn a_record_bills_each_run() { let (pool, track) = account(); let mut record = Record::default(); - for sequence in 0..4 { - record.delivered.insert(sequence); + for key in 0..4 { + record.delivered.insert(key); } record.settle(|| track.clone(), &mut (), None).unwrap(); assert_eq!(pool.used(), 0, "a lone prefix bills nothing"); - for sequence in (10..1_000).step_by(2) { - record.reported.insert(sequence..sequence + 1, Report::Dropped); + for key in (10..1_000).step_by(2) { + record.reported.insert(key..key + 1, Report::Dropped); } record.settle(|| track.clone(), &mut (), None).unwrap(); assert_eq!(pool.used(), 495 * RUN_OVERHEAD); @@ -473,8 +473,8 @@ mod test { } /// Under a debt its track could not pay, a record folds what is final into its floor: - /// handed-out groups and reports nothing reopens, however many sequences a run spans. - /// Folded sequences count as handed out and reported. + /// handed-out groups and reports nothing reopens, however many keys a run spans. + /// Folded keys count as handed out and reported. #[test] fn a_debt_folds_the_final_prefix() { let (pool, track) = account(); @@ -499,7 +499,7 @@ mod test { } /// The group a read hands out stays above the fold: its publisher has yet to tell its wire - /// of it, and takes a sequence below the floor as told already. + /// of it, and takes a key below the floor as told already. #[test] fn a_debt_folds_short_of_the_handed_group() { let (pool, track) = account(); @@ -537,8 +537,8 @@ mod test { record.reported.insert(1..2, Report::Dropped); record.reported.insert(2..3, promise); // Enough runs above it that ending the reader frees more than evicting a group. - for sequence in (5..1_000).step_by(2) { - record.reported.insert(sequence..sequence + 1, Report::Dropped); + for key in (5..1_000).step_by(2) { + record.reported.insert(key..key + 1, Report::Dropped); } record.settle(|| track.clone(), &mut (), None).unwrap(); owe(&track, pool.used()); @@ -553,7 +553,7 @@ mod test { } } - /// A run given up and not reported yet, or a sequence nothing settled, is no fold's to + /// A run given up and not reported yet, or a key nothing settled, is no fold's to /// forget either; a debt the fold pays leaves the reader running. #[test] fn a_debt_the_fold_pays_leaves_the_reader() { @@ -585,8 +585,8 @@ mod test { let mut small = Record::default(); small.reported.insert(1..2, stale); let mut large = Record::default(); - for sequence in (1..1_000).step_by(2) { - large.reported.insert(sequence..sequence + 1, stale); + for key in (1..1_000).step_by(2) { + large.reported.insert(key..key + 1, stale); } small.settle(|| track.clone(), &mut (), None).unwrap(); large.settle(|| track.clone(), &mut (), None).unwrap(); diff --git a/rs/moq-net/src/model/resume.rs b/rs/moq-net/src/model/resume.rs index 8f72f801ca6..79adef2bf81 100644 --- a/rs/moq-net/src/model/resume.rs +++ b/rs/moq-net/src/model/resume.rs @@ -33,7 +33,7 @@ //! content. //! //! A group is handed out once however many copies deliver it: the reader skips any -//! sequence its [`track::Subscriber`] already handed out, the same record that keeps a +//! key its [`track::Subscriber`] already handed out, the same record that keeps a //! direct reader from taking a re-created group twice. //! //! A reader polled for drops reports what the serving copy gave up, other than a route @@ -53,7 +53,7 @@ use crate::{Error, Result, StreamError, group, stats, track}; use super::delivered::Delivered; use super::idle::{Idle, Walk}; -use super::subscription::{Accounted, Anchor, Position, Resolved, Subscription, Unheld, Window}; +use super::subscription::{Accounted, Anchor, Containing, Holdings, Position, Resolved, Subscription, Unheld, Window}; use crate::ledger; /// What the front says about a logical track: the copy to read now, and how it ended. @@ -431,13 +431,13 @@ impl Consumer { /// track's state, which the fetched group holds while it is read. pub(super) fn fetch_group( &self, - sequence: u64, + key: u64, options: track::FetchOptions, logical: kio::Consumer, ) -> Fetching { Fetching { route: self.clone(), - sequence, + key, options, logical, pending: None, @@ -472,6 +472,33 @@ impl Consumer { } } +/// Ranges a copy resolved, for a reader that holds no groups of its own: each time anchor +/// names the group the copy resolved it to, so it is that group's start, not a time to +/// look for among groups held. +struct Placed; + +impl Holdings for Placed { + fn resolves(&self) -> bool { + false + } + + fn containing(&self, time: u64) -> Containing { + Containing::Group(time) + } + + fn next_start(&self, _: u64) -> Option { + None + } + + fn back(&self, latest: u64, back: u64) -> Option { + (back == 0).then_some(latest) + } + + fn complete(&self) -> bool { + true + } +} + /// Whether `copy` lost its route, so an error read from it is no verdict on what it carried. /// A terminal track verdict, unlike a delivery failure, still judges its content. fn copy_failed(copy: &track::Consumer) -> bool { @@ -545,10 +572,20 @@ impl Copy { } fn update(&mut self, reader: &Subscription) { + self.reopen(); let wire = self.wire(reader); self.send(wire); } + /// Read this copy again: one that ran out under the reader's old ranges, budget, or + /// bounds may hold a group the new ones admit, and runs out again at once otherwise. A + /// copy that failed stays failed. + fn reopen(&mut self) { + if matches!(self.done, Some(Ok(()))) { + self.done = None; + } + } + /// Ask for `wire` instead, answered or not. fn send(&mut self, wire: Subscription) { let _ = match &mut self.sub { @@ -658,17 +695,17 @@ impl Copy { // A group still read from it, or that it holds or may yet get within the grace a late // group is waited for: a failure of the copy that group is read from continues here. let lands = self.since.is_some() && !self.expired(grace, reader.waiter); - let backs = |sequence: u64| { + let backs = |key: u64| { // A group at or past where it was asked to start may land even before its first // arrival; without a floor, only what follows its first arrival can. - let asked = self.floor.is_some_and(|floor| floor <= sequence); - self.track.peek_group(sequence).is_some() + let asked = self.floor.is_some_and(|floor| floor <= key); + self.track.peek_group(key).is_some() || (lands && self .track - .may_land(sequence..sequence.saturating_add(1), asked, grace, None, reader.waiter)) + .may_land(key..key.saturating_add(1), asked, grace, None, reader.waiter)) }; - match Arc::strong_count(&self.lease) > 1 || reader.out.iter().any(|(sequence, _)| backs(*sequence)) { + match Arc::strong_count(&self.lease) > 1 || reader.out.iter().any(|(key, _)| backs(*key)) { true => Retains::Keep, false => Retains::No, } @@ -760,7 +797,7 @@ impl Mirror { impl Subscriber { /// Reserve the lowest next group not in `delivered` across copies, without handing it - /// out: in arrival order, or in sequence order for `ordered`. + /// out: in arrival order, or in key order for `ordered`. /// Unless it `parks`, a bounded reader ends once its window has nothing left on any copy; /// see [`Reader::exhausted`]. /// @@ -870,7 +907,23 @@ impl Subscriber { /// The lowest group at or above `from` the reader's ranges cover; see /// [`track::Subscriber::next_in_range`]. pub(super) fn next_in_range(&self, from: u64) -> Option { - self.reader().window().runs(from).next().map(|run| *run.start()) + let mut reader = self.reader(); + reader.window(); + // The serving route's copy holds what its publisher skipped or gave up. + let serving = reader.copies.last().map(|copy| copy.track.clone()); + track::first_open(reader.window.runs(from), |key| { + serving.as_ref().and_then(|track| track.hole_end(key, true)) + }) + } + + /// Whether the serving route's copy holds every group of `run` declared skipped; see + /// [`track::Subscriber::is_skipped`]. + pub(super) fn is_skipped(&self, run: impl Iterator>) -> bool { + let serving = self.reader().copies.last().map(|copy| copy.track.clone()); + let Some(serving) = serving else { + return false; + }; + track::first_open(run, |key| serving.hole_end(key, false)).is_none() } /// The groups the reader's ranges name from `from` on; see @@ -901,7 +954,7 @@ impl Subscriber { self.reader().reoffer_stale() } - /// Poll for sequences the serving copy gave up that no copy may still deliver; see + /// Poll for keys the serving copy gave up that no copy may still deliver; see /// [`track::Subscriber::poll_drop`]. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(super) fn poll_drop( @@ -929,7 +982,7 @@ impl Subscriber { }) } - /// The newest sequence any copy has. + /// The newest key any copy has. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(super) fn latest(&self) -> Option { self.reader().latest() @@ -946,7 +999,7 @@ impl Subscriber { route.poll_largest(seen, bounds, waiter) } - /// The final sequence and the error the logical track ended with, once the serving copy + /// The final key and the error the logical track ended with, once the serving copy /// completed and its groups drained to that end. pub(super) fn end_error(&self) -> Option<(u64, Error)> { let reader = self.reader(); @@ -959,7 +1012,7 @@ impl Subscriber { { return None; } - Some((reader.declared?.final_sequence()?, reader.declared_error.clone()?)) + Some((reader.declared?.final_key()?, reader.declared_error.clone()?)) } /// The error the logical track's declared end carries, reached or not. @@ -1017,8 +1070,8 @@ pub(super) struct Candidate { } impl Candidate { - pub(super) fn sequence(&self) -> u64 { - self.reserved.sequence() + pub(super) fn key(&self) -> u64 { + self.reserved.key() } pub(super) fn ordered(&self) -> bool { @@ -1058,7 +1111,7 @@ struct Reader { groups: (u64, Option), /// Whether groups were ever asked for; until then the reader keeps no replaced copy. groups_read: bool, - /// The groups handed out, by sequence, held while each is read. + /// The groups handed out, by key, held while each is read. out: Vec<(u64, Weak<()>)>, /// Runs the serving copy gave up that the reader has yet to report; see /// [`Reader::poll_drop`]. @@ -1081,10 +1134,19 @@ impl Reader { break; } let copy = &mut self.copies[index]; + let capped = copy.end().is_some(); let resolved = match &mut copy.sub { Sub::Ready(sub) => sub.resolved_ranges(), Sub::Pending(_) => continue, }; + // A failover cap at or below where a range starts empties the copy's range, which + // then says nothing of where the reader's own range starts. + let resolved: Vec<_> = resolved + .into_iter() + .map(|resolved| { + resolved.filter(|resolved| !capped || resolved.end.is_none_or(|end| end > resolved.start)) + }) + .collect(); self.resolve(&resolved); } &self.window @@ -1095,6 +1157,8 @@ impl Reader { fn resolve(&mut self, resolved: &[Option]) { // Once placed, relative/time starts stay pinned even while a time end settles. let placed = (!self.window.is_pending()).then(|| self.window.resolved()); + // A range the copy left unresolved keeps its anchors, which only a copy can place. + let mut unplaced = false; let ranges: Vec<_> = self .mirrored .ranges @@ -1102,25 +1166,37 @@ impl Reader { .enumerate() .map(|(index, range)| { let Some(resolved) = resolved.get(index).copied().flatten() else { + unplaced |= !range.is_empty() + && (matches!(range.start, Anchor::Relative(_) | Anchor::Time(1..)) + || matches!(range.end, Some(Anchor::Relative(_)))); return *range; }; let previous = placed.as_ref().and_then(|placed| placed[index]); - let start = previous.map_or(resolved.start, |previous| previous.start); + let (start, floored) = previous.map_or((resolved.start, resolved.floored), |previous| { + (previous.start, previous.floored) + }); let end = previous.and_then(|previous| previous.end).or(resolved.end); track::Range { start: match range.start { - Anchor::Relative(_) | Anchor::Time(_) => Anchor::Absolute(start), + Anchor::Relative(_) | Anchor::Time(_) if floored => Anchor::Time(start), + // One that reached back to the oldest group still does, as on a plain reader. + Anchor::Relative(_) | Anchor::Time(_) => Anchor::Oldest, start => start, }, - // A time end the copy has not settled stays one: its cap is not the end. end: match (range.end, end) { - (Some(Anchor::Relative(_) | Anchor::Time(_)), Some(end)) => Some(Anchor::Absolute(end)), + (Some(Anchor::Relative(_) | Anchor::Time(_)), Some(end)) => match end.checked_sub(1) { + Some(last) => Some(Anchor::Time(last)), + None => return track::Range::EMPTY, + }, (end, _) => end, }, } }) .collect(); - let window = Window::new(&ranges, None, &Unheld).expect("an unheld hop refuses nothing"); + if unplaced { + return; + } + let window = Window::new(&ranges, None, &Placed).expect("a placed hop refuses nothing"); if !window.is_pending() { self.window = window; } @@ -1272,18 +1348,16 @@ impl Reader { match self.declared { None => self.declared = Some(end), // FETCH can name only the group boundary just after a mid-group end. - Some(declared) - if declared.frame == 0 && end.frame != 0 && declared.final_sequence() == end.final_sequence() => - { + Some(declared) if declared.frame == 0 && end.frame != 0 && declared.final_key() == end.final_key() => { self.declared = Some(end); } Some(_) => {} } // A copy over a version with no end error declares the same end cleanly. Only a copy // at the end that stands says how the track ended there. - let same = self.declared.is_some_and(|declared| { - declared == end || end.frame == 0 && end.final_sequence() == declared.final_sequence() - }); + let same = self + .declared + .is_some_and(|declared| declared == end || end.frame == 0 && end.final_key() == declared.final_key()); if same && self.declared_error.is_none() { self.declared_error = copy.track.declared_error(); } @@ -1310,7 +1384,7 @@ impl Reader { // Only a replaced copy is judged against the edge, which costs the serving track's // lock: a reader following one route never pays it. let edge = match self.copies.iter().any(|copy| copy.replaced) { - true => self.copies.last().and_then(|copy| copy.track.latest_sequence()), + true => self.copies.last().and_then(|copy| copy.track.latest_key()), false => None, }; let reader = Retain { @@ -1355,11 +1429,11 @@ impl Reader { } let until = match self .declared - .and_then(Position::final_sequence) - .or_else(|| serving.track.final_sequence()) + .and_then(Position::final_key) + .or_else(|| serving.track.final_key()) { Some(end) => end.checked_sub(1), - None => self.newest.max(serving.track.latest_sequence()), + None => self.newest.max(serving.track.latest_key()), }; let mirrored = &self.mirrored; for copy in &mut self.copies { @@ -1372,16 +1446,16 @@ impl Reader { true } - /// The copy at `index` hands out `sequence`. The serving copy's first group bounds the + /// The copy at `index` hands out `key`. The serving copy's first group bounds the /// copies it replaced just below it: what they hold under it may be groups the new route /// never has, and what is at or above it the new route delivers. Never below a group /// already handed out. A group it replays, already handed out, bounds nothing: the /// replaced copies may still hold the next. Nor does a backfill of the serving route. - fn observe(&mut self, index: usize, sequence: u64) { + fn observe(&mut self, index: usize, key: u64) { if self.copies[index].replaced || Some(self.copies[index].generation) != self.generation { return; } - let bound = self.newest.max(sequence.checked_sub(1)); + let bound = self.newest.max(key.checked_sub(1)); for copy in &mut self.copies[..index] { if copy.replaced && copy.until.is_none() { copy.until = Some(bound); @@ -1391,20 +1465,30 @@ impl Reader { } } - fn deliverable(copy: &Copy, sequence: u64, delivered: &Delivered) -> bool { - let replaced = copy - .until - .is_some_and(|until| until.is_none_or(|until| sequence > until)); - !replaced && !delivered.contains(sequence) + /// Whether the window names a group from `floor` up to `key` the reader lacks. + fn owes_below(&mut self, floor: u64, key: u64, delivered: &Delivered) -> bool { + self.window() + .runs(floor) + .take_while(|run| *run.start() < key) + .any(|run| { + !delivered + .gaps(*run.start()..run.end().saturating_add(1).min(key)) + .is_empty() + }) + } + + fn deliverable(copy: &Copy, key: u64, delivered: &Delivered) -> bool { + let replaced = copy.until.is_some_and(|until| until.is_none_or(|until| key > until)); + !replaced && !delivered.contains(key) } fn hand_out(&mut self, index: usize, group: group::Consumer) -> group::Consumer { - let sequence = group.sequence; - self.newest = self.newest.max(Some(sequence)); + let key = group.start; + self.newest = self.newest.max(Some(key)); // The group won over any stale report on it, whichever copy made it, so no update // reopens it. - if let Some(end) = sequence.checked_add(1) { - self.reported_stale.remove(sequence..end); + if let Some(end) = key.checked_add(1) { + self.reported_stale.remove(key..end); } // Another copy holding the group, now or once it lands there, is where a failure // continues it, so it stays while the group is read; see [`Copy::retains`]. Recorded @@ -1415,11 +1499,11 @@ impl Reader { self.out.reserve(self.out.len().max(1)); } let out = Arc::new(()); - self.out.push((sequence, Arc::downgrade(&out))); + self.out.push((key, Arc::downgrade(&out))); let copy = &self.copies[index]; let handoff = group.handoff(); group.with_recover(Recover { - sequence, + key, reader: Some(self.this.clone()), max_delay: None, route: self.route.watch(), @@ -1447,12 +1531,9 @@ impl Reader { unreachable!("a candidate comes from a copy that answered") }; let mut group = sub.take(candidate.reserved)?; - self.observe(index, group.sequence); + self.observe(index, group.start); // A mid-group end caps the group it falls in, as it caps a direct track's. - if let Some(end) = self - .declared - .filter(|end| end.frame != 0 && end.group == group.sequence) - { + if let Some(end) = self.declared.filter(|end| end.frame != 0 && end.group == group.start) { group = group.with_end(end.frame); } Ok(self.hand_out(index, group)) @@ -1519,19 +1600,27 @@ impl Reader { match res { Poll::Pending => break, Poll::Ready(Ok(Some(reserved))) => { - let sequence = reserved.sequence(); - if !Self::deliverable(&self.copies[index], sequence, delivered) { + let key = reserved.key(); + // A copy resolves a time start against the groups it holds, which may place + // it below where the reader's window did: the window decides. + let deliverable = Self::deliverable(&self.copies[index], key, delivered); + if !deliverable || !self.window().contains(key) { + // Skipping an ordered cursor past a group the window still owes below + // would lose it, though it may yet land on this copy: leave it there. + if ordered && deliverable && self.owes_below(floor, key, delivered) { + break; + } let Sub::Ready(sub) = &mut self.copies[index].sub else { unreachable!() }; if ordered { - sub.skip_ordered(sequence + 1); + sub.skip_ordered(key + 1); } else { let _ = sub.take(reserved); } continue; } - if lowest.as_ref().is_none_or(|lowest| sequence <= lowest.sequence()) { + if lowest.as_ref().is_none_or(|lowest| key <= lowest.key()) { lowest = Some(Candidate { lease: self.copies[index].lease.clone(), reserved, @@ -1616,8 +1705,9 @@ impl Reader { replaced: false, done: None, sub: Sub::Ready(sub), + floor, .. - }) if Some(*generation) == self.generation => sub.exhausted_for(taken, waiter), + }) if Some(*generation) == self.generation => sub.exhausted_for(taken, *floor, waiter), _ => false, } } @@ -1887,6 +1977,7 @@ impl Reader { fn set_range(&mut self, start: u64, end: Option, delivered: &Delivered) { let passed = std::mem::replace(&mut self.groups, (start, end)).0; for copy in &mut self.copies { + copy.reopen(); let start = copy.start(start); if let Sub::Ready(sub) = &mut copy.sub { sub.set_range((Bound::Included(start), end.map_or(Bound::Unbounded, Bound::Excluded))); @@ -1987,7 +2078,7 @@ impl Reader { .ranges .iter() .map(|range| match range.start { - Anchor::Absolute(start) if replaced.is_none_or(|replaced| start > replaced) => start, + Anchor::Time(start) if replaced.is_none_or(|replaced| start > replaced) => start, _ => floor, }) .min() @@ -2000,7 +2091,7 @@ impl Reader { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn latest(&self) -> Option { - self.copies.iter().filter_map(|copy| copy.track.latest_sequence()).max() + self.copies.iter().filter_map(|copy| copy.track.latest_key()).max() } } @@ -2009,7 +2100,7 @@ impl Reader { /// Held by a group handed out from a logical track. It is consulted only once the /// group's copy fails or stalls, so a passthrough read never touches it. pub(super) struct Recover { - sequence: u64, + key: u64, /// The reader it was handed out to, followed onto each new route; `None` for a fetch. reader: Option>>, /// A fetch's own age budget, asked of each route and judged while waiting on one. @@ -2049,7 +2140,7 @@ enum Fetch { impl Clone for Recover { fn clone(&self) -> Self { Self { - sequence: self.sequence, + key: self.key, reader: self.reader.clone(), max_delay: self.max_delay, route: self.route.clone(), @@ -2099,7 +2190,7 @@ impl Recover { tracing::debug!( broadcast = %self.copy.broadcast().path, track = self.copy.name(), - group = self.sequence, + group = self.key, frame = index, cause = failed.map_or_else(|| "stalled behind a newer route".to_owned(), ToString::to_string), fetch = self.reader.is_none(), @@ -2120,7 +2211,7 @@ impl Recover { tracing::debug!( broadcast = %self.copy.broadcast().path, track = self.copy.name(), - group = self.sequence, + group = self.key, frame = index, %err, read_generation = self.generation, @@ -2233,7 +2324,7 @@ impl Recover { } if res.is_pending() && let Some(budget) = self.poll_budget(waiter) - && serving.poll_stale(self.sequence, budget, waiter).is_ready() + && serving.poll_stale(self.key, budget, waiter).is_ready() { return Poll::Ready(Err(Error::Old)); } @@ -2250,8 +2341,8 @@ impl Recover { if copy.generation == self.generation { return None; } - let group = copy.track.peek_group(self.sequence)?; - lend_head(&self.handoff, self.sequence, ©.track, &group); + let group = copy.track.peek_group(self.key)?; + lend_head(&self.handoff, self.key, ©.track, &group); group.at(index, byte).map(|group| Replacement { group, generation: copy.generation, @@ -2274,9 +2365,9 @@ impl Recover { // A newer copy's subscription delivered it: continue from there. The copy that // failed delivered it already, so its subscription will not again. let same = generation == self.generation; - match serving.poll_group(self.sequence, waiter) { + match serving.poll_group(self.key, waiter) { Poll::Ready(Ok(Some(group))) if !same => { - lend_head(&self.handoff, self.sequence, serving, &group); + lend_head(&self.handoff, self.key, serving, &group); if let Some(group) = group.at(index, byte) { return Poll::Ready(Ok(Replacement { group, @@ -2345,7 +2436,7 @@ impl Recover { }; self.fetch = Some(( generation, - Fetch::Pending(Box::new(serving.fetch_group(self.sequence, options).into_inner())), + Fetch::Pending(Box::new(serving.fetch_group(self.key, options).into_inner())), )); } let Some((_, fetch)) = &mut self.fetch else { @@ -2357,7 +2448,7 @@ impl Recover { Fetch::Refused(err) => err.clone(), Fetch::Pending(pending) => match ready!(kio::Task::poll(&mut **pending, waiter)) { Ok(group) => { - lend_head(&self.handoff, self.sequence, serving, &group); + lend_head(&self.handoff, self.key, serving, &group); if let Some(group) = group.at(index, byte) { return Poll::Ready(Ok(Replacement { group, @@ -2382,7 +2473,7 @@ impl Recover { // A refusal is a verdict only for a copy that failed, and once the serving // subscription will not deliver the group either. let same = generation == self.generation; - if failed.is_some() && (same || serving.poll_group(self.sequence, waiter).is_ready()) { + if failed.is_some() && (same || serving.poll_group(self.key, waiter).is_ready()) { return Poll::Ready(Err(err)); } @@ -2409,7 +2500,7 @@ impl Recover { tracing::debug!( broadcast = %self.copy.broadcast().path, track = self.copy.name(), - group = self.sequence, + group = self.key, frame = replacement.group.index(), read_generation = self.generation, adopted_generation = replacement.generation, @@ -2436,13 +2527,13 @@ impl Recover { } } -/// Lend the head `handoff` holds to `serving`'s copy of group `sequence`, `group`, when that +/// Lend the head `handoff` holds to `serving`'s copy of group `key`, `group`, when that /// starts later: the fetch for the rest asks only from the reader's frame. -fn lend_head(handoff: &group::Handoff, sequence: u64, serving: &track::Consumer, group: &group::Consumer) { +fn lend_head(handoff: &group::Handoff, key: u64, serving: &track::Consumer, group: &group::Consumer) { if group.first_frame() > 0 && let Some(head) = handoff.head() { - serving.adopt_head(sequence, &head); + serving.adopt_head(key, &head); } } @@ -2456,7 +2547,7 @@ pub(super) struct Replacement { /// A fetch of one group through whichever route serves the logical track. pub(super) struct Fetching { route: Consumer, - sequence: u64, + key: u64, options: track::FetchOptions, logical: kio::Consumer, /// The fetch in flight, and the copy it went to. @@ -2565,7 +2656,7 @@ impl kio::Task for Fetching { if self.pending.as_ref().is_none_or(|asked| asked.target != target) { self.pending = Some(Asked { target, - fetch: copy.fetch_range(self.sequence, self.options.clone()), + fetch: copy.fetch_range(self.key, self.options.clone()), answer: None, }); } @@ -2599,7 +2690,7 @@ impl kio::Task for Fetching { .with_end(self.options.end); let handoff = group.handoff(); Poll::Ready(Ok(group.with_recover(Recover { - sequence: self.sequence, + key: self.key, reader: None, max_delay: self.options.options.max_delay, route: self.route.watch(), @@ -2697,11 +2788,11 @@ mod test { let source = copy(); routes.serve(source.consume()); let mut sub = subscribe(if routed { &logical } else { &source }, HOUR); - let mut history = source.create_group(group::Info { sequence: 0 }).unwrap(); + let mut history = source.create_group(group::Info { start: 0 }).unwrap(); history.write_frame(ts(0), b"history".as_ref()).unwrap(); - history.finish().unwrap(); + history.close().unwrap(); let mut finished = recv(&mut sub); - let mut live = source.create_group(group::Info { sequence: 1 }).unwrap(); + let mut live = source.create_group(group::Info { start: 1 }).unwrap(); live.write_frame(ts(1), b"live".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"live".to_vec())); @@ -2724,7 +2815,7 @@ mod test { )); let result = sub .poll_recv_group(&waiter) - .map(|result| result.map(|group| group.map(|group| group.sequence))); + .map(|result| result.map(|group| group.map(|group| group.start))); assert!( matches!(result, Poll::Ready(Err(Error::GroupTooLarge))), "routed={routed}: {result:?}" @@ -2749,16 +2840,16 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(1)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"first".as_ref()).unwrap(); let mut held = recv(&mut sub); assert_eq!(read(&mut held), Some(b"first".to_vec())); let b = copy(); let dynamic = b.requests(); - let mut newer = b.create_group(group::Info { sequence: 1 }).unwrap(); + let mut newer = b.create_group(group::Info { start: 1 }).unwrap(); newer.write_frame(ts(1), b"newer".as_ref()).unwrap(); - newer.finish().unwrap(); + newer.close().unwrap(); routes.serve(b.consume()); open.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); @@ -2775,10 +2866,10 @@ mod test { ); if arrives { - let mut late = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut late = b.create_group(group::Info { start: 0 }).unwrap(); late.write_frame(ts(0), b"first".as_ref()).unwrap(); late.write_frame(ts(1), b"continued".as_ref()).unwrap(); - late.finish().unwrap(); + late.close().unwrap(); assert_eq!(read(&mut held), Some(b"continued".to_vec())); assert_eq!(read(&mut held), None); } else { @@ -2807,7 +2898,7 @@ mod test { routes.serve(a.consume()); let mut quick = subscribe(&logical, short); let mut patient = subscribe(&logical, hour); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"first".as_ref()).unwrap(); let mut quick_held = recv(&mut quick); let mut patient_held = recv(&mut patient); @@ -2816,9 +2907,9 @@ mod test { let b = copy(); let dynamic = b.requests(); - let mut newer = b.create_group(group::Info { sequence: 1 }).unwrap(); + let mut newer = b.create_group(group::Info { start: 1 }).unwrap(); newer.write_frame(ts(1), b"newer".as_ref()).unwrap(); - newer.finish().unwrap(); + newer.close().unwrap(); routes.serve(b.consume()); open.abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); @@ -2850,10 +2941,10 @@ mod test { ); drop(quick_held); drop(quick); - let mut late = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut late = b.create_group(group::Info { start: 0 }).unwrap(); late.write_frame(ts(0), b"first".as_ref()).unwrap(); late.write_frame(ts(1), b"continued".as_ref()).unwrap(); - late.finish().unwrap(); + late.close().unwrap(); assert_eq!( read(&mut patient_held), Some(b"continued".to_vec()), @@ -2930,13 +3021,13 @@ mod test { let mut source = copy(); routes.serve(source.consume()); let mut sub = subscribe(&logical, HOUR); - let mut whole = source.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = source.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"a".as_ref()).unwrap(); whole.write_frame(ts(1), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); - let mut last = source.create_group(group::Info { sequence: 1 }).unwrap(); + whole.close().unwrap(); + let mut last = source.create_group(group::Info { start: 2 }).unwrap(); last.write_frame(ts(2), b"c".as_ref()).unwrap(); - source.finish_at(Position { group: 1, frame: 1 }, None).unwrap(); + source.finish_at(Position { group: 2, frame: 1 }, None).unwrap(); let mut first = recv(&mut sub); assert_eq!(read(&mut first), Some(b"a".to_vec())); assert_eq!(read(&mut first), Some(b"b".to_vec())); @@ -2984,7 +3075,7 @@ mod test { ); assert!(matches!(sub.end_error(), Some(Error::App(7)))); assert!(!sub.ended_with_error(), "group 0 is still to read"); - assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).start, 0); assert!(matches!( sub.poll_recv_group(&kio::Waiter::noop()), Poll::Ready(Err(Error::App(7))) @@ -3012,7 +3103,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); write(&a, first.group - 1); - assert_eq!(recv(&mut sub).sequence, first.group - 1); + assert_eq!(recv(&mut sub).start, first.group - 1); a.finish_at(first, None).unwrap(); let mut b = copy(); b.finish_at(next, None).unwrap(); @@ -3062,7 +3153,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); write(&a, 0); - assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).start, 0); let b = copy(); routes.serve(b.consume()); let wake = Arc::new(Wake::default()); @@ -3107,7 +3198,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); write(&a, 4); - assert_eq!(recv(&mut sub).sequence, 4); + assert_eq!(recv(&mut sub).start, 4); a.finish_at(Position::group(5), None).unwrap(); let mut b = copy(); b.finish_at(Position::group(6), Some(Error::App(7))).unwrap(); @@ -3131,7 +3222,7 @@ mod test { let b = copy(); routes.serve(b.consume()); write(&b, 4); - assert_eq!(recv(&mut sub).sequence, 4); + assert_eq!(recv(&mut sub).start, 4); write(&b, 5); write(&b, 6); let _ = drain(&mut sub, false); @@ -3153,14 +3244,14 @@ mod test { a.finish_at(Position { group: 4, frame: 2 }, None).unwrap(); let b = copy(); routes.serve(b.consume()); - let mut group = b.create_group(group::Info { sequence: 4 }).unwrap(); + let mut group = b.create_group(group::Info { start: 4 }).unwrap(); group.write_frame(ts(0), b"x".as_ref()).unwrap(); let mut reading = recv(&mut sub); // Frames past the end arrive only once the group is handed out. for frame in 1..4 { group.write_frame(ts(frame), b"x".as_ref()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(read(&mut reading), Some(b"x".to_vec())); assert_eq!(read(&mut reading), Some(b"x".to_vec())); assert_eq!(read(&mut reading), None); @@ -3211,21 +3302,21 @@ mod test { } // Returning to a route must not leave its old replacement cap in place. write(&tracks[1], 1); - assert_eq!(recv(&mut sub).sequence, 1); + assert_eq!(recv(&mut sub).start, 1); routes.serve(tracks[0].consume()); write(&tracks[0], 2); - assert_eq!(recv(&mut sub).sequence, 2); + assert_eq!(recv(&mut sub).start, 2); drop(held); } #[kio::test] async fn a_returning_route_keeps_its_relative_end() { - returning_route(Anchor::Relative(991)).await; + returning_route(Anchor::Relative(2)).await; } #[kio::test] async fn a_returning_route_keeps_its_resolved_time_end() { - returning_route(Anchor::Time(ts(10))).await; + returning_route(Anchor::Time(10)).await; } /// Its old cap bounds a replaced copy's wire, not the reader's resolved end. Reusing @@ -3235,15 +3326,15 @@ mod test { let logical = logical(&routes); let a = copy(); // Both anchors resolve the reader's exclusive end at 11. - for sequence in 2..=11 { - write(&a, sequence); + for key in 2..=11 { + write(&a, key); } write(&a, 1001); routes.serve(a.consume()); let subscription = Subscription::default() .with_delay(Delay::default().with_max(HOUR)) .with_ranges([track::Range { - start: Anchor::Absolute(2), + start: Anchor::Time(2), end: Some(end), }]); let mut sub = logical @@ -3253,7 +3344,7 @@ mod test { .unwrap() .with_range(2..3); let held = recv(&mut sub); - assert_eq!(held.sequence, 2); + assert_eq!(held.start, 2); assert_eq!(sub.next_in_range(11), None, "the reader resolved its end"); sub.set_range(4..9); @@ -3264,8 +3355,8 @@ mod test { assert_eq!(copies(&routes), 2, "the held group keeps A's capped copy"); routes.serve(a.consume()); - for sequence in [1, 12] { - write(&a, sequence); + for key in [1, 12] { + write(&a, key); } let mut resumed = drain(&mut sub, false); resumed.sort_unstable(); @@ -3310,21 +3401,25 @@ mod test { let logical = logical(&routes); let a = copy(); routes.serve(a.consume()); + // Held before the reader subscribes, so the second range starts at group 2 itself. + write(&a, 0); + write(&a, 2); let subscription = Subscription::default() .with_delay(Delay::default().with_max(HOUR)) - .with_ranges([track::Range::groups(0..1), track::Range::groups(2..3)]); + .with_ranges([track::Range::groups(0..1), track::Range::groups(2..4)]); let mut sub = logical .consume() .subscribe(subscription) .now_or_never() .unwrap() .unwrap(); - write(&a, 0); - assert_eq!(drain(&mut sub, false), [0]); + let mut read = drain(&mut sub, false); + read.sort_unstable(); + assert_eq!(read, [0, 2]); let b = copy(); routes.serve(b.consume()); - write(&b, 2); - assert_eq!(drain(&mut sub, false), [2]); + write(&b, 3); + assert_eq!(drain(&mut sub, false), [3]); assert_eq!( copies(&routes), 1, @@ -3398,10 +3493,10 @@ mod test { let logical = logical(&routes); let resumed = logical.consume(); let source = copy(); - let mut warm = source.create_group(group::Info { sequence: 2 }).unwrap(); + let mut warm = source.create_group(group::Info { start: 2 }).unwrap(); warm.write_frame(crate::Timestamp::from_millis(0).unwrap(), b"2-0".as_ref()) .unwrap(); - warm.finish().unwrap(); + warm.close().unwrap(); routes.serve(source.consume()); assert!(resumed.is_live(), "a serving live copy is live"); @@ -3424,9 +3519,9 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::ZERO); - let mut group = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = a.create_group(group::Info { start: 0 }).unwrap(); group.write_frame(ts(0), b"a".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -3445,7 +3540,7 @@ mod test { fn a_front_reader_reports_its_ranges_as_a_plain_reader_does() { let range = track::Range { start: track::Anchor::Relative(1), - end: Some(track::Anchor::Absolute(6)), + end: Some(track::Anchor::Time(5)), }; let subscription = Subscription::default() .with_delay(Delay::default().with_max(HOUR)) @@ -3472,13 +3567,15 @@ mod test { write(&a, sequence); } assert_eq!(drain(&mut sub, false), [0, 1, 2, 3], "routed={routed}"); - assert_eq!(sub.floor(), Some(0), "routed={routed}"); + // Counted back from group 0, the first one either reader saw, it reaches back to + // the oldest group rather than naming one. + assert_eq!(sub.floor(), None, "routed={routed}"); assert_eq!( sub.resolved(), [Some(Resolved { start: 0, end: Some(6), - floored: true, + floored: false, })], "routed={routed}" ); @@ -3506,11 +3603,11 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(if routed { &logical } else { &a }, Duration::from_secs(1)); for second in 0..4 { - let mut group = a.create_group(group::Info { sequence: second }).unwrap(); + let mut group = a.create_group(group::Info { start: second * 1000 }).unwrap(); group.write_frame(ts(second * 1000), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - assert_eq!(drain(&mut sub, true), [2, 3], "routed={routed}"); + assert_eq!(drain(&mut sub, true), [2000, 3000], "routed={routed}"); assert_eq!(sub.take_stale().groups, 2, "routed={routed}"); assert_eq!(sub.take_stale().groups, 0, "routed={routed}: counted once"); } @@ -3525,7 +3622,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); open.write_frame(ts(1), b"b".as_ref()).unwrap(); let mut reading = recv(&mut sub); @@ -3534,14 +3631,14 @@ mod test { // The replacement holds the whole group; the replaced route stays quiet. let b = copy(); - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); for (ms, frame) in [(0, b"a"), (1, b"b"), (2, b"c")] { whole.write_frame(ts(ms), frame.as_ref()).unwrap(); } routes.serve(b.consume()); assert_eq!(read(&mut reading), Some(b"c".to_vec())); - whole.finish().unwrap(); + whole.close().unwrap(); assert_eq!(read(&mut reading), None); assert!( sub.recv_group().now_or_never().is_none(), @@ -3560,7 +3657,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -3570,7 +3667,7 @@ mod test { open.write_frame(ts(1), b"b".as_ref()).unwrap(); assert_eq!(read(&mut reading), Some(b"b".to_vec())); - open.finish().unwrap(); + open.close().unwrap(); assert_eq!(read(&mut reading), None); } @@ -3592,7 +3689,7 @@ mod test { .map(|index| format!("{index}{}", "z".repeat(64)).into()) .collect(); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), frames[0].clone()).unwrap(); open.write_frame(ts(1), frames[1].clone()).unwrap(); let mut reading = recv(&mut sub); @@ -3605,10 +3702,10 @@ mod test { routes.serve(b.consume()); assert!(reading.read_frame().now_or_never().is_none()); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - assert_eq!((request.sequence(), request.frame_start()), (0, 2)); + assert_eq!((request.key(), request.frame_start()), (0, 2)); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(2), frames[2].clone()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut reading).as_ref(), Some(&frames[2])); assert_eq!(read(&mut reading), None); assert_eq!(read_cached(&b.consume(), 0), frames); @@ -3629,13 +3726,13 @@ mod test { let mut sub = subscribe(&logical, Duration::from_secs(10)); let frame = || bytes::Bytes::from("z".repeat(64)); - let mut group = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = a.create_group(group::Info { start: 0 }).unwrap(); group.write_frame(ts(0), frame()).unwrap(); let slice = super::super::flate::Encoder::new().frame(&frame()); let header = crate::model::frame::Header { info: crate::model::frame::Info { size: slice.len() as u64, - timestamp: Some(ts(1)), + timestamp: ts(1), }, fields: None, inflated: Some(frame().len() as u64 + 1), @@ -3643,7 +3740,7 @@ mod test { let mut corrupt = group.create_object(header).unwrap(); corrupt.write(slice).unwrap(); let group = corrupt.finish().unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(frame().to_vec())); @@ -3671,7 +3768,7 @@ mod test { let mut sub = subscribe(&logical, Duration::from_secs(10)); let frame = || bytes::Bytes::from("z".repeat(64)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), frame()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(frame().to_vec())); @@ -3689,14 +3786,14 @@ mod test { // Shrink the pool so the next write reclaims the held head. let other = track::Producer::new(broadcast, "other", crate::track::Info::timed()); pool.resize(pool.used().saturating_sub(1)); - let mut filler = other.append_group().unwrap(); + let mut filler = other.append_next().unwrap(); filler.write_frame(ts(0), frame()).unwrap(); - filler.finish().unwrap(); + filler.close().unwrap(); assert!(handoff.head().is_none(), "the shrink reclaimed the head"); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(1), frame()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); let res = reading.read_frame().now_or_never().expect("fails instead of spinning"); assert!(matches!(res, Err(Error::Dropped)), "{res:?}"); } @@ -3710,11 +3807,11 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let open = a.create_group(group::Info { start: 0 }).unwrap(); let mut frame = open .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), }) .unwrap(); frame.write(b"ab".as_ref()).unwrap(); @@ -3729,7 +3826,7 @@ mod test { drop(frame); a.clone().abort(Error::Dropped).unwrap(); let b = copy(); - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"abcd".as_ref()).unwrap(); routes.serve(b.consume()); @@ -3751,11 +3848,11 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let open = a.create_group(group::Info { start: 0 }).unwrap(); let mut frame = open .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), }) .unwrap(); frame.write(b"ab".as_ref()).unwrap(); @@ -3790,7 +3887,7 @@ mod test { .create_lead( crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), } .into(), ) @@ -3798,7 +3895,7 @@ mod test { lead.write(b"cd".as_ref()).unwrap(); let mut rest = lead.finish().unwrap(); rest.write_frame(ts(1), b"e".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!( &chunks.read_chunk().now_or_never().unwrap().unwrap().unwrap()[..], @@ -3819,11 +3916,11 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let open = a.create_group(group::Info { start: 0 }).unwrap(); let mut frame = open .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), }) .unwrap(); frame.write(b"ab".as_ref()).unwrap(); @@ -3846,7 +3943,7 @@ mod test { let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(0), b"abcd".as_ref()).unwrap(); rest.write_frame(ts(1), b"e".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!( &chunks.read_chunk().now_or_never().unwrap().unwrap().unwrap()[..], @@ -3868,7 +3965,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -3915,7 +4012,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -3925,7 +4022,7 @@ mod test { for _ in 0..3 { assert!(reading.read_frame().now_or_never().is_none()); } - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); for (ms, frame) in [(0, b"a"), (1, b"b")] { whole.write_frame(ts(ms), frame.as_ref()).unwrap(); } @@ -3946,7 +4043,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -3974,7 +4071,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -4003,7 +4100,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -4011,10 +4108,10 @@ mod test { assert!(reading.read_frame().now_or_never().is_none()); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - assert_eq!((request.sequence(), request.frame_start()), (0, 1)); + assert_eq!((request.key(), request.frame_start()), (0, 1)); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(1), b"b".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); a.abort(Error::Stream(StreamError::App(5))).unwrap(); assert_eq!(read(&mut reading), Some(b"b".to_vec())); @@ -4031,7 +4128,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -4040,10 +4137,10 @@ mod test { assert!(reading.read_frame().now_or_never().is_none()); let b = copy(); - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"a".as_ref()).unwrap(); whole.write_frame(ts(1), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); routes.serve(b.consume()); assert_eq!(read(&mut reading), Some(b"b".to_vec())); assert_eq!(read(&mut reading), None); @@ -4059,7 +4156,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -4081,7 +4178,7 @@ mod test { let dynamic = a.requests(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -4098,10 +4195,10 @@ mod test { assert!(reading.read_frame().now_or_never().is_none(), "still waiting"); let b = copy(); - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"a".as_ref()).unwrap(); whole.write_frame(ts(1), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); routes.serve(b.consume()); assert_eq!(read(&mut reading), Some(b"b".to_vec())); } @@ -4121,7 +4218,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = if fetched { logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap() @@ -4154,10 +4251,10 @@ mod test { // Another route holds it: the group continues there. let b = copy(); - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"a".as_ref()).unwrap(); whole.write_frame(ts(1), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); routes.serve(b.consume()); assert_eq!(read(&mut reading), Some(b"b".to_vec()), "fetched={fetched}"); assert_eq!(read(&mut reading), None, "fetched={fetched}"); @@ -4173,7 +4270,7 @@ mod test { let logical = logical(&routes); let a = copy(); routes.serve(a.consume()); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut fetched = logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(); assert_eq!( @@ -4190,7 +4287,7 @@ mod test { assert!(fetched.read_frame().now_or_never().is_none()); let mut rest = pending.accept(None).unwrap(); rest.write_frame(ts(1), b"b".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut fetched), Some(b"b".to_vec())); assert_eq!(read(&mut fetched), None); } @@ -4227,7 +4324,7 @@ mod test { let copy = request.accept(crate::track::Info::timed()); let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); group.write_frame(ts(0), b"early".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert!( (&mut fetching).now_or_never().is_none(), "a group is never handed out before its copy's info passes" @@ -4254,7 +4351,7 @@ mod test { let wrong = refused.accept(crate::track::Info::timed()); let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); group.write_frame(ts(0), b"wrong".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); drop(staged); drop(wrong); @@ -4269,7 +4366,7 @@ mod test { let copy = request.accept(crate::track::Info::timed()); let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); group.write_frame(ts(0), b"right".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _logical = logical.accept(crate::track::Info::timed()); routes.splice(copy.consume(), staged); let mut fetched = fetching.now_or_never().expect("spliced").unwrap(); @@ -4377,7 +4474,7 @@ mod test { let _staged_copy = request.accept(crate::track::Info::timed()); let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); group.write_frame(ts(0), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert!((&mut fetching).now_or_never().is_none()); let _logical = logical.accept(crate::track::Info::timed()); @@ -4395,7 +4492,7 @@ mod test { .unwrap(); let mut group = fill.accept(0, crate::track::Info::timed()).unwrap(); group.write_frame(ts(0), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut fetched = fetching.now_or_never().expect("answered").unwrap(); assert_eq!(read(&mut fetched), Some(b"x".to_vec())); } @@ -4581,7 +4678,7 @@ mod test { write(&a, 0); assert_eq!(drain(&mut sub, false), [0]); let (groups, given_up) = sub.reoffer_groups(0..3); - assert_eq!(groups.iter().map(|group| group.sequence).collect::>(), [0]); + assert_eq!(groups.iter().map(|group| group.start).collect::>(), [0]); assert_eq!(given_up, std::iter::once(1..3).collect::>()); } @@ -4741,9 +4838,9 @@ mod test { /// Read a fetched group from frame 0, which must already be cached. #[cfg(feature = "async")] - fn read_cached(track: &track::Consumer, sequence: u64) -> Vec> { + fn read_cached(track: &track::Consumer, key: u64) -> Vec> { let mut group = track - .fetch_group(sequence, None) + .fetch_group(key, None) .now_or_never() .expect("the whole group is cached") .unwrap(); @@ -4761,7 +4858,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); open.write_frame(ts(1), b"b".as_ref()).unwrap(); let mut reading = recv(&mut sub); @@ -4779,10 +4876,10 @@ mod test { assert_eq!(read(&mut reading), Some(b"b".to_vec())); assert!(reading.read_frame().now_or_never().is_none()); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - assert_eq!((request.sequence(), request.frame_start()), (0, 2)); + assert_eq!((request.key(), request.frame_start()), (0, 2)); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(2), b"c".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut reading), Some(b"c".to_vec())); assert_eq!(read(&mut reading), None); @@ -4805,7 +4902,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -4818,7 +4915,7 @@ mod test { assert_eq!(request.frame_start(), 1); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(1), b"b".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut reading), Some(b"b".to_vec())); assert_eq!(read(&mut reading), None); @@ -4832,7 +4929,7 @@ mod test { let routes = Producer::new(); let a = copy(); routes.serve(a.consume()); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut fetched = logical(&routes) .consume() @@ -4851,7 +4948,7 @@ mod test { assert_eq!(request.frame_start(), 1); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(1), b"b".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut fetched), Some(b"b".to_vec())); assert_eq!(read(&mut fetched), None); let _whole = b.consume().fetch_group(0, None); @@ -4869,7 +4966,7 @@ mod test { let routes = Producer::new(); let a = copy(); routes.serve(a.consume()); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut fetched = logical(&routes) .consume() @@ -4894,7 +4991,7 @@ mod test { let routes = Producer::new(); let a = track::Producer::new(Arc::new(broadcast::Info::default()), "video", info.clone()); routes.serve(a.consume()); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); let mut drain = open.consume(); // Two frames that compress to almost nothing, but inflate past the tail together. for _ in 0..2 { @@ -4920,17 +5017,17 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); // B holds the whole group, then its route fails, keeping it. let b = copy(); - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"a".as_ref()).unwrap(); whole.write_frame(ts(1), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); routes.serve(b.consume()); assert!( sub.poll_recv_group(&kio::Waiter::noop()).is_pending(), @@ -4942,8 +5039,8 @@ mod test { // C lacks the group; reading later ones from it lets go of what nothing needs, however // many it hands out meanwhile. let c = copy(); - for sequence in 1..9 { - write(&c, sequence); + for key in 1..9 { + write(&c, key); } routes.serve(c.consume()); assert_eq!(drain(&mut sub, false), (1..9).collect::>()); @@ -4965,18 +5062,18 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); let b = copy(); - for (track, sequences) in [(&a, 0..2), (&b, 0..3)] { - for sequence in sequences { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for (track, keys) in [(&a, 0..2), (&b, 0..3)] { + for key in keys { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(ts(key), b"x".as_ref()).unwrap(); + group.close().unwrap(); } } - assert_eq!(recv(&mut sub).sequence, 0); - assert_eq!(recv(&mut sub).sequence, 1); + assert_eq!(recv(&mut sub).start, 0); + assert_eq!(recv(&mut sub).start, 1); routes.serve(b.consume()); - assert_eq!(recv(&mut sub).sequence, 2); + assert_eq!(recv(&mut sub).start, 2); assert!(sub.recv_group().now_or_never().is_none()); } @@ -5019,8 +5116,8 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - for sequence in 0..5 { - write(&a, sequence); + for key in 0..5 { + write(&a, key); } sub.set_range(..3); assert!(sub.largest().now_or_never().is_none(), "the largest is past the cap"); @@ -5028,11 +5125,11 @@ mod test { assert_eq!(sub.largest().now_or_never(), Some(Some(Position::group(4)))); } - /// A front's reader created with a floor above the serving copy's edge reports nothing - /// below it, as a plain reader does. + /// A front's reader whose start time is past the serving copy's edge starts at the group + /// containing it, so it reports that copy's largest, as a plain reader of the copy does. #[test] #[cfg(feature = "async")] - fn largest_honours_the_subscription_floor_on_the_serving_copy() { + fn largest_honours_the_subscription_start_on_the_serving_copy() { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -5042,14 +5139,16 @@ mod test { .with_range(8..); let mut sub = logical .consume() - .subscribe(subscription) + .subscribe(subscription.clone()) .now_or_never() .unwrap() .unwrap(); - for sequence in 0..5 { - write(&a, sequence); + for key in 0..5 { + write(&a, key); } - assert!(sub.largest().now_or_never().is_none(), "the largest is below the floor"); + let mut plain = a.consume().subscribe(subscription).now_or_never().unwrap().unwrap(); + assert_eq!(plain.largest().now_or_never(), Some(Some(Position::group(4)))); + assert_eq!(sub.largest().now_or_never(), Some(Some(Position::group(4)))); write(&a, 8); assert_eq!(sub.largest().now_or_never(), Some(Some(Position::group(8)))); } @@ -5178,11 +5277,11 @@ mod test { /// A budget that never writes off a group in these tests. const HOUR: Duration = Duration::from_secs(3600); - /// Write a finished one-frame group, presented at its sequence in milliseconds. - fn write(track: &track::Producer, sequence: u64) { - let mut group = track.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + /// Write a finished one-frame group, presented at its key in milliseconds. + fn write(track: &track::Producer, key: u64) { + let mut group = track.create_group(group::Info { start: key }).unwrap(); + group.write_frame(ts(key), b"x".as_ref()).unwrap(); + group.close().unwrap(); } /// The group a peek reserves, to take later. @@ -5195,16 +5294,16 @@ mod test { /// Every group the reader hands out right now. fn drain(sub: &mut track::Subscriber, ordered: bool) -> Vec { - let mut sequences = Vec::new(); + let mut keys = Vec::new(); loop { let res = match ordered { true => sub.poll_next_group(&kio::Waiter::noop()), false => sub.poll_recv_group(&kio::Waiter::noop()), }; let Poll::Ready(Ok(Some(group))) = res else { - return sequences; + return keys; }; - sequences.push(group.sequence); + keys.push(group.start); } } @@ -5217,8 +5316,8 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR).with_range(4..); - for sequence in 0..6 { - write(&a, sequence); + for key in 0..6 { + write(&a, key); } assert_eq!(drain(&mut sub, false), vec![4, 5]); @@ -5229,6 +5328,52 @@ mod test { assert_eq!(drain(&mut sub, false), Vec::::new(), "nothing goes out twice"); } + /// A reader that ran out of a finished track under a cap and ranges short of its last + /// group reads that group once both widen past it, whichever widens last, as a + /// publisher's reader does when an update reaches for the rest of an ended track. + #[test] + fn a_widened_reader_reads_a_finished_tracks_group_it_ran_out_before() { + for ranges_first in [false, true] { + let routes = Producer::new(); + let logical = logical(&routes); + let mut a = copy(); + routes.serve(a.consume()); + write(&a, 407); + a.finish_at(Position::group(408), None).unwrap(); + let history = Subscription::default() + .with_delay(Delay::default().with_max(HOUR)) + .with_ranges([track::Range { + start: track::Anchor::Time(0), + end: Some(track::Anchor::Time(406)), + }]); + let sub = logical.consume().subscribe(history).now_or_never().unwrap().unwrap(); + let mut sub = sub.with_range(0..407); + let Poll::Ready(ended) = sub.poll_recv_group(&kio::Waiter::noop()) else { + panic!("the history read ends") + }; + assert!(ended.unwrap().is_none(), "ranges_first={ranges_first}"); + + let everything = sub.subscription().with_ranges([track::Range { + start: track::Anchor::Oldest, + end: None, + }]); + let mut sub = match ranges_first { + true => { + sub.update(everything).unwrap(); + assert_eq!(drain(&mut sub, false), Vec::::new(), "the cap still ends at 407"); + sub.with_range(0..) + } + false => { + let mut sub = sub.with_range(0..); + assert_eq!(drain(&mut sub, false), Vec::::new(), "the ranges still end at 407"); + sub.update(everything).unwrap(); + sub + } + }; + assert_eq!(drain(&mut sub, false), [407], "ranges_first={ranges_first}"); + } + } + /// Lowering a resumed reader's floor offers the groups below the old floor that only a /// copy it already let go cached: the serving route is asked for just those, whether it /// caches them or gets them once asked. An ordered reader never goes back, so it asks @@ -5242,15 +5387,15 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR).with_range(4..); - for sequence in 0..6 { - write(&a, sequence); + for key in 0..6 { + write(&a, key); } assert_eq!(drain(&mut sub, ordered), [4, 5]); let b = copy(); if cached { - for sequence in 0..6 { - write(&b, sequence); + for key in 0..6 { + write(&b, key); } } routes.serve(b.consume()); @@ -5265,8 +5410,8 @@ mod test { assert_eq!(copies(&routes), if ordered { 1 } else { 2 }, "asked once: {case}"); if !cached { assert_eq!(drain(&mut sub, ordered), Vec::::new()); - for sequence in 1..4 { - write(&b, sequence); + for key in 1..4 { + write(&b, key); } } let lowered: &[u64] = if ordered { &[] } else { &[1, 2, 3] }; @@ -5283,16 +5428,16 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&outer, HOUR); - for sequence in 0..4 { - write(&a, sequence); + for key in 0..4 { + write(&a, key); } assert_eq!(drain(&mut sub, true), [0, 1, 2, 3]); let nested_routes = Producer::new(); let nested = logical(&nested_routes); let b = copy(); - for sequence in 0..4 { - write(&b, sequence); + for key in 0..4 { + write(&b, key); } nested_routes.serve(b.consume()); routes.serve(nested.consume()); @@ -5302,8 +5447,8 @@ mod test { // The nested reader has handed nothing out itself. Its containing reader's // past prefix must also release the replaced copy when its serving copy ends. let c = copy(); - for sequence in 0..4 { - write(&c, sequence); + for key in 0..4 { + write(&c, key); } let c = c.finish().unwrap(); nested_routes.serve(c.consume()); @@ -5358,16 +5503,16 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(if nested { &outer } else { &inner }, HOUR); sub.set_range(..0); - for sequence in 0..4 { - write(&a, sequence); + for key in 0..4 { + write(&a, key); } assert!( drain(&mut sub, true).is_empty(), "subscribe the nested reader before switching" ); let b = copy(); - for sequence in 0..4 { - write(&b, sequence); + for key in 0..4 { + write(&b, key); } routes.serve(b.consume()); assert!(drain(&mut sub, true).is_empty()); @@ -5390,8 +5535,8 @@ mod test { ); } let c = c.accept(crate::track::Info::timed()); - for sequence in 0..5 { - write(&c, sequence); + for key in 0..5 { + write(&c, key); } assert_eq!(drain(&mut sub, true), [4], "a newly ready copy inherits the floor"); let _c = c.finish().unwrap(); @@ -5433,7 +5578,7 @@ mod test { true => sub.poll_next_group(&kio::Waiter::noop()), false => sub.poll_recv_group(&kio::Waiter::noop()), }; - let res = res.map(|res| res.map(|group| group.map(|group| group.sequence))); + let res = res.map(|res| res.map(|group| group.map(|group| group.start))); assert!( matches!(res, Poll::Ready(Ok(None))), "{case}: the read ends, got {res:?}" @@ -5471,8 +5616,8 @@ mod test { .unwrap(); write(&a, 0); write(&a, 1); - assert_eq!(recv(&mut sub).sequence, 0); - assert_eq!(recv(&mut sub).sequence, 1); + assert_eq!(recv(&mut sub).start, 0); + assert_eq!(recv(&mut sub).start, 1); routes.park(); assert!(sub.poll_recv_group(&kio::Waiter::noop()).is_pending()); routes.serve(a.consume()); @@ -5551,7 +5696,7 @@ mod test { kio::clock::sleep(budget).await; let res = sub .poll_recv_group(&fresh()) - .map(|res| res.map(|group| group.map(|group| group.sequence))); + .map(|res| res.map(|group| group.map(|group| group.start))); match ends { true => assert!(matches!(res, Poll::Ready(Ok(None))), "{case}: {res:?}"), false => assert!(res.is_pending(), "{case}: {res:?}"), @@ -5562,7 +5707,7 @@ mod test { /// A replaced copy is bounded only once the new copy delivers its first group: a group /// buffered on it must be read before a later group on the serving copy. #[test] - fn sequence_reads_take_a_replaced_copys_lower_buffered_group() { + fn key_reads_take_a_replaced_copys_lower_buffered_group() { replaced_copy_reads_lowest(true); } @@ -5617,18 +5762,18 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(if nested { &outer } else { &inner }, HOUR); - let mut one = a.create_group(group::Info { sequence: 1 }).unwrap(); + let mut one = a.create_group(group::Info { start: 1 }).unwrap(); one.write_frame(ts(1), b"x".as_ref()).unwrap(); - let one = one.finish().unwrap(); + let one = one.close().unwrap(); write(&a, 3); let reserved = peek(&mut sub, ordered); - assert_eq!(reserved.sequence(), 1); + assert_eq!(reserved.key(), 1); one.abort(Error::Evicted).unwrap(); let b = copy(); routes.serve(b.consume()); write(&b, 2); let group = sub.take(reserved).unwrap(); - assert_eq!(group.sequence, 1, "ordered={ordered}, nested={nested}"); + assert_eq!(group.start, 1, "ordered={ordered}, nested={nested}"); assert_eq!(drain(&mut sub, ordered), [2], "ordered={ordered}, nested={nested}"); } } @@ -5656,11 +5801,11 @@ mod test { write(&a, 3); } drain(&mut sub, ordered); - let mut two = a.create_group(group::Info { sequence: 2 }).unwrap(); + let mut two = a.create_group(group::Info { start: 2 }).unwrap(); two.write_frame(ts(2), b"x".as_ref()).unwrap(); - let two = two.finish().unwrap(); + let two = two.close().unwrap(); let reserved = peek(&mut sub, ordered); - assert_eq!(reserved.sequence(), 2, "{case}"); + assert_eq!(reserved.key(), 2, "{case}"); two.abort(Error::Evicted).unwrap(); match outer_flips { true => { @@ -5672,7 +5817,7 @@ mod test { routes.serve(a.consume()); } } - assert_eq!(sub.take(reserved).unwrap().sequence, 2, "{case}"); + assert_eq!(sub.take(reserved).unwrap().start, 2, "{case}"); write(&a, 4); assert_eq!(drain(&mut sub, ordered), [4], "{case}"); } @@ -5885,7 +6030,7 @@ mod test { let b = copy(); routes.serve(b.consume()); - let mut failing = b.create_group(group::Info { sequence: 1 }).unwrap(); + let mut failing = b.create_group(group::Info { start: 1 }).unwrap(); failing.write_frame(ts(1), b"a".as_ref()).unwrap(); let noop = kio::Waiter::noop(); let res = match ordered { @@ -5899,10 +6044,10 @@ mod test { assert_eq!(drain(&mut sub, ordered), Vec::::new()); // The old route finishes group 1 only now, and the new route's copy of it fails. - let mut whole = a.create_group(group::Info { sequence: 1 }).unwrap(); + let mut whole = a.create_group(group::Info { start: 1 }).unwrap(); whole.write_frame(ts(1), b"a".as_ref()).unwrap(); whole.write_frame(ts(2), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); failing .abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); @@ -5929,7 +6074,7 @@ mod test { let c = copy(); routes.serve(c.consume()); - let mut failing = c.create_group(group::Info { sequence: 1 }).unwrap(); + let mut failing = c.create_group(group::Info { start: 1 }).unwrap(); failing.write_frame(ts(1), b"a".as_ref()).unwrap(); let noop = kio::Waiter::noop(); let res = match ordered { @@ -5943,10 +6088,10 @@ mod test { assert_eq!(drain(&mut sub, ordered), Vec::::new()); // b's first group lands only now, and c's copy of it fails. - let mut whole = b.create_group(group::Info { sequence: 1 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 1 }).unwrap(); whole.write_frame(ts(1), b"a".as_ref()).unwrap(); whole.write_frame(ts(2), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); failing .abort(Error::Stream(StreamError::Session(SessionError::Cancel))) .unwrap(); @@ -6006,7 +6151,7 @@ mod test { write(&b, if first_group { 3 } else { 2 }); routes.serve(b.consume()); if first_group { - assert_eq!(recv(&mut sub).sequence, 3); + assert_eq!(recv(&mut sub).start, 3); } let _b = b.finish().unwrap(); sub.set_range(..1); @@ -6118,7 +6263,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -6127,10 +6272,10 @@ mod test { assert!(reading.read_frame().now_or_never().is_none(), "no verdict yet"); let b = copy(); - let mut whole = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut whole = b.create_group(group::Info { start: 0 }).unwrap(); whole.write_frame(ts(0), b"a".as_ref()).unwrap(); whole.write_frame(ts(1), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); routes.serve(b.consume()); assert_eq!(read(&mut reading), Some(b"b".to_vec())); assert_eq!(read(&mut reading), None); @@ -6154,17 +6299,17 @@ mod test { routes.serve(c.consume()); // Every route holds group 1: a whole, b cut off by its route failing, c failing after a frame. - let mut whole = a.create_group(group::Info { sequence: 1 }).unwrap(); + let mut whole = a.create_group(group::Info { start: 1 }).unwrap(); whole.write_frame(ts(1), b"a".as_ref()).unwrap(); whole.write_frame(ts(2), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); - let mut partial = b.create_group(group::Info { sequence: 1 }).unwrap(); + whole.close().unwrap(); + let mut partial = b.create_group(group::Info { start: 1 }).unwrap(); partial.write_frame(ts(1), b"a".as_ref()).unwrap(); b.clone().abort(Error::Dropped).unwrap(); - let mut failing = c.create_group(group::Info { sequence: 1 }).unwrap(); + let mut failing = c.create_group(group::Info { start: 1 }).unwrap(); failing.write_frame(ts(1), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); - assert_eq!(reading.sequence, 1); + assert_eq!(reading.start, 1); // a's route fails too, after it finished the group. a.clone().abort(Error::Dropped).unwrap(); failing @@ -6194,11 +6339,11 @@ mod test { routes.serve(b.consume()); // Both routes hold group 1: the old one whole, the serving one failing after a frame. - let mut whole = a.create_group(group::Info { sequence: 1 }).unwrap(); + let mut whole = a.create_group(group::Info { start: 1 }).unwrap(); whole.write_frame(ts(1), b"a".as_ref()).unwrap(); whole.write_frame(ts(2), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); - let mut failing = b.create_group(group::Info { sequence: 1 }).unwrap(); + whole.close().unwrap(); + let mut failing = b.create_group(group::Info { start: 1 }).unwrap(); failing.write_frame(ts(1), b"a".as_ref()).unwrap(); let mut failing = Some(failing); if early { @@ -6216,7 +6361,7 @@ mod test { let Poll::Ready(Ok(Some(mut reading))) = res else { panic!("expected group 1"); }; - assert_eq!(reading.sequence, 1); + assert_eq!(reading.start, 1); // Reading on must not let go of the replaced copy while the group still needs it. assert_eq!(drain(&mut sub, ordered), Vec::::new()); if let Some(failing) = failing { @@ -6246,11 +6391,11 @@ mod test { let b = copy(); routes.serve(b.consume()); - let mut whole = a.create_group(group::Info { sequence: 1 }).unwrap(); + let mut whole = a.create_group(group::Info { start: 1 }).unwrap(); whole.write_frame(ts(1), b"a".as_ref()).unwrap(); whole.write_frame(ts(2), b"b".as_ref()).unwrap(); - whole.finish().unwrap(); - let mut failing = b.create_group(group::Info { sequence: 1 }).unwrap(); + whole.close().unwrap(); + let mut failing = b.create_group(group::Info { start: 1 }).unwrap(); failing.write_frame(ts(1), b"a".as_ref()).unwrap(); let res = match ordered { true => sub.poll_next_group(&kio::Waiter::noop()), @@ -6289,7 +6434,7 @@ mod test { routes.serve(b.consume()); // The serving route has nothing yet, so group 8 comes from the replaced one. - let mut open = a.create_group(group::Info { sequence: 8 }).unwrap(); + let mut open = a.create_group(group::Info { start: 8 }).unwrap(); open.write_frame(ts(8), b"a".as_ref()).unwrap(); let res = match ordered { true => sub.poll_next_group(&kio::Waiter::noop()), @@ -6298,14 +6443,14 @@ mod test { let Poll::Ready(Ok(Some(mut reading))) = res else { panic!("expected group 8"); }; - assert_eq!(reading.sequence, 8); + assert_eq!(reading.start, 8); assert_eq!(read(&mut reading), Some(b"a".to_vec())); // The replaced route gives it up, so the rest is fetched from the serving route. open.abort(Error::Stream(StreamError::DeliveryTimeout)).unwrap(); assert!(reading.read_frame().now_or_never().is_none()); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - assert_eq!((request.sequence(), request.frame_start()), (8, 1)); + assert_eq!((request.key(), request.frame_start()), (8, 1)); let mut rest = request.accept(None).unwrap(); assert!(reading.read_frame().now_or_never().is_none()); @@ -6316,7 +6461,7 @@ mod test { rest.write_frame(ts(9), b"b".as_ref()).unwrap(); rest.write_frame(ts(10), b"c".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut reading), Some(b"b".to_vec()), "ordered={ordered}"); assert_eq!(read(&mut reading), Some(b"c".to_vec())); assert_eq!(read(&mut reading), None); @@ -6363,7 +6508,7 @@ mod test { routes.serve(a.consume()); let budget = Duration::from_secs(1); let mut sub = subscribe(&logical, budget); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -6403,7 +6548,7 @@ mod test { routes.serve(b.consume()); write(&b, 1); let held = recv(&mut sub); - assert_eq!(held.sequence, 1); + assert_eq!(held.start, 1); assert_eq!(drain(&mut sub, false), Vec::::new()); assert_eq!(copies(&routes), 2, "group 1 may still land on the replaced copy"); @@ -6447,8 +6592,8 @@ mod test { let b = copy(); routes.serve(b.consume()); - for sequence in 2..=2000 { - write(&b, sequence); + for key in 2..=2000 { + write(&b, key); } assert_eq!(drain(&mut sub, false), (2..=2000).collect::>()); assert_eq!(sub.delivered_runs(), 2, "the gap at 1 is within the budget"); @@ -6473,8 +6618,8 @@ mod test { let b = copy(); routes.serve(b.consume()); - for sequence in 1..=2000 { - write(&b, sequence); + for key in 1..=2000 { + write(&b, key); } assert_eq!(drain(&mut sub, true), (1..=2000).collect::>()); write(&b, 0); @@ -6491,7 +6636,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); write(&a, 0); - let open = a.create_group(group::Info { sequence: 1 }).unwrap(); + let open = a.create_group(group::Info { start: 1 }).unwrap(); assert_eq!(drain(&mut sub, false), [0, 1]); open.abort(Error::Cancel).unwrap(); @@ -6515,19 +6660,19 @@ mod test { let (serving, other) = (&tracks[round as usize % 2], &tracks[1 - round as usize % 2]); let groups = round * 10..round * 10 + 10; // The serving route brings half the round, the reader takes it, then the rest. - for sequence in groups.clone().take(5) { - write(serving, sequence); + for key in groups.clone().take(5) { + write(serving, key); } seen.extend(drain(&mut sub, false)); - for sequence in groups.clone().skip(5) { - write(serving, sequence); + for key in groups.clone().skip(5) { + write(serving, key); } // The other route takes over and the replaced copy hands out the rest; the // other's own copy of the round lands only afterwards, newest first. routes.serve(other.consume()); seen.extend(drain(&mut sub, false)); - for sequence in groups.clone().rev() { - write(other, sequence); + for key in groups.clone().rev() { + write(other, key); } seen.extend(drain(&mut sub, false)); assert_eq!(sub.delivered_runs(), 1, "round {round}"); @@ -6547,7 +6692,7 @@ mod test { routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); sub.set_range(..=0); - let mut held = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut held = a.create_group(group::Info { start: 0 }).unwrap(); held.write_frame(ts(0), b"a".as_ref()).unwrap(); let Poll::Ready(Ok(Some(reading))) = sub.poll_recv_group(&kio::Waiter::noop()) else { panic!("group 0 is handed out") @@ -6559,8 +6704,8 @@ mod test { let b = copy(); routes.serve(b.consume()); - for sequence in [2, 0, 3, 1] { - write(&b, sequence); + for key in [2, 0, 3, 1] { + write(&b, key); } assert_eq!(drain(&mut sub, false), Vec::::new()); @@ -6572,7 +6717,7 @@ mod test { assert_eq!(drain(&mut sub, false), Vec::::new()); // The replaced copy ends with the held group, declaring no end: another route continues the track. - held.finish().unwrap(); + held.close().unwrap(); a.clone().abort(Error::Dropped).unwrap(); drop(reading); assert_eq!(drain(&mut sub, false), Vec::::new()); @@ -6592,18 +6737,18 @@ mod test { // Groups present 1ms apart, so the budget spans ten of them. let mut sub = subscribe(&logical, Duration::from_millis(10)); let mut most = 0; - for sequence in 0..3000u64 { - let track = &tracks[(sequence / 500) as usize % 2]; + for key in 0..3000u64 { + let track = &tracks[(key / 500) as usize % 2]; // Every 500 groups the other route takes over. - if sequence % 500 == 0 { + if key % 500 == 0 { routes.serve(track.consume()); } // Every third group never arrives. - if sequence % 3 == 1 { + if key % 3 == 1 { continue; } - write(track, sequence); - assert_eq!(drain(&mut sub, false), [sequence]); + write(track, key); + assert_eq!(drain(&mut sub, false), [key]); most = most.max(sub.delivered_runs()); } // The floor and the four gaps a budget spans, twice over across a route change: the @@ -6628,19 +6773,19 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut group = a.create_group(group::Info { sequence: 5 }).unwrap(); + let mut group = a.create_group(group::Info { start: 5 }).unwrap(); group.write_frame(ts(5), b"x".as_ref()).unwrap(); - group.finish().unwrap(); - assert_eq!(recv(&mut sub).sequence, 5); + group.close().unwrap(); + assert_eq!(recv(&mut sub).start, 5); let b = copy(); routes.serve(b.consume()); // Subscribe the replacement before it has anything, so the update moves a ready copy. assert!(sub.recv_group().now_or_never().is_none()); - for sequence in [2, 6] { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for key in [2, 6] { + let mut group = b.create_group(group::Info { start: key }).unwrap(); + group.write_frame(ts(key), b"x".as_ref()).unwrap(); + group.close().unwrap(); } sub.set_range(..); @@ -6660,8 +6805,8 @@ mod test { let ranges = [ track::Range::groups(0..1), track::Range { - start: track::Anchor::Time(ts(10_000)), - end: Some(track::Anchor::Time(ts(11_000))), + start: track::Anchor::Time(10_000), + end: Some(track::Anchor::Time(11_000)), }, ]; let subscription = Subscription::default() @@ -6673,30 +6818,30 @@ mod test { .now_or_never() .unwrap() .unwrap(); - for sequence in 0..13 { - let mut group = a.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 1000), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 0..13 { + let mut group = a.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"x".as_ref()).unwrap(); + group.close().unwrap(); } - assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).start, 0); let b = copy(); routes.serve(b.consume()); // The read subscribes the replacement while the replaced copy still serves. let mut rest = Vec::new(); while let Some(Ok(Some(group))) = sub.recv_group().now_or_never() { - rest.push(group.sequence); + rest.push(group.start); } - for sequence in 0..13 { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 1000), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 0..13 { + let mut group = b.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"x".as_ref()).unwrap(); + group.close().unwrap(); } while let Some(Ok(Some(group))) = sub.recv_group().now_or_never() { - rest.push(group.sequence); + rest.push(group.start); } rest.sort(); - assert_eq!(rest, vec![10, 11]); + assert_eq!(rest, vec![10_000, 11_000]); } /// A replacement updated to a time range it cannot resolve yet asks for that range, @@ -6714,7 +6859,7 @@ mod test { let copy = &mut reader.copies[0]; copy.floor = Some(0); let ranges = vec![track::Range { - start: track::Anchor::Time(ts(10_000)), + start: track::Anchor::Time(10_000), end: None, }]; let updated = Subscription::default().with_ranges(ranges.clone()); @@ -6747,7 +6892,7 @@ mod test { end: None, }]); copy.update(&updated); - assert_eq!(copy.wire(&updated).ranges[0].start, Anchor::Absolute(2)); + assert_eq!(copy.wire(&updated).ranges[0].start, Anchor::Time(2)); } /// A replacement asked for a range reaching back past where the reader got to, while @@ -6761,9 +6906,9 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(30)); - let mut five = a.create_group(group::Info { sequence: 5 }).unwrap(); + let mut five = a.create_group(group::Info { start: 5000 }).unwrap(); five.write_frame(ts(5000), b"x".as_ref()).unwrap(); - assert_eq!(recv(&mut sub).sequence, 5); + assert_eq!(recv(&mut sub).start, 5000); let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); routes.serve(b.consume()); @@ -6777,16 +6922,16 @@ mod test { assert!(sub.recv_group().now_or_never().is_none()); let b = b.accept(crate::track::Info::timed()); - for sequence in 0..7 { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 1000), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 0..7 { + let mut group = b.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"x".as_ref()).unwrap(); + group.close().unwrap(); } let mut rest = Vec::new(); while let Some(Ok(Some(group))) = sub.recv_group().now_or_never() { - rest.push(group.sequence); + rest.push(group.start); } - assert_eq!(rest, vec![0, 1, 2, 3, 4, 6]); + assert_eq!(rest, vec![0, 1000, 2000, 3000, 4000, 6000]); } /// Ranges widened while only an unanswered replacement can place them keep the switch's @@ -6802,7 +6947,7 @@ mod test { let subscription = Subscription::default() .with_delay(Delay::default().with_max(HOUR)) .with_ranges([track::Range { - start: track::Anchor::Absolute(5), + start: track::Anchor::Time(5), end: None, }]); let mut sub = logical @@ -6827,8 +6972,8 @@ mod test { assert_eq!(drain(&mut sub, false), Vec::::new()); let b = b.accept(crate::track::Info::timed()); - for sequence in 0..5 { - write(&b, sequence); + for key in 0..5 { + write(&b, key); } let mut rest = drain(&mut sub, false); rest.sort_unstable(); @@ -6838,8 +6983,8 @@ mod test { } /// A route's backfill widened by a later resume point asks for the wider span even once - /// it answered for the narrower one: a range ending relative to the latest group keeps the - /// reader's end, not the backfill's earlier cap. + /// it answered for the narrower one: a bounded range keeps the reader's end, not the + /// backfill's earlier cap. #[kio::test] async fn a_widened_backfill_reaches_past_its_earlier_cap() { let routes = Producer::new(); @@ -6852,8 +6997,8 @@ mod test { let subscription = Subscription::default() .with_delay(Delay::default().with_max(HOUR)) .with_ranges([track::Range { - start: track::Anchor::Absolute(0), - end: Some(track::Anchor::Relative(990)), + start: track::Anchor::Time(0), + end: Some(track::Anchor::Time(10)), }]); let mut sub = logical .consume() @@ -6878,8 +7023,8 @@ mod test { // B returns at 6, widening its backfill past 2. routes.serve(b.consume()); assert_eq!(drain(&mut sub, false), Vec::::new()); - for sequence in [1, 3, 5] { - write(&b, sequence); + for key in [1, 3, 5] { + write(&b, key); } let mut late = drain(&mut sub, false); late.sort_unstable(); @@ -6898,26 +7043,26 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(30)); - let mut zero = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut zero = a.create_group(group::Info { start: 0 }).unwrap(); zero.write_frame(ts(0), b"x".as_ref()).unwrap(); - assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).start, 0); let b = track::Request::new(Arc::new(broadcast::Info::default()), "video"); routes.serve(b.consume()); // The read subscribes the replacement, which has not answered yet. assert!(sub.recv_group().now_or_never().is_none()); let ranges = [track::Range { - start: track::Anchor::Time(ts(10_000)), + start: track::Anchor::Time(10_000), end: None, }]; sub.update(sub.subscription().with_ranges(ranges)).unwrap(); assert!(sub.recv_group().now_or_never().is_none()); let b = b.accept(crate::track::Info::timed()); - for sequence in 0..13 { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 1000), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 0..13 { + let mut group = b.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"x".as_ref()).unwrap(); + group.close().unwrap(); } while let Some(Ok(Some(_))) = sub.recv_group().now_or_never() {} @@ -6926,7 +7071,7 @@ mod test { assert!(sub.recv_group().now_or_never().is_none()); let start = c.subscription().expect("subscribed").ranges[0].start; assert!( - matches!(start, track::Anchor::Absolute(group) if group >= 10), + matches!(start, track::Anchor::Time(group) if group >= 10_000), "the replacement asks from where b resolved the time: {start:?}" ); } @@ -6956,7 +7101,7 @@ mod test { .now_or_never() .unwrap() .unwrap(); - assert_eq!(recv(&mut sub).sequence, 8); + assert_eq!(recv(&mut sub).start, 8); let (b, _requests) = pending_copy(); routes.serve(b.consume()); @@ -6965,7 +7110,7 @@ mod test { routes.serve(c.consume()); assert_eq!( c.subscription().expect("subscribed").ranges[0].start, - track::Anchor::Absolute(8) + track::Anchor::Time(8) ); } @@ -6982,7 +7127,7 @@ mod test { routes.serve(a.consume()); let range = track::Range { start: track::Anchor::Relative(0), - end: Some(track::Anchor::Time(ts(1_000_000))), + end: Some(track::Anchor::Time(1_000_000)), }; let subscription = Subscription::default() .with_delay(Delay::default().with_max(HOUR)) @@ -7017,9 +7162,9 @@ mod test { a.received(); routes.serve(a.consume()); let mut sub = subscribe(&front, Duration::from_secs(30)); - let mut zero = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut zero = a.create_group(group::Info { start: 0 }).unwrap(); zero.write_frame(ts(0), b"x".as_ref()).unwrap(); - assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).start, 0); let b = copy(); b.received(); @@ -7031,14 +7176,14 @@ mod test { }]; sub.update(sub.subscription().with_ranges(ranges)).unwrap(); assert!(sub.recv_group().now_or_never().is_none()); - for sequence in 1..4 { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 1000), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 1..4 { + let mut group = b.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"x".as_ref()).unwrap(); + group.close().unwrap(); } // Asked before b held any group, the relative start resolves at its first. let group = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1000); // Served by a copy that can't resolve it, the same range fails. let routes = Producer::new(); @@ -7070,20 +7215,20 @@ mod test { assert_eq!(a.subscription().unwrap().priority, 3); } - /// Capping a replaced copy never replaces a time end it has not settled: the copy keeps - /// filtering by time, so a group below the cap that starts past the end stays out. + /// Capping a replaced copy past a time end keeps the end: a group below the cap that + /// starts past the end stays out. #[test] #[cfg(feature = "async")] fn capping_a_replaced_copy_keeps_its_time_end() { let routes = Producer::new(); let a = copy(); routes.serve(a.consume()); - let mut zero = a.create_group(group::Info { sequence: 0 }).unwrap(); - zero.write_frame(ts(3000), b"x".as_ref()).unwrap(); - zero.finish().unwrap(); + let mut first = a.create_group(group::Info { start: 3000 }).unwrap(); + first.write_frame(ts(3000), b"x".as_ref()).unwrap(); + first.close().unwrap(); let range = track::Range { - start: track::Anchor::Absolute(0), - end: Some(track::Anchor::Time(ts(3500))), + start: track::Anchor::Time(3000), + end: Some(track::Anchor::Time(3500)), }; let subscription = Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(30))) @@ -7093,23 +7238,25 @@ mod test { let (groups, mirrored) = (reader.groups, reader.mirrored.clone()); let old = &mut reader.copies[0]; old.replaced = true; - old.until = Some(Some(4)); + old.until = Some(Some(3800)); old.update(&mirrored); - assert_eq!(old.wire(&mirrored).ranges, vec![range], "the time end outlives the cap"); + assert_eq!( + old.wire(&mirrored).ranges, + vec![range], + "the time end is inside the cap" + ); - for (sequence, millis) in [(1, 3600), (2, 3700)] { - let mut group = a.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(millis), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + for key in [3600, 3700] { + write(&a, key); } let Poll::Ready(Ok(copy)) = old.poll_ready(groups, &kio::Waiter::noop()) else { panic!("the copy answered"); }; let mut read = Vec::new(); while let Some(Ok(Some(group))) = copy.recv_group().now_or_never() { - read.push(group.sequence); + read.push(group.start); } - assert_eq!(read, vec![0]); + assert_eq!(read, vec![3000]); } /// Bounding a replaced copy keeps its relative start where the copy resolved it, though @@ -7142,14 +7289,14 @@ mod test { assert_eq!( old.wire(&mirrored).ranges, [track::Range { - start: track::Anchor::Absolute(8), - end: Some(track::Anchor::Absolute(13)), + start: track::Anchor::Time(8), + end: Some(track::Anchor::Time(12)), }] ); } #[test] - fn a_replaced_copy_at_the_last_sequence_has_no_overflow_cap() { + fn a_replaced_copy_at_the_last_key_has_no_overflow_cap() { for (last, end) in [(Some(u64::MAX), None), (None, Some(0))] { let routes = Producer::new(); let a = copy(); @@ -7174,20 +7321,20 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let mut first = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut first = a.create_group(group::Info { start: 0 }).unwrap(); first.write_frame(ts(0), b"a0".as_ref()).unwrap(); - assert_eq!(recv(&mut sub).sequence, 0); + assert_eq!(recv(&mut sub).start, 0); let b = copy(); routes.serve(b.consume()); // Both routes go on to group 1; only the serving one's copy counts. for (track, payload) in [(&a, b"a1"), (&b, b"b1")] { - let mut group = track.create_group(group::Info { sequence: 1 }).unwrap(); + let mut group = track.create_group(group::Info { start: 1 }).unwrap(); group.write_frame(ts(1), payload.as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let mut next = recv(&mut sub); - assert_eq!(next.sequence, 1); + assert_eq!(next.start, 1); assert_eq!(read(&mut next), Some(b"b1".to_vec())); } @@ -7199,7 +7346,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(2)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -7208,14 +7355,14 @@ mod test { // stays up but quiet. let b = copy(); routes.serve(b.consume()); - for sequence in 1..=2 { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 1000), b"b".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 1..=2 { + let mut group = b.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"b".as_ref()).unwrap(); + group.close().unwrap(); } assert!(reading.read_frame().now_or_never().is_none(), "not stale yet"); - let mut group = b.create_group(group::Info { sequence: 3 }).unwrap(); + let mut group = b.create_group(group::Info { start: 3000 }).unwrap(); group.write_frame(ts(3000), b"b".as_ref()).unwrap(); assert!(matches!(reading.read_frame().now_or_never(), Some(Err(Error::Old)))); } @@ -7239,8 +7386,9 @@ mod test { let a = untimed(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(1)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); - open.write_frame(None, b"a".as_ref()).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); + open.write_frame(crate::Timestamp::unscaled(open.start), b"a".as_ref()) + .unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -7248,10 +7396,12 @@ mod test { let pool = b.broadcast().pool.clone(); // A driver collects the replacement's pool, so its wall deadlines fire. pool.step(Duration::ZERO); - let write = |sequence| { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, b"b".as_ref()).unwrap(); - group.finish().unwrap(); + let write = |key| { + let mut group = b.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), b"b".as_ref()) + .unwrap(); + group.close().unwrap(); }; if warm { (1..=3).for_each(write); @@ -7313,23 +7463,23 @@ mod test { else { panic!("a group to reserve") }; - assert_eq!(sub.take(candidate).unwrap().sequence, 5); + assert_eq!(sub.take(candidate).unwrap().start, 5); for sequence in 11..=20 { write(&a, sequence); } let start = || routes.subscription(Subscription::default(), 1).ranges[0].start; - assert_eq!(start(), track::Anchor::Absolute(5)); + assert_eq!(start(), track::Anchor::Time(5)); // Changed, but not yet mirrored into the copies. if let Ok(mut changed) = preferences.write() { *changed = ranges(0); } - assert_eq!(start(), track::Anchor::Absolute(20)); + assert_eq!(start(), track::Anchor::Time(20)); sub.mirror().sync(); let (b, _requests) = pending_copy(); routes.serve(b.consume()); - assert_eq!(start(), track::Anchor::Absolute(20)); + assert_eq!(start(), track::Anchor::Time(20)); if let Ok(mut changed) = preferences.write() { *changed = ranges(3); } @@ -7348,7 +7498,7 @@ mod test { let mut live: Vec<_> = (0..5).map(|_| subscribe(&front, HOUR)).collect(); write(&a, 5); for sub in &mut live { - assert_eq!(recv(sub).sequence, 5); + assert_eq!(recv(sub).start, 5); } let demand = front.subscription().unwrap(); assert_eq!(routes.subscription(demand.clone(), 5).start(), Some(5)); @@ -7371,14 +7521,14 @@ mod test { let mut first = subscribe(&front, HOUR); let mut second = subscribe(&front, HOUR); write(&a, 5); - assert_eq!(recv(&mut first).sequence, 5); + assert_eq!(recv(&mut first).start, 5); let demand = front.subscription().unwrap(); // One reader has not read anything yet, so it still needs the oldest range. assert_eq!(routes.subscription(demand.clone(), 2).start(), None); - assert_eq!(recv(&mut second).sequence, 5); + assert_eq!(recv(&mut second).start, 5); assert_eq!(routes.subscription(demand.clone(), 2).start(), Some(5)); write(&a, 10); - assert_eq!(recv(&mut first).sequence, 10); + assert_eq!(recv(&mut first).start, 10); assert_eq!(routes.subscription(demand.clone(), 2).start(), Some(5)); let pending = front .consume() @@ -7406,7 +7556,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let _open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let _open = a.create_group(group::Info { start: 0 }).unwrap(); let mut reading = recv(&mut sub); let wake = Arc::new(Wake::default()); let waiter = kio::Waiter::new(wake.clone().into()); @@ -7422,7 +7572,7 @@ mod test { struct Held { group: group::Consumer, copy: track::Producer, - /// The replacement's groups, by sequence. + /// The replacement's groups, by key. groups: std::collections::BTreeMap, wake: Arc, waiter: kio::Waiter, @@ -7437,7 +7587,7 @@ mod test { #[cfg(feature = "async")] impl Held { - /// `groups` are the replacement's `(sequence, first timestamp)`, written before the + /// `groups` are the replacement's `(key, first timestamp)`, written before the /// switch is noticed, so its subscription has passed the held group. #[cfg(feature = "async")] fn new(groups: &[(u64, Option)]) -> Self { @@ -7446,14 +7596,14 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_millis(100)); - for (sequence, ms) in [(0, 0), (1, 1000)] { - let mut group = a.create_group(group::Info { sequence }).unwrap(); + for (key, ms) in [(0, 0), (1, 1000)] { + let mut group = a.create_group(group::Info { start: key }).unwrap(); group.write_frame(ts(ms), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - assert_eq!(recv(&mut sub).sequence, 0); - assert_eq!(recv(&mut sub).sequence, 1); - let mut open = a.create_group(group::Info { sequence: 2 }).unwrap(); + assert_eq!(recv(&mut sub).start, 0); + assert_eq!(recv(&mut sub).start, 1); + let mut open = a.create_group(group::Info { start: 2 }).unwrap(); open.write_frame(ts(2000), b"held".as_ref()).unwrap(); let mut group = recv(&mut sub); assert_eq!(read(&mut group), Some(b"held".to_vec())); @@ -7473,12 +7623,12 @@ mod test { routes.serve(b.consume()); let groups = groups .iter() - .map(|&(sequence, ms)| { - let mut group = b.create_group(group::Info { sequence }).unwrap(); + .map(|&(key, ms)| { + let mut group = b.create_group(group::Info { start: key }).unwrap(); if let Some(ms) = ms { group.write_frame(ts(ms), b"b".as_ref()).unwrap(); } - (sequence, group) + (key, group) }) .collect(); @@ -7518,8 +7668,8 @@ mod test { self.wake.0.load(std::sync::atomic::Ordering::Relaxed) } - fn group(&self, sequence: u64) -> group::Producer { - self.copy.create_group(group::Info { sequence }).unwrap() + fn group(&self, key: u64) -> group::Producer { + self.copy.create_group(group::Info { start: key }).unwrap() } } @@ -7602,11 +7752,11 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let open = a.create_group(group::Info { start: 0 }).unwrap(); let mut frame = open .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), }) .unwrap(); frame.write(b"ab".as_ref()).unwrap(); @@ -7618,14 +7768,14 @@ mod test { ); let b = copy(); - let mut replacement = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut replacement = b.create_group(group::Info { start: 0 }).unwrap(); routes.serve(b.consume()); assert!(chunks.read_chunk().now_or_never().is_none()); if !arrives { - for sequence in 1..=4 { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 10_000), b"new".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 1..=4 { + let mut group = b.create_group(group::Info { start: key }).unwrap(); + group.write_frame(ts(key * 10_000), b"new".as_ref()).unwrap(); + group.close().unwrap(); } assert!(matches!(chunks.read_chunk().now_or_never(), Some(Err(Error::Old)))); continue; @@ -7652,11 +7802,11 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(2)); - let open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let open = a.create_group(group::Info { start: 0 }).unwrap(); let mut frame = open .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), }) .unwrap(); frame.write(b"a".as_ref()).unwrap(); @@ -7665,11 +7815,11 @@ mod test { assert!(chunks.read_chunk().now_or_never().unwrap().is_ok()); let b = copy(); - let replacement = b.create_group(group::Info { sequence: 0 }).unwrap(); + let replacement = b.create_group(group::Info { start: 0 }).unwrap(); let mut replacement_frame = replacement .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), }) .unwrap(); replacement_frame.write(b"ab".as_ref()).unwrap(); @@ -7677,10 +7827,10 @@ mod test { assert_eq!(&chunks.read_chunk().now_or_never().unwrap().unwrap().unwrap()[..], b"b"); // Transfer the group cursor too, while its next frame is still pending. assert!(reading.read_frame().now_or_never().is_none()); - for sequence in 1..=4 { - let mut next = b.create_group(group::Info { sequence }).unwrap(); - next.write_frame(ts(sequence * 1000), b"x".as_ref()).unwrap(); - next.finish().unwrap(); + for key in 1..=4 { + let mut next = b.create_group(group::Info { start: key }).unwrap(); + next.write_frame(ts(key * 1000), b"x".as_ref()).unwrap(); + next.close().unwrap(); } if partial { assert!(matches!(chunks.read_chunk().now_or_never(), Some(Err(Error::Old)))); @@ -7705,9 +7855,9 @@ mod test { let _request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); wake.0.store(false, std::sync::atomic::Ordering::Relaxed); let b = copy(); - let mut group = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = b.create_group(group::Info { start: 0 }).unwrap(); group.write_frame(ts(0), b"b".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); routes.serve(b.consume()); assert!(wake.0.load(std::sync::atomic::Ordering::Relaxed)); assert!(matches!(kio::Task::poll(&mut *fetching, &waiter), Poll::Ready(Ok(_)))); @@ -7736,7 +7886,7 @@ mod test { for frame in 0..4u8 { group.write_frame(ts(0), vec![frame]).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let Poll::Ready(Ok(mut group)) = kio::Task::poll(&mut *fetching, &waiter) else { panic!("the fill did not resolve the fetch"); }; @@ -7753,7 +7903,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(2)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -7761,10 +7911,10 @@ mod test { let _dynamic = b.requests(); routes.serve(b.consume()); assert!(reading.read_frame().now_or_never().is_none()); - for sequence in 1..=4 { - let mut group = b.create_group(group::Info { sequence }).unwrap(); - group.write_frame(ts(sequence * 1000), b"b".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 1..=4 { + let mut group = b.create_group(group::Info { start: key * 1000 }).unwrap(); + group.write_frame(ts(key * 1000), b"b".as_ref()).unwrap(); + group.close().unwrap(); } assert!(matches!(reading.read_frame().now_or_never(), Some(Err(Error::Old)))); } @@ -7775,7 +7925,7 @@ mod test { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); - let group = a.create_group(group::Info { sequence: 0 }).unwrap(); + let group = a.create_group(group::Info { start: 0 }).unwrap(); group .set_wire(group::Wire { sparse: false, @@ -7791,7 +7941,7 @@ mod test { .create_object(crate::frame::Header { info: crate::frame::Info { size: 1, - timestamp: Some(ts(0)), + timestamp: ts(0), }, fields: Some(Arc::new(super::super::object::Fields { id: 6, @@ -7811,7 +7961,7 @@ mod test { .create_object(crate::frame::Header { info: crate::frame::Info { size: 2, - timestamp: Some(ts(1)), + timestamp: ts(1), }, fields: Some(Arc::new(super::super::object::Fields { id: 8, @@ -7843,7 +7993,7 @@ mod test { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); - let mut group = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = a.create_group(group::Info { start: 0 }).unwrap(); group .set_wire(group::Wire { sparse: false, @@ -7859,7 +8009,7 @@ mod test { let header = crate::frame::Header { info: crate::frame::Info { size: payload.len() as u64, - timestamp: Some(ts(id)), + timestamp: ts(id), }, fields: Some(Arc::new(super::super::object::Fields { id, @@ -7889,7 +8039,7 @@ mod test { #[cfg(feature = "async")] fn joined_past_a_gap(objects: &[(u64, &'static [u8])]) -> track::Producer { let b = copy(); - let mut group = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = b.create_group(group::Info { start: 0 }).unwrap(); group .set_wire(group::Wire { sparse: false, @@ -7906,7 +8056,7 @@ mod test { .create_object(crate::frame::Header { info: crate::frame::Info { size: payload.len() as u64, - timestamp: Some(ts(id)), + timestamp: ts(id), }, fields: Some(Arc::new(super::super::object::Fields { id, @@ -7918,7 +8068,7 @@ mod test { frame.write(bytes::Bytes::from_static(payload)).unwrap(); group = frame.finish().unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); b } @@ -7931,11 +8081,11 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&front, Duration::from_secs(10)); - let open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let open = a.create_group(group::Info { start: 0 }).unwrap(); let mut frame = open .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(0)), + timestamp: ts(0), }) .unwrap(); frame.write(b"ab".as_ref()).unwrap(); @@ -7960,7 +8110,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&front, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); open.write_frame(ts(1), b"b".as_ref()).unwrap(); let mut reading = recv(&mut sub); @@ -7981,7 +8131,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&front, Duration::from_secs(10)); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); let b = joined_past_a_gap(&[(6, b"x"), (7, b"y"), (8, b"z")]); @@ -8008,7 +8158,7 @@ mod test { fn a_fetched_group_wakes_when_the_route_changes() { let routes = Producer::new(); let a = copy(); - let _open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let _open = a.create_group(group::Info { start: 0 }).unwrap(); routes.serve(a.consume()); let mut reading = logical(&routes) .consume() @@ -8033,7 +8183,7 @@ mod test { let dynamic = a.requests(); routes.serve(a.consume()); - let mut arriving = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut arriving = a.create_group(group::Info { start: 0 }).unwrap(); arriving.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut fetched = logical(&routes) .consume() @@ -8048,10 +8198,10 @@ mod test { assert!(fetched.read_frame().now_or_never().is_none()); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - assert_eq!((request.sequence(), request.frame_start()), (0, 1)); + assert_eq!((request.key(), request.frame_start()), (0, 1)); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(1), b"b".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut fetched), Some(b"b".to_vec())); assert_eq!(read(&mut fetched), None); } @@ -8088,7 +8238,7 @@ mod test { let dynamic = a.requests(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = match fetched { true => logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(), @@ -8124,7 +8274,7 @@ mod test { let a_requests = a.requests(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = match fetched { true => logical.consume().fetch_group(0, None).now_or_never().unwrap().unwrap(), @@ -8135,7 +8285,7 @@ mod test { let b = copy(); let b_requests = b.requests(); routes.serve(b.consume()); - let mut repeat = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut repeat = b.create_group(group::Info { start: 0 }).unwrap(); repeat.write_frame(ts(0), b"a".as_ref()).unwrap(); open.abort(Error::WrongSize).unwrap(); assert!(reading.read_frame().now_or_never().is_none(), "fetched={fetched}"); @@ -8166,7 +8316,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, HOUR); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let mut reading = recv(&mut sub); assert_eq!(read(&mut reading), Some(b"a".to_vec())); @@ -8179,7 +8329,7 @@ mod test { let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); let mut rest = request.accept(None).unwrap(); rest.write_frame(ts(1), b"b".as_ref()).unwrap(); - rest.finish().unwrap(); + rest.close().unwrap(); assert_eq!(read(&mut reading), Some(b"b".to_vec()), "{code:?}"); assert_eq!(read(&mut reading), None, "{code:?}"); } @@ -8193,7 +8343,7 @@ mod test { let routes = Producer::new(); let a = copy(); routes.serve(a.consume()); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let logical = logical(&routes); @@ -8215,7 +8365,7 @@ mod test { let routes = Producer::new(); let a = copy(); routes.serve(a.consume()); - let mut open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let mut open = a.create_group(group::Info { start: 0 }).unwrap(); open.write_frame(ts(0), b"a".as_ref()).unwrap(); let budget = Duration::from_millis(500); @@ -8236,10 +8386,10 @@ mod test { assert_eq!((request.frame_start(), request.max_delay()), (1, Some(budget))); // The new route runs a full second past the group's successor while its fetch is unanswered. - for (sequence, ms) in [(1, 1000), (2, 2000)] { - let mut group = b.create_group(group::Info { sequence }).unwrap(); + for (key, ms) in [(1, 1000), (2, 2000)] { + let mut group = b.create_group(group::Info { start: key }).unwrap(); group.write_frame(ts(ms), b"x".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } assert!(matches!(fetched.read_frame().now_or_never(), Some(Err(Error::Old)))); } @@ -8252,7 +8402,7 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let _open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let _open = a.create_group(group::Info { start: 0 }).unwrap(); let mut reading = recv(&mut sub); assert!(reading.read_frame().now_or_never().is_none()); routes.end(Err(Error::Unroutable)); @@ -8318,8 +8468,8 @@ mod test { let b = copy(); routes.serve(b.consume()); - for sequence in 1..=5 { - write(&b, sequence); + for key in 1..=5 { + write(&b, key); } assert!(!drain(&mut sub, false).is_empty()); { @@ -8369,7 +8519,7 @@ mod test { routes.serve(a.consume()); } let subscription = Subscription::default().with_ranges([track::Range { - start: Anchor::Absolute(0), + start: Anchor::Time(0), end: Some(Anchor::Oldest), }]); let mut sub = logical @@ -8382,7 +8532,7 @@ mod test { assert!( matches!(result, Poll::Ready(Err(Error::InvalidRange))), "served={served}: {:?}", - result.map(|r| r.map(|g| g.map(|g| g.sequence))) + result.map(|r| r.map(|g| g.map(|g| g.start))) ); } @@ -8390,7 +8540,7 @@ mod test { #[test] #[cfg(feature = "async")] fn a_resumed_reader_reports_starts_within_its_time_end() { - for start in [Anchor::Absolute(0), Anchor::Relative(0)] { + for start in [Anchor::Time(0), Anchor::Relative(0)] { let routes = Producer::new(); let logical = logical(&routes); let a = copy(); @@ -8400,7 +8550,7 @@ mod test { .with_delay(Delay::default().with_max(HOUR)) .with_ranges([track::Range { start, - end: Some(Anchor::Time(ts(1))), + end: Some(Anchor::Time(1)), }]); let mut sub = logical .consume() @@ -8410,8 +8560,8 @@ mod test { .unwrap(); let _ = sub.poll_start(&kio::Waiter::noop()); assert_eq!(sub.starts(0), [Some(0)]); - for sequence in 1..3 { - write(&a, sequence); + for key in 1..3 { + write(&a, key); } assert_eq!(sub.starts(10), [None]); assert_eq!(sub.starts(0), [Some(0)], "the start stays pinned while the end settles"); @@ -8419,7 +8569,7 @@ mod test { } /// A resumed reader names the next group its ranges cover, so a reader waiting on each - /// group in sequence walks over the gap between two ranges instead of waiting for it. + /// group in key walks over the gap between two ranges instead of waiting for it. #[test] #[cfg(feature = "async")] fn a_resumed_reader_names_the_next_group_in_range() { @@ -8436,8 +8586,8 @@ mod test { .now_or_never() .unwrap() .unwrap(); - for sequence in [0, 5, 10] { - write(&a, sequence); + for key in [0, 5, 10] { + write(&a, key); } let mut read = drain(&mut sub, false); read.sort(); @@ -8475,10 +8625,10 @@ mod test { let a = copy(); routes.serve(a.consume()); let mut sub = subscribe(&logical, Duration::from_secs(10)); - let _open = a.create_group(group::Info { sequence: 0 }).unwrap(); + let _open = a.create_group(group::Info { start: 0 }).unwrap(); let mut reading = recv(&mut sub); let b = copy(); - let mut replacement = b.create_group(group::Info { sequence: 0 }).unwrap(); + let mut replacement = b.create_group(group::Info { start: 0 }).unwrap(); replacement.write_frame(ts(0), b"b".as_ref()).unwrap(); routes.serve(b.consume()); assert_eq!(read(&mut reading), Some(b"b".to_vec())); @@ -8519,7 +8669,7 @@ mod test { let mut sub = subscribe(&logical, Duration::from_secs(if widen { 2 } else { 10 })); let b = copy(); // Both routes carry the same content, including the unfinished payload. - let writers = [&a, &b].map(|source| source.create_group(group::Info { sequence: 0 }).unwrap()); + let writers = [&a, &b].map(|source| source.create_group(group::Info { start: 0 }).unwrap()); // Each group's writer, held open by its unfinished frame when chunked. let _writers: Vec<_> = writers .into_iter() @@ -8531,7 +8681,7 @@ mod test { let mut frame = group .create_frame(crate::frame::Info { size: 4, - timestamp: Some(ts(1)), + timestamp: ts(1), }) .unwrap(); frame.write(b"ab".as_ref()).unwrap(); @@ -8583,10 +8733,10 @@ mod test { 7, "recovered={recovered}, widen={widen}, chunked={chunked}: preference update did not reach the copy" ); - for sequence in 1..=4 { - let mut next = source.create_group(group::Info { sequence }).unwrap(); - next.write_frame(ts(sequence * 1000), b"next".as_ref()).unwrap(); - next.finish().unwrap(); + for key in 1..=4 { + let mut next = source.create_group(group::Info { start: key }).unwrap(); + next.write_frame(ts(key * 1000), b"next".as_ref()).unwrap(); + next.close().unwrap(); } let result = reading.poll(&waiter); if widen { @@ -8646,7 +8796,7 @@ mod test { routes.serve(copies[0].consume()); let mut writers: Vec<_> = copies .iter() - .map(|copy| Some(copy.create_group(group::Info { sequence: 0 }).unwrap())) + .map(|copy| Some(copy.create_group(group::Info { start: 0 }).unwrap())) .collect(); writers[0] .as_mut() @@ -8696,7 +8846,7 @@ mod test { writer.write_frame(ts(0), b"first".as_ref()).unwrap(); } writer.write_frame(ts(1), b"second".as_ref()).unwrap(); - writer.finish().unwrap(); + writer.close().unwrap(); } Event::Update => control .update( diff --git a/rs/moq-net/src/model/resume_model.rs b/rs/moq-net/src/model/resume_model.rs index c2be6b697f3..f07b714f359 100644 --- a/rs/moq-net/src/model/resume_model.rs +++ b/rs/moq-net/src/model/resume_model.rs @@ -1,6 +1,6 @@ //! Resume checked against a model of one logical track. //! -//! Random sequences of route flips, route failures, give-ups, ranges, cache expiry, and +//! Random keys of route flips, route failures, give-ups, ranges, cache expiry, and //! reader joins and leaves drive an origin's fronts on the simulated clock. Every reader's //! groups, frames, drops, and end are checked against what one plain cursor on the logical //! track would see: every group its range and budget allow, each once, in its order. A @@ -73,7 +73,7 @@ impl Budget { #[derive(Clone, Copy, Debug)] struct Spec { - /// Sequence order through [`track::Ordered`] rather than arrival order. + /// Key order through [`track::Ordered`] rather than arrival order. ordered: bool, budget: Budget, /// The subscription's one group range, or the default. @@ -294,6 +294,17 @@ impl Reader { within(self.ranges) && within(self.bounds) } + /// Whether the reader may be handed `group`: inside its bounds and before its range's + /// end. The range starts at the group containing its start time among those the copy + /// that resolved it held, which may be a lower one, so a group below the start may be + /// read too, though only one at or past where the range resolved (`resolved`). + fn admits(&self, group: u64, resolved: Option) -> bool { + let ranges = self.ranges.is_none_or(|(start, end)| { + group < end && (group >= start || resolved.is_none_or(|resolved| resolved <= group)) + }); + ranges && self.bounds.is_none_or(|(start, end)| (start..end).contains(&group)) + } + /// Everything observed, to tell whether a poll found anything new. fn observed(&self) -> (usize, usize, bool, usize) { let frames = self @@ -484,7 +495,7 @@ impl Model { } let parts = copy.groups.entry(group).or_default(); if !parts.iter().any(|part| part.from == 0) - && let Ok(writer) = copy.track.create_group(group::Info { sequence: group }) + && let Ok(writer) = copy.track.create_group(group::Info { start: group }) { parts.push(Part { from: 0, @@ -514,7 +525,7 @@ impl Model { part.written += 1; } if part.written == frames(group) { - part.writer.take().unwrap().finish().unwrap(); + part.writer.take().unwrap().close().unwrap(); publisher.complete.insert(group); } let most = publisher.written.entry(group).or_default(); @@ -647,37 +658,41 @@ impl Model { match polled { Poll::Pending => break, Poll::Ready(Ok(Some(group))) => { - let sequence = group.sequence; - if reader.reading.contains_key(&sequence) { + let key = group.start; + if reader.reading.contains_key(&key) { return Err(fail( index, - format!("group {sequence} handed out twice: {:?}", reader.handed), + format!("group {key} handed out twice: {:?}", reader.handed), )); } - if reader.spec.ordered && reader.handed.last().is_some_and(|last| *last >= sequence) { + if reader.spec.ordered && reader.handed.last().is_some_and(|last| *last >= key) { return Err(fail( index, - format!("group {sequence} out of order after {:?}", reader.handed), + format!("group {key} out of order after {:?}", reader.handed), )); } - if !reader.covers(sequence) { + // Only an arrival cursor says where its range resolved. + let resolved = match reader.cursor.as_mut().unwrap() { + Cursor::Arrival(sub) => { + sub.resolved().first().copied().flatten().map(|resolved| resolved.start) + } + Cursor::Ordered(_) => None, + }; + if !reader.admits(key, resolved) { return Err(fail( index, format!( - "group {sequence} outside ranges {:?} and bounds {:?}", + "group {key} outside ranges {:?} (resolved from {resolved:?}) and bounds {:?}", reader.ranges, reader.bounds ), )); } - if !instance.written.contains_key(&sequence) { - return Err(fail( - index, - format!("group {sequence} handed out before any route had it"), - )); + if !instance.written.contains_key(&key) { + return Err(fail(index, format!("group {key} handed out before any route had it"))); } - reader.handed.push(sequence); + reader.handed.push(key); reader.reading.insert( - sequence, + key, Reading { consumer: Some(group), frames: 0, @@ -729,7 +744,7 @@ impl Model { } } - for (sequence, read) in &mut reader.reading { + for (key, read) in &mut reader.reading { let Some(consumer) = &mut read.consumer else { continue; }; @@ -737,12 +752,12 @@ impl Model { match consumer.poll_read_frame(&waiter) { Poll::Pending => break, Poll::Ready(Ok(Some(frame))) => { - let expected = payload(reader.instance, *sequence, read.frames); - if frame.payload[..] != expected[..] || read.frames >= frames(*sequence) { + let expected = payload(reader.instance, *key, read.frames); + if frame.payload[..] != expected[..] || read.frames >= frames(*key) { return Err(fail( index, format!( - "group {sequence} frame {} read {:?}, expected {:?}", + "group {key} frame {} read {:?}, expected {:?}", read.frames, String::from_utf8_lossy(&frame.payload), String::from_utf8_lossy(&expected) @@ -752,10 +767,10 @@ impl Model { read.frames += 1; } Poll::Ready(Ok(None)) => { - if read.frames != frames(*sequence) || !instance.complete.contains(sequence) { + if read.frames != frames(*key) || !instance.complete.contains(key) { return Err(fail( index, - format!("group {sequence} ended cleanly after {} frames", read.frames), + format!("group {key} ended cleanly after {} frames", read.frames), )); } read.end = Some(Ok(())); @@ -892,11 +907,11 @@ impl Model { let _ = write!(problems, "missing groups {missing:?}; "); } if reader.spec.budget == Budget::Hour { - for (sequence, read) in &reader.reading { - if carried(*sequence) && !instance.given_up.contains(sequence) && !matches!(read.end, Some(Ok(()))) { + for (key, read) in &reader.reading { + if carried(*key) && !instance.given_up.contains(key) && !matches!(read.end, Some(Ok(()))) { let _ = write!( problems, - "group {sequence} stopped at frame {} ({:?}); ", + "group {key} stopped at frame {} ({:?}); ", read.frames, read.end ); } @@ -999,6 +1014,41 @@ fn regressions() { drops, }; let cases = [ + // A time start on a received copy that never reports its live edge waits for a group + // rather than reading everything, so it stops at its end. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(true, Budget::Hour, Some((2, 3)), false), + }, + Op::Join { + reader: 1, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Write { + instance: 0, + slot: 0, + group: None, + frames: 1, + }, + Op::Write { + instance: 0, + slot: 0, + group: None, + frames: 1, + }, + Op::Write { + instance: 0, + slot: 0, + group: Some(11), + frames: 1, + }, + Op::Up { instance: 0, slot: 1 }, + Op::End { instance: 0, slot: 0 }, + ], + finish: false, + }, // An ordered reader whose range a finished track already passed ends. Case { ops: vec![Op::Join { @@ -1023,6 +1073,28 @@ fn regressions() { ], finish: true, }, + // A copy replaced before its first group, whose cap empties the range it was asked, + // says nothing of where a later reader's time start resolves: the serving copy does. + Case { + ops: vec![ + Op::Join { + reader: 0, + spec: spec(false, Budget::Hour, None, false), + }, + Op::Up { instance: 0, slot: 2 }, + Op::Join { + reader: 1, + spec: spec(false, Budget::Hour, Some((1, 2)), false), + }, + Op::Write { + instance: 0, + slot: 2, + group: None, + frames: 1, + }, + ], + finish: false, + }, // Local bounds below the subscription's range leave no drops to report. Case { ops: vec![ diff --git a/rs/moq-net/src/model/skipped.rs b/rs/moq-net/src/model/skipped.rs index 6d142aa5f2c..74f0bb70474 100644 --- a/rs/moq-net/src/model/skipped.rs +++ b/rs/moq-net/src/model/skipped.rs @@ -6,7 +6,7 @@ use std::time::Duration; use rangemap::RangeInclusiveMap; /// The groups a reader skipped as stale and never handed out, with the budget each was -/// judged under, as runs of consecutive sequences. +/// judged under, as runs of consecutive keys. /// /// Only a widening reverses a stale verdict, so a run comes back once the budget grows /// past the one it was judged under. A run is kept only while one of its groups is cached: @@ -25,21 +25,21 @@ struct Runs { } impl Skipped { - /// Record `sequence` as skipped under `budget`. - pub fn insert(&mut self, sequence: u64, budget: Duration) { + /// Record `key` as skipped under `budget`. + pub fn insert(&mut self, key: u64, budget: Duration) { let skipped = self.0.get_or_insert_default(); if skipped.runs.is_empty() || budget < skipped.narrowest { skipped.narrowest = budget; } - skipped.runs.insert(sequence..=sequence, budget); + skipped.runs.insert(key..=key, budget); } - /// Forget `sequence`, now handed out. - pub fn remove(&mut self, sequence: u64) { + /// Forget `key`, now handed out. + pub fn remove(&mut self, key: u64) { if let Some(skipped) = &mut self.0 && !skipped.runs.is_empty() { - skipped.runs.remove(sequence..=sequence); + skipped.runs.remove(key..=key); } } @@ -93,7 +93,7 @@ impl Skipped { } } - /// The lowest sequence skipped. + /// The lowest key skipped. pub fn lowest(&self) -> Option { let skipped = self.0.as_ref()?; skipped.runs.first_range_value().map(|(run, _)| *run.start()) @@ -126,12 +126,12 @@ impl Skipped { } } - /// One past the last sequence of the run holding `sequence`, if it was skipped. - pub fn run_end(&self, sequence: u64) -> Option { + /// One past the last key of the run holding `key`, if it was skipped. + pub fn run_end(&self, key: u64) -> Option { let skipped = self.0.as_ref()?; skipped .runs - .get_key_value(&sequence) + .get_key_value(&key) .map(|(run, _)| run.end().saturating_add(1)) } @@ -181,8 +181,8 @@ mod test { #[test] fn a_hand_out_splits_a_run() { let mut skipped = Skipped::default(); - for sequence in 3..=5 { - skipped.insert(sequence, NARROW); + for key in 3..=5 { + skipped.insert(key, NARROW); } skipped.remove(4); assert_eq!(skipped.widened(WIDE), vec![3..=3, 5..=5]); @@ -191,8 +191,8 @@ mod test { #[test] fn prune_drops_runs_with_nothing_cached() { let mut skipped = Skipped::default(); - for sequence in [1, 3, 5, 7] { - skipped.insert(sequence, NARROW); + for key in [1, 3, 5, 7] { + skipped.insert(key, NARROW); } // The lowest runs go until one still holds a group. skipped.prune(4, |run| *run.start() < 3); diff --git a/rs/moq-net/src/model/socket.rs b/rs/moq-net/src/model/socket.rs index b873a228de8..4fe146bb3a2 100644 --- a/rs/moq-net/src/model/socket.rs +++ b/rs/moq-net/src/model/socket.rs @@ -130,7 +130,10 @@ fn subscription() -> Subscription { Subscription::default() .with_order(Order::Ascending) .with_delay(Delay::default().with_max(None)) - .with_range(0..) + .with_ranges([track::Range { + start: track::Anchor::Time(0), + end: None, + }]) } /// What an endpoint's handles share. diff --git a/rs/moq-net/src/model/subscription.rs b/rs/moq-net/src/model/subscription.rs index 687825d0c77..984a08420bf 100644 --- a/rs/moq-net/src/model/subscription.rs +++ b/rs/moq-net/src/model/subscription.rs @@ -3,14 +3,12 @@ use std::{ time::Duration, }; -use crate::Timestamp; - mod runs; mod window; pub(super) use runs::{Layered, Runs}; pub(crate) use window::Accounted; pub(crate) use window::Resolved; -pub(super) use window::{Containing, Held, Holdings, Seen, Unheld, Window, later}; +pub(super) use window::{Containing, Holdings, Unheld, Window}; /// Subscriber-side preferences for receiving a track. /// @@ -159,18 +157,18 @@ impl Delay { /// therefore needs a budget that covers it. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub enum Order { - /// Oldest first, draining a backlog in sequence order. + /// Oldest first, draining a backlog in key order. Ascending, /// Newest first, so congestion sheds the backlog rather than the live edge. #[default] Descending, } -/// Where one end of a [`Range`] falls: a fixed group, a count of groups back from the -/// latest one, the oldest group, or a presentation time. +/// Where one end of a [`Range`] falls: a count of groups back from the latest one, the +/// oldest group, or a time, which names the group containing it. /// /// Every anchor names a whole group: a subscription always starts at a group's first frame, -/// since a group is the unit of decodability. +/// since a group is the unit of decodability, and every end includes its group. /// /// A relative or time anchor resolves at the hop that serves it, when it starts serving /// and again whenever the subscriber replaces its ranges. Before any group exists a @@ -178,18 +176,16 @@ pub enum Order { /// copy waiting on a fresh upstream answer counts from the live feed's latest group once /// it answers, never from the groups an ended feed left in its cache. /// -/// Over moq-lite-07, a copy resolves them from the reported live edge and media time. -/// A Rust moq-transport copy resolves a relative anchor from the Largest Object its live -/// join reports, and accepts time anchors its cache can already resolve, translating both -/// to group bounds upstream; any other time anchor is refused. Older moq-lite versions -/// accept absolute and [`Anchor::Oldest`] starts but refuse time anchors and relative -/// anchors other than [`Range::live`]'s with [`Error::Unsupported`](crate::Error::Unsupported) -/// rather than guess from a partial copy. +/// Over moq-lite-07, a copy resolves relative anchors from the reported live edge. A Rust +/// moq-transport copy resolves a relative anchor from the Largest Object its live join +/// reports. Older moq-lite versions accept time and [`Anchor::Oldest`] starts but refuse +/// relative anchors other than [`Range::live`]'s with +/// [`Error::Unsupported`](crate::Error::Unsupported) rather than guess from a partial copy. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum Anchor { - /// The group with this sequence. - Absolute(u64), - /// The group this many before the latest, clamped to group 0: `Relative(0)` is the latest group. + /// The group this many before the latest, counting the groups the serving hop holds: + /// `Relative(0)` is the latest group. One counting back past every group held reaches + /// back to the oldest as a start, and ends before them all as an end. Relative(u64), /// The oldest group still around, as far back as [`Delay::max`] allows, naming no floor. /// @@ -197,32 +193,32 @@ pub enum Anchor { /// with [`Error::InvalidRange`](crate::Error::InvalidRange), since no hop can tell /// which group is the oldest still around anywhere upstream. Oldest, - /// The group containing this presentation time: the newest one starting at or before it. + /// The group containing this time, in the track's timescale: the newest one starting at + /// or before it. Groups are keyed by their start, so a group's start names exactly it. /// /// Resolved from the starts of the groups the serving hop holds. When none starts at /// or before it, a hop whose [`Requests`](crate::track::Requests) handler serves its /// history looks the start up there (see [`Fill::time`](crate::track::Fill::time)), /// reading nothing until it answers; otherwise, or when the handler can't tell, a start - /// reaches back to the oldest group. While no group has started at all it waits for one - /// that has, reading nothing of its range meanwhile. An end settles once a group starting - /// after it is known and every sequence below that one is accounted for (started, - /// dropped, or never to start). Until then reads stop at the first held group that - /// may still start, so at the live edge at most that one group past the end is read - /// before its first frame shows where it starts. An untimed track - /// refuses it with [`Error::TimestampMismatch`](crate::Error::TimestampMismatch). - Time(Timestamp), + /// reaches back to the oldest group. While no group is held at all it waits for one, + /// reading nothing of its range meanwhile. An end includes every group starting at or + /// before it. + /// + /// On a scale-less track (no [`Info::timescale`](crate::track::Info::timescale)) the + /// time is a bare start, in the same units as the group starts. + Time(u64), } -/// A span of a track to deliver: from `start` up to `end`, or onward when `end` is `None`. +/// A span of a track to deliver: from `start` through `end`, or onward when `end` is `None`. /// -/// An absolute end is exclusive, like the end of a [`std::ops::Range`], so `Absolute(2).. -/// Absolute(6)` delivers groups 2 through 5. Relative and time ends include their group, so -/// `Absolute(0)..Relative(0)` delivers through the latest group. +/// Both ends include their group: `Time(2)..=Time(6)` delivers every group starting from the +/// one containing time 2 through the one containing time 6, and `Oldest..=Relative(0)` +/// delivers through the latest group. A time end before its time start is empty. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct Range { /// Where delivery starts. pub start: Anchor, - /// Where delivery stops, or `None` to keep following the live edge. + /// Where delivery stops, including its group, or `None` to keep following the live edge. pub end: Option, } @@ -237,7 +233,7 @@ impl Range { /// Everything from the oldest group still around onward, the default. /// - /// Unlike `Absolute(0)..` this names no floor: a wire whose publisher resolves the start + /// Unlike `Time(0)..` this names no floor: a wire whose publisher resolves the start /// from [`Delay::max`] reaches as far back as it allows, and an older wire /// starts at the live edge, filling nothing below it. pub fn oldest() -> Self { @@ -247,30 +243,14 @@ impl Range { } } - /// The whole groups in `groups`, such as `2..=5`, `2..`, or `..6`. - /// - /// An omitted start is [`Self::oldest`]'s, and an inclusive end past the last group is - /// no end. - pub fn groups(groups: impl RangeBounds) -> Self { - let unbounded_start = matches!(groups.start_bound(), Bound::Unbounded); - let (start, end) = sequence_bounds(groups); - Self { - start: match unbounded_start { - true => Self::oldest().start, - false => Anchor::Absolute(start), - }, - end: end.map(Anchor::Absolute), - } - } - - /// Whether a wire without ranges carries this range upstream: absolute anchors, and - /// the live and oldest starts it spells with no floor. + /// Whether a wire without ranges carries this range upstream: time anchors, as group + /// IDs, and the live and oldest starts it spells with no floor. pub(super) fn is_wire(&self) -> bool { let start = match self.start { - Anchor::Oldest | Anchor::Absolute(_) | Anchor::Relative(0) => true, - Anchor::Relative(_) | Anchor::Time(_) => false, + Anchor::Oldest | Anchor::Time(_) | Anchor::Relative(0) => true, + Anchor::Relative(_) => false, }; - start && matches!(self.end, None | Some(Anchor::Absolute(_))) + start && matches!(self.end, None | Some(Anchor::Time(_))) } /// Whether this range certainly delivers nothing, wherever the latest group lands. @@ -280,19 +260,43 @@ impl Range { pub(super) fn is_empty(&self) -> bool { match (self.start, self.end) { (_, Some(Anchor::Oldest)) => true, - (Anchor::Absolute(start), Some(Anchor::Absolute(end))) => end <= start, + (Anchor::Time(start), Some(Anchor::Time(end))) => end < start, _ => false, } } + + /// A range holding nothing: nothing sorts below group 0, so an exclusive end there holds + /// nothing, whatever the start. + pub(crate) const EMPTY: Self = Self { + start: Anchor::Time(1), + end: Some(Anchor::Time(0)), + }; + + /// The range from group `start` up to the exclusive group `end`, or onward for `None`: + /// what a legacy wire's bounds become. `None` when it holds nothing, which an exclusive + /// end of 0 says whatever the start. + #[cfg(any(feature = "async", test))] + pub(crate) fn span(start: Option, end: Option) -> Option { + let end = match end { + Some(end) => Some(Anchor::Time(end.checked_sub(1)?)), + None => None, + }; + let range = Self { + start: start.map_or(Anchor::Oldest, Anchor::Time), + end, + }; + (!range.is_empty()).then_some(range) + } } /// The union of `ranges` in a canonical order, without empty or redundant ranges: the /// reference for the track's incremental union. /// -/// Absolute spans merge where they overlap or touch. The rest can't be placed without the -/// latest group, so only an open range drops what it certainly covers: the open absolute -/// span drops ranges starting inside it, and the farthest open relative range drops -/// relative ranges starting no farther back. [`Range::oldest`] covers everything. +/// Time spans merge where they overlap or touch, as the group starts they cover. The rest +/// can't be placed without the latest group, so only an open range drops what it certainly +/// covers: the open time span drops ranges starting inside it, and the farthest open +/// relative range drops relative ranges starting no farther back. [`Range::oldest`] covers +/// everything. #[cfg(test)] pub(super) fn union(ranges: impl IntoIterator) -> Vec { let ranges: Vec = ranges.into_iter().filter(|range| !range.is_empty()).collect(); @@ -303,8 +307,8 @@ pub(super) fn union(ranges: impl IntoIterator) -> Vec { let mut spans: Vec<(u64, Option)> = ranges .iter() .filter_map(|range| match (range.start, range.end) { - (Anchor::Absolute(start), None) => Some((start, None)), - (Anchor::Absolute(start), Some(Anchor::Absolute(end))) => Some((start, Some(end))), + (Anchor::Time(start), None) => Some((start, None)), + (Anchor::Time(start), Some(Anchor::Time(end))) => Some((start, end.checked_add(1))), _ => None, }) .collect(); @@ -330,18 +334,16 @@ pub(super) fn union(ranges: impl IntoIterator) -> Vec { let mut union: Vec = runs .into_iter() .map(|(start, end)| Range { - start: Anchor::Absolute(start), - end: end.map(Anchor::Absolute), + start: Anchor::Time(start), + end: end.map(|end| Anchor::Time(end - 1)), }) .collect(); union.extend(ranges.into_iter().filter(|range| match (range.start, range.end) { - (Anchor::Absolute(_), None | Some(Anchor::Absolute(_))) | (Anchor::Relative(_), None) => false, - (Anchor::Absolute(start), _) => tail.is_none_or(|tail| start < tail), + (Anchor::Time(_), None | Some(Anchor::Time(_))) | (Anchor::Relative(_), None) => false, + (Anchor::Time(start), _) => tail.is_none_or(|tail| start < tail), (Anchor::Relative(back), _) => farthest.is_none_or(|farthest| back > farthest), // Nothing but the open oldest range, handled above, covers the oldest group. (Anchor::Oldest, _) => true, - // A time can't be placed against the others without the hop's groups. - (Anchor::Time(_), _) => true, })); union.extend(farthest.map(|back| Range { start: Anchor::Relative(back), @@ -388,27 +390,21 @@ impl Subscription { self } - /// Request the whole groups in `groups` as the only range; see [`Range::groups`]. - /// Returns `self` for chaining. - pub fn with_range(self, groups: impl RangeBounds) -> Self { - self.with_ranges([Range::groups(groups)]) - } - /// Set which pending group the publisher sends first, returning `self` for chaining. pub fn with_order(mut self, order: Order) -> Self { self.order = order; self } - /// The lowest absolute group any range asks for, or `None` when one counts back from - /// the latest group (or none is asked for). + /// The lowest group any range names by time, or `None` when one counts back from the + /// latest group or reaches back to the oldest (or none is asked for). /// /// The form a wire without ranges carries: a floor below which nothing is requested. #[cfg(any(feature = "async", test))] pub(crate) fn start(&self) -> Option { let mut floor: Option = None; - for range in &self.ranges { - let Anchor::Absolute(start) = range.start else { + for range in self.ranges.iter().filter(|range| !range.is_empty()) { + let Anchor::Time(start) = range.start else { return None; }; floor = Some(floor.map_or(start, |floor| floor.min(start))); @@ -416,18 +412,18 @@ impl Subscription { floor } - /// The exclusive end group past every range, or `None` when one is open or counts back - /// from the latest group. An empty list is the empty range, ending at group 0. + /// The exclusive end past every range, or `None` when one is open or counts back from + /// the latest group. An empty list, or one of empty ranges, ends at group 0. /// /// The form a wire without ranges carries, alongside [`Self::start`]. #[cfg(any(feature = "async", test))] pub(crate) fn end(&self) -> Option { let mut end = Some(0); - for range in &self.ranges { - let Some(Anchor::Absolute(next)) = range.end else { + for range in self.ranges.iter().filter(|range| !range.is_empty()) { + let Some(Anchor::Time(last)) = range.end else { return None; }; - end = end.max(Some(next)); + end = end.max(Some(last.checked_add(1)?)); } end } @@ -450,20 +446,16 @@ impl Subscription { #[cfg(any(feature = "async", test))] fn with_span(self, start: Option, end: Option) -> Self { - self.with_ranges([Range { - start: start.map_or(Range::oldest().start, Anchor::Absolute), - end: end.map(Anchor::Absolute), - }]) + self.with_ranges(Range::span(start, end)) } /// These preferences with every range raised to start no earlier than `floor` and - /// stop no later than `end`, the exclusive end, without ever widening one. + /// stop before `end`, the exclusive end, without ever widening one. /// /// `resolved` names where each range resolved, by index, so a relative or time anchor /// becomes its own group, raised to `floor` or capped at `end`. A start not resolved /// keeps its anchor, since the floor may lie before it, so the caller drops what it - /// resolves to below `floor`; an end not resolved becomes the bound, except a time end, - /// which stays so the caller enforces `end` apart from it. + /// resolves to below `floor`; an end not resolved becomes the bound. pub(super) fn clamped(&self, floor: Option, end: Option, resolved: &[Option]) -> Self { let clamping = floor.is_some() || end.is_some(); let ranges = self @@ -474,40 +466,33 @@ impl Subscription { let resolved = resolved.get(index).copied().flatten().filter(|_| clamping); let start = match (range.start, resolved) { (Anchor::Oldest | Anchor::Relative(_) | Anchor::Time(_), Some(resolved)) => { - Anchor::Absolute(resolved.start) + Anchor::Time(resolved.start) } (start, _) => start, }; + let start = match (start, floor) { + (Anchor::Time(start), Some(floor)) => Anchor::Time(start.max(floor)), + (start, _) => start, + }; let requested = match (range.end, resolved.and_then(|resolved| resolved.end)) { - (Some(Anchor::Relative(_) | Anchor::Time(_)), Some(resolved)) => Some(Anchor::Absolute(resolved)), + (Some(Anchor::Relative(_) | Anchor::Time(_)), Some(resolved)) => match resolved.checked_sub(1) { + Some(last) => Some(Anchor::Time(last)), + None => return Range::EMPTY, + }, (requested, _) => requested, }; - Range { - start: match (start, floor) { - (Anchor::Absolute(start), Some(floor)) => Anchor::Absolute(start.max(floor)), - (start, _) => start, - }, - end: match (requested, end) { - (requested, None) => requested, - (Some(Anchor::Absolute(requested)), Some(end)) => Some(Anchor::Absolute(requested.min(end))), - // Folding the cap in would end the copy's time filter for good, so the - // caller enforces the cap itself and the copy keeps the time. - (Some(Anchor::Time(pts)), Some(_)) => Some(Anchor::Time(pts)), - (_, Some(end)) => Some(Anchor::Absolute(end)), - }, - } + let end = match (requested, end.map(|end| end.checked_sub(1))) { + (requested, None) => requested, + (_, Some(None)) => return Range::EMPTY, + (Some(Anchor::Time(requested)), Some(Some(last))) => Some(Anchor::Time(requested.min(last))), + (_, Some(Some(last))) => Some(Anchor::Time(last)), + }; + Range { start, end } }) .collect(); Self { ranges, ..self.clone() } } - /// Whether any range ends at a presentation time, which an untimed track refuses. - pub(super) fn has_time(&self) -> bool { - self.ranges - .iter() - .any(|range| matches!(range.start, Anchor::Time(_)) || matches!(range.end, Some(Anchor::Time(_)))) - } - /// Whether a subscriber joins at the live edge: a range counting back from the latest /// group and following it. A fixed start resumes a known group instead, such as a route /// switch picking up where the reader stopped. @@ -518,6 +503,34 @@ impl Subscription { } } +/// Group spans for unit tests whose groups start at consecutive keys, where a span of +/// starts counts groups. +#[cfg(test)] +impl Range { + /// The groups starting in `range`. + pub(crate) fn groups(range: impl RangeBounds) -> Self { + let start = match range.start_bound() { + Bound::Included(start) => Some(*start), + Bound::Excluded(start) => Some(start.saturating_add(1)), + Bound::Unbounded => None, + }; + let end = match range.end_bound() { + Bound::Included(end) => end.checked_add(1), + Bound::Excluded(end) => Some(*end), + Bound::Unbounded => None, + }; + Self::span(start, end).unwrap_or(Self::EMPTY) + } +} + +#[cfg(test)] +impl Subscription { + /// These preferences reading only the groups starting in `range`. + pub(crate) fn with_range(self, range: impl RangeBounds) -> Self { + self.with_ranges([Range::groups(range)]) + } +} + /// The most demanding request across subscribers, folded in one at a time: the reference /// the track's counted aggregate is checked against. #[cfg(test)] @@ -542,7 +555,7 @@ impl Combined { (a, b) => a.or(b), }; combined.ranges.extend_from_slice(&sub.ranges); - // Sequence-first prioritization is enabled only when every subscriber wants it. + // Key-first prioritization is enabled only when every subscriber wants it. if let Order::Descending = sub.order { combined.order = Order::Descending; } @@ -557,7 +570,7 @@ impl Combined { } } -/// A frame-precise point in a track: a group sequence and a frame index within it. +/// A frame-precise point in a track: a group, by its start, and a frame index within it. /// /// Ordered lexicographically, so comparing positions is the same as comparing groups /// and only falling back to frames within one. This is the model's counterpart of the @@ -567,14 +580,20 @@ impl Combined { /// means nothing on its own. Carrying them together makes "frame 5 of nothing" /// unrepresentable rather than merely undefined. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[non_exhaustive] pub struct Position { - /// The group sequence. + /// Where the group starts, in its track's timescale. pub group: u64, /// The frame index within the group, numbered from 0 in write order. pub frame: u64, } impl Position { + /// Frame `frame` of the group starting at `group`. + pub fn new(group: u64, frame: u64) -> Self { + Self { group, frame } + } + /// The first frame of `group`. /// /// As an exclusive end this means "everything before `group`"; as a start it means @@ -619,7 +638,7 @@ impl Position { /// The first group this exclusive end excludes whole, `None` past the last group: the /// boundary a wire that cannot name a frame carries as the track's end. - pub(crate) fn final_sequence(self) -> Option { + pub(crate) fn final_key(self) -> Option { self.group.checked_add(u64::from(self.frame != 0)) } } @@ -636,7 +655,7 @@ fn exclusive_end(bound: Bound<&u64>) -> Option { } // Normalize discrete ranges once, including the empty range above the last index. -pub(super) fn sequence_bounds(range: impl RangeBounds) -> (u64, Option) { +pub(super) fn key_bounds(range: impl RangeBounds) -> (u64, Option) { let start = match range.start_bound() { Bound::Included(&start) => start, Bound::Excluded(&start) => match start.checked_add(1) { @@ -648,9 +667,9 @@ pub(super) fn sequence_bounds(range: impl RangeBounds) -> (u64, Option (start, exclusive_end(range.end_bound())) } -/// Whether `sequence` is strictly below an exclusive cap. `None` is unbounded. -pub(super) fn before_end(sequence: u64, end: Option) -> bool { - end.is_none_or(|end| sequence < end) +/// Whether `key` is strictly below an exclusive cap. `None` is unbounded. +pub(super) fn before_end(key: u64, end: Option) -> bool { + end.is_none_or(|end| key < end) } #[cfg(test)] @@ -665,8 +684,37 @@ mod tests { combined.finish() } - fn absolute(group: u64) -> Anchor { - Anchor::Absolute(group) + fn time(start: u64) -> Anchor { + Anchor::Time(start) + } + + /// Groups from `start` up to the exclusive `end`, or onward for `None`. + fn groups(start: u64, end: impl Into>) -> Range { + Range { + start: Anchor::Time(start), + end: end.into().map(|end| Anchor::Time(end - 1)), + } + } + + /// A hop holding every group from 0 through the latest, as contiguous starts do. + struct Contiguous; + + impl Holdings for Contiguous { + fn containing(&self, time: u64) -> Containing { + Containing::Group(time) + } + + fn next_start(&self, from: u64) -> Option { + Some(from) + } + + fn back(&self, latest: u64, back: u64) -> Option { + latest.checked_sub(back) + } + + fn complete(&self) -> bool { + false + } } fn range(start: Anchor, end: impl Into>) -> Range { @@ -677,20 +725,19 @@ mod tests { combine(subscriptions).unwrap().ranges } - /// A group range spells both ends at once, in whichever form the caller has. + /// A legacy span becomes inclusive time anchors: an exclusive end of 0 holds nothing, + /// whatever the start, and an end of 1 holds group 0. #[test] - fn group_ranges_build_absolute_anchors() { - assert_eq!(Range::groups(2..=5), range(absolute(2), absolute(6))); - assert_eq!(Range::groups(2..6), range(absolute(2), absolute(6))); - assert_eq!(Range::groups(..6), range(Range::oldest().start, absolute(6))); - assert_eq!(Range::groups(2..), range(absolute(2), None)); - assert_eq!(Range::groups(..), Range::oldest()); - - // Through the last group is unbounded. - assert_eq!(Range::groups(..=u64::MAX).end, None); - - let sub = Subscription::default().with_range(2..=5).with_range(..); - assert_eq!(sub, Subscription::default()); + fn legacy_spans_become_inclusive_time_ranges() { + assert_eq!(Range::span(Some(2), Some(6)), Some(range(time(2), time(5)))); + assert_eq!(Range::span(None, Some(6)), Some(range(Anchor::Oldest, time(5)))); + assert_eq!(Range::span(Some(2), None), Some(range(time(2), None))); + assert_eq!(Range::span(None, None), Some(Range::oldest())); + assert_eq!(Range::span(Some(0), Some(0)), None); + assert_eq!(Range::span(None, Some(0)), None); + assert_eq!(Range::span(Some(5), Some(5)), None); + assert_eq!(Range::span(Some(0), Some(1)), Some(range(time(0), time(0)))); + assert!(Range::EMPTY.is_empty()); } #[test] @@ -711,21 +758,18 @@ mod tests { // A cursor cap is the first index it withholds; an inclusive bound at the last // index withholds nothing. - assert_eq!(sequence_bounds(..0), (0, Some(0))); - assert_eq!(sequence_bounds(..=5), (0, Some(6))); - assert_eq!(sequence_bounds(..=u64::MAX), (0, None)); - assert_eq!(sequence_bounds(..), (0, None)); - assert_eq!( - sequence_bounds((Bound::Unbounded, Bound::Included(5))), - sequence_bounds(..6) - ); + assert_eq!(key_bounds(..0), (0, Some(0))); + assert_eq!(key_bounds(..=5), (0, Some(6))); + assert_eq!(key_bounds(..=u64::MAX), (0, None)); + assert_eq!(key_bounds(..), (0, None)); + assert_eq!(key_bounds((Bound::Unbounded, Bound::Included(5))), key_bounds(..6)); } /// A relative start counts back from the latest group and clamps at group 0. Before any /// group exists only the farthest offset resolves, since it is group 0 regardless. #[test] fn relative_starts_resolve_against_the_latest_group() { - let floor = |sub: &Subscription, latest| Window::new(&sub.ranges, latest, &Unheld).unwrap().floor(); + let floor = |sub: &Subscription, latest| Window::new(&sub.ranges, latest, &Contiguous).unwrap().floor(); let live = Subscription::default().with_ranges([Range::live()]); assert_eq!(floor(&live, Some(7)), Some(7)); @@ -738,35 +782,35 @@ mod tests { assert_eq!(floor(&Subscription::default(), None), Some(0)); // The lowest range wins, and one unresolved range leaves the whole floor unresolved. - let mixed = Subscription::default().with_ranges([range(absolute(4), None), Range::live()]); + let mixed = Subscription::default().with_ranges([range(time(4), None), Range::live()]); assert_eq!(floor(&mixed, Some(3)), Some(3)); assert_eq!(floor(&mixed, Some(9)), Some(4)); assert_eq!(floor(&mixed, None), None); // Nothing asked for floors nowhere, and an empty range asks for nothing. assert_eq!(floor(&Subscription::default().with_ranges([]), None), Some(0)); - let empty = Subscription::default().with_ranges([Range::groups(5..5), Range::groups(10..)]); + let empty = Subscription::default().with_ranges([groups(5, 5), groups(10, None)]); assert_eq!(floor(&empty, None), Some(10)); } - /// The bounds a wire without ranges carries: the lowest absolute start and the highest - /// absolute end, either one lost to a range that counts back from the latest group. + /// The bounds a wire without ranges carries: the lowest time start and the highest + /// exclusive time end, either one lost to a range that counts back from the latest group. #[test] fn bounds_project_the_ranges_for_older_wires() { let sub = Subscription::default(); assert_eq!((sub.start(), sub.end()), (None, None)); - let sub = Subscription::default().with_ranges([Range::groups(2..4), Range::groups(7..9)]); + let sub = Subscription::default().with_ranges([groups(2, 4), groups(7, 9)]); assert_eq!((sub.start(), sub.end()), (Some(2), Some(9))); - // An explicit group 0 is a floor, unlike the default. - let sub = Subscription::default().with_range(0..); + // An explicit start at 0 is a floor, unlike the default. + let sub = Subscription::default().with_ranges([groups(0, None)]); assert_eq!((sub.start(), sub.end()), (Some(0), None)); - let sub = Subscription::default().with_ranges([Range::groups(2..4), Range::live()]); + let sub = Subscription::default().with_ranges([groups(2, 4), Range::live()]); assert_eq!((sub.start(), sub.end()), (None, None)); - let sub = Subscription::default().with_ranges([range(absolute(2), Anchor::Relative(0))]); + let sub = Subscription::default().with_ranges([range(time(2), Anchor::Relative(0))]); assert_eq!((sub.start(), sub.end()), (Some(2), None)); // Nothing asked for is the empty range. @@ -778,35 +822,32 @@ mod tests { assert_eq!(sub.with_start(None).with_end(None), Subscription::default()); } - /// The aggregate is the union: overlapping and adjacent absolute ranges merge, an open + /// The aggregate is the union: overlapping and adjacent time ranges merge, an open /// range absorbs what starts inside it, and the oldest range absorbs everything. #[test] fn combined_ranges_are_the_union() { let sub = |ranges: &[Range]| Subscription::default().with_ranges(ranges.iter().copied()); assert_eq!( - ranges(&[sub(&[Range::groups(2..5)]), sub(&[Range::groups(4..8)])]), - vec![Range::groups(2..8)] + ranges(&[sub(&[groups(2, 5)]), sub(&[groups(4, 8)])]), + vec![groups(2, 8)] ); assert_eq!( - ranges(&[sub(&[Range::groups(2..5)]), sub(&[Range::groups(5..8)])]), - vec![Range::groups(2..8)], + ranges(&[sub(&[groups(2, 5)]), sub(&[groups(5, 8)])]), + vec![groups(2, 8)], "adjacent ranges merge" ); assert_eq!( - ranges(&[sub(&[Range::groups(2..5)]), sub(&[Range::groups(6..8)])]), - vec![Range::groups(2..5), Range::groups(6..8)], + ranges(&[sub(&[groups(2, 5)]), sub(&[groups(6, 8)])]), + vec![groups(2, 5), groups(6, 8)], "a gap stays a gap" ); assert_eq!( - ranges(&[ - sub(&[Range::groups(2..5), Range::groups(9..)]), - sub(&[Range::groups(4..)]) - ]), - vec![Range::groups(2..)] + ranges(&[sub(&[groups(2, 5), groups(9, None)]), sub(&[groups(4, None)])]), + vec![groups(2, None)] ); assert_eq!( - ranges(&[sub(&[Range::groups(2..5)]), Subscription::default()]), + ranges(&[sub(&[groups(2, 5)]), Subscription::default()]), vec![Range::oldest()] ); @@ -821,25 +862,23 @@ mod tests { vec![range(Anchor::Relative(3), None)] ); - // An absolute floor above the live edge does not hide the live edge: both stay. + // A time floor above the live edge does not hide the live edge: both stay, relative + // first. assert_eq!( - ranges(&[sub(&[Range::groups(4..)]), sub(&[Range::live()])]), - vec![Range::groups(4..), Range::live()] + ranges(&[sub(&[groups(4, None)]), sub(&[Range::live()])]), + vec![Range::live(), groups(4, None)] ); // Empty ranges ask for nothing. - assert_eq!(ranges(&[sub(&[Range::groups(5..5)])]), Vec::::new()); - assert_eq!( - ranges(&[sub(&[]), sub(&[Range::groups(1..2)])]), - vec![Range::groups(1..2)] - ); + assert_eq!(ranges(&[sub(&[groups(5, 5)])]), Vec::::new()); + assert_eq!(ranges(&[sub(&[]), sub(&[groups(1, 2)])]), vec![groups(1, 2)]); } /// Combining is order-independent and stops changing once it covers everyone. #[test] fn combined_ranges_settle() { - let a = Subscription::default().with_ranges([Range::groups(2..5), Range::live()]); - let b = Subscription::default().with_ranges([Range::groups(4..9)]); + let a = Subscription::default().with_ranges([groups(2, 5), Range::live()]); + let b = Subscription::default().with_ranges([groups(4, 9)]); assert_eq!(combine(&[a.clone(), b.clone()]), combine(&[b.clone(), a.clone()])); let combined = combine(&[a.clone(), b]); diff --git a/rs/moq-net/src/model/subscription/runs.rs b/rs/moq-net/src/model/subscription/runs.rs index e60e6f43bb3..4d5b0906481 100644 --- a/rs/moq-net/src/model/subscription/runs.rs +++ b/rs/moq-net/src/model/subscription/runs.rs @@ -1,4 +1,4 @@ -//! Sets of group sequences held as sorted runs, and the layered set a [`Window`](super::Window) +//! Sets of group keys held as sorted runs, and the layered set a [`Window`](super::Window) //! reads: fixed runs shared across its moves, with the few runs its time ends walk on top. //! //! A subscription may name thousands of ranges, yet an unsettled time end moves its cap on @@ -25,13 +25,9 @@ impl Runs { Self(set.into_iter().collect()) } - pub fn iter(&self) -> std::slice::Iter<'_, RangeInclusive> { - self.0.iter() - } - - pub fn contains(&self, sequence: u64) -> bool { - let index = self.0.partition_point(|run| *run.end() < sequence); - self.0.get(index).is_some_and(|run| *run.start() <= sequence) + pub fn contains(&self, key: u64) -> bool { + let index = self.0.partition_point(|run| *run.end() < key); + self.0.get(index).is_some_and(|run| *run.start() <= key) } /// The lowest run reaching into `lo..=hi`. @@ -54,25 +50,6 @@ impl Runs { .iter() .map(move |run| (*run.start()).max(lo)..=(*run.end()).min(hi)) } - - /// Add `run`, merging the runs it overlaps or touches. Costs a search and a shift. - pub fn insert(&mut self, run: RangeInclusive) { - if run.is_empty() { - return; - } - let (mut start, mut end) = run.into_inner(); - // The first run reaching `start - 1` or later, and the first starting past `end + 1`. - let from = self.0.partition_point(|run| run.end().saturating_add(1) < start); - let to = self - .0 - .partition_point(|run| *run.start() <= end.saturating_add(1)) - .max(from); - if let (Some(first), Some(last)) = (self.0[from..to].first(), self.0[from..to].last()) { - start = start.min(*first.start()); - end = end.max(*last.end()); - } - self.0.splice(from..to, std::iter::once(start..=end)); - } } /// A set of groups held as shared fixed runs with a layer of runs over them, which may @@ -112,25 +89,13 @@ impl Layered { &self.layer } - /// Hold `fixed` under the layer from now on. The runs already held stay shared when they - /// are the same, so what shares them still differs from this only in the layer. - pub fn set_fixed(&mut self, fixed: Runs) { - if *self.fixed != fixed { - self.fixed = Arc::new(fixed); - } - } - - pub fn set_layer(&mut self, layer: Runs) { - self.layer = Arc::new(layer); - } - /// Whether both hold the same fixed runs, so they differ only in their layers. pub fn shares_fixed(&self, other: &Self) -> bool { Arc::ptr_eq(&self.fixed, &other.fixed) } - pub fn contains(&self, sequence: u64) -> bool { - self.fixed.contains(sequence) || self.layer.contains(sequence) + pub fn contains(&self, key: u64) -> bool { + self.fixed.contains(key) || self.layer.contains(key) } /// The one set holding runs, when the other holds none: its runs are the union's, each @@ -286,7 +251,7 @@ mod tests { (0..next(count + 1)) .map(|_| { let start = next(span); - // Now and then a run reaching the last sequence. + // Now and then a run reaching the last key. let end = match next(8) { 0 => u64::MAX, _ => start + next(4), @@ -354,29 +319,15 @@ mod tests { let (lo, hi) = (next(30), 20 + next(30)); let gained: RangeInclusiveSet = new.gained(&old, lo, hi).unwrap().into_iter().collect(); let expected: Vec = (lo..=hi) - .filter(|sequence| new.contains(*sequence) && !old.contains(*sequence)) + .filter(|key| new.contains(*key) && !old.contains(*key)) .collect(); - let found: Vec = (lo..=hi).filter(|sequence| gained.contains(sequence)).collect(); + let found: Vec = (lo..=hi).filter(|key| gained.contains(key)).collect(); assert_eq!(found, expected, "round {round}: {old:?} to {new:?} in {lo}..={hi}"); } let unshared = Layered::new(Arc::new(Runs::default()), Runs::default()); assert!(unshared.gained(&Layered::default(), 0, 9).is_none()); } - /// Inserting a run merges what it overlaps or touches, as a set of the runs would. - #[test] - fn inserted_runs_merge_as_a_set() { - let mut next = generator(); - for _ in 0..2000 { - let runs = random_runs(&mut next, 8, 40); - let mut inserted = Runs::default(); - for run in &runs { - inserted.insert(run.clone()); - } - assert_eq!(inserted, Runs::new(runs.clone()), "{runs:?}"); - } - } - /// A run minus a set is the parts of it the set doesn't hold. #[test] fn a_run_minus_runs_is_its_gaps() { @@ -385,7 +336,7 @@ mod tests { let runs = Runs::new(random_runs(&mut next, 6, 40)); let start = next(40); let run = start..=start + next(10); - let expected: Vec = run.clone().filter(|sequence| !runs.contains(*sequence)).collect(); + let expected: Vec = run.clone().filter(|key| !runs.contains(*key)).collect(); let found: Vec = minus(run.clone(), &runs).flatten().collect(); assert_eq!(found, expected, "{run:?} minus {runs:?}"); } diff --git a/rs/moq-net/src/model/subscription/window.rs b/rs/moq-net/src/model/subscription/window.rs index b0c0fa58e30..88a10efc891 100644 --- a/rs/moq-net/src/model/subscription/window.rs +++ b/rs/moq-net/src/model/subscription/window.rs @@ -1,17 +1,13 @@ -//! A subscriber's own ranges, resolved to the group sequences it reads. +//! A subscriber's own ranges, resolved to the group starts it reads. //! //! The aggregate a publisher sees is the union of every subscriber's ranges, so it says //! what arrives, not what one subscriber asked for. A [`Window`] is the per-subscriber //! side: resolved at the hop serving the subscriber, it filters that subscriber's reads. -use std::{ - collections::{BTreeMap, BTreeSet}, - ops, - sync::Arc, -}; +use std::{ops, sync::Arc}; use super::{Anchor, Layered, Range, Runs}; -use crate::{Error, Result, Timestamp}; +use crate::{Error, Result}; /// One range resolved to groups: from `start` up to the exclusive `end`, `None` while it /// stays open. @@ -43,14 +39,8 @@ impl Accounted for rangemap::RangeMap { } } -/// When a serving hop's groups start, for resolving time anchors. -/// -/// Each query answers from ordered indexes, so a time end that walks them forward a step -/// at a time pays for each group or run it passes once, however long it stays unsettled. +/// The groups a serving hop holds, by start, for resolving relative and time anchors. pub(in crate::model) trait Holdings { - /// Whether the track carries timestamps at all. - fn timed(&self) -> bool; - /// Whether the groups are a copy received over a wire that can't resolve relative or /// time anchors as the publisher would, so only what [`Range::is_wire`] covers reads. fn drops_anchors(&self) -> bool { @@ -65,74 +55,46 @@ pub(in crate::model) trait Holdings { } /// Whether a [`Requests`](crate::track::Requests) handler serving the track's history - /// may look up a time its groups can't resolve: a time start before every group with a - /// known start then waits on that lookup, and a sequence the live feed never delivered, - /// or delivered and the cache evicted, may still arrive from that history. + /// may look up a time its groups can't resolve: a time start before every group held + /// then waits on that lookup. fn looks_up(&self) -> bool { false } - /// Where `pts` falls among the groups with a known start. - fn containing(&self, pts: Timestamp) -> Containing; - - /// The lowest group at or above `from` known to start, with where it starts. - fn next_start(&self, from: u64) -> Option<(u64, Timestamp)>; + /// Where `time` falls among the groups held. + fn containing(&self, time: u64) -> Containing; - /// The lowest held group at or above `from`, with what it says about its start. - fn next_held(&self, from: u64) -> Option<(u64, Held)>; + /// The lowest group held starting at or after `from`. + fn next_start(&self, from: u64) -> Option; - /// The exclusive end of the run of sequences from `sequence` that can no longer arrive - /// (dropped, given up, or below where the feed starts), `None` while `sequence` may. - fn gone(&self, sequence: u64) -> Option; + /// The group `back` groups before `latest`, which counts as the first, among the groups + /// held: `None` once fewer are held than that. + fn back(&self, latest: u64, back: u64) -> Option; - /// Whether nothing more can arrive or start here, so a time end has nothing to wait - /// on and is left uncapped. + /// Whether nothing more can arrive here, so a time start nothing contains reaches back + /// to the oldest group rather than wait. fn complete(&self) -> bool; - /// The runs of sequences whose arrivals or drops changed since `seen`, which this then - /// moves on to now, or `None` once anything else a parked walk reads may have changed - /// (see [`Window::parked`]): the declared start, completion, whether it - /// [looks up](Self::looks_up) its history, or a copy the live feed never delivered, which - /// may hold a group or give one a start. `None` by default, which wakes every parked walk. - fn changes(&self, seen: &mut Seen) -> Option>> { - let _ = seen; - None + /// Whether the producer declared every key in `keys` skipped, so no group starts there + /// even in the history a handler serves. + fn never_produced(&self, keys: ops::Range) -> bool { + keys.is_empty() } } -/// How far a [`Window`] has taken in its [`Holdings::changes`], in whatever marks the -/// holdings keep: the revisions of the indexes reporting changed runs, and what they report -/// no runs for. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub(in crate::model) struct Seen { - pub runs: [u64; 2], - pub global: Option<[u64; 4]>, -} - -/// Where a presentation time falls among the groups a hop holds. +/// Where a time falls among the groups a hop holds. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(in crate::model) enum Containing { /// The newest group starting at or before it. Group(u64), - /// Every group with a known start begins after it. + /// Every group held starts after it. Before, - /// No group has a known start. + /// No group is held. Unknown, } -/// What a held group says about where it starts. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(in crate::model) enum Held { - /// Its frame 0 presented at this time. - Started(Timestamp), - /// It holds frame 0 and has not presented it yet. - Pending, - /// It never will: it ended empty, was aborted, or is held from past frame 0. - Never, -} - /// A hop that holds no groups of its own, such as a reader following route copies that -/// resolve its ranges for it: a time end there filters nothing. +/// resolve its ranges for it. pub(in crate::model) struct Unheld; impl Holdings for Unheld { @@ -140,24 +102,16 @@ impl Holdings for Unheld { false } - fn timed(&self) -> bool { - true - } - - fn containing(&self, _: Timestamp) -> Containing { + fn containing(&self, _: u64) -> Containing { Containing::Unknown } - fn next_start(&self, _: u64) -> Option<(u64, Timestamp)> { - None - } - - fn next_held(&self, _: u64) -> Option<(u64, Held)> { + fn next_start(&self, _: u64) -> Option { None } - fn gone(&self, _: u64) -> Option { - None + fn back(&self, latest: u64, back: u64) -> Option { + (back == 0).then_some(latest) } fn complete(&self) -> bool { @@ -165,19 +119,7 @@ impl Holdings for Unheld { } } -/// Whether `time` is strictly after `other`, as instants: exact across scales, unlike -/// [`Timestamp`]'s `Ord`, which breaks a tie between scales. -pub(in crate::model) fn later(time: Timestamp, other: Timestamp) -> bool { - instant(time, other).is_gt() -} - -/// How `time` orders against `other` as instants, exact across scales. -fn instant(time: Timestamp, other: Timestamp) -> std::cmp::Ordering { - (u128::from(time.value()) * u128::from(other.scale().as_u64())) - .cmp(&(u128::from(other.value()) * u128::from(time.scale().as_u64()))) -} - -/// A subscriber's [`Range`]s resolved to the group sequences it reads. +/// A subscriber's [`Range`]s resolved to the group starts it reads. #[derive(Debug)] pub(in crate::model) struct Window { /// What these were resolved from, so only replacing the ranges resolves again. @@ -186,33 +128,13 @@ pub(in crate::model) struct Window { /// The spans still looking up their time start (see [`Self::found`]), which read nothing /// meanwhile: kept apart so a probe finds them without visiting every span. looking: Vec, - /// The spans whose time start waits on a group starting, which read nothing meanwhile. - /// Their end resolved when asked, so a relative one counts back from that latest group - /// however many arrive while they wait. + /// The spans whose time start waits on a group arriving, which read only the groups from + /// that time meanwhile. waiting: Vec, - /// The unsettled time ends, grouped by walk. - walks: Vec, - /// The walks waiting out a gap below the group their ends fall before, which move only - /// once a sequence from the gap up to that group arrives or is given up, so a settle - /// wakes only the walks its [`Holdings::changes`] reach. Ends left in different gaps of - /// a sparse feed never merge, so walking each of them on every settle would cost the - /// ranges. Keyed by where each [`Parked`] starts. - parked: BTreeMap, - /// How far the parked walks have taken in the holdings' changes. - seen: Seen, - /// The groups the spans cover, an unsettled time end through its cap: every settled span - /// and parked walk in the fixed runs, shared with what reads them, and one run per walk - /// not parked layered over them, so a walk that moves rebuilds only the layer. + /// The groups the spans cover, shared with what reads them. groups: Layered, - /// The part of `groups` a miss may ask for, layered the same way: see [`Self::missable`]. + /// The part of `groups` a miss may ask for: see [`Self::missable`]. missable: Layered, - /// The groups resolved for good: the spans with a settled end, which the fixed runs hold - /// with the parked walks' runs. - resolved: Runs, - resolved_missable: Runs, - /// The fixed runs no longer hold the settled spans and parked walks: rebuilt once the - /// settle under way is done, which a settle or a walk parking or waking does. - stale: bool, /// A relative anchor waits on a first group, so nothing is filtered yet. pending: bool, /// Nothing is left to resolve: see [`Self::is_settled`]. Kept so a read checks it once @@ -227,28 +149,21 @@ pub(in crate::model) struct Window { all_named: bool, } -/// Parked walks whose waits overlap, so a change reaching any of them wakes them all: the -/// parked groups are disjoint, each found by one search. -#[derive(Debug)] -struct Parked { - /// The exclusive end of what they wait on, from the key it is parked under. - end: u64, - walks: Vec, -} - /// One non-empty range, resolved. #[derive(Clone, Copy, Debug)] struct Span { /// Which of the ranges it came from. index: usize, start: u64, - end: End, + /// The exclusive end, `None` past every group. + end: Option, /// Whether the start names a group, rather than reaching back to the oldest one. floored: bool, /// A time start the groups held can't place, looked up from the track's history: the /// span reads nothing until the lookup answers. looking: Option, - /// A time start no group has started for yet: the span reads nothing until one has. + /// A time start the groups held can't place yet: the span reads only the groups from + /// that time until it can. waiting: bool, } @@ -256,332 +171,15 @@ struct Span { /// group held, which holds the group containing it if the track's history does. #[derive(Clone, Copy, Debug)] struct Looking { - pts: Timestamp, + time: u64, below: u64, } -#[derive(Clone, Copy, Debug)] -enum End { - /// Before this group. - Before(u64), - /// Past every group. - Open, - /// Before the first group starting after this time, once a [`Walk`] settles it. - Time(Timestamp), -} - -/// Unsettled time ends whose walks agree, so one walk serves them all. -/// -/// Ends that start alike walk the same groups until one starts between their times, so -/// the ends sharing a walk differ only in when they end. A group starting at time `t` -/// splits off the ends before `t`, which walk on, and settle, as a walk of their own. -/// Walks that come to agree merge. Each end settles once, and a settle costs the walks -/// rather than the ends. -/// -/// Timestamps never rewind, so an end falls before the lowest group starting past it, once -/// every sequence between it and the newest group starting at or before it is accounted -/// for. Every position below only moves forward, so each group or run of sequences is -/// passed once, whatever a peer leaves missing. -#[derive(Debug)] -struct Walk { - /// The ends sharing it, earliest first. - ends: BTreeSet, - trail: Trail, - /// The lowest group any of its spans reads, and the lowest a miss asks for: each span - /// reads on from its start up to the cap, so the walk's groups are one run from here. - low: u64, - low_reach: u64, - /// The cap when the settle under way began, inherited by the walks split off since, so - /// each knows whether the groups it reads moved. - was: Option, -} - -/// An unsettled time end: the span it ends, ordered by when, as instants. -#[derive(Clone, Copy, Debug)] -struct Ending { - pts: Timestamp, - span: usize, -} - -impl Ord for Ending { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - instant(self.pts, other.pts).then(self.span.cmp(&other.span)) - } -} - -impl PartialOrd for Ending { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl PartialEq for Ending { - fn eq(&self, other: &Self) -> bool { - self.cmp(other).is_eq() - } -} - -impl Eq for Ending {} - -/// How far a [`Walk`] got: the same for every end sharing it, so walks merge once theirs -/// are equal. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -struct Trail { - /// The lowest group the walk has yet to put in range for good. Each of its spans starts - /// here, or at or below `known`, past which [`Walk::fix`] moves it. - floor: u64, - /// The lowest group a miss asks for in what the walk reads: a time start before every - /// group held asks none below the oldest, since which of them it reaches is unknown - /// until they start. - reach: u64, - /// The newest group known to start at or before every end: it and everything below it - /// in the spans is in range. - known: Option, - /// The lowest group known to start after every end: where they fall once settled. - after: Option, - /// Where the walk over held groups resumes: those below it started, or never will. - held: u64, - /// Where settling resumes: every sequence from above `known` up to it is accounted for. - accounted: u64, - /// The exclusive cap on what is read meanwhile, `None` for none. - limit: Option, - /// The group the cap admits on speculation: the first above `known` that may still - /// start, which may start past the ends. - speculative: Option, -} - -impl Trail { - fn new(start: u64, reach: u64) -> Self { - Self { - floor: start, - reach, - known: None, - after: None, - held: start, - accounted: start, - limit: Some(start), - speculative: None, - } - } - - /// The lowest sequence not known to be in range, `None` once the last sequence is. - /// - /// `known` is only ever found at or above every span's start, so it alone decides once - /// set. - fn unknown(&self) -> Option { - match self.known { - Some(known) => known.checked_add(1), - None => Some(self.floor), - } - } -} - -impl Walk { - /// Whether every end is before `time`, so a group starting then lies past each. - /// - /// When only some are, those leave as a walk of their own from here, queued in `forks` - /// to walk on, and this one keeps the rest: `false`. - fn past(&mut self, time: Timestamp, forks: &mut Vec) -> bool { - let (Some(first), Some(last)) = (self.ends.first(), self.ends.last()) else { - return false; - }; - if later(time, last.pts) { - return true; - } - if !later(time, first.pts) { - return false; - } - let rest = self.ends.split_off(&Ending { pts: time, span: 0 }); - forks.push(Walk { - ends: std::mem::replace(&mut self.ends, rest), - trail: self.trail, - low: self.low, - low_reach: self.low_reach, - was: self.was, - }); - false - } - - /// Walk the held groups on from where the last call stopped, splitting off into `forks` - /// the ends a group starts between. Returns where the ends fall once they settle. - /// - /// A split-off walks again from the state it split at: each call resumes where the - /// last stopped, so it takes the other branch there, passing nothing twice. - fn advance(&mut self, holdings: &impl Holdings, forks: &mut Vec) -> Option { - // The starts index passes a group that may still start, so a stalled one never - // keeps a later group known to start inside the ends from being read. - while let Some(from) = self.trail.unknown() - && let Some((sequence, at)) = holdings.next_start(from) - && self.trail.after.is_none_or(|after| sequence < after) - { - match self.past(at, forks) { - true => self.trail.after = Some(sequence), - false => self.trail.known = Some(sequence), - } - } - - // The first sequence above `known` that may still start is the one read on - // speculation: a held group yet to start, or a missing one that may arrive. Only a - // group starting inside the ends is jumped to, since it puts all below it in range. - self.trail.speculative = None; - while let Some(from) = self.trail.unknown() { - let at = self.trail.held.max(from); - if self.trail.after.is_some_and(|after| at >= after) { - break; - } - let next = holdings - .next_held(at) - .filter(|(sequence, _)| self.trail.after.is_none_or(|after| *sequence < after)); - // A later group starting inside some ends but not others splits them first: those - // it starts past account for `at` on their own. - let jump = match next { - Some((sequence, Held::Started(time))) if sequence != at => { - (!self.past(time, forks)).then_some(sequence) - } - _ => None, - }; - match next { - Some((sequence, held)) if sequence == at => match held { - Held::Started(time) => match self.past(time, forks) { - true => self.trail.after = Some(at), - false => self.trail.known = Some(at), - }, - Held::Never => {} - Held::Pending => { - self.trail.held = at; - self.trail.speculative = Some(at); - break; - } - }, - Some(_) if jump.is_some() => self.trail.known = jump, - // Past every held group: the next to arrive is the speculative one, judged - // once a settle sees it, before it is read. - None if self.trail.after.is_none() => { - self.trail.held = at; - break; - } - _ => match holdings.gone(at) { - Some(end) if end > at => { - self.trail.held = end; - continue; - } - _ => { - self.trail.held = at; - self.trail.speculative = (!holdings.complete()).then_some(at); - break; - } - }, - } - // Nothing sits above the last sequence, so the walk ends on it. - let Some(next) = at.checked_add(1) else { break }; - self.trail.held = next; - } - self.trail.limit = match (self.trail.speculative, self.trail.after) { - _ if holdings.complete() => None, - (Some(speculative), _) => speculative.checked_add(1), - (None, Some(after)) => Some(after), - // Every held group is accounted for: the next to arrive is the speculative one. - (None, None) => None, - }; - - // Settle once every sequence up to the first group starting past the ends can no - // longer start inside them. - let after = self.trail.after?; - // Everything at or below `known` is in range, and `after` lies above it. - self.trail.accounted = self.trail.accounted.max(self.trail.unknown().unwrap_or(after)); - while self.trail.accounted < after { - let next = holdings.next_held(self.trail.accounted); - match next { - Some((sequence, held)) if sequence == self.trail.accounted => match held { - Held::Pending => return None, - // It started past the ends since the walk above stopped on it: everything - // below is accounted for, so they fall here. - Held::Started(at) if self.past(at, forks) => { - self.trail.after = Some(sequence); - return self.trail.after; - } - _ => self.trail.accounted += 1, - }, - _ => { - let below = next.map_or(after, |(sequence, _)| sequence.min(after)); - self.trail.accounted = match holdings.complete() { - true => below, - false => holdings.gone(self.trail.accounted)?.min(below), - }; - } - } - } - self.trail.after - } - - /// Walk on only from above `known`, what the walk has put in range for good staying in - /// its run from [`Self::low`]. Walks that started apart then agree once they get here. - fn fix(&mut self) { - let Some(known) = self.trail.known else { return }; - if self.trail.floor > known { - return; - } - self.trail.floor = known.saturating_add(1); - self.trail.reach = self.trail.reach.max(self.trail.floor); - } - - /// The walk's groups now, and the part a miss may ask for: one run each, from where its - /// spans start up to its cap. Everything from `low` up to `known` is in range for good, - /// and the cap never falls below it. - fn runs(&self) -> (Option>, Option>) { - let last = match self.trail.limit { - Some(end) => end.checked_sub(1).filter(|last| *last >= self.low), - None => Some(u64::MAX), - }; - let Some(last) = last else { - return (None, None); - }; - let missable = (self.low_reach <= last).then_some(self.low_reach..=last); - (Some(self.low..=last), missable) - } - - /// What this walk waits on, if it can park: settling stopped at a gap that may still - /// arrive, below the group its ends fall before, holding from above `known` up to that - /// group only groups below the gap that never start. - /// - /// Until a sequence there arrives or is given up, the walk reads the same answers. One - /// arriving late, even into a run given up, is in the holdings' changes, so the walk - /// wakes then. Costs the groups held there, which never start. - fn waits(&self, holdings: &impl Holdings) -> Option> { - let (gap, after, from) = (self.trail.accounted, self.trail.after?, self.trail.unknown()?); - if gap >= after || holdings.complete() || holdings.gone(gap).is_some() { - return None; - } - let mut at = from; - while let Some((sequence, held)) = holdings.next_held(at) - && sequence < after - { - if sequence >= gap || held != Held::Never { - return None; - } - at = sequence + 1; - } - Some(from..after) - } - - /// Take on `other`'s ends, its trail being this one's. Both runs end where the trails - /// put them, so the merged runs start at the lower of each. - fn merge(&mut self, mut other: Walk) { - self.low = self.low.min(other.low); - self.low_reach = self.low_reach.min(other.low_reach); - if other.ends.len() > self.ends.len() { - std::mem::swap(&mut self.ends, &mut other.ends); - } - self.ends.extend(other.ends); - } -} - impl Window { /// Resolve `ranges`, counting relative anchors back from `latest`. /// /// Refuses an [`Anchor::Oldest`] end with [`Error::InvalidRange`] where the hop - /// [resolves](Holdings::resolves) ranges itself, a time anchor on an - /// untimed track with [`Error::TimestampMismatch`], and an anchor a received copy's wire + /// [resolves](Holdings::resolves) ranges itself, and an anchor a received copy's wire /// drops with [`Error::Unsupported`]. pub fn new(ranges: &[Range], latest: Option, holdings: &impl Holdings) -> Result { let mut window = Self { @@ -589,14 +187,8 @@ impl Window { spans: Vec::new(), looking: Vec::new(), waiting: Vec::new(), - walks: Vec::new(), - parked: BTreeMap::new(), - seen: Seen::default(), groups: Layered::default(), missable: Layered::default(), - resolved: Runs::default(), - resolved_missable: Runs::default(), - stale: false, pending: false, settled: false, covered: None, @@ -607,7 +199,7 @@ impl Window { Ok(window) } - /// Whether nothing is left to resolve as groups arrive and start. + /// Whether nothing is left to resolve as groups arrive. pub fn is_settled(&self) -> bool { self.settled } @@ -617,8 +209,13 @@ impl Window { self.pending } + /// Whether a time start still waits on the groups held to place it. + pub fn is_waiting(&self) -> bool { + !self.waiting.is_empty() + } + /// Resolve what the groups now held settle: relative anchors once `latest` exists, and - /// time starts and ends as groups start. Returns whether the groups in range moved. + /// time starts once a group arrives. Returns whether the groups in range moved. pub fn settle(&mut self, latest: Option, holdings: &impl Holdings) -> bool { if self.pending { if latest.is_none() { @@ -628,27 +225,24 @@ impl Window { let _ = self.place(latest, holdings); return !self.pending; } - let mut moved = !self.waiting.is_empty() && self.start_waiting(holdings); - if !self.walks.is_empty() || !self.parked.is_empty() { - moved |= self.walk(holdings); - } - if moved || self.stale { - self.rebuild(); + let moved = !self.waiting.is_empty() && self.start_waiting(holdings); + if moved { + self.rebuild(holdings); } moved } - /// Start the spans whose time start waits on a group starting, once one has, or once - /// nothing can start any more. Returns whether any did. + /// Start the spans whose time start waits on a group, once one arrived, or once nothing + /// can arrive any more. Returns whether any did. fn start_waiting(&mut self, holdings: &impl Holdings) -> bool { let mut started = false; for position in std::mem::take(&mut self.waiting) { - let Anchor::Time(pts) = self.ranges[self.spans[position].index].start else { - unreachable!("only a time start waits on a group starting"); + let Anchor::Time(time) = self.ranges[self.spans[position].index].start else { + unreachable!("only a time start waits on a group"); }; - let (start, floored) = match holdings.containing(pts) { + let (start, floored) = match holdings.containing(time) { Containing::Group(group) => (group, true), - // The group holding it is older than any held, or nothing held can start. + // The group holding it is older than any held, or nothing can arrive. Containing::Before => (0, false), Containing::Unknown if holdings.complete() => (0, false), Containing::Unknown => { @@ -657,191 +251,29 @@ impl Window { } }; started = true; - self.start_span(position, pts, start, floored, holdings); + self.start_span(position, time, start, floored, holdings); } if started { - self.stale = true; self.name(); } started } - /// Start the span at `position`, whose time start `pts` waited, at `start`: looked up + /// Start the span at `position`, whose time start `time` waited, at `start`: looked up /// where the groups held can't place it, as when first resolved, and otherwise reading - /// its groups, its time end walking from there. - fn start_span(&mut self, position: usize, pts: Timestamp, start: u64, floored: bool, holdings: &impl Holdings) { + /// its groups. + fn start_span(&mut self, position: usize, time: u64, start: u64, floored: bool, holdings: &impl Holdings) { let span = &mut self.spans[position]; (span.start, span.floored, span.waiting) = (start, floored, false); - if let Some(below) = unresolved(start, floored, holdings) { - span.looking = Some(Looking { pts, below }); + if let Some(below) = unresolved(time, start, floored, holdings) { + span.looking = Some(Looking { time, below }); self.looking.push(position); - return; - } - let reach = match floored { - true => start, - false => holdings.next_start(0).map_or(start, |(oldest, _)| oldest), - }; - let limit = match span.end { - End::Before(end) => Some(end), - End::Open => None, - End::Time(end) => { - self.walks.push(Walk { - ends: BTreeSet::from([Ending { - pts: end, - span: position, - }]), - trail: Trail::new(start, reach), - low: start, - low_reach: reach, - was: Some(start), - }); - return; - } - }; - if let Some(last) = limit.map_or(Some(u64::MAX), |end| end.checked_sub(1)) - && last >= start - { - self.resolved.insert(start..=last); - if reach <= last { - self.resolved_missable.insert(reach..=last); - } } } - /// Walk each unsettled time end on, settling those that can be, and merge the walks - /// that come to agree. Returns whether the groups in range moved. - fn walk(&mut self, holdings: &impl Holdings) -> bool { - let mut moved = false; - let mut walks = std::mem::take(&mut self.walks); - self.wake(holdings, &mut walks); - for walk in &mut walks { - walk.was = walk.trail.limit; - } - let mut kept = Vec::with_capacity(walks.len()); - while let Some(mut walk) = walks.pop() { - match walk.advance(holdings, &mut walks) { - Some(end) => { - // What settles joins the fixed runs, which readers share: copied once per - // settle under way, since a settle is once per end. - (moved, self.stale) = (true, true); - if walk.low < end { - self.resolved.insert(walk.low..=end - 1); - } - if walk.low_reach < end { - self.resolved_missable.insert(walk.low_reach..=end - 1); - } - for ending in walk.ends { - self.spans[ending.span].end = End::Before(end); - } - } - None => { - moved |= walk.trail.limit != walk.was; - walk.fix(); - match walk.waits(holdings) { - Some(waits) => self.park(waits, walk), - None => kept.push(walk), - } - } - } - } - kept.sort_unstable_by_key(|walk| walk.trail); - for walk in kept { - match self.walks.last_mut() { - Some(last) if last.trail == walk.trail => last.merge(walk), - _ => self.walks.push(walk), - } - } - moved - } - - /// Move into `walks` the parked walks a change since the last settle reaches: all of - /// them once the holdings can't say which. - fn wake(&mut self, holdings: &impl Holdings, walks: &mut Vec) { - if self.parked.is_empty() { - // Kept current, so the first walk to park wakes only on what changes after it. - let _ = holdings.changes(&mut self.seen); - return; - } - let woke = walks.len(); - match holdings.changes(&mut self.seen) { - None => walks.extend( - std::mem::take(&mut self.parked) - .into_values() - .flat_map(|parked| parked.walks), - ), - Some(changed) => { - for run in changed { - for (_, parked) in self.overlapping(run) { - walks.extend(parked.walks); - } - } - } - } - // The fixed runs held those it woke. - self.stale |= walks.len() > woke; - } - - /// Park `walk`, waiting on `waits`, with every parked walk whose wait overlaps it. - fn park(&mut self, waits: ops::Range, walk: Walk) { - self.stale = true; - let mut parked = Parked { - end: waits.end, - walks: vec![walk], - }; - let mut start = waits.start; - for overlapping in self.overlapping(waits.clone()) { - start = start.min(overlapping.0); - parked.end = parked.end.max(overlapping.1.end); - parked.walks.extend(overlapping.1.walks); - } - self.parked.insert(start, parked); - } - - /// Take out the parked walks whose waits overlap `run`. - fn overlapping(&mut self, run: ops::Range) -> Vec<(u64, Parked)> { - if run.is_empty() { - return Vec::new(); - } - // They are disjoint: the one holding the run's start, then each starting inside it. - let below = self - .parked - .range(..=run.start) - .next_back() - .filter(|(_, parked)| parked.end > run.start) - .map(|(start, _)| *start); - let inside = self - .parked - .range(run.start.saturating_add(1)..run.end) - .map(|(start, _)| *start); - let starts: Vec = below.into_iter().chain(inside).collect(); - starts - .into_iter() - .filter_map(|start| self.parked.remove(&start).map(|parked| (start, parked))) - .collect() - } - - /// The groups an unsettled time end reads on speculation, which may start past it: a - /// read parked on the cap wakes when one of them starts. - /// - /// A parked walk waits on a group arriving, never one starting, and an arrival changes - /// the track a read parks on anyway, so only the walks not parked count, here and in - /// [`Self::watch_starts`]. - pub fn speculative(&self) -> impl Iterator + '_ { - self.walks.iter().filter_map(|walk| walk.trail.speculative) - } - - /// The lowest sequence whose start could still move an unsettled time start, or a time - /// end not parked, `None` without one or once the last sequence is in range. - pub fn watch_starts(&self) -> Option { - if !self.waiting.is_empty() { - return Some(0); - } - self.walks.iter().filter_map(|walk| walk.trail.unknown()).min() - } - - /// Whether group `sequence` is in range. Everything is while a relative anchor waits. - pub fn contains(&self, sequence: u64) -> bool { - self.pending || self.groups.contains(sequence) + /// Whether group `key` is in range. Everything is while a relative anchor waits. + pub fn contains(&self, key: u64) -> bool { + self.pending || self.groups.contains(key) } /// The lowest group in range, `None` while an anchor waits. Nothing in range floors @@ -872,16 +304,15 @@ impl Window { } /// Each range resolved, in the order given: `None` for one whose start still waits, or - /// still looking up its start. One empty as given, or that only resolves empty, ends at - /// or below its start. An unsettled time end is reported open, never at its cap. + /// still looking up its start. One empty as given ends at or below its start. pub fn resolved(&self) -> Vec> { let mut resolved: Vec<_> = self .ranges .iter() .map(|range| match (range.start, range.end) { - (Anchor::Absolute(start), Some(Anchor::Absolute(end))) if range.is_empty() => Some(Resolved { + (Anchor::Time(start), Some(Anchor::Time(_))) if range.is_empty() => Some(Resolved { start, - end: Some(end), + end: Some(start), floored: true, }), _ => None, @@ -890,10 +321,7 @@ impl Window { for span in self.spans.iter().filter(|span| span.looking.is_none() && !span.waiting) { resolved[span.index] = Some(Resolved { start: span.start, - end: match span.end { - End::Before(end) => Some(end), - End::Open | End::Time(_) => None, - }, + end: span.end, floored: span.floored, }); } @@ -914,7 +342,7 @@ impl Window { } /// The exclusive end past every group in range, `None` when one stays open or an anchor - /// waits. An unsettled time end counts as open: its cap only holds reads back meanwhile. + /// waits. pub fn end(&self) -> Option { if !self.settled { return None; @@ -965,7 +393,8 @@ impl Window { } /// The groups in range from `from` on that a miss may ask for, lowest first: all of - /// them but those below the oldest group held when a time start fell before it. + /// them but those starting before a time start that fell before every group held, or + /// below the oldest group held when a relative start counted back past them all. pub fn missable(&self, from: u64) -> impl Iterator> + '_ { self.from(&self.missable, from) } @@ -1002,21 +431,20 @@ impl Window { /// The time starts still looked up, each with the exclusive end of the run below every /// group held, which holds the group containing it if the track's history does. - pub fn lookups(&self) -> impl Iterator + '_ { + pub fn lookups(&self) -> impl Iterator + '_ { self.looking .iter() - .filter_map(|span| self.spans[*span].looking.map(|looking| (looking.pts, looking.below))) + .filter_map(|span| self.spans[*span].looking.map(|looking| (looking.time, looking.below))) } - /// The lookup of time start `pts` answered with the group containing it, or the oldest - /// group of the history when `pts` is older than all of it: its spans start there, once - /// that group has started. `None`, or an answer that never started, means the history - /// couldn't tell, so they start where the groups held say. Returns whether the groups in - /// range moved. - pub fn found(&mut self, pts: Timestamp, group: Option, holdings: &impl Holdings) -> bool { + /// The lookup of time start `time` answered with the group containing it, or the oldest + /// group of the history when `time` is older than all of it: its spans start there, once + /// that group is held. `None`, or an answer not held, means the history couldn't tell, so + /// they start where the groups held say. Returns whether the groups in range moved. + pub fn found(&mut self, time: u64, group: Option, holdings: &impl Holdings) -> bool { let mut found = Vec::new(); self.looking.retain(|span| match self.spans[*span].looking { - Some(looking) if looking.pts == pts => { + Some(looking) if looking.time == time => { found.push(*span); false } @@ -1025,52 +453,19 @@ impl Window { if found.is_empty() { return false; } - let started = group.filter(|group| holdings.next_start(*group).is_some_and(|(found, _)| found == *group)); + let held = group.filter(|group| holdings.next_start(*group) == Some(*group)); for index in found { let span = &mut self.spans[index]; - let Some(looking) = span.looking.take() else { continue }; - let (start, floored, reach) = match (started, holdings.containing(pts)) { - (Some(group), _) | (None, Containing::Group(group)) => (group, true, group), - (None, _) => ( - 0, - false, - holdings.next_start(0).map_or(looking.below, |(oldest, _)| oldest), - ), - }; - (span.start, span.floored) = (start, floored); - let limit = match span.end { - End::Before(end) => Some(end), - End::Open => None, - // It walks from where the span reaches: a start the history couldn't place asks - // nothing below the oldest group held, so none of those may hold the end open. - End::Time(end) => { - self.walks.push(Walk { - ends: BTreeSet::from([Ending { pts: end, span: index }]), - trail: Trail { - held: reach, - accounted: reach, - ..Trail::new(start, reach) - }, - low: start, - low_reach: reach, - was: Some(start), - }); - continue; - } - }; - if let Some(last) = limit.map_or(Some(u64::MAX), |end| end.checked_sub(1)) - && last >= start - { - self.resolved.insert(start..=last); - if reach <= last { - self.resolved_missable.insert(reach..=last); - } + if span.looking.take().is_none() { + continue; } + (span.start, span.floored) = match (held, holdings.containing(time)) { + (Some(group), _) | (None, Containing::Group(group)) => (group, true), + (None, _) => (0, false), + }; } - self.stale = true; self.name(); - self.walk(holdings); - self.rebuild(); + self.rebuild(holdings); true } @@ -1078,29 +473,15 @@ impl Window { if holdings.resolves() && self.ranges.iter().any(|range| range.end == Some(Anchor::Oldest)) { return Err(Error::InvalidRange); } - let time = |anchor: Option| matches!(anchor, Some(Anchor::Time(_))); - if !holdings.timed() - && self - .ranges - .iter() - .any(|range| time(Some(range.start)) || time(range.end)) - { - return Err(Error::TimestampMismatch); - } if holdings.drops_anchors() && !self.ranges.iter().all(Range::is_wire) { return Err(Error::Unsupported); } self.spans.clear(); self.looking.clear(); self.waiting.clear(); - self.walks.clear(); - self.parked.clear(); - let (mut fixed, mut fixed_missable) = (Vec::new(), Vec::new()); self.pending = false; - // The time ends that start alike walk alike, until a group starts between them. - let mut walks = std::collections::BTreeMap::<(u64, u64), BTreeSet>::new(); for (index, range) in self.ranges.iter().enumerate().filter(|(_, range)| !range.is_empty()) { - let Some(end) = end(range.end, latest) else { + let Some(end) = end(range.end, latest, holdings) else { self.pending = true; continue; }; @@ -1110,7 +491,8 @@ impl Window { self.pending = true; continue; } - // Reads nothing until a group starts; see [`Self::start_waiting`]. + // Reads only the groups from its time until the groups held place it; see + // [`Self::start_waiting`]. Start::Waiting => { self.waiting.push(self.spans.len()); self.spans.push(Span { @@ -1126,71 +508,23 @@ impl Window { }; // A time start the groups held can't place reads nothing until its lookup answers. let looking = match range.start { - Anchor::Time(pts) => unresolved(start, floored, holdings).map(|below| Looking { pts, below }), + Anchor::Time(time) => unresolved(time, start, floored, holdings).map(|below| Looking { time, below }), _ => None, }; if looking.is_some() { self.looking.push(self.spans.len()); - self.spans.push(Span { - index, - start, - end, - floored, - looking, - waiting: false, - }); - continue; - } - // The lowest group a miss asks for: a time start before every group held asks none - // below the oldest, since which of them it reaches is unknown until they start. - let reach = match range.start { - Anchor::Time(_) if !floored => holdings.next_start(0).map_or(start, |(oldest, _)| oldest), - _ => start, - }; - let limit = match end { - End::Before(end) => Some(end), - End::Open => None, - End::Time(pts) => { - let span = self.spans.len(); - walks.entry((start, reach)).or_default().insert(Ending { pts, span }); - Some(start) - } - }; - if let Some(last) = limit.map_or(Some(u64::MAX), |end| end.checked_sub(1)) - && last >= start - { - fixed.push(start..=last); - if reach <= last { - fixed_missable.push(reach..=last); - } } self.spans.push(Span { index, start, end, floored, - looking: None, + looking, waiting: false, }); } self.name(); - self.resolved = Runs::new(fixed); - self.resolved_missable = Runs::new(fixed_missable); - self.groups = Layered::new(Arc::new(self.resolved.clone()), Runs::default()); - self.missable = Layered::new(Arc::new(self.resolved_missable.clone()), Runs::default()); - self.stale = false; - self.walks = walks - .into_iter() - .map(|((start, reach), ends)| Walk { - ends, - trail: Trail::new(start, reach), - low: start, - low_reach: reach, - was: Some(start), - }) - .collect(); - self.walk(holdings); - self.rebuild(); + self.rebuild(holdings); Ok(()) } @@ -1205,39 +539,49 @@ impl Window { self.all_named = self.spans.iter().all(|span| span.floored); } - /// The groups in range: the fixed ones, and the run each walk not parked reads now - /// layered over them. Costs those walks, and the fixed runs only once a settle or a walk - /// parking or waking changed them. - fn rebuild(&mut self) { + /// The groups in range: every span that knows where it starts, through its end, and + /// every span waiting on a time start from that time. + fn rebuild(&mut self, holdings: &impl Holdings) { self.covered = None; - // A span still looking up or waiting on its start reads nothing until it knows which - // group that is. - self.settled = !self.pending - && self.walks.is_empty() - && self.parked.is_empty() - && self.looking.is_empty() - && self.waiting.is_empty(); - let (groups, missable): (Vec<_>, Vec<_>) = self.walks.iter().map(Walk::runs).unzip(); - self.groups.set_layer(Runs::new(groups.into_iter().flatten())); - self.missable.set_layer(Runs::new(missable.into_iter().flatten())); - if std::mem::take(&mut self.stale) { - let parked = || { - self.parked - .values() - .flat_map(|parked| parked.walks.iter()) - .map(Walk::runs) + // A span still looking up its start reads nothing until it knows which group that is. + self.settled = !self.pending && self.looking.is_empty() && self.waiting.is_empty(); + let (mut groups, mut missable) = (Vec::new(), Vec::new()); + for span in self.spans.iter().filter(|span| span.looking.is_none()) { + let Some(last) = span.end.map_or(Some(u64::MAX), |end| end.checked_sub(1)) else { + continue; }; - let groups = self - .resolved - .iter() - .cloned() - .chain(parked().filter_map(|(groups, _)| groups)); - let missable = - (self.resolved_missable.iter().cloned()).chain(parked().filter_map(|(_, missable)| missable)); - let (groups, missable) = (Runs::new(groups), Runs::new(missable)); - self.groups.set_fixed(groups); - self.missable.set_fixed(missable); + // A span waiting on its time start reads from that time meanwhile, and may ask a miss + // there, as one reaching back past every group held does: whichever group contains + // it starts at or before it, so every group from there on is in range. + if span.waiting { + if let Anchor::Time(time) = self.ranges[span.index].start + && time <= last + { + groups.push(time..=last); + missable.push(time..=last); + } + continue; + } + if last < span.start { + continue; + } + groups.push(span.start..=last); + // A time start before every group held reaches back to the group containing it, + // which is unknown here, so it asks no miss below its time: every key from there + // on is in range, the key at the time itself being the containing group if it + // exists. A relative one counting back past every group held asks none below the + // oldest, since which of them it reaches is unknown. + let reach = match (self.ranges[span.index].start, span.floored) { + (Anchor::Time(time), false) => time, + (Anchor::Relative(_), false) => holdings.next_start(0).unwrap_or(span.start), + _ => span.start, + }; + if reach <= last { + missable.push(reach..=last); + } } + self.groups = Layered::new(Arc::new(Runs::new(groups)), Runs::default()); + self.missable = Layered::new(Arc::new(Runs::new(missable)), Runs::default()); } } @@ -1246,7 +590,8 @@ enum Start { /// At this group, and whether it names that group rather than reaching back to the /// oldest one. At(u64, bool), - /// A time start no group has started for yet, so it reads nothing. + /// A time start the groups held can't place yet, so it reads only the groups from that + /// time. Waiting, /// Waiting on a latest group, reading everything meanwhile. Pending, @@ -1254,57 +599,67 @@ enum Start { fn start(anchor: Anchor, latest: Option, holdings: &impl Holdings) -> Start { match anchor { - Anchor::Absolute(start) => Start::At(start, true), // Nothing sits below group 0, so the oldest start needs no latest group. Anchor::Oldest => Start::At(0, false), - Anchor::Relative(back) => latest.map_or(Start::Pending, |latest| Start::At(latest.saturating_sub(back), true)), - Anchor::Time(pts) => match holdings.containing(pts) { + Anchor::Relative(back) => match latest { + None => Start::Pending, + Some(latest) => match holdings.back(latest, back) { + Some(group) => Start::At(group, true), + // Fewer groups are held than it counts back, so reach back to the oldest. + None => Start::At(0, false), + }, + }, + // No group starts before 0, so time 0 names group 0 whatever is held. + Anchor::Time(0) => Start::At(0, true), + Anchor::Time(time) => match holdings.containing(time) { Containing::Group(group) => Start::At(group, true), // The group holding it is older than any held, so reach back to the oldest. Containing::Before => Start::At(0, false), // Before any group, as for a relative anchor, which a hop that holds none (such as // a reader following route copies) leaves to the copies to resolve. - Containing::Unknown if latest.is_none() => Start::Pending, - // Nothing can start any more, so nothing held reaches past it. + Containing::Unknown if latest.is_none() && !holdings.resolves() => Start::Pending, + // Nothing can arrive any more, so nothing held reaches past it. Containing::Unknown if holdings.complete() => Start::At(0, false), - // Nothing has started yet, so wait for a group that has, rather than guess the - // latest: groups arrive before the frames that show where they start, so a copy - // can see the live edge before the groups its publisher resolved the time to. + // The groups held can't place it, so wait for one that can rather than guess from the + // latest: a copy can see the live edge before the groups its publisher resolved the + // time to. Containing::Unknown => Start::Waiting, }, } } -/// The exclusive end of the run a lookup of a time start resolved to `start` searches, when -/// the groups held can't tell which group contains it and a handler may: it is before every -/// group known to start, or in a gap after `start`, the newest known to start at or before it. -/// Even a gap the live feed accounted for may sit in the history once evicted. -fn unresolved(start: u64, floored: bool, holdings: &impl Holdings) -> Option { +/// The exclusive end of the run a lookup of time start `time`, resolved to `start`, searches, +/// when the groups held can't tell which group contains it and a handler may: it is before +/// every group held, or a group the cache evicted may start after `start`, the newest held +/// at or before it, and still at or before `time`. Keys the producer skipped hold none. +fn unresolved(time: u64, start: u64, floored: bool, holdings: &impl Holdings) -> Option { if !holdings.looks_up() { return None; } if !floored { - return holdings.next_start(0).map(|(oldest, _)| oldest); + return holdings.next_start(0); } let next = start.checked_add(1)?; - let (after, _) = holdings.next_start(next)?; - (after > next).then_some(after) + let after = holdings.next_start(next)?; + let gap = next..after.min(time.saturating_add(1)); + (!holdings.never_produced(gap)).then_some(after) } -/// Where a range stops, or `None` while it waits on a latest group. A time end's walk -/// starts once its start is known. -fn end(anchor: Option, latest: Option) -> Option { +/// Where a range stops, exclusive, or `None` while it waits on a latest group. `Some(None)` +/// is past every group. +fn end(anchor: Option, latest: Option, holdings: &impl Holdings) -> Option> { Some(match anchor { - None => End::Open, - Some(Anchor::Absolute(end)) => End::Before(end), - Some(Anchor::Relative(back)) => latest? - .saturating_sub(back) - .checked_add(1) - .map_or(End::Open, End::Before), - Some(Anchor::Time(pts)) => End::Time(pts), + None => None, + // Every end includes its group. + Some(Anchor::Time(time)) => time.checked_add(1), + Some(Anchor::Relative(back)) => match holdings.back(latest?, back) { + Some(group) => group.checked_add(1), + // It counts back past every group held, so it ends before them all. + None => Some(0), + }, // Refused where groups are held; a hop that holds none lets its copies refuse it, and // delivers nothing meanwhile. - Some(Anchor::Oldest) => End::Before(0), + Some(Anchor::Oldest) => Some(0), }) } @@ -1312,168 +667,83 @@ fn end(anchor: Option, latest: Option) -> Option { mod tests { use super::*; - use rangemap::RangeInclusiveSet; - - use std::cell::Cell; - - fn at(millis: u64) -> Timestamp { - Timestamp::from_millis(millis).unwrap() - } + use std::collections::BTreeSet; - /// A hop's groups by sequence, the runs that can no longer arrive, and how many held - /// groups and starts a walk asked about. - /// - /// Reports changes as a track does: a group arriving and a run given up, never a group - /// starting or ending empty, which its ledger never sees. + /// The group starts a hop holds, the key runs its producer skipped, and whether a + /// handler serves its history. #[derive(Default)] - struct Groups { - held: std::collections::BTreeMap, - gone: Vec>, + struct Keys { + held: BTreeSet, + skipped: Vec>, complete: bool, looks_up: bool, - asked: Cell, - /// How many fetched copies gave a group held past its frame 0 a start, which a - /// track counts rather than logs. - fetched: Cell, - /// Every change so far, oldest first, and what was last logged: the held sequences - /// and how many runs were given up. - log: std::cell::RefCell<(Vec>, BTreeSet, usize)>, } - impl Groups { - fn new(held: impl IntoIterator) -> Self { + impl Keys { + fn new(held: impl IntoIterator) -> Self { Self { held: held.into_iter().collect(), ..Self::default() } } - - fn started(sequence: u64, millis: u64) -> (u64, Held) { - (sequence, Held::Started(at(millis))) - } } - impl Holdings for Groups { - fn timed(&self) -> bool { - true - } - + impl Holdings for Keys { fn looks_up(&self) -> bool { self.looks_up } - fn containing(&self, pts: Timestamp) -> Containing { - let mut stamped = false; - for (&sequence, held) in self.held.iter().rev() { - let Held::Started(start) = held else { continue }; - stamped = true; - if !later(*start, pts) { - return Containing::Group(sequence); - } - } - match stamped { - true => Containing::Before, - false => Containing::Unknown, + fn containing(&self, time: u64) -> Containing { + match self.held.range(..=time).next_back() { + Some(key) => Containing::Group(*key), + None if self.held.is_empty() => Containing::Unknown, + None => Containing::Before, } } - fn next_start(&self, from: u64) -> Option<(u64, Timestamp)> { - self.asked.set(self.asked.get() + 1); - self.held.range(from..).find_map(|(&sequence, held)| match held { - Held::Started(start) => Some((sequence, *start)), - _ => None, - }) - } - - fn next_held(&self, from: u64) -> Option<(u64, Held)> { - self.asked.set(self.asked.get() + 1); - self.held - .range(from..) - .next() - .map(|(&sequence, held)| (sequence, *held)) + fn next_start(&self, from: u64) -> Option { + self.held.range(from..).next().copied() } - fn gone(&self, sequence: u64) -> Option { - self.gone - .iter() - .find(|gone| gone.contains(&sequence)) - .map(|gone| gone.end) + fn back(&self, latest: u64, back: u64) -> Option { + let Some(back) = back.checked_sub(1) else { + return Some(latest); + }; + self.held.range(..latest).rev().nth(back as usize).copied() } fn complete(&self) -> bool { self.complete } - fn changes(&self, seen: &mut Seen) -> Option>> { - let mut log = self.log.borrow_mut(); - let (log, held, gone) = &mut *log; - let arrived = self.held.keys().filter(|sequence| !held.contains(sequence)); - log.extend(arrived.map(|sequence| *sequence..sequence.saturating_add(1))); - log.extend(self.gone[*gone..].iter().cloned()); - *held = self.held.keys().copied().collect(); - *gone = self.gone.len(); - let from = usize::try_from(seen.runs[0]).unwrap(); - let changed = log[from..].to_vec(); - let global = [ - u64::from(self.complete), - self.fetched.get(), - u64::from(self.looks_up), - 0, - ]; - let primed = seen.global == Some(global); - *seen = Seen { - runs: [log.len() as u64, 0], - global: Some(global), - }; - primed.then_some(changed) + fn never_produced(&self, keys: ops::Range) -> bool { + keys.is_empty() + || self + .skipped + .iter() + .any(|run| run.start <= keys.start && keys.end <= run.end) } } - struct Untimed; - - impl Holdings for Untimed { - fn timed(&self) -> bool { - false - } - - fn containing(&self, _: Timestamp) -> Containing { - Containing::Unknown - } - - fn next_start(&self, _: u64) -> Option<(u64, Timestamp)> { - None - } - - fn next_held(&self, _: u64) -> Option<(u64, Held)> { - None - } - - fn gone(&self, _: u64) -> Option { - None - } - - fn complete(&self) -> bool { - false - } - } - - fn time(millis: u64) -> Anchor { - Anchor::Time(Timestamp::from_millis(millis).unwrap()) - } - fn range(start: Anchor, end: impl Into>) -> Range { Range { start, end: end.into() } } - fn covered(window: &Window, upto: u64) -> Vec { - (0..upto).filter(|sequence| window.contains(*sequence)).collect() + fn time(start: u64) -> Anchor { + Anchor::Time(start) + } + + fn covered(window: &Window, keys: impl IntoIterator) -> Vec { + keys.into_iter().filter(|key| window.contains(*key)).collect() } /// Disjoint ranges filter to exactly their groups; the gap between them is out. #[test] fn ranges_cover_only_their_groups() { - let window = Window::new(&[Range::groups(2..4), Range::groups(7..)], Some(9), &Untimed).unwrap(); - assert_eq!(covered(&window, 10), vec![2, 3, 7, 8, 9]); + let keys = Keys::new(0..10); + let ranges = [range(time(2), time(3)), range(time(7), None)]; + let window = Window::new(&ranges, Some(9), &keys).unwrap(); + assert_eq!(covered(&window, 0..10), vec![2, 3, 7, 8, 9]); assert_eq!(window.floor(), Some(2)); assert_eq!(window.end(), None); assert_eq!(window.runs(3).collect::>(), vec![3..=3, 7..=u64::MAX]); @@ -1500,513 +770,138 @@ mod tests { assert_eq!(window.starts(5), [None, Some(7)]); // Nothing asked for is nothing in range, floored nowhere. - let window = Window::new(&[], None, &Untimed).unwrap(); - assert_eq!(covered(&window, 4), Vec::::new()); + let window = Window::new(&[], None, &keys).unwrap(); + assert_eq!(covered(&window, 0..4), Vec::::new()); assert_eq!((window.floor(), window.end()), (Some(0), Some(0))); } - /// Relative anchors count back from the latest group, both ends whole groups, and wait - /// for a first group when there is none. + /// A time start names the group containing it, the newest starting at or before it, and + /// a time end includes every group starting at or before it: sparse starts need no walk. #[test] - fn relative_anchors_count_back_from_the_latest_group() { - let window = Window::new(&[range(Anchor::Relative(3), Anchor::Relative(1))], Some(7), &Untimed).unwrap(); - assert_eq!(covered(&window, 10), vec![4, 5, 6]); + fn time_anchors_name_groups_by_their_starts() { + let keys = Keys::new([0, 33, 66, 99]); + let window = Window::new(&[range(time(50), time(70))], Some(99), &keys).unwrap(); + assert_eq!(covered(&window, [0, 33, 66, 99]), vec![33, 66]); + assert!(window.is_settled()); + assert_eq!( + window.resolved(), + vec![Some(Resolved { + start: 33, + end: Some(71), + floored: true + })] + ); + + // A start exactly on a group names that group, and an end on one includes it. + let window = Window::new(&[range(time(33), time(66))], Some(99), &keys).unwrap(); + assert_eq!(covered(&window, [0, 33, 66, 99]), vec![33, 66]); + + // An end before its start is empty, and so is one ending before every group. + let window = Window::new(&[range(time(66), time(33))], Some(99), &keys).unwrap(); + assert_eq!(covered(&window, [0, 33, 66, 99]), Vec::::new()); + + // Before every start held: reach back to the group containing it, naming no floor. + let keys = Keys::new([10, 20]); + let window = Window::new(&[range(time(5), None)], Some(20), &keys).unwrap(); + assert_eq!(covered(&window, [10, 20]), vec![10, 20]); + assert_eq!((window.floor(), window.start(), window.named()), (Some(0), None, None)); + assert_eq!(window.missable(0).next(), Some(5..=u64::MAX), "nothing below its time"); + } - let mut window = Window::new(&[Range::live()], None, &Untimed).unwrap(); + /// Relative anchors count back through the groups held, however sparse their starts, + /// and wait for a first group when there is none. + #[test] + fn relative_anchors_count_groups_back_from_the_latest() { + let keys = Keys::new([0, 33, 66, 99]); + let window = Window::new(&[range(Anchor::Relative(2), Anchor::Relative(1))], Some(99), &keys).unwrap(); + assert_eq!(covered(&window, [0, 33, 66, 99]), vec![33, 66]); + assert_eq!(window.start(), Some(33)); + + // Counting back past every group held reaches back to the oldest as a start, and ends + // before them all as an end. + let window = Window::new(&[range(Anchor::Relative(10), None)], Some(99), &keys).unwrap(); + assert_eq!(covered(&window, [0, 33, 66, 99]), vec![0, 33, 66, 99]); + assert_eq!(window.start(), None); + let window = Window::new(&[range(Anchor::Oldest, Anchor::Relative(10))], Some(99), &keys).unwrap(); + assert_eq!(covered(&window, [0, 33, 66, 99]), Vec::::new()); + + let mut window = Window::new(&[Range::live()], None, &Keys::default()).unwrap(); assert!(window.is_pending() && window.contains(3) && window.floor().is_none()); - assert!(!window.settle(None, &Untimed)); - assert!(window.settle(Some(5), &Untimed)); - assert_eq!(covered(&window, 8), vec![5, 6, 7]); + assert!(!window.settle(None, &Keys::default())); + let keys = Keys::new([5]); + assert!(window.settle(Some(5), &keys)); + assert_eq!(covered(&window, 0..8), vec![5, 6, 7]); assert!(window.is_settled()); // The oldest start needs no latest group, and names no floor. - let window = Window::new(&[Range::oldest()], None, &Untimed).unwrap(); + let window = Window::new(&[Range::oldest()], None, &keys).unwrap(); assert_eq!((window.floor(), window.start()), (Some(0), None)); - // Any count back, however far, waits on a latest group and names the group it clamps - // to, as a peer's largest counts must agree. + // Any count back, however far, waits on a latest group. for back in [u64::MAX, u64::MAX - 1] { - let open = Range { - start: Anchor::Relative(back), - end: None, - }; + let open = range(Anchor::Relative(back), None); assert!(!open.is_wire()); - assert!(Window::new(&[open], None, &Untimed).unwrap().is_pending()); - assert_eq!(Window::new(&[open], Some(7), &Untimed).unwrap().start(), Some(0)); + assert!(Window::new(&[open], None, &keys).unwrap().is_pending()); + assert_eq!(Window::new(&[open], Some(5), &keys).unwrap().start(), None); } } /// No hop can place the oldest group as an end, so it is refused. #[test] fn an_oldest_end_is_refused() { - let ranges = [range(Anchor::Absolute(0), Anchor::Oldest)]; + let ranges = [range(time(0), Anchor::Oldest)]; assert!(matches!( - Window::new(&ranges, Some(3), &Untimed), + Window::new(&ranges, Some(3), &Keys::new(0..4)), Err(Error::InvalidRange) )); + // A hop holding no groups leaves the refusal to the copies that do. + assert!(Window::new(&ranges, Some(3), &Unheld).is_ok()); } /// The named floor is the lowest start, unless a range reaches back to the oldest /// group or still waits on a first one; an explicit group 0 names one. #[test] fn the_named_start_is_the_lowest_unless_one_reaches_back() { - let start = |ranges: &[Range], latest| Window::new(ranges, latest, &Untimed).unwrap().start(); - assert_eq!(start(&[Range::groups(4..), Range::live()], Some(9)), Some(4)); - assert_eq!(start(&[Range::groups(4..), Range::live()], Some(2)), Some(2)); - assert_eq!(start(&[Range::groups(0..)], None), Some(0)); - assert_eq!(start(&[Range::groups(4..), Range::live()], None), None); - assert_eq!(start(&[Range::groups(4..), Range::oldest()], Some(9)), None); - assert_eq!(start(&[Range::groups(4..), Range::groups(..2)], Some(9)), None); - } - - /// A time start is the group containing it, and an end stops just past that group once - /// a later group is known to start after it with nothing between left to arrive. - #[test] - fn time_anchors_resolve_from_group_starts() { - let groups = Groups::new([ - Groups::started(2, 2000), - Groups::started(3, 3000), - Groups::started(4, 4000), - Groups::started(5, 5000), - (6, Held::Pending), - ]); - let window = Window::new(&[range(time(3500), time(4500))], Some(6), &groups).unwrap(); - assert_eq!(covered(&window, 8), vec![3, 4]); - assert!(window.is_settled()); - assert_eq!( - window.resolved(), - vec![Some(Resolved { - start: 3, - end: Some(5), - floored: true - })] - ); - - // Group 6 may yet start at 5000 too: read on speculation, the end left open. - let window = Window::new(&[range(time(3500), time(5000))], Some(6), &groups).unwrap(); - assert!(!window.is_settled()); - assert_eq!(covered(&window, 9), vec![3, 4, 5, 6]); - assert_eq!( - window.resolved(), - vec![Some(Resolved { - start: 3, - end: None, - floored: true - })] - ); - assert_eq!(window.end(), None, "the cap is not the end"); - assert_eq!(window.speculative().collect::>(), vec![6]); - - // Before every start held: reach back to the oldest. - let window = Window::new(&[range(time(1000), time(2500))], Some(6), &groups).unwrap(); - assert_eq!(covered(&window, 8), vec![0, 1, 2]); - - // Nothing started yet: a time start waits on a first group, then on one starting. - let mut window = Window::new(&[range(time(1000), None)], None, &Groups::default()).unwrap(); - assert!(window.is_pending()); - let mut held = Groups::new([(4, Held::Pending)]); - assert!(window.settle(Some(4), &held)); - assert!(!window.contains(4), "4 has not started"); - held.held.insert(4, Held::Started(at(1000))); - assert!(window.settle(Some(4), &held)); - assert_eq!(window.floor(), Some(4)); + let keys = Keys::new(0..10); + let start = |ranges: &[Range], latest| Window::new(ranges, latest, &keys).unwrap().start(); + assert_eq!(start(&[range(time(4), None), Range::live()], Some(9)), Some(4)); + assert_eq!(start(&[range(time(4), None), Range::live()], Some(2)), Some(2)); + assert_eq!(start(&[range(time(0), None)], None), Some(0)); + assert_eq!(start(&[range(time(4), None), Range::live()], None), None); + assert_eq!(start(&[range(time(4), None), Range::oldest()], Some(9)), None); } - /// An end settles only once every sequence between the newest group starting inside it - /// and the first starting past it is accounted for. - #[test] - fn a_time_end_settles_once_nothing_below_its_end_can_arrive() { - let mut groups = Groups::new([Groups::started(3, 3000), Groups::started(6, 6000)]); - let mut window = Window::new(&[range(time(3000), time(4000))], Some(6), &groups).unwrap(); - assert!(!window.is_settled(), "4 and 5 may still arrive inside the end"); - assert_eq!( - covered(&window, 8), - vec![3, 4], - "only 4, the first that may still arrive, on speculation, and never 6" - ); - - // 4 arrives yet to start: still the one read on speculation, so 5 waits. - groups.held.insert(4, Held::Pending); - assert!(!window.settle(Some(6), &groups), "4 was already the speculative one"); - assert_eq!(covered(&window, 8), vec![3, 4]); - groups.held.insert(4, Held::Started(at(3500))); - assert!(window.settle(Some(6), &groups)); - assert!(!window.is_settled(), "5 may still arrive"); - groups.gone.push(5..6); - assert!(window.settle(Some(6), &groups)); - assert!(window.is_settled()); - assert_eq!(covered(&window, 8), vec![3, 4, 5]); - - // A group that never starts proves nothing, so it never holds the end open. - let groups = Groups::new([Groups::started(3, 3000), (4, Held::Never), Groups::started(5, 5000)]); - let window = Window::new(&[range(time(3000), time(4000))], Some(5), &groups).unwrap(); - assert!(window.is_settled()); - assert_eq!(covered(&window, 8), vec![3, 4]); - } - - /// An unsettled time end reads at most one group on speculation: the first above the - /// newest known to start inside it that may still start. A group that never starts - /// does not take that place, and a stalled one does not hold back a later group known - /// to start inside the end. - #[test] - fn an_unsettled_time_end_reads_one_group_on_speculation() { - let mut groups = Groups::new([(3, Held::Pending), (4, Held::Pending), (5, Held::Pending)]); - let mut window = Window::new(&[range(Anchor::Absolute(3), time(3500))], Some(5), &groups).unwrap(); - assert_eq!(covered(&window, 8), vec![3]); - - groups.held.insert(3, Held::Started(at(3000))); - assert!(window.settle(Some(5), &groups)); - assert_eq!(covered(&window, 8), vec![3, 4]); - - // 4 never starts, so 5 is the one read on speculation. - groups.held.insert(4, Held::Never); - assert!(window.settle(Some(5), &groups)); - assert_eq!(covered(&window, 8), vec![3, 4, 5]); - - // 5 stalls, 6 starts inside the end: 5 is in range then, and 7 is speculative. - groups.held.insert(6, Held::Started(at(3400))); - groups.held.insert(7, Held::Pending); - assert!(window.settle(Some(7), &groups)); - assert_eq!(covered(&window, 9), vec![3, 4, 5, 6, 7]); - - // 7 starts past the end, which settles with 5 still stalled below it. - groups.held.insert(7, Held::Started(at(3600))); - assert!(window.settle(Some(7), &groups)); - assert!(window.is_settled()); - assert_eq!(covered(&window, 9), vec![3, 4, 5, 6]); - - // Every held group accounted for: the next to arrive is the speculative one. - let groups = Groups::new([Groups::started(3, 3000)]); - let window = Window::new(&[range(time(3000), time(3500))], Some(3), &groups).unwrap(); - assert!(window.contains(4)); - - // Nothing more can arrive or start: uncapped. - let mut groups = Groups::new([Groups::started(3, 3000), (4, Held::Pending), (5, Held::Pending)]); - groups.complete = true; - let window = Window::new(&[range(time(3000), time(3500))], Some(5), &groups).unwrap(); - assert_eq!(covered(&window, 7), vec![3, 4, 5, 6]); - } - - /// A peer that leaves an early sequence missing and keeps sending later groups costs a - /// time end each group once, not a rescan of everything since the gap on every settle. - #[test] - fn a_time_end_walks_each_group_once_behind_a_missing_sequence() { - let mut groups = Groups::new([Groups::started(0, 0)]); - let mut window = Window::new(&[range(time(0), time(1_000_000))], Some(0), &groups).unwrap(); - let count = 2_000; - for sequence in 2..count { - groups.held.insert(sequence, Held::Started(at(sequence * 10))); - window.settle(Some(sequence), &groups); - } - assert!(!window.is_settled(), "group 1 never arrived"); - assert!(window.contains(count - 1)); - let asked = groups.asked.get(); - assert!(asked < 8 * count as usize, "{asked} queries for {count} groups"); - } - - /// Time ends that start alike share one walk, so a settle costs the walk, not the ends. - /// A group starting between them splits off and settles the ends before it. - #[test] - fn time_ends_that_start_alike_share_one_walk() { - let ends = 1_000; - let mut groups = Groups::new([Groups::started(0, 0)]); - let ranges: Vec = (0..ends).map(|i| range(time(0), time(1_000_000 + i))).collect(); - let mut window = Window::new(&ranges, Some(0), &groups).unwrap(); - assert_eq!(window.walks.len(), 1); - - let count = 200; - let asked = groups.asked.get(); - for sequence in 1..count { - groups.held.insert(sequence, Held::Started(at(sequence))); - window.settle(Some(sequence), &groups); - } - let asked = groups.asked.get() - asked; - assert!(asked < 8 * count as usize, "{asked} queries for {count} groups"); - assert_eq!(window.walks.len(), 1); - assert!(!window.is_settled() && window.contains(count)); - - // Group 200 starts inside the later half of the ends: the earlier half settles before it. - groups.held.insert(count, Held::Started(at(1_000_000 + ends / 2))); - assert!(window.settle(Some(count), &groups)); - assert_eq!(window.walks.len(), 1); - let resolved = window.resolved(); - let split = usize::try_from(ends / 2).unwrap(); - assert!(resolved[..split].iter().all(|r| *r - == Some(Resolved { - start: 0, - end: Some(count), - floored: true, - }))); - assert!(resolved[split..].iter().all(|r| *r - == Some(Resolved { - start: 0, - end: None, - floored: true - }))); - - // Once a group starts past every end, they all settle. - groups.held.insert(count + 1, Held::Started(at(2_000_000))); - assert!(window.settle(Some(count + 1), &groups)); - assert!(window.is_settled() && window.walks.is_empty()); - assert_eq!(covered(&window, count + 3), (0..=count).collect::>()); - } - - /// Shared walks resolve every range as walking it alone would, as groups arrive, start, - /// never start, or go missing in any order, with ends in mixed scales. - #[test] - fn shared_walks_match_each_end_walked_alone() { - // A small deterministic generator, so a failure replays. - let seed = Cell::new(0x2545_f491_4f6c_dd1d_u64); - let next = |bound: u64| { - let mut x = seed.get(); - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - seed.set(x); - x % bound - }; - let micros = |value| Anchor::Time(Timestamp::new(value, crate::Timescale::MICRO).unwrap()); - const SEQUENCES: u64 = 16; - for round in 0..400 { - // Each group starts in its own ten milliseconds, so starts ascend with the sequence. - let starts: Vec = (0..SEQUENCES).map(|sequence| sequence * 10 + next(10)).collect(); - let pts = |end: bool| match next(3) { - 0 => micros(next(SEQUENCES * 10_000)), - // Exactly a group's start, in either scale. - 1 if next(2) == 0 => time(starts[next(SEQUENCES) as usize]), - 1 => micros(starts[next(SEQUENCES) as usize] * 1000), - _ => time(next(SEQUENCES * 10) + u64::from(end) * 5), - }; - let ranges: Vec = (0..1 + next(12)) - .map(|_| { - let start = match next(3) { - 0 => Anchor::Absolute(next(SEQUENCES)), - 1 => Anchor::Relative(next(4)), - _ => pts(false), - }; - let end = match next(6) { - 0 => None, - 1 => Some(Anchor::Absolute(next(SEQUENCES + 2))), - _ => Some(pts(true)), - }; - range(start, end) - }) - .collect(); - - let mut groups = Groups::default(); - for _ in 0..next(6) { - let sequence = next(SEQUENCES); - groups - .held - .insert(sequence, Held::Started(at(starts[sequence as usize]))); - } - let latest = Some(next(SEQUENCES)); - let mut window = Window::new(&ranges, latest, &groups).unwrap(); - let mut alone: Vec = ranges - .iter() - .map(|range| Window::new(std::slice::from_ref(range), latest, &groups).unwrap()) - .collect(); - - for step in 0..40 { - let sequence = next(SEQUENCES); - let started = Held::Started(at(starts[sequence as usize])); - match next(6) { - 0 | 1 => { - groups.held.entry(sequence).or_insert(Held::Pending); - } - 2 | 3 => { - if groups.held.get(&sequence).is_none_or(|held| *held == Held::Pending) { - groups.held.insert(sequence, started); - } - } - 4 => match groups.held.get(&sequence) { - None | Some(Held::Pending) => { - groups.held.insert(sequence, Held::Never); - } - // A fetched head gives a group held past its frame 0 a start. - Some(Held::Never) if next(2) == 0 => { - groups.held.insert(sequence, started); - groups.fetched.set(groups.fetched.get() + 1); - } - _ => {} - }, - _ => { - let end = sequence + 1 + next(3); - if groups.held.range(sequence..end).next().is_none() { - groups.gone.push(sequence..end); - } - } - } - groups.complete |= step == 39 && round % 4 == 0; - - let before: Vec = covered(&window, SEQUENCES + 4); - let moved = window.settle(latest, &groups); - for window in &mut alone { - window.settle(latest, &groups); - } - let context = || format!("round {round} step {step}: {ranges:?} over {:?}", groups.held); - - let union: Vec = (0..SEQUENCES + 4) - .filter(|sequence| alone.iter().any(|window| window.contains(*sequence))) - .collect(); - let now = covered(&window, SEQUENCES + 4); - assert_eq!(now, union, "{}", context()); - assert!(moved || now == before, "{}: moved unreported", context()); - - let mut missable = RangeInclusiveSet::new(); - for run in alone.iter().flat_map(|window| window.missable(0)) { - missable.insert(run); - } - assert!(window.missable(0).eq(missable.iter().cloned()), "{}", context()); - - let resolved: Vec<_> = alone.iter().map(|window| window.resolved()[0]).collect(); - assert_eq!(window.resolved(), resolved, "{}", context()); - assert_eq!( - window.is_settled(), - alone.iter().all(Window::is_settled), - "{}", - context() - ); - let speculative: std::collections::BTreeSet = window.speculative().collect(); - let each: std::collections::BTreeSet = alone.iter().flat_map(Window::speculative).collect(); - assert_eq!(speculative, each, "{}", context()); - assert_eq!( - window.watch_starts(), - alone.iter().filter_map(Window::watch_starts).min(), - "{}", - context() - ); - } - } - } - - /// Groups whose `racing` one starts past the end after the first time it is asked - /// about, as when its first frame lands between the walks of one settle. - struct Racing { - groups: Groups, - racing: u64, - asked: Cell, - } - - impl Holdings for Racing { - fn timed(&self) -> bool { - true - } - - fn containing(&self, pts: Timestamp) -> Containing { - self.groups.containing(pts) - } - - fn next_start(&self, from: u64) -> Option<(u64, Timestamp)> { - self.groups.next_start(from) - } - - fn next_held(&self, from: u64) -> Option<(u64, Held)> { - let next = self.groups.next_held(from); - if next.is_some_and(|(sequence, _)| sequence == self.racing) && self.asked.replace(true) { - return Some((self.racing, Held::Started(at(6000)))); - } - next - } - - fn gone(&self, sequence: u64) -> Option { - self.groups.gone(sequence) - } - - fn complete(&self) -> bool { - false - } - } - - #[test] - fn a_time_end_settles_on_a_group_that_starts_between_its_walks() { - let racing = Racing { - groups: Groups::new([Groups::started(0, 0), (1, Held::Pending), Groups::started(2, 7000)]), - racing: 1, - asked: Cell::new(false), - }; - let window = Window::new(&[range(Anchor::Absolute(0), time(5000))], None, &racing).unwrap(); - assert_eq!( - window.resolved(), - vec![Some(Resolved { - start: 0, - end: Some(1), - floored: true - })] - ); - } - - #[test] - fn a_missing_sequence_past_a_time_end_is_the_one_read_on_speculation() { - // 1 and 2 are missing, and 3 never starts: the walk may not pass the gap, or every - // sequence in it that arrives later is read past the end. - let mut groups = Groups::new([Groups::started(0, 3000), (3, Held::Never)]); - let mut window = Window::new(&[range(time(3000), time(3500))], Some(3), &groups).unwrap(); - assert_eq!(covered(&window, 5), vec![0, 1]); - assert_eq!(window.speculative().collect::>(), vec![1]); - groups.held.insert(2, Held::Pending); - groups.held.insert(1, Held::Pending); - window.settle(Some(3), &groups); - assert_eq!(covered(&window, 5), vec![0, 1], "2 waits on 1 starting"); - groups.held.insert(1, Held::Started(at(3600))); - window.settle(Some(3), &groups); - assert_eq!( - window.resolved(), - vec![Some(Resolved { - start: 0, - end: Some(1), - floored: true - })] - ); - } - - #[test] - fn a_time_end_stops_walking_at_the_last_sequence() { - let groups = Groups::new([Groups::started(u64::MAX, 3000)]); - let window = Window::new(&[range(Anchor::Absolute(u64::MAX), time(5000))], None, &groups).unwrap(); - assert_eq!( - window.resolved(), - vec![Some(Resolved { - start: u64::MAX, - end: None, - floored: true, - })] - ); - assert!(window.contains(u64::MAX)); - assert_eq!(window.watch_starts(), None, "nothing above the last sequence can start"); - } - - /// A time start no held group has started for yet reads nothing of its range, rather - /// than guess the latest group, and resolves once one starts. A track that can start + /// A time start with no group held yet reads only the groups from its time, rather than + /// guess the latest group, and resolves once a group arrives. A track that can receive /// nothing more reaches back to the oldest group. #[test] - fn a_time_start_waits_for_a_group_to_start() { - let pending = (2..10).map(|sequence| (sequence, Held::Pending)); - let mut groups = Groups::new(pending.clone()); - let ranges = [ - range(time(2500), None), - range(Anchor::Absolute(20), Anchor::Absolute(21)), - ]; - let mut window = Window::new(&ranges, Some(9), &groups).unwrap(); + fn a_time_start_waits_for_a_group() { + let mut keys = Keys::default(); + let ranges = [range(time(2500), None), range(Anchor::Relative(0), Anchor::Relative(0))]; + let mut window = Window::new(&ranges, Some(9), &keys).unwrap(); assert!(!window.is_pending() && !window.is_settled()); - assert_eq!(covered(&window, 22), vec![20], "only the other range reads"); + assert_eq!(covered(&window, 0..22), vec![9], "only the other range reads below it"); + assert_eq!( + covered(&window, 2498..2502), + vec![2500, 2501], + "the groups from the time are in range whatever contains it" + ); assert_eq!((window.floor(), window.start(), window.end()), (None, None, None)); assert_eq!( window.resolved(), vec![ None, Some(Resolved { - start: 20, - end: Some(21), + start: 9, + end: Some(10), floored: true, }) ] ); - assert_eq!(window.watch_starts(), Some(0), "any group starting may resolve it"); - assert!(!window.settle(Some(9), &groups), "nothing started"); + assert!(!window.settle(Some(9), &keys), "nothing arrived"); - groups.held.insert(3, Held::Started(at(3000))); - assert!(window.settle(Some(9), &groups)); + keys.held.insert(3000); + assert!(window.settle(Some(3000), &keys)); assert_eq!( window.resolved()[0], Some(Resolved { @@ -2014,25 +909,25 @@ mod tests { end: None, floored: false }), - "3 starts past it" + "3000 starts past it" ); - groups.held.insert(2, Held::Started(at(2000))); - let window = Window::new(&ranges, Some(9), &groups).unwrap(); + keys.held.insert(2000); + let window = Window::new(&ranges, Some(3000), &keys).unwrap(); assert_eq!( window.resolved()[0], Some(Resolved { - start: 2, + start: 2000, end: None, floored: true }) ); assert!(window.is_settled()); - let groups = Groups { + let keys = Keys { complete: true, - ..Groups::new(pending) + ..Keys::default() }; - let window = Window::new(&ranges, Some(9), &groups).unwrap(); + let window = Window::new(&ranges, Some(9), &keys).unwrap(); assert_eq!( window.resolved()[0], Some(Resolved { @@ -2048,25 +943,18 @@ mod tests { #[test] fn a_waiting_time_start_keeps_its_relative_end() { for latest in [None, Some(9)] { - let mut groups = Groups::default(); + let mut keys = Keys::default(); let ranges = [range(time(2500), Anchor::Relative(0))]; - let mut window = Window::new(&ranges, latest, &groups).unwrap(); + let mut window = Window::new(&ranges, latest, &keys).unwrap(); assert_eq!(window.is_pending(), latest.is_none()); - groups.held.extend((2..10).map(|sequence| (sequence, Held::Pending))); - window.settle(Some(9), &groups); + keys.held.extend([2000, 3000]); + assert!(window.settle(Some(3000), &keys)); assert!(!window.is_pending()); - groups.held.insert(10, Held::Pending); - assert!(!window.settle(Some(10), &groups), "nothing started"); - - groups.held.insert(2, Held::Started(at(2000))); - assert!(window.settle(Some(10), &groups)); + keys.held.insert(4000); + assert!(!window.settle(Some(4000), &keys), "resolved already"); assert_eq!( - window.resolved()[0], - Some(Resolved { - start: 2, - end: Some(10), - floored: true, - }) + window.resolved()[0].map(|resolved| (resolved.start, resolved.end)), + Some((2000, Some(latest.map_or(3001, |latest| latest + 1)))) ); } } @@ -2075,7 +963,9 @@ mod tests { /// cap: a lower floor or a wider cap reaches groups no check has seen. #[test] fn coverage_resumes_within_what_was_checked() { - let mut window = Window::new(&[Range::groups(0..2), Range::groups(4..6)], None, &Untimed).unwrap(); + let keys = Keys::new(0..6); + let ranges = [range(time(0), time(1)), range(time(4), time(5))]; + let mut window = Window::new(&ranges, None, &keys).unwrap(); let accounted = rangemap::RangeSet::from_iter([0..1, 4..6]); assert!(window.covered_by(4, None, &accounted)); assert!( @@ -2096,7 +986,7 @@ mod tests { /// behind where it stopped. #[test] fn coverage_resumes_within_a_range() { - let mut window = Window::new(&[Range::groups(0..8)], None, &Untimed).unwrap(); + let mut window = Window::new(&[range(time(0), time(7))], None, &Keys::new(0..8)).unwrap(); let mut accounted = rangemap::RangeSet::from_iter(std::iter::once(0..3)); assert!(!window.covered_by(0, None, &accounted)); assert_eq!(window.covered, Some((0, 3)), "stops at the first missing group"); @@ -2111,7 +1001,8 @@ mod tests { /// A map keyed by the groups accounted for counts as a set of them, whatever its values. #[test] fn coverage_reads_a_map_of_the_groups_accounted_for() { - let mut window = Window::new(&[Range::groups(0..2), Range::groups(4..6)], None, &Untimed).unwrap(); + let ranges = [range(time(0), time(1)), range(time(4), time(5))]; + let mut window = Window::new(&ranges, None, &Keys::new(0..6)).unwrap(); let mut accounted = rangemap::RangeMap::new(); accounted.insert(0..2, 'a'); accounted.insert(4..5, 'b'); @@ -2120,48 +1011,37 @@ mod tests { assert!(window.covered_by(0, None, &accounted)); } - #[test] - fn an_untimed_track_refuses_time_anchors() { - assert!(matches!( - Window::new(&[range(time(0), None)], None, &Untimed), - Err(Error::TimestampMismatch) - )); - assert!(matches!( - Window::new(&[range(Anchor::Relative(0), time(0))], None, &Untimed), - Err(Error::TimestampMismatch) - )); - } - /// With a handler serving the history, a time start the groups held can't place is looked - /// up and reads nothing meanwhile: before every start, or in a gap after the newest group - /// starting at or before it, even one the live feed accounted for, since an evicted group - /// still sits in the history. The answer places it; none leaves what the groups held say. + /// up and reads nothing meanwhile: before every start, or with a group the cache evicted + /// possibly starting between the newest held at or before it and the time. The answer + /// places it; none leaves what the groups held say. #[test] fn a_time_start_the_groups_cant_place_is_looked_up() { - let mut groups = Groups::new([Groups::started(4, 4000), Groups::started(9, 9000)]); - groups.gone.push(5..9); - groups.looks_up = true; - let mut window = Window::new(&[range(time(8500), None)], Some(9), &groups).unwrap(); - assert_eq!(window.lookups().collect::>(), [(at(8500), 9)]); - assert!(!window.is_settled() && !window.contains(9) && window.resolved() == [None]); - groups.held.insert(8, Held::Started(at(8000))); - assert!(window.found(at(8500), Some(8), &groups)); + let mut keys = Keys::new([4000, 9000]); + keys.looks_up = true; + let mut window = Window::new(&[range(time(8500), None)], Some(9000), &keys).unwrap(); + assert_eq!(window.lookups().collect::>(), [(8500, 9000)]); + assert!(!window.is_settled() && !window.contains(9000) && window.resolved() == [None]); + keys.held.insert(8000); + assert!(window.found(8500, Some(8000), &keys)); assert_eq!( window.resolved(), [Some(Resolved { - start: 8, + start: 8000, end: None, floored: true })] ); assert!(window.is_settled()); - let window = Window::new(&[range(time(8500), None)], Some(9), &groups).unwrap(); - assert_eq!(window.lookups().count(), 0, "group 8 is held, with 9 after it"); + let window = Window::new(&[range(time(8500), None)], Some(9000), &keys).unwrap(); + assert_eq!(window.lookups().count(), 1, "something may start between 8000 and 8500"); + let window = Window::new(&[range(time(8000), None)], Some(9000), &keys).unwrap(); + assert_eq!(window.lookups().count(), 0, "8000 starts exactly there"); - let mut window = Window::new(&[range(time(500), None)], Some(9), &groups).unwrap(); - assert_eq!(window.lookups().collect::>(), [(at(500), 4)]); - assert!(window.found(at(500), None, &groups)); + let mut window = Window::new(&[range(time(500), None)], Some(9000), &keys).unwrap(); + assert_eq!(window.lookups().collect::>(), [(500, 4000)]); + assert!(window.found(500, None, &keys)); assert_eq!( window.resolved(), [Some(Resolved { @@ -2170,24 +1050,36 @@ mod tests { floored: false })] ); - assert!(window.contains(4) && !window.missable(0).any(|run| *run.start() < 4)); + assert!(window.contains(4000) && !window.missable(0).any(|run| *run.start() < 500)); + } + + /// Keys the producer skipped hold no group, even in the history, so a start in a skipped + /// span between two held groups is placed without a lookup. + #[test] + fn a_skipped_gap_needs_no_lookup() { + let mut keys = Keys::new([0, 33, 66]); + keys.looks_up = true; + keys.skipped = vec![1..33, 34..66]; + let window = Window::new(&[range(time(50), None)], Some(66), &keys).unwrap(); + assert_eq!(window.lookups().count(), 0); + assert_eq!(window.resolved()[0].map(|resolved| resolved.start), Some(33)); } - /// A time end on a start the history couldn't place walks from the oldest group held, - /// since nothing asks for those below it: none holds the end open. + /// A time end on a start the history couldn't place ends where it says, since time ends + /// name their groups by start. #[test] - fn an_unanswered_lookup_settles_a_time_end() { - let mut groups = Groups::new([Groups::started(4, 4000), Groups::started(9, 9000)]); - groups.looks_up = true; - let mut window = Window::new(&[range(time(500), Some(time(3000)))], Some(9), &groups).unwrap(); - assert_eq!(window.lookups().collect::>(), [(at(500), 4)]); - assert!(window.found(at(500), None, &groups)); + fn an_unanswered_lookup_settles_with_its_time_end() { + let mut keys = Keys::new([4000, 9000]); + keys.looks_up = true; + let mut window = Window::new(&[range(time(500), time(3000))], Some(9000), &keys).unwrap(); + assert_eq!(window.lookups().collect::>(), [(500, 4000)]); + assert!(window.found(500, None, &keys)); assert!(window.is_settled()); assert_eq!( window.resolved(), [Some(Resolved { start: 0, - end: Some(4), + end: Some(3001), floored: false })] ); @@ -2196,14 +1088,14 @@ mod tests { /// Without a handler serving the history, the groups held decide alone. #[test] fn a_time_start_is_not_looked_up_without_a_history() { - let groups = Groups::new([Groups::started(4, 4000), Groups::started(9, 9000)]); - let window = Window::new(&[range(time(8500), None), range(time(500), None)], Some(9), &groups).unwrap(); + let keys = Keys::new([4000, 9000]); + let window = Window::new(&[range(time(8500), None), range(time(500), None)], Some(9000), &keys).unwrap(); assert_eq!(window.lookups().count(), 0); assert_eq!( window.resolved(), [ Some(Resolved { - start: 4, + start: 4000, end: None, floored: true }), diff --git a/rs/moq-net/src/model/time.rs b/rs/moq-net/src/model/time.rs index a91d5955e3e..d1c86c7710f 100644 --- a/rs/moq-net/src/model/time.rs +++ b/rs/moq-net/src/model/time.rs @@ -16,7 +16,8 @@ const fn bounded(value: u128, max: u64) -> Option { /// Returned when a [`Timestamp`] operation would exceed its range (`2^61 - 1`), a /// [`Timescale`] would exceed the QUIC varint range (`2^62 - 1`), scale conversion or -/// arithmetic would overflow, or arithmetic mixes timestamps with mismatched scales. +/// arithmetic would overflow, arithmetic mixes timestamps with mismatched scales, or a +/// conversion needs the scale an unscaled [`Timestamp`] lacks. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] #[error("time overflow")] pub struct TimeOverflow; @@ -125,16 +126,16 @@ impl std::fmt::Display for Timescale { } } -/// A timestamp in a track's timescale (units per second). +/// A timestamp in a track's timescale (units per second), or a scale-less track's group key. /// /// All timestamps within a track are relative, so zero for one track is not zero for another. -/// The underlying value is at most `2^61 - 1`, the most every moq-lite version can encode as -/// a group's first frame; the scale is carried alongside so frames from different +/// A scaled value is at most `2^61 - 1`, the most every moq-lite version can encode as a +/// group's first frame, while an unscaled key spans every group key; the scale is carried alongside so frames from different /// sources can be compared and converted without lossy detours through a single fixed scale. /// -/// The scale is a [`Timescale`] (always non-zero), so unit conversions (`as_secs`, `as_millis`, -/// etc.) are infallible. Use [`Option`] at call sites that need a "missing" sentinel -/// instead of relying on a magic value. +/// A scale-less track has no time: each frame's timestamp is [`Self::unscaled`], whose value +/// is its group's key. Anything that needs a scale ([`Self::convert`], `as_secs`, `as_millis`, +/// and so on) is [`TimeOverflow`] on one, and it orders against no scaled timestamp. /// /// # An instant, not a number /// @@ -151,16 +152,19 @@ impl std::fmt::Display for Timescale { /// - [`Eq`] / [`Hash`] are **structural** (field-wise): `from_secs(1) != from_millis(1000)`, /// because they encode as different `(value, scale)` pairs on the wire. Two timestamps are /// equal only when both their value and scale match. -/// - [`Ord`] is **temporal**: it cross-multiplies scales, so `from_millis(1000)` orders after -/// `from_millis(999)` and `from_secs(1)` slots in between. When a cross-scale comparison is -/// otherwise a tie, it breaks by `(scale, value)` to stay consistent with `Eq`. +/// - [`PartialOrd`] is **temporal**: it cross-multiplies scales, so `from_millis(1000)` orders +/// after `from_millis(999)` and `from_secs(1)` slots in between. When a cross-scale comparison +/// is otherwise a tie, it breaks by `(scale, value)` to stay consistent with `Eq`. A scaled +/// and an unscaled timestamp don't compare: `partial_cmp` is `None`, so `<` and `>` are both +/// false. There is no [`Ord`]. /// -/// So `from_secs(1).cmp(&from_millis(1000))` is *not* `Equal`, and neither is `==` true. If you -/// want "same instant regardless of encoding", compare after a [`Self::convert`] to a common scale. +/// So `from_secs(1).partial_cmp(&from_millis(1000))` is *not* `Equal`, and neither is `==` true. +/// If you want "same instant regardless of encoding", compare after a [`Self::convert`] to a +/// common scale. #[derive(Clone, Copy, PartialEq, Eq, Hash)] pub struct Timestamp { value: u64, - scale: Timescale, + scale: Option, } impl Timestamp { @@ -176,9 +180,23 @@ impl Timestamp { /// Construct a timestamp directly from a raw value at the given scale. /// Returns [`TimeOverflow`] if `value` exceeds `2^61 - 1`. pub const fn new(value: u64, scale: Timescale) -> Result { - match bounded(value as u128, MAX_TIMESTAMP) { - Some(value) => Ok(Self { value, scale }), - None => Err(TimeOverflow), + Self::bounded(value, Some(scale)) + } + + /// A scale-less track's timestamp: `key` is its group's key, with no time. + pub const fn unscaled(key: u64) -> Self { + Self { + value: key, + scale: None, + } + } + + /// `value` at `scale`, bounded to `2^61 - 1` when it carries a time. + const fn bounded(value: u64, scale: Option) -> Result { + match (scale, bounded(value as u128, MAX_TIMESTAMP)) { + (None, _) => Ok(Self { value, scale }), + (Some(_), Some(value)) => Ok(Self { value, scale }), + (Some(_), None) => Err(TimeOverflow), } } @@ -216,13 +234,13 @@ impl Timestamp { Self::new(nanos, Timescale::NANO) } - /// The raw value in the timestamp's own scale. + /// The raw value in the timestamp's own scale, or the group key when unscaled. pub const fn value(self) -> u64 { self.value } - /// The scale (units per second) attached to this timestamp. - pub const fn scale(self) -> Timescale { + /// The scale (units per second) attached to this timestamp, or `None` when unscaled. + pub const fn scale(self) -> Option { self.scale } @@ -231,17 +249,29 @@ impl Timestamp { self.value == 0 } - /// Re-express this timestamp at a new scale. Returns [`TimeOverflow`] if the new - /// value would exceed `2^61 - 1`. + /// The scale, or [`TimeOverflow`] when unscaled: a group key has no time to convert. + const fn scaled(self) -> Result, TimeOverflow> { + match self.scale { + Some(scale) => Ok(scale.0), + None => Err(TimeOverflow), + } + } + + /// Re-express this timestamp at a new scale. Returns [`TimeOverflow`] if it is unscaled, + /// or if the new value would exceed `2^61 - 1`. pub const fn convert(self, new_scale: Timescale) -> Result { - if self.scale.0.get() == new_scale.0.get() { + let scale = match self.scaled() { + Ok(scale) => scale.get(), + Err(err) => return Err(err), + }; + if scale == new_scale.0.get() { return Ok(self); } match (self.value as u128).checked_mul(new_scale.0.get() as u128) { - Some(scaled) => match bounded(scaled / self.scale.0.get() as u128, MAX_TIMESTAMP) { + Some(scaled) => match bounded(scaled / scale as u128, MAX_TIMESTAMP) { Some(value) => Ok(Self { value, - scale: new_scale, + scale: Some(new_scale), }), None => Err(TimeOverflow), }, @@ -249,39 +279,54 @@ impl Timestamp { } } - /// The value re-expressed at `target` as a `u128`. - pub const fn as_scale(self, target: Timescale) -> u128 { - self.value as u128 * target.0.get() as u128 / self.scale.0.get() as u128 + /// The value re-expressed at `target` as a `u128`, or [`TimeOverflow`] when unscaled. + pub const fn as_scale(self, target: Timescale) -> Result { + match self.scaled() { + Ok(scale) => Ok(self.value as u128 * target.0.get() as u128 / scale.get() as u128), + Err(err) => Err(err), + } } - /// The value re-expressed in seconds. - pub const fn as_secs(self) -> u64 { - self.value / self.scale.0.get() + /// The value re-expressed in seconds, or [`TimeOverflow`] when unscaled. + pub const fn as_secs(self) -> Result { + match self.scaled() { + Ok(scale) => Ok(self.value / scale.get()), + Err(err) => Err(err), + } } - /// The value re-expressed in milliseconds. - pub const fn as_millis(self) -> u128 { + /// The value re-expressed in milliseconds, or [`TimeOverflow`] when unscaled. + pub const fn as_millis(self) -> Result { self.as_scale(Timescale::MILLI) } - /// The value re-expressed in microseconds. - pub const fn as_micros(self) -> u128 { + /// The value re-expressed in microseconds, or [`TimeOverflow`] when unscaled. + pub const fn as_micros(self) -> Result { self.as_scale(Timescale::MICRO) } - /// The value re-expressed in nanoseconds. - pub const fn as_nanos(self) -> u128 { + /// The value re-expressed in nanoseconds, or [`TimeOverflow`] when unscaled. + pub const fn as_nanos(self) -> Result { self.as_scale(Timescale::NANO) } + /// Whether both carry the same scale, or both none. + const fn same_scale(self, rhs: Self) -> bool { + match (self.scale, rhs.scale) { + (Some(a), Some(b)) => a.0.get() == b.0.get(), + (None, None) => true, + _ => false, + } + } + /// Add two timestamps. Returns [`TimeOverflow`] if the sum exceeds `2^61 - 1` or /// if the scales differ. pub const fn checked_add(self, rhs: Self) -> Result { - if self.scale.0.get() != rhs.scale.0.get() { + if !self.same_scale(rhs) { return Err(TimeOverflow); } match self.value.checked_add(rhs.value) { - Some(result) => Self::new(result, self.scale), + Some(result) => Self::bounded(result, self.scale), None => Err(TimeOverflow), } } @@ -289,23 +334,45 @@ impl Timestamp { /// Subtract `rhs` from `self`. Returns [`TimeOverflow`] if `rhs > self` or if the /// scales differ. pub const fn checked_sub(self, rhs: Self) -> Result { - if self.scale.0.get() != rhs.scale.0.get() { + if !self.same_scale(rhs) { return Err(TimeOverflow); } match self.value.checked_sub(rhs.value) { - Some(result) => Self::new(result, self.scale), + Some(result) => Self::bounded(result, self.scale), None => Err(TimeOverflow), } } +} - /// Current point on the local monotonic clock, in milliseconds ([`Timescale::MILLI`]). - /// - /// A convenience for publishers stamping their own frames, and the only model API - /// that reads the local clock; drivers stamp frames from the instant they are - /// polled with instead. There is deliberately no inverse (a [`Timestamp`] is - /// relative and jittered, never a clock). - pub fn now() -> Self { - crate::model::clock::now().into() +impl Timestamp { + /// This timestamp when it carries a time, or `None` for a scale-less track's key. + pub(crate) const fn timed(self) -> Option { + match self.scale { + Some(_) => Some(self), + None => None, + } + } + + /// A received frame's time: the `timestamp` its wire carried, or on a scale-less track, + /// which carries none, its group's `key`. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn received(timestamp: Option, key: u64) -> Self { + match timestamp { + Some(timestamp) => timestamp, + None => Self::unscaled(key), + } + } + + /// The later of two timestamps of one track, which share a scale; `self` when they + /// don't compare. + pub(crate) fn later(self, other: Self) -> Self { + if other > self { other } else { self } + } + + /// The earlier of two timestamps of one track, which share a scale; `self` when they + /// don't compare. + pub(crate) fn earlier(self, other: Self) -> Self { + if other < self { other } else { self } } } @@ -317,23 +384,31 @@ impl TryFrom for Timestamp { match bounded(duration.as_nanos(), MAX_TIMESTAMP) { Some(value) => Ok(Self { value, - scale: Timescale::NANO, + scale: Some(Timescale::NANO), }), None => Err(TimeOverflow), } } } -impl From for std::time::Duration { - fn from(time: Timestamp) -> Self { - let nanos = time.as_nanos(); - std::time::Duration::new(time.as_secs(), (nanos % 1_000_000_000) as u32) +impl TryFrom for std::time::Duration { + type Error = TimeOverflow; + + /// The time since the track's zero, or [`TimeOverflow`] when unscaled. + fn try_from(time: Timestamp) -> Result { + let nanos = time.as_nanos()?; + Ok(std::time::Duration::new( + time.as_secs()?, + (nanos % 1_000_000_000) as u32, + )) } } impl std::fmt::Debug for Timestamp { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let nanos = self.as_nanos(); + let Ok(nanos) = self.as_nanos() else { + return write!(f, "#{}", self.value); + }; // Choose the largest unit where we don't need decimal places. if nanos.is_multiple_of(1_000_000_000) { @@ -349,28 +424,30 @@ impl std::fmt::Debug for Timestamp { } impl PartialOrd for Timestamp { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for Timestamp { /// Temporal comparison, normalizing across scales (see the type-level docs for how /// this relates to structural `Eq`). /// - /// - Equal scales compare raw values directly. + /// - Equal scales, or two unscaled keys, compare raw values directly. /// - Otherwise cross-multiplies in 128-bit so e.g. `1s > 2ms` orders correctly. /// - A would-be cross-scale tie (e.g. `from_secs(1)` vs `from_millis(1000)`) breaks by - /// `(scale, value)`, keeping `Ord` consistent with the field-wise `Eq`/`Hash`. - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - if self.scale.0.get() == other.scale.0.get() { - return self.value.cmp(&other.value); + /// `(scale, value)`, keeping the order consistent with the field-wise `Eq`/`Hash`. + /// - A scaled and an unscaled timestamp are `None`. + fn partial_cmp(&self, other: &Self) -> Option { + let (lhs_scale, rhs_scale) = match (self.scale, other.scale) { + (None, None) => return Some(self.value.cmp(&other.value)), + (Some(lhs), Some(rhs)) => (lhs.0.get(), rhs.0.get()), + _ => return None, + }; + if lhs_scale == rhs_scale { + return Some(self.value.cmp(&other.value)); } - let lhs = self.value as u128 * other.scale.0.get() as u128; - let rhs = other.value as u128 * self.scale.0.get() as u128; - lhs.cmp(&rhs) - .then_with(|| self.scale.0.get().cmp(&other.scale.0.get())) - .then_with(|| self.value.cmp(&other.value)) + let lhs = self.value as u128 * rhs_scale as u128; + let rhs = other.value as u128 * lhs_scale as u128; + Some( + lhs.cmp(&rhs) + .then_with(|| lhs_scale.cmp(&rhs_scale)) + .then_with(|| self.value.cmp(&other.value)), + ) } } @@ -397,35 +474,35 @@ mod tests { #[test] fn test_from_secs() { let time = Timestamp::from_secs(5).unwrap(); - assert_eq!(time.scale(), Timescale::SECOND); - assert_eq!(time.as_secs(), 5); - assert_eq!(time.as_millis(), 5000); - assert_eq!(time.as_micros(), 5_000_000); - assert_eq!(time.as_nanos(), 5_000_000_000); + assert_eq!(time.scale(), Some(Timescale::SECOND)); + assert_eq!(time.as_secs().unwrap(), 5); + assert_eq!(time.as_millis().unwrap(), 5000); + assert_eq!(time.as_micros().unwrap(), 5_000_000); + assert_eq!(time.as_nanos().unwrap(), 5_000_000_000); } #[test] fn test_from_millis() { let time = Timestamp::from_millis(5000).unwrap(); - assert_eq!(time.scale(), Timescale::MILLI); - assert_eq!(time.as_secs(), 5); - assert_eq!(time.as_millis(), 5000); + assert_eq!(time.scale(), Some(Timescale::MILLI)); + assert_eq!(time.as_secs().unwrap(), 5); + assert_eq!(time.as_millis().unwrap(), 5000); } #[test] fn test_from_micros() { let time = Timestamp::from_micros(5_000_000).unwrap(); - assert_eq!(time.scale(), Timescale::MICRO); - assert_eq!(time.as_secs(), 5); - assert_eq!(time.as_micros(), 5_000_000); + assert_eq!(time.scale(), Some(Timescale::MICRO)); + assert_eq!(time.as_secs().unwrap(), 5); + assert_eq!(time.as_micros().unwrap(), 5_000_000); } #[test] fn test_from_nanos() { let time = Timestamp::from_nanos(5_000_000_000).unwrap(); - assert_eq!(time.scale(), Timescale::NANO); - assert_eq!(time.as_secs(), 5); - assert_eq!(time.as_nanos(), 5_000_000_000); + assert_eq!(time.scale(), Some(Timescale::NANO)); + assert_eq!(time.as_secs().unwrap(), 5); + assert_eq!(time.as_nanos().unwrap(), 5_000_000_000); } #[test] @@ -478,16 +555,16 @@ mod tests { fn test_convert_to_finer() { let time_ms = Timestamp::from_millis(5000).unwrap(); let time_us = time_ms.convert(Timescale::MICRO).unwrap(); - assert_eq!(time_us.scale(), Timescale::MICRO); - assert_eq!(time_us.as_micros(), 5_000_000); + assert_eq!(time_us.scale(), Some(Timescale::MICRO)); + assert_eq!(time_us.as_micros().unwrap(), 5_000_000); } #[test] fn test_convert_to_coarser() { let time_ms = Timestamp::from_millis(5000).unwrap(); let time_s = time_ms.convert(Timescale::SECOND).unwrap(); - assert_eq!(time_s.scale(), Timescale::SECOND); - assert_eq!(time_s.as_secs(), 5); + assert_eq!(time_s.scale(), Some(Timescale::SECOND)); + assert_eq!(time_s.as_secs().unwrap(), 5); } #[test] @@ -495,7 +572,7 @@ mod tests { // 1234 ms = 1.234 s, rounds down to 1 s let time_ms = Timestamp::from_millis(1234).unwrap(); let time_s = time_ms.convert(Timescale::SECOND).unwrap(); - assert_eq!(time_s.as_secs(), 1); + assert_eq!(time_s.as_secs().unwrap(), 1); } #[test] @@ -519,8 +596,8 @@ mod tests { let a = Timestamp::from_millis(1000).unwrap(); let b = Timestamp::from_millis(2000).unwrap(); let c = a.checked_add(b).unwrap(); - assert_eq!(c.as_millis(), 3000); - assert_eq!(c.scale(), Timescale::MILLI); + assert_eq!(c.as_millis().unwrap(), 3000); + assert_eq!(c.scale(), Some(Timescale::MILLI)); } #[test] @@ -544,7 +621,7 @@ mod tests { let zero_ms = Timestamp::from_millis(0).unwrap(); assert!(zero_ms.is_zero()); assert_ne!(Timestamp::ZERO, zero_ms); - assert_ne!(Timestamp::ZERO.cmp(&zero_ms), std::cmp::Ordering::Equal); + assert_ne!(Timestamp::ZERO.partial_cmp(&zero_ms), Some(std::cmp::Ordering::Equal)); } #[test] @@ -573,9 +650,9 @@ mod tests { // Temporally-equivalent timestamps with different representations are NOT // Equal under cmp: derived Eq compares fields, and Ord must agree. let one_sec_b = Timestamp::from_millis(1000).unwrap(); - assert_ne!(one_sec.cmp(&one_sec_b), std::cmp::Ordering::Equal); + assert_ne!(one_sec.partial_cmp(&one_sec_b), Some(std::cmp::Ordering::Equal)); assert_ne!(one_sec, one_sec_b); - assert_eq!(one_sec.cmp(&one_sec), std::cmp::Ordering::Equal); + assert_eq!(one_sec.partial_cmp(&one_sec), Some(std::cmp::Ordering::Equal)); // Mixed-scale sort lands in correct temporal order. let mut items = [ @@ -583,7 +660,7 @@ mod tests { Timestamp::from_millis(500).unwrap(), Timestamp::from_micros(1_500_000).unwrap(), ]; - items.sort(); + items.sort_by(|a, b| a.partial_cmp(b).unwrap()); assert_eq!(items[0], Timestamp::from_millis(500).unwrap()); assert_eq!(items[1], Timestamp::from_micros(1_500_000).unwrap()); assert_eq!(items[2], Timestamp::from_secs(2).unwrap()); @@ -593,10 +670,10 @@ mod tests { fn test_duration_conversion() { let duration = std::time::Duration::from_secs(5); let time: Timestamp = duration.try_into().unwrap(); - assert_eq!(time.scale(), Timescale::NANO); - assert_eq!(time.as_secs(), 5); + assert_eq!(time.scale(), Some(Timescale::NANO)); + assert_eq!(time.as_secs().unwrap(), 5); - let duration_back: std::time::Duration = time.into(); + let duration_back: std::time::Duration = time.try_into().unwrap(); assert_eq!(duration_back.as_secs(), 5); } @@ -615,6 +692,36 @@ mod tests { assert_eq!(format!("{:?}", t), "1ms"); } + /// A scale-less key carries no time: anything needing a scale fails, it orders only + /// against other keys, and arithmetic stays among keys. + #[test] + fn test_unscaled_has_no_time() { + let key = Timestamp::unscaled(7); + assert_eq!((key.value(), key.scale()), (7, None)); + assert!(key.convert(Timescale::MILLI).is_err()); + assert!(key.as_millis().is_err()); + assert!(key.as_secs().is_err()); + assert!(std::time::Duration::try_from(key).is_err()); + // A key spans every group key, past where a time stops. + assert_eq!(Timestamp::unscaled(u64::MAX - 1).value(), u64::MAX - 1); + assert!( + Timestamp::unscaled(u64::MAX) + .checked_add(Timestamp::unscaled(1)) + .is_err() + ); + + let millis = Timestamp::from_millis(7).unwrap(); + assert_ne!(key, millis); + assert_eq!(key.partial_cmp(&millis), None); + assert_eq!(millis.partial_cmp(&key), None, "incomparable either way"); + assert!(key.checked_add(millis).is_err()); + + let next = Timestamp::unscaled(8); + assert!(key < next); + assert_eq!(key.checked_add(Timestamp::unscaled(1)).unwrap(), next); + assert_eq!(format!("{key:?}"), "#7"); + } + #[test] fn test_custom_scale_convert() { // 120 units at 60Hz = 2 seconds, expressed at 1000Hz = 2000 ms. @@ -623,7 +730,7 @@ mod tests { .unwrap() .convert(Timescale::MILLI) .unwrap(); - assert_eq!(t.scale(), Timescale::MILLI); - assert_eq!(t.as_millis(), 2000); + assert_eq!(t.scale(), Some(Timescale::MILLI)); + assert_eq!(t.as_millis().unwrap(), 2000); } } diff --git a/rs/moq-net/src/model/timed.rs b/rs/moq-net/src/model/timed.rs deleted file mode 100644 index 71bdc7f7b69..00000000000 --- a/rs/moq-net/src/model/timed.rs +++ /dev/null @@ -1,63 +0,0 @@ -use bytes::Bytes; - -use crate::Timestamp; - -/// A value to publish, and optionally when it was captured. -/// -/// Converts from a bare payload (bytes, or a `&V` to serialize), which stays untimed. -#[derive(Debug, Clone, Copy)] -pub struct Timed

{ - /// The value to publish. - pub value: P, - - /// When the value was captured. `None` leaves it untimed. - pub at: Option, -} - -impl

Timed

{ - /// Set when the value was captured. - pub fn at(mut self, at: Timestamp) -> Self { - self.at = Some(at); - self - } -} - -impl> From for Timed { - fn from(value: B) -> Self { - Self { - value: value.into(), - at: None, - } - } -} - -impl<'a, V> From<&'a V> for Timed<&'a V> { - fn from(value: &'a V) -> Self { - Self { value, at: None } - } -} - -impl From for Timed { - fn from(value: u64) -> Self { - Self { value, at: None } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_bare_payload_converts_untimed() { - let timed: Timed = vec![1u8, 2].into(); - assert_eq!(timed.value, Bytes::from_static(&[1, 2])); - assert_eq!(timed.at, None); - - let stamped = Timed::from(&b"x"[..]).at(Timestamp::from_millis(5).unwrap()); - assert_eq!(stamped.at, Some(Timestamp::from_millis(5).unwrap())); - - let value = 7u32; - let timed: Timed<&u32> = (&value).into(); - assert_eq!(*timed.value, 7); - } -} diff --git a/rs/moq-net/src/model/track.rs b/rs/moq-net/src/model/track.rs index c60d07284aa..fe00dfecd4f 100644 --- a/rs/moq-net/src/model/track.rs +++ b/rs/moq-net/src/model/track.rs @@ -1,9 +1,9 @@ //! A track is a collection of semi-reliable and semi-ordered streams, split into a [Producer] and [Subscriber] handle. //! -//! A [Producer] creates streams with a sequence number and priority. -//! The sequence number is used to determine the order of streams, while the priority is used to determine which stream to transmit first. +//! A [Producer] creates streams with a key and priority. +//! The key is used to determine the order of streams, while the priority is used to determine which stream to transmit first. //! This may seem counter-intuitive, but is designed for live streaming where the newest streams may be higher priority. -//! A cloned [Producer] can be used to create streams in parallel, but will error if a duplicate sequence number is used. +//! A cloned [Producer] can be used to create streams in parallel, but will error if a duplicate key is used. //! //! A [Subscriber] may not receive all streams in order or at all. //! These streams are meant to be transmitted over congested networks and the key to MoQ Transport is to not block on them. @@ -23,7 +23,7 @@ use super::record::{Pending, Record, Report}; use crate::ledger::{self, Ledger}; pub(crate) use super::subscription::Resolved; -use super::subscription::{Accounted, Containing, Held, Holdings, Layered, Runs, Seen, Window, later}; +use super::subscription::{Accounted, Containing, Holdings, Layered, Runs, Window}; pub use super::subscription::{Anchor, Delay, Order, Position, Range, Subscription}; use rangemap::{RangeInclusiveSet, RangeMap, RangeSet}; @@ -280,21 +280,30 @@ impl Info { /// `timestamp` at this track's [`timescale`](Self::timescale). /// - /// A track is all timed or all untimed, so a frame must match it: a timed frame on an - /// untimed track, or an untimed one on a timed track, is [`Error::TimestampMismatch`], - /// as is a timestamp the track's scale can't hold. A writer that changes state before - /// its frame reaches the track, such as signing or encrypting it, rescales first so a - /// refused timestamp changes nothing. - pub fn rescale(&self, timestamp: Option) -> Result> { - match (timestamp, self.timescale) { - (Some(timestamp), Some(timescale)) => Ok(Some( - timestamp.convert(timescale).map_err(|_| Error::TimestampMismatch)?, - )), - (None, None) => Ok(None), + /// A track is all timed or all scale-less, so a frame must match it: a scaled timestamp + /// on a scale-less track, or an [unscaled](Timestamp::unscaled) one on a timed track, is + /// [`Error::TimestampMismatch`], as is a timestamp the track's scale can't hold. A writer + /// that changes state before its frame reaches the track, such as signing or encrypting + /// it, rescales first so a refused timestamp changes nothing. + pub fn rescale(&self, timestamp: Timestamp) -> Result { + match (timestamp.scale(), self.timescale) { + (Some(_), Some(timescale)) => timestamp.convert(timescale).map_err(|_| Error::TimestampMismatch), + (None, None) => Ok(timestamp), _ => Err(Error::TimestampMismatch), } } + /// [`Self::rescale`] for a frame of group `key`: a scale-less track's frame carries its + /// group's key, so any other is [`Error::TimestampMismatch`]. A writer that changes state + /// before its frame reaches the group checks this first, as with [`Self::rescale`]. + pub fn rescale_in(&self, timestamp: Timestamp, key: u64) -> Result { + let timestamp = self.rescale(timestamp)?; + match timestamp.scale() { + None if timestamp.value() != key => Err(Error::TimestampMismatch), + _ => Ok(timestamp), + } + } + /// The original moq-transport publisher's draft, whose timestamps may rewind, or /// `None` for a native moq-lite track. pub(crate) fn ietf(&self) -> Option { @@ -509,7 +518,7 @@ enum Start { impl Start { /// The first group the live feed serves, or the floor asked while that is pending. - fn sequence(self) -> Option { + fn key(self) -> Option { match self { Self::Unknown => None, Self::Pending(floor) => floor, @@ -529,12 +538,9 @@ enum Anchors { /// still in flight below it (see [`End::tail`]): its readers' relative and time anchors are /// refused (see [`Holdings::drops_anchors`]). Refused, - /// A received copy whose wire tells it what relative and time anchors resolve against, - /// so they resolve here as at the publisher. + /// A received copy whose wire tells it what relative anchors count back from, and carries + /// time anchors as group starts, so they resolve here as at the publisher. Wire, - /// A received copy that can translate only time anchors its cache has already settled. A - /// relative one waits for the live edge its wire reports, as under [`Self::Wire`]. - Cached, } impl Anchors { @@ -568,13 +574,13 @@ pub(crate) struct TrackState { // track's largest position once it was reported: anything received past that outdates it. object_edge: Option<((u64, u64), Position)>, - // Cached groups by sequence: the single source of truth for what is cached. The - // two orderings below hold bare sequences and validate against this map, so a + // Cached groups by key: the single source of truth for what is cached. The + // two orderings below hold bare keys and validate against this map, so a // removed or replaced group turns their entries into discarded-on-pop hints. // Ordered so an in-range subscriber can seek to its cursor instead of scanning // every cached group for each delivery. lookup: BTreeMap, - // The sequences in `lookup` the live feed holds ([`Slot::visible`]): the searches for + // The keys in `lookup` the live feed holds ([`Slot::visible`]): the searches for // live content walk these, so fetched backfill between them costs nothing. visible: BTreeSet, contents: group::Contents, @@ -587,23 +593,23 @@ pub(crate) struct TrackState { // the readers riding them; see [`Self::board`]. Held weakly, so the last rider leaving // frees one, and pruned as they are looked up. flights: BTreeMap>, - // The copies of each sequence still arriving that another received copy waits on, once per + // The copies of each key still arriving that another received copy waits on, once per // [`Hold`], which a reader of that copy waits out when it fails; see [`successor`]. holds: BTreeMap>, - // Publisher-produced groups in arrival order as (sequence, stamp), walked by + // Publisher-produced groups in arrival order as (key, stamp), walked by // subscriptions; an entry only resolves while its stamp matches the slot's. // Fetched backfill (`insert_group_request`) is deliberately absent: it is - // served by sequence, never replayed to arrival-order subscribers. A fetched + // served by key, never replayed to arrival-order subscribers. A fetched // head of a live group is the exception, served through that group's entry. arrival: VecDeque<(u64, u32)>, - // Eviction order under memory pressure as (sequence, stamp): every cached + // Eviction order under memory pressure as (key, stamp): every cached // group except the protected latest. `pay_debt` scans victims from the front; // groups accessed more recently than the pool-wide average rotate to the back // instead of dying, decoupling eviction order from arrival order. Entries are // hints that only resolve while their stamp matches the slot's, so a re-served - // sequence can't accumulate duplicate hints that alias its replacement. + // key can't accumulate duplicate hints that alias its replacement. // Eviction is deliberately approximate: a bounded scan per write. evict: VecDeque<(u64, u32)>, @@ -617,33 +623,28 @@ pub(crate) struct TrackState { // absolute cursor to an index. offset: usize, - // The highest sequence number successfully appended to the track, which can run ahead + // The highest key successfully appended to the track, which can run ahead // of any cached group. - max_sequence: Option, + max_key: Option, // The newest group the publisher produced, which fetched backfill never moves: where // relative starts count back from (see `latest_live`). newest_produced: Option, - // The checked starts cached groups hold, by sequence (see [`group::Origin::Bound`]), so a + // The checked starts cached groups hold, by key (see [`group::Origin::Bound`]), so a // first frame finds its neighbors without walking groups not yet stamped. Filled through // the read guard a first frame holds, so it needs its own lock, never contended. bounds: Mutex>, - // Where each cached group's frame 0 presented, by sequence, received groups too: what - // a time anchor resolves against without walking groups that have not started. Filled - // through the read guard a first frame holds, like `bounds`. - starts: Mutex>, - - // The sequence and start of the highest-sequence group that left the cache holding a - // checked start (see [`group::Origin::Bound`]). Eviction is not a sequence prefix, so a + // The key and start of the highest-key group that left the cache holding a + // checked start (see [`group::Origin::Bound`]). Eviction is not a key prefix, so a // group above it is checked against it as well as its cached neighbors; one at or below // it may have lost its lower bound. One entry, so nothing grows with the cache. evicted: Option<(u64, Timestamp)>, - // The sequence of the newest cached group: the live edge, protected from + // The key of the newest cached group: the live edge, protected from // eviction by never entering the eviction order until no producer remains. - // Tracked separately from `max_sequence`, which a reserved sequence can push past + // Tracked separately from `max_key`, which a reserved key can push past // any cached group. latest_group: Option, @@ -655,7 +656,7 @@ pub(crate) struct TrackState { // last group does not lose where the track ended. outcome: Outcome, - // The sequences the live feed delivered, and the drops a reclaim forgot: what a pending + // The keys the live feed delivered, and the drops a reclaim forgot: what a pending // tail is judged by. Fetched copies never count, and eviction never touches it. A gap folds // away a subscription's grace after a later arrival opened it, as the session's tail // does, so this holds only the gaps still worth waiting for. @@ -665,7 +666,7 @@ pub(crate) struct TrackState { // pool would. arrivals_bill: cache::Extra, - // The first sequence the live feed serves, as the publisher declared it (the wire's + // The first key the live feed serves, as the publisher declared it (the wire's // SUBSCRIBE_START). Lower groups are not waited for, though one may still arrive late // or be fetched. start: Start, @@ -677,17 +678,26 @@ pub(crate) struct TrackState { // budget, but neither this copy's timestamps nor a fill can judge an older group against // a reader's budget. A widened budget reaches lower groups through the feed's re-offer. fills_from_start: bool, - // Sequences the upstream publisher gave up, with why: an incoming SUBSCRIBE_SKIP or + // Whether readers start a time anchor at the group starting there, held or not: the + // wire's publisher starts one at the first group at or after it, and a copy holding part + // of the track cannot tell which earlier group contains the time. + starts_at_keys: bool, + // Where the upstream publisher started each of a lite-07 copy's live wire ranges, by the + // time it asked from: a copy holding part of the track cannot tell which group contains + // a time it holds no group at, but its publisher reports where it resolved one. Replaced + // whole as the ranges change, so it holds at most one entry per live range. + time_starts: BTreeMap, + // Keys the upstream publisher gave up, with why: an incoming SUBSCRIBE_SKIP or // SUBSCRIBE_DROP, or a producer cut. Kept apart from the cache so reclaiming a slot // cannot erase a disposition a relay must forward, and as ranges so a wide one costs // one entry. drops: RangeMap, // What this track's own publisher produced since its last `append_group`, which skips - // every sequence below it that never arrived and was not given up: the producer moved on. + // every key below it that never arrived and was not given up: the producer moved on. // Arrivals cannot say so, since a gap folds into them once nobody waits for it. Only - // sequences at or above `appended` are kept, so it holds the runs created since then. + // keys at or above `appended` are kept, so it holds the runs created since then. produced: RangeSet, - // One past the last `append_group`: every sequence below it arrived, was given up, or + // One past the last `append_group`: every key below it arrived, was given up, or // was skipped. appended: Option, // Every group the ranges of a received copy's current upstream terms name, which its @@ -710,7 +720,7 @@ pub(crate) struct TrackState { // What `drops` and `drops_born` bill to the pool: [`RUN_OVERHEAD`] and [`BORN_OVERHEAD`] // per entry. Paying // eviction debt sheds the lowest runs, and the expiry scan forgets those older than the - // pool's window; either leaves their sequences unknown again. + // pool's window; either leaves their keys unknown again. drops_bill: cache::Extra, // The runs reclaimed from `drops`, for readers that judged them to judge again. forgotten: miss::Forgotten, @@ -761,7 +771,7 @@ pub(crate) struct TrackState { } /// The starts a group's first frame must fall between: a timed track never holds a group -/// whose start is earlier than the one before it. Read under the track lock and checked +/// whose start is at or before the one before it. Read under the track lock and checked /// under it, so no two groups stamped concurrently can each accept a rewind. #[derive(Clone, Copy, Debug, Default)] pub(super) struct Neighbors { @@ -772,11 +782,10 @@ pub(super) struct Neighbors { } impl Neighbors { - /// Refuse a `start` that rewinds past a neighbor's with [`Error::TimestampMismatch`], - /// or say how the group holds it. An equal start is allowed: a keyframe may land at the - /// previous group's start. + /// Refuse a `start` that rewinds past a neighbor's, or ties one, with + /// [`Error::TimestampMismatch`], or say how the group holds it: starts strictly increase. pub(super) fn admit(&self, start: Timestamp) -> Result { - if self.floor.is_some_and(|floor| start < floor) || self.ceiling.is_some_and(|ceiling| start > ceiling) { + if self.floor.is_some_and(|floor| start <= floor) || self.ceiling.is_some_and(|ceiling| start >= ceiling) { return Err(Error::TimestampMismatch); } Ok(if self.late { @@ -796,7 +805,7 @@ struct Slot { group: group::Handle, // Incarnation stamp, echoed by this slot's arrival entry (if any). A re-served - // sequence (an aborted group re-created by the publisher or re-fetched as + // key (an aborted group re-created by the publisher or re-fetched as // backfill) gets a fresh stamp, so a historical arrival entry can't resolve to // the replacement and deliver it twice or at the wrong position. stamp: u32, @@ -967,27 +976,6 @@ impl Slot { } } - /// Where the group starts in presentation time, from a live copy holding frame 0. A - /// copy joined mid-group stamps a later frame, which proves nothing about the start. - /// - /// Only for time anchors. `unverified` counts a start accepted below the eviction - /// watermark too, which a track whose history a local handler serves trusts (see - /// [`Holdings::looks_up`]): no remote publisher wrote it. - fn start(&self, unverified: bool) -> Option { - self.copies() - .filter(|group| group.live_first_frame() == Some(0)) - .find_map(|group| match unverified { - true => group.first_timestamp(), - false => group.timestamp(), - }) - } - - /// Whether the group has no [`Self::start`] yet but may still get one: a copy holding - /// frame 0 has not presented it. - fn may_start(&self, unverified: bool) -> bool { - self.start(unverified).is_none() && self.copies().any(group::Handle::may_start) - } - /// The checked start of whichever copy began at frame 0. fn bound(&self) -> Option { self.copies().find_map(group::Handle::bound) @@ -1004,7 +992,7 @@ impl Slot { /// Heap the track keeps per cached group, excluding the group itself /// ([`group::CACHE_OVERHEAD`]). /// -/// One [`Slot`] under its sequence in `lookup`, its sequence in `visible`, its start in +/// One [`Slot`] under its key in `lookup`, its key in `visible`, its start in /// `starts` once frame 0 arrives, plus a hint in each of `arrival` and `evict`. Doubled /// because the containers run half empty in the worst case: a /// `BTreeMap` node sits between half and fully packed, and a `VecDeque` holds up to @@ -1024,13 +1012,9 @@ pub(super) const RUN_OVERHEAD: u64 = /// [`RUN_OVERHEAD`] is. pub(super) const BORN_OVERHEAD: u64 = 2 * size_of::<(ops::Range, u64)>() as u64; -/// Where the groups this hop holds start, for resolving time anchors: ordered lookups in -/// the starts index and the cache, so no query walks more than the run it answers. +/// The groups this hop holds, for resolving relative and time anchors: ordered lookups in +/// the cache, so no query walks more than the groups it answers. impl Holdings for TrackState { - fn timed(&self) -> bool { - self.info.as_ref().is_none_or(|info| info.timescale.is_some()) - } - fn drops_anchors(&self) -> bool { self.anchors == Anchors::Refused } @@ -1040,138 +1024,92 @@ impl Holdings for TrackState { !self.anchors.received() && self.fills.read().has_handlers() } - fn containing(&self, pts: Timestamp) -> Containing { - let starts = self.starts.lock().unwrap_or_else(PoisonError::into_inner); - let (Some((&first, &first_at)), Some((&last, &last_at))) = (starts.first_key_value(), starts.last_key_value()) - else { - return Containing::Unknown; - }; - if later(first_at, pts) { - return Containing::Before; + fn containing(&self, time: u64) -> Containing { + if self.starts_at_keys { + return Containing::Group(time); } - if !later(last_at, pts) { - return Containing::Group(last); + if self.anchors == Anchors::Wire { + return self.containing_received(time); } - // Timestamps never rewind, so the starts ascend with the sequence: bisect the - // sequences between the newest start known at or before `pts` and one known after. - let (mut best, mut low, mut high) = (first, first, last); - while high - low > 1 { - let mid = low + (high - low) / 2; - match starts.range(low + 1..=mid).next_back() { - Some((&sequence, &at)) if later(at, pts) => high = sequence, - Some((&sequence, _)) => (best, low) = (sequence, mid), - None => low = mid, - } + match self.lookup.range(..=time).next_back() { + Some((&key, _)) => Containing::Group(key), + // Every key at or before it is gone, so whichever group held it will never arrive. + None if self.holes_cover(0..time.saturating_add(1), |_| true) => Containing::Before, + None if self.lookup.is_empty() => Containing::Unknown, + None => Containing::Before, } - Containing::Group(best) } - fn next_start(&self, from: u64) -> Option<(u64, Timestamp)> { - let starts = self.starts.lock().unwrap_or_else(PoisonError::into_inner); - starts.range(from..).next().map(|(&sequence, &at)| (sequence, at)) + fn next_start(&self, from: u64) -> Option { + self.lookup.range(from..).next().map(|(&key, _)| key) } - fn next_held(&self, from: u64) -> Option<(u64, Held)> { - let (&sequence, slot) = self.lookup.range(from..).next()?; - let held = match slot.start(self.looks_up()) { - Some(at) => Held::Started(at), - None if slot.may_start(self.looks_up()) => Held::Pending, - None => Held::Never, + fn back(&self, latest: u64, back: u64) -> Option { + let Some(back) = back.checked_sub(1) else { + return Some(latest); }; - Some((sequence, held)) - } - - fn gone(&self, sequence: u64) -> Option { - let dropped = self.drops.get_key_value(&sequence).map(|(dropped, _)| dropped.end); - // The history a handler serves may still hold what the live feed never delivered, or - // delivered and the cache evicted, starting anywhere: only a drop, or a reader's own - // refused fills (see `ReaderHoldings`), says it is gone. - if self.looks_up() { - return dropped; - } - // Without a declared start, the first arrival stands in for it, as the tail's does. - let floor = self.start.sequence().or(self.arrivals.first()).unwrap_or(0); - if sequence < floor { - return Some(floor); - } - dropped.or_else(|| self.arrivals.run_end(sequence)) + // Bounded by the cache: a huge count walks it once and reaches back to the oldest. + let back = usize::try_from(back).ok()?; + self.lookup.range(..latest).rev().nth(back).map(|(&key, _)| key) } fn complete(&self) -> bool { self.outcome.is_final() } - fn changes(&self, seen: &mut Seen) -> Option>> { - let global = [ - self.start.sequence().map_or(0, |start| start.saturating_add(1)), - u64::from(self.complete()), - self.fetched, - // A handler appearing or leaving changes which starts count and what is gone. - u64::from(self.looks_up()), - ]; - let (ledger, drops) = (self.arrivals.changes(), &self.drops_changes); - let changed = (seen.global == Some(global)) - .then(|| ledger.since(seen.runs[0]).chain(drops.since(seen.runs[1])).collect()); - *seen = Seen { - runs: [ledger.revision(), drops.revision()], - global: Some(global), - }; - changed + fn never_produced(&self, keys: ops::Range) -> bool { + // Skip runs coalesce, so a gap between two held groups is one run unless the cache + // evicted a group inside it. + self.holes_cover(keys, |hole| *hole == Hole::Skip) } } -/// Where a cursor's groups start, for resolving its time anchors: its track's, with the -/// runs its own fills refused or it gave up counted gone, so a time end walking history a -/// handler serves passes what that history lacks. -struct ReaderHoldings<'a> { - state: &'a TrackState, - misses: Option<&'a miss::Misses>, - /// What the reader's own fills gave up since its window last took it in (see - /// [`miss::Misses::take_gone`]): changes a parked time end wakes on beside the track's. - newly_gone: RangeSet, -} - -impl Holdings for ReaderHoldings<'_> { - fn timed(&self) -> bool { - self.state.timed() - } - - fn drops_anchors(&self) -> bool { - self.state.drops_anchors() - } - - fn looks_up(&self) -> bool { - self.state.looks_up() - } - - fn containing(&self, pts: Timestamp) -> Containing { - self.state.containing(pts) - } - - fn next_start(&self, from: u64) -> Option<(u64, Timestamp)> { - self.state.next_start(from) - } - - fn next_held(&self, from: u64) -> Option<(u64, Held)> { - self.state.next_held(from) - } - - fn gone(&self, sequence: u64) -> Option { - self.state.gone(sequence).or_else(|| self.misses?.gone(sequence)) - } - - fn complete(&self) -> bool { - self.state.complete() +impl TrackState { + /// [`Holdings::containing`] on a lite-07 copy, which holds only part of the track: a group + /// held before `time` contains it only once nothing between them is still to arrive, so + /// until then it waits for the group starting there, the gap's holes, a group past it, + /// or where the publisher reports starting the range it asked from `time`. A group past + /// it says the feed went by: one between still in flight, a stream overtaken, would + /// only add that group to the reader's range. + fn containing_received(&self, time: u64) -> Containing { + if self.lookup.contains_key(&time) { + return Containing::Group(time); + } + match self.time_starts.get(&time) { + Some(&start) if start <= time => return Containing::Group(start), + // The publisher serves nothing at or before it, as it does for a time older than + // every group it holds. + Some(_) => return Containing::Before, + None => {} + } + match self.lookup.range(..=time).next_back() { + Some((&key, _)) + if self.holes_cover(key.saturating_add(1)..time.saturating_add(1), |_| true) + || self.lookup.range(time..).next().is_some() => + { + Containing::Group(key) + } + None if self.holes_cover(0..time.saturating_add(1), |_| true) => Containing::Before, + _ => Containing::Unknown, + } } - fn changes(&self, seen: &mut Seen) -> Option>> { - let mut changed = self.state.changes(seen)?; - changed.extend(self.newly_gone.iter().cloned()); - Some(changed) + /// Whether drops of a kind `kind` accepts cover every key in `keys`. An empty run is + /// covered, though it may overlap a drop around its start. + fn holes_cover(&self, keys: ops::Range, kind: impl Fn(&Hole) -> bool) -> bool { + if keys.is_empty() { + return true; + } + let mut at = keys.start; + for (run, hole) in self.drops.overlapping(&keys) { + if !kind(hole) || run.start > at { + return false; + } + at = run.end; + } + at >= keys.end } -} -impl TrackState { /// The upstream publisher's Largest Object, unless a frame or group received since its /// report outdates it, even one the cache has since evicted. fn object_edge(&self) -> Option<(u64, u64)> { @@ -1182,7 +1120,7 @@ impl TrackState { .then_some(edge) } - /// A consumer joined at the live edge: ask for a group past every sequence written when + /// A consumer joined at the live edge: ask for a group past every key written when /// the newest group's bytes after its first frame exceed [`Info::refresh`] times that /// frame. /// @@ -1209,17 +1147,17 @@ impl TrackState { Self::refresh(state); } - /// A reader of group `sequence` reached a frame its copy shed: ask for a new group as + /// A reader of group `key` reached a frame its copy shed: ask for a new group as /// a join over [`Info::refresh`] does, if it is the newest and the track opts in. An /// older group's reader moves on to the groups after it instead. - pub(super) fn shed_read(state: kio::Ref<'_, TrackState>, sequence: u64) { + pub(super) fn shed_read(state: kio::Ref<'_, TrackState>, key: u64) { let opted = state.info.as_ref().is_some_and(|info| info.refresh.is_some()); - if opted && state.latest_group == Some(sequence) { + if opted && state.latest_group == Some(key) { Self::refresh(state); } } - /// Ask for a group past every sequence written, releasing the read guard before + /// Ask for a group past every key written, releasing the read guard before /// locking the demand. fn refresh(state: kio::Ref<'_, TrackState>) { let subscriptions = state.subscriptions.clone(); @@ -1228,14 +1166,14 @@ impl TrackState { wakes.wake(); } - /// The starts a first frame of group `sequence` must fall between: the nearest checked - /// starts cached on either side, since groups land out of sequence order, and the + /// The starts a first frame of group `key` must fall between: the nearest checked + /// starts cached on either side, since groups land out of key order, and the /// eviction watermark. A group at or below the watermark is late: its floor may be gone, /// so the watermark caps it instead. /// /// Usually one group lookup each way; only unstarted or partial neighbors are skipped. /// `None` on an [unchecked](Info::unchecked) track, whose timestamps may rewind. - pub(super) fn neighbors(&self, sequence: u64) -> Option { + pub(super) fn neighbors(&self, key: u64) -> Option { if self .info .as_ref() @@ -1244,20 +1182,26 @@ impl TrackState { return None; } let bounds = self.bounds.lock().unwrap_or_else(PoisonError::into_inner); - let below = bounds.range(..sequence).next_back().map(|(_, start)| *start); + let below = bounds.range(..key).next_back().map(|(_, start)| *start); let above = bounds - .range((Bound::Excluded(sequence), Bound::Unbounded)) + .range((Bound::Excluded(key), Bound::Unbounded)) .next() .map(|(_, start)| *start); drop(bounds); Some(match self.evicted { - Some((evicted, start)) if sequence <= evicted => Neighbors { + // The evicted group itself, fetched again, starts where it did. + Some((evicted, _)) if key == evicted => Neighbors { + floor: below, + ceiling: above, + late: true, + }, + Some((evicted, start)) if key < evicted => Neighbors { floor: below, - ceiling: Some(above.map_or(start, |above| above.min(start))), + ceiling: Some(above.map_or(start, |above| above.earlier(start))), late: true, }, Some((_, start)) => Neighbors { - floor: Some(below.map_or(start, |below| below.max(start))), + floor: Some(below.map_or(start, |below| below.later(start))), ceiling: above, late: false, }, @@ -1269,63 +1213,46 @@ impl TrackState { }) } - /// Group `sequence` wrote its first frame: index where it starts, and share a - /// just-checked `bound` with its neighbors. - pub(super) fn first_frame(&self, sequence: u64, bound: Option) { + /// Group `key` wrote its first frame: share a just-checked `bound` with its neighbors. + pub(super) fn first_frame(&self, key: u64, bound: Option) { if let Some(bound) = bound { - self.share_bound(sequence, bound); + self.share_bound(key, bound); } - self.index_start(sequence); - } - - /// Index where cached group `sequence` starts, read back through its slot so only the - /// cached copy holding frame 0 counts, and wake the reads a time end parks on it. On a - /// track whose history a handler serves, a start accepted unverified counts: the fills - /// below the eviction watermark that a time range walks are all unverified. - fn index_start(&self, sequence: u64) { - let Some(start) = self.lookup.get(&sequence).and_then(|slot| slot.start(self.looks_up())) else { - return; - }; - self.starts - .lock() - .unwrap_or_else(PoisonError::into_inner) - .insert(sequence, start); - self.cache.wakes().started(sequence); } - /// Record group `sequence`'s just-checked `start` as a bound for its neighbors. A group + /// Record group `key`'s just-checked `start` as a bound for its neighbors. A group /// stamped after it left the cache bounds nothing. - fn share_bound(&self, sequence: u64, start: Timestamp) { - if self.lookup.get(&sequence).and_then(Slot::bound) != Some(start) { + fn share_bound(&self, key: u64, start: Timestamp) { + if self.lookup.get(&key).and_then(Slot::bound) != Some(start) { return; } self.bounds .lock() .unwrap_or_else(PoisonError::into_inner) - .insert(sequence, start); + .insert(key, start); } - /// Take group `sequence` out of the cache, raising the eviction watermark to its start. - fn uncache(&mut self, sequence: u64) -> Option { - let slot = self.remove_slot(sequence)?; + /// Take group `key` out of the cache, raising the eviction watermark to its start. + fn uncache(&mut self, key: u64) -> Option { + let slot = self.remove_slot(key)?; if let Some(start) = slot.bound() - && self.evicted.is_none_or(|(evicted, _)| sequence > evicted) + && self.evicted.is_none_or(|(evicted, _)| key > evicted) { - self.evicted = Some((sequence, start)); + self.evicted = Some((key, start)); } Some(slot) } /// [`Self::uncache`] a servable group to free the cache and abort it with `err`, recording /// it for the readers that judged it cached to judge again. - fn evict_group(&mut self, sequence: u64, err: Error) { - let slot = self.uncache(sequence).expect("a cached group"); + fn evict_group(&mut self, key: u64, err: Error) { + let slot = self.uncache(key).expect("a cached group"); slot.abort(err); let landed = self .inboxes .get_mut() .unwrap_or_else(PoisonError::into_inner) - .evicted(sequence, slot.cached); + .evicted(key, slot.cached); if landed { self.evictions += 1; } @@ -1333,8 +1260,8 @@ impl TrackState { /// [`Self::uncache`] a group that another copy displaces rather than one that ends, so a /// teardown still ends the copies its readers hold. - fn displace(&mut self, sequence: u64) { - if let Some(slot) = self.uncache(sequence) { + fn displace(&mut self, key: u64) { + if let Some(slot) = self.uncache(key) { for group in slot.copies() { self.detach(group); } @@ -1350,7 +1277,7 @@ impl TrackState { } } - /// Record sequences the upstream publisher gave up, with why. A skip is final and the + /// Record keys the upstream publisher gave up, with why. A skip is final and the /// most authoritative verdict, so a drop never replaces one; see [`Self::skip_groups`]. /// Returns the runs recorded. fn drop_groups(&mut self, groups: ops::Range, hole: Hole) -> RangeSet { @@ -1373,16 +1300,28 @@ impl TrackState { runs } - /// Whether the publisher declared `sequence` skipped: never produced, which is final. - fn skipped(&self, sequence: u64) -> bool { - self.drops.get(&sequence).is_some_and(|hole| *hole == Hole::Skip) + /// Whether the publisher declared `key` skipped: never produced, which is final. + fn skipped(&self, key: u64) -> bool { + self.drops.get(&key).is_some_and(|hole| *hole == Hole::Skip) + } + + /// The exclusive end of the run at `key` that no group will fill: one the publisher + /// declared skipped, or with `dropped`, also one given up for good. A stale drop is not: + /// a wider budget takes it back. + fn hole_end(&self, key: u64, dropped: bool) -> Option { + let (run, hole) = self.drops.get_key_value(&key)?; + let gone = match hole { + Hole::Skip => true, + Hole::Drop { stale, .. } => dropped && stale.is_none(), + }; + gone.then_some(run.end) } /// Skip what of `groups` holds no cached group, replacing any drop of it, final or stale: - /// "never produced" is the most authoritative verdict on a sequence. A reader a drop + /// "never produced" is the most authoritative verdict on a key. A reader a drop /// already ended stays ended, and a relay forwards the skip over the drop it reported. /// A group that arrived is never overwritten, since it was produced after all. Returns - /// the cached sequences left alone. See [`Producer::skip_groups`]. + /// the cached keys left alone. See [`Producer::skip_groups`]. fn skip_groups(&mut self, groups: ops::Range) -> RangeSet { let mut cached = RangeSet::new(); if groups.is_empty() { @@ -1390,10 +1329,10 @@ impl TrackState { } let mut left = RangeSet::new(); left.insert(groups.clone()); - for (sequence, _) in self.lookup.range(groups.clone()) { - if let Some(end) = sequence.checked_add(1) { - left.remove(*sequence..end); - cached.insert(*sequence..end); + for (key, _) in self.lookup.range(groups.clone()) { + if let Some(end) = key.checked_add(1) { + left.remove(*key..end); + cached.insert(*key..end); } } // Already skipped: nothing changes, so no reader rescans. @@ -1408,16 +1347,16 @@ impl TrackState { cached } - /// Settle an append at `sequence`: skip every sequence below it that this track's own + /// Settle an append at `key`: skip every key below it that this track's own /// publisher never produced, from just past its previous append (before the first, from /// the lowest it produced), since appending says it will not go back. The skip replaces a /// drop given there, as [`Self::skip_groups`] does. - fn appended(&mut self, sequence: u64) { + fn appended(&mut self, key: u64) { if let Some(from) = self.appended.or(self.produced.iter().next().map(|run| run.start)) - && from < sequence + && from < key { let mut left = RangeSet::new(); - left.insert(from..sequence); + left.insert(from..key); for run in self.produced.iter() { left.remove(run.clone()); } @@ -1426,7 +1365,7 @@ impl TrackState { } } self.produced.clear(); - self.appended = sequence.checked_add(1); + self.appended = key.checked_add(1); } /// Forget every run held as stale; see [`Producer::supersede_stale`]. @@ -1498,7 +1437,7 @@ impl TrackState { } /// Forget the runs of the drop set given up longer than the pool's expiry window before - /// `now`: their sequences are unknown again, as a shed run's are. Walks the set only once + /// `now`: their keys are unknown again, as a shed run's are. Walks the set only once /// its oldest run is due, which a quarter of the window separates from the next. fn expire_drops(&mut self, now: u64) { if !self.drops_expired(now) { @@ -1547,7 +1486,7 @@ impl TrackState { } /// Forget the lowest runs of the drop set until the debt is paid or `cap` bytes are - /// freed, returning the bytes freed. Their sequences are unknown again: a reader judges + /// freed, returning the bytes freed. Their keys are unknown again: a reader judges /// them by its budget, or fills them by range. fn shed_drops(&mut self, cap: u64) -> u64 { let mut paid = 0u64; @@ -1607,22 +1546,22 @@ impl TrackState { /// Remove an aborted slot, keeping a cut as a drop: reclaiming it must not erase a /// disposition a relay still has to forward. - fn reclaim_aborted(&mut self, sequence: u64) { - if let Some(slot) = self.uncache(sequence) + fn reclaim_aborted(&mut self, key: u64) { + if let Some(slot) = self.uncache(key) && slot.visible && let Some(err) = slot.verdict() - && let Some(end) = sequence.checked_add(1) + && let Some(end) = key.checked_add(1) { - self.drop_groups(sequence..end, Hole::drop(StreamError::from(err))); + self.drop_groups(key..end, Hole::drop(StreamError::from(err))); } } - /// Why `sequence` was given up, if it was: cut while cached, or dropped. - fn given_up(&self, sequence: u64) -> Option { - if let Some(err) = self.lookup.get(&sequence).and_then(Slot::cut) { + /// Why `key` was given up, if it was: cut while cached, or dropped. + fn given_up(&self, key: u64) -> Option { + if let Some(err) = self.lookup.get(&key).and_then(Slot::cut) { return Some(err.clone()); } - self.drops.get(&sequence).map(|hole| Error::Stream(hole.error())) + self.drops.get(&key).map(|hole| Error::Stream(hole.error())) } /// Whether a Producer was minted, committed or not. @@ -1716,15 +1655,15 @@ impl TrackState { fn poll_recv_group( &self, index: usize, - min_sequence: u64, + min_key: u64, window: &Window, ) -> Poll>> { let start = index.saturating_sub(self.offset); let readable = self.serves_bounded(window); - for (i, (sequence, stamp)) in self.arrival.iter().enumerate().skip(start).filter(|_| readable) { + for (i, (key, stamp)) in self.arrival.iter().enumerate().skip(start).filter(|_| readable) { // A cut group is handed out too, so its reader sees why it ended. - if *sequence >= min_sequence - && let Some(slot) = self.lookup.get(sequence) + if *key >= min_key + && let Some(slot) = self.lookup.get(key) && slot.stamp == *stamp && slot.is_read() { @@ -1732,7 +1671,7 @@ impl TrackState { } } - // TODO once we have drop notifications, check if index == final_sequence. + // TODO once we have drop notifications, check if index == final_key. if self.is_complete() { Poll::Ready(Ok(None)) } else if let Some(err) = self.outcome.aborted() { @@ -1742,8 +1681,8 @@ impl TrackState { } } - /// Find the smallest-sequence cached group in `window` satisfying - /// `next_sequence <= seq < end_sequence (if set)`. Used by + /// Find the smallest-key cached group in `window` satisfying + /// `next_key <= seq < end_key (if set)`. Used by /// [`Subscriber::next_group`] so the range can be widened (or unset) /// after the fact and previously-skipped cached groups become available /// without scanning past them in arrival order. @@ -1753,8 +1692,8 @@ impl TrackState { /// the track is finalized and no further in-range group is possible. fn poll_next_in_range( &self, - next_sequence: u64, - end_sequence: Option, + next_key: u64, + end_key: Option, window: &Window, ) -> Poll>> { // Nothing more can arrive once the track was sealed (dropped with its end @@ -1773,29 +1712,29 @@ impl TrackState { // Pending (not None) so the consumer is parked rather than told the stream // is over, unless the track declared its end: a finished track ends a reader // with nothing left in range, as the arrival path does. An empty range - // (`end == 0`) parks even at the first sequence. - let cap = match (end_sequence, window.end()) { + // (`end == 0`) parks even at the first key. + let cap = match (end_key, window.end()) { (Some(cap), Some(end)) => Some(cap.min(end)), (cap, end) => cap.or(end), }; if let Some(cap) = cap - && cap <= next_sequence + && cap <= next_key { - if closed || self.final_sequence().is_some() { + if closed || self.final_key().is_some() { return Poll::Ready(end()); } return Poll::Pending; } let best = window - .runs(next_sequence) + .runs(next_key) .filter(|_| self.serves_bounded(window)) .find_map(|run| { // A cut group is handed out too, so its reader sees why it ended. self.lookup .range(run) .map(|(_, slot)| slot) - .take_while(|slot| super::subscription::before_end(slot.group.sequence, end_sequence)) + .take_while(|slot| super::subscription::before_end(slot.group.start, end_key)) .find(|slot| slot.serves_end() || slot.cut().is_some()) .map(Slot::serving) }); @@ -1809,28 +1748,28 @@ impl TrackState { } // No in-range group is cached. Decide whether more could ever arrive. - // `final_sequence` is one past the last possible sequence. If our + // `final_key` is one past the last possible key. If our // floor is already at/past it, nothing else can land in range. // A closed track produces nothing more either, so a gap below its // boundary is the end, not a wait, as is one in a complete track, which // ends the arrival path too. Cached in-range groups were returned above, // so an aborted track drains what finished first. // This is the cursor the ordered reader uses. - if closed || self.is_complete() || self.final_sequence().is_some_and(|fin| next_sequence >= fin) { + if closed || self.is_complete() || self.final_key().is_some_and(|fin| next_key >= fin) { return Poll::Ready(end()); } Poll::Pending } - /// The cached group for `sequence`, but only when it holds every frame from + /// The cached group for `key`, but only when it holds every frame from /// `frame_start` onward. /// /// A group cached from a frame-bounded subscription starts partway in, and serving /// it to someone who asked for the whole group would silently hand back a tail. It /// is a miss instead, so the fetch goes upstream for the frames that are missing. - fn covering_group(&self, sequence: u64, frame_start: u64) -> Option<&group::Handle> { + fn covering_group(&self, key: u64, frame_start: u64) -> Option<&group::Handle> { self.covering_range( - sequence, + key, group::Bounds { start: frame_start, byte: 0, @@ -1839,8 +1778,8 @@ impl TrackState { ) } - fn covering_range(&self, sequence: u64, frames: group::Bounds) -> Option<&group::Handle> { - self.lookup.get(&sequence)?.copies().find(|group| group.covers(frames)) + fn covering_range(&self, key: u64, frames: group::Bounds) -> Option<&group::Handle> { + self.lookup.get(&key)?.copies().find(|group| group.covers(frames)) } /// The local freshness bound, or `None` when unknown or unlimited: the publisher's max @@ -1862,11 +1801,11 @@ impl TrackState { } /// The live edge a subscription served `servable` measures drift against: the - /// highest-sequence group in it that has presented a frame, or `None` while nothing + /// highest-key group in it that has presented a frame, or `None` while nothing /// stamped is servable (an unstamped track drives no media-time expiry; the wall /// clock alone judges it). /// - /// One scan answers a whole poll. The highest-sequence edge in range is above + /// One scan answers a whole poll. The highest-key edge in range is above /// every candidate below it and above none at or past it. Recomputing per candidate /// would make walking a backlog of N groups off in one poll cost O(N^2). /// @@ -1881,30 +1820,27 @@ impl TrackState { if !slot.is_shown() { return None; } - // The map is ordered by sequence, so the first stamped group from the + // The map is ordered by key, so the first stamped group from the // back is the newest content that exists. - let timestamp = slot.group.timestamp()?; + let timestamp = self.started_at(slot)?; Some(PresentationEdge { - sequence: slot.group.sequence, + key: slot.group.start, stamp: slot.stamp, timestamp: slot.group.latest().unwrap_or(timestamp), }) }) } - /// This track's own edge within `servable`, for measuring drift. - /// - /// None on an untimed track, where nothing is ever stale: its groups carry no - /// timestamp, so looking for one would scan the whole cache on every poll. + /// This track's own edge within `servable`, for measuring drift. A scale-less track's + /// edge is its newest shown group's key, so its max age counts bare units. fn drift_edge(&self, servable: &Servable) -> Edge { - let untimed = self.info.as_ref().is_some_and(|info| info.timescale.is_none()); Edge { - presentation: if untimed { None } else { self.live_edge(servable) }, + presentation: self.live_edge(servable), servable: servable.clone(), } } - /// The live feed's cached groups within `range`, by sequence: never fetched backfill. + /// The live feed's cached groups within `range`, by key: never fetched backfill. fn live(&self, range: impl ops::RangeBounds) -> impl DoubleEndedIterator { // Without backfill every cached group is the feed's, so the cache is walked as is. match self.visible.len() == self.lookup.len() { @@ -1912,7 +1848,7 @@ impl TrackState { false => LiveWalk::Indexed( self.visible .range(range) - .map(|sequence| (sequence, self.lookup.get(sequence).expect("an indexed live slot"))), + .map(|key| (key, self.lookup.get(key).expect("an indexed live slot"))), ), } } @@ -1995,8 +1931,8 @@ impl TrackState { match leap { Leap::Found(found) => return Some(found), Leap::Empty => return None, - Leap::Past(sequence) if rev => hi = sequence, - Leap::Past(sequence) => lo = sequence, + Leap::Past(key) if rev => hi = key, + Leap::Past(key) => lo = key, Leap::Exhausted if rev => hi = start.checked_sub(1)?, Leap::Exhausted => lo = end.checked_add(1)?, } @@ -2004,14 +1940,14 @@ impl TrackState { None } - /// Whether `sequence` still holds the servable incarnation `stamp`. - fn holds(&self, sequence: u64, stamp: u32) -> bool { + /// Whether `key` still holds the servable incarnation `stamp`. + fn holds(&self, key: u64, stamp: u32) -> bool { self.lookup - .get(&sequence) + .get(&key) .is_some_and(|slot| slot.stamp == stamp && slot.serves_end()) } - /// The furthest presentation time the group at `sequence` could still reach: where + /// The furthest presentation time the group at `key` could still reach: where /// its immediate successor begins, or `None` when nothing proves where it stops. /// /// An upper bound, deliberately. A frame's duration is not on the wire, so a group's @@ -2020,23 +1956,60 @@ impl TrackState { /// servable successor's start, so an unstamped successor leaves the reach unbounded /// until it presents its first frame, and so does having no successor in `servable` /// at all. - fn reach(&self, sequence: u64, servable: &Servable) -> Option { - self.successor(sequence, servable)?.group.timestamp() + fn reach(&self, key: u64, servable: &Servable) -> Option { + if let Some(end) = self.own_end(key) { + return Some(end); + } + self.started_at(self.successor(key, servable)?) + } + + /// A key, or a group's end, in presentation time: in the track's timescale, or unscaled + /// on a scale-less track, over the whole key range, whose max age counts bare units as + /// milliseconds (see [`PresentationEdge::behind`]). + fn presented(&self, value: u64) -> Option { + match self.info.as_ref()?.timescale { + Some(scale) => Timestamp::new(value, scale).ok(), + None => Some(Timestamp::unscaled(value)), + } + } + + /// Where the group in `slot` starts presenting: its first frame's timestamp, or its key + /// on a scale-less track, whose frames carry none. + fn started_at(&self, slot: &Slot) -> Option { + match self.info.as_ref()?.timescale { + Some(_) => slot.group.timestamp(), + None => self.presented(slot.group.start), + } } - /// The group after `sequence` that bounds it: the first servable one in `servable` whose + /// Where group `key` ends, once its producer finished it whole: what bounds its reach + /// without a successor (see [`group::Producer::finish`]). + fn own_end(&self, key: u64) -> Option { + self.presented(self.lookup.get(&key)?.group.end()?) + } + + /// When group `key`, finished whole, stops presenting on the wall clock: its arrival plus + /// its span, which stands in for a successor arriving, so the last group of a paused or + /// finished track ages out without one. + fn ended_at(&self, key: u64) -> Option { + let slot = self.lookup.get(&key)?; + let span = self.presented(slot.group.end()?.checked_sub(key)?)?; + slot.arrived.checked_add(age(span)?) + } + + /// The group after `key` that bounds it: the first servable one in `servable` whose /// first frame opened or that finished, stamped or not, empty or not. - fn successor(&self, sequence: u64, servable: &Servable) -> Option<&Slot> { - self.find_servable(servable, sequence.checked_add(1)?, false, |slot| { + fn successor(&self, key: u64, servable: &Servable) -> Option<&Slot> { + self.find_servable(servable, key.checked_add(1)?, false, |slot| { slot.is_shown().then_some(slot) }) } - /// The group after `sequence` that starts its wall clock: like [`Self::successor`], but + /// The group after `key` that starts its wall clock: like [`Self::successor`], but /// counting a group before its first frame. Its header arrived, so a relay ages the group /// before it as the origin does, even if its first frame never comes. - fn arrival(&self, sequence: u64, servable: &Servable) -> Option<&Slot> { - self.find_servable(servable, sequence.checked_add(1)?, false, |slot| { + fn arrival(&self, key: u64, servable: &Servable) -> Option<&Slot> { + self.find_servable(servable, key.checked_add(1)?, false, |slot| { slot.is_live().then_some(slot) }) } @@ -2058,21 +2031,25 @@ impl TrackState { } } - /// Whether `sequence`'s successor arrived a full `budget` ago. + /// Whether `key`'s successor arrived a full `budget` ago. /// /// The wall-clock twin of [`Self::reach`], re-read the same way rather than latched: /// a group that lands between the two restarts the clock, and an aborted successor /// hands it to the next one. The clock is read only once there is a successor, /// so judging the newest group, the common case, costs no clock read. - fn superseded(&self, sequence: u64, servable: &Servable, budget: Duration) -> bool { - self.arrival(sequence, servable) - .is_some_and(|successor| self.aged(successor.arrived, budget)) + fn superseded(&self, key: u64, servable: &Servable, budget: Duration) -> bool { + match self.arrival(key, servable) { + Some(successor) => self.aged(successor.arrived, budget), + None => self.ended_at(key).is_some_and(|ended| self.aged(ended, budget)), + } } - /// Whether a full `budget` passed since a successor `arrived`. + /// Whether a full `budget` passed since a successor `arrived`, or since a group finished + /// whole stops presenting, which may still be ahead: a zero budget ages it only once + /// that comes. fn aged(&self, arrived: crate::time::Instant, budget: Duration) -> bool { let now = self.cache.pool().now_wall(); - now.saturating_duration_since(arrived) >= budget + now >= arrived && now.saturating_duration_since(arrived) >= budget } /// When a group was superseded, given `arrival`, its cached successor in `servable`: when @@ -2093,20 +2070,20 @@ impl TrackState { let mut last = superseded.0.lock().unwrap_or_else(PoisonError::into_inner); if let Some(recorded) = &*last && !recorded.fate.failed() - && servable.contains(recorded.sequence) - && arrival.is_none_or(|slot| slot.group.sequence > recorded.sequence) + && servable.contains(recorded.key) + && arrival.is_none_or(|slot| slot.group.start > recorded.key) { return Some(recorded.arrived); } *last = arrival.map(|slot| Successor { - sequence: slot.group.sequence, + key: slot.group.start, arrived: slot.arrived, fate: slot.group.fate(), }); arrival.map(|slot| slot.arrived) } - /// Whether the group at `sequence` has drifted further behind `edge` than `budget` + /// Whether the group at `key` has drifted further behind `edge` than `budget` /// tolerates, so a subscriber should skip it rather than hand it over. /// /// Presentation time measures a group by how far it could still *reach*, not by how far @@ -2116,15 +2093,17 @@ impl TrackState { /// delivering, so a group is abandoned only once everything it could still present /// falls outside the budget. /// - /// A group's reach is bounded by its immediate successor (see [`Self::reach`]): it - /// cannot present past where the next group begins. The candidate itself needs no - /// timestamp: an empty group is bounded by its stamped successor the same way. + /// A group's reach is bounded by its own end once finished whole, else by its immediate + /// successor (see [`Self::reach`]): it cannot present past where the next group begins. + /// The candidate itself needs no timestamp: an empty group is bounded by its stamped + /// successor the same way. /// /// The wall clock judges it too: a group is also stale once its successor arrived a /// full `budget` ago (see [`Self::superseded`]), so a peer whose timestamps stop - /// advancing, an untimed track, or an empty successor that never presents cannot pin - /// a superseded group. Media time still convicts first when a burst arrives at once. - /// The newest group has no successor, so neither clock ever convicts it. + /// advancing or an empty successor that never presents cannot pin a superseded group. + /// Media time still convicts first when a burst arrives at once. A group finished whole + /// ages from its arrival plus its span instead when no successor arrived, so the last + /// group of a paused or finished track ages out; one with neither is never convicted. /// /// The bound is exclusive, so the comparison is `>=` rather than `>`: the freshest frame /// a group could still hold sits just *below* its reach, so an age equal to the budget @@ -2133,13 +2112,12 @@ impl TrackState { /// /// The edge must sit strictly above the candidate. The live edge is never late /// against itself, and a group received from past its first frame carries a later - /// frame's timestamp on a low sequence without being an edge at all. + /// frame's timestamp on a low key without being an edge at all. /// /// A group below [`Self::cache_floor`] is stale whether or not the cache still holds /// it: an eviction between selecting it and this verdict must not hand it out. - fn is_stale(&self, sequence: u64, edge: &Edge, budget: Duration) -> bool { - sequence < self.cache_floor(budget) - || (self.lookup.contains_key(&sequence) && self.drifted(sequence, edge, budget)) + fn is_stale(&self, key: u64, edge: &Edge, budget: Duration) -> bool { + key < self.cache_floor(budget) || (self.lookup.contains_key(&key) && self.drifted(key, edge, budget)) } /// A finite subscription cannot admit a cache whose reach its resumed feed cannot @@ -2148,22 +2126,22 @@ impl TrackState { self.live_floor.filter(|_| budget != Duration::MAX).unwrap_or(0) } - /// Whether group `sequence`, cached or not, has a successor a full `budget` behind + /// Whether group `key`, cached or not, has a successor a full `budget` behind /// the edge, or one that arrived a full `budget` ago. - fn drifted(&self, sequence: u64, edge: &Edge, budget: Duration) -> bool { - if self.superseded(sequence, &edge.servable, budget) { + fn drifted(&self, key: u64, edge: &Edge, budget: Duration) -> bool { + if self.superseded(key, &edge.servable, budget) { return true; } - self.live_above(sequence, edge) - .is_some_and(|live| live.behind(self.reach(sequence, &edge.servable), budget)) + self.live_above(key, edge) + .is_some_and(|live| live.behind(self.reach(key, &edge.servable), budget)) } - /// The edge that could convict group `sequence`: above it, and still holding the + /// The edge that could convict group `key`: above it, and still holding the /// servable incarnation it was resolved from. It was resolved under an earlier lock, /// and failing safe (delivering) is right, since the next poll resolves a fresh edge. - fn live_above(&self, sequence: u64, edge: &Edge) -> Option { + fn live_above(&self, key: u64, edge: &Edge) -> Option { edge.presentation - .filter(|live| live.sequence > sequence && self.holds(live.sequence, live.stamp)) + .filter(|live| live.key > key && self.holds(live.key, live.stamp)) } /// [`Self::drifted`] within `servable`, parking `waiter` on exactly what else can move the @@ -2171,12 +2149,12 @@ impl TrackState { /// the media deadline, and the wall clock crossing the successor's arrival plus /// `budget`. Parking on every track change instead wakes each of N parked reads per append. /// - /// The caller parks on a group landing above `sequence` first, in the way that fits + /// The caller parks on a group landing above `key` first, in the way that fits /// whether this track holds it. A reader holding the group passes its `superseded`, so /// the wall clock outlives the successor's eviction; see [`Self::superseded_at`]. fn poll_drifted( &self, - sequence: u64, + key: u64, servable: &Servable, budget: Duration, superseded: Option<&Superseded>, @@ -2185,8 +2163,12 @@ impl TrackState { // Unbounded until a group lands above. Registered before judging, so a racing abort // is observed or wakes us, and armed for whichever successor this pass selected, so // re-selecting re-arms it. - let arrival = self.watch_abort(waiter, |state| state.arrival(sequence, servable)); - let Some(arrived) = self.superseded_at(arrival, servable, superseded) else { + let arrival = self.watch_abort(waiter, |state| state.arrival(key, servable)); + // A group finished whole ages from its own end without one; see [`Self::ended_at`]. + let Some(arrived) = self + .superseded_at(arrival, servable, superseded) + .or_else(|| self.ended_at(key)) + else { return false; }; if let Some(deadline) = arrived.checked_add(budget) { @@ -2196,15 +2178,18 @@ impl TrackState { return true; } let reach = loop { + if let Some(end) = self.own_end(key) { + break end; + } // An abort can change reach even after the successor is stamped. Unbounded // until a group shows above, or the successor presents its first frame. - let Some(successor) = self.watch_abort(waiter, |state| state.successor(sequence, servable)) else { + let Some(successor) = self.watch_abort(waiter, |state| state.successor(key, servable)) else { return false; }; - let successor = &successor.group; - if let Some(reach) = successor.timestamp() { + if let Some(reach) = self.started_at(successor) { break reach; } + let successor = &successor.group; if successor.poll_started(waiter).is_pending() || successor.timestamp().is_none() { return false; } @@ -2219,12 +2204,12 @@ impl TrackState { // re-resolve if it already landed. Terminates: an aborted group is never the // edge again, so each pass resolves a lower one. if let Some(live) = edge.presentation - && let Some(slot) = self.lookup.get(&live.sequence) + && let Some(slot) = self.lookup.get(&live.key) && slot.group.poll_aborted(waiter).is_ready() { continue; } - return self.drifted(sequence, &edge, budget); + return self.drifted(key, &edge, budget); } } @@ -2232,10 +2217,10 @@ impl TrackState { /// once it can never be served. A missing group is a failure ([`Error::NotFound`]), not an /// end-of-stream. The handler side (a refusal, or no [`Requests`] at all) lives /// in the track's fills; [`Fetching`] polls both. - fn poll_fetch_cached(&self, sequence: u64, frames: group::Bounds) -> Poll> { + fn poll_fetch_cached(&self, key: u64, frames: group::Bounds) -> Poll> { // A group aborted between the lookup and the consume (an abandoned fetch cut // short) is a miss, not a hit that fails on its first read. - if let Some(group) = self.covering_range(sequence, frames) + if let Some(group) = self.covering_range(key, frames) && let Some(consumer) = group.try_consume() { // A cache hit refreshes the group: it resets both its age (expiry keys @@ -2249,8 +2234,8 @@ impl TrackState { return Poll::Ready(Err(err.clone())); } - // Past the final sequence: the group can never exist. - if self.final_sequence().is_some_and(|fin| sequence >= fin) { + // Past the final key: the group can never exist. + if self.final_key().is_some_and(|fin| key >= fin) { return Poll::Ready(Err(Error::NotFound)); } @@ -2316,14 +2301,14 @@ impl TrackState { let start = scan.start % len; let mut retained = 0; for step in 0..len.min(scan.width) { - let (sequence, stamp) = self.evict[(start + step) % len]; - let Some(slot) = self.lookup.get(&sequence) else { + let (key, stamp) = self.evict[(start + step) % len]; + let Some(slot) = self.lookup.get(&key) else { continue; }; if slot.stamp != stamp { continue; } - if slot.is_spent() || (!self.protects(sequence) && slot.is_expired(&scan)) { + if slot.is_spent() || (!self.protects(key) && slot.is_expired(&scan)) { return true; } retained += 1; @@ -2337,11 +2322,11 @@ impl TrackState { || self .arrival .front() - .is_some_and(|(sequence, stamp)| !self.is_current(*sequence, *stamp)) + .is_some_and(|(key, stamp)| !self.is_current(*key, *stamp)) || self .evict .front() - .is_some_and(|(sequence, stamp)| !self.is_current(*sequence, *stamp)) + .is_some_and(|(key, stamp)| !self.is_current(*key, *stamp)) || self.evict.len() > 2 * self.lookup.len() + EVICT_SLACK } @@ -2349,8 +2334,8 @@ impl TrackState { /// [`Self::evict_expired_scan`] takes: abort the groups in place, releasing their /// frames, and retire expired cuts, leaving entries that every read path skips. pub(super) fn expire_closed(&self, scan: ExpiryScan) { - for (sequence, stamp) in &self.evict { - let Some(slot) = self.lookup.get(sequence) else { + for (key, stamp) in &self.evict { + let Some(slot) = self.lookup.get(key) else { continue; }; if slot.stamp != *stamp || (slot.is_aborted() && slot.cut().is_none()) || !slot.is_expired(&scan) { @@ -2372,8 +2357,8 @@ impl TrackState { let start = scan.start % len; let mut retained = 0; for step in 0..len.min(scan.width) { - let (sequence, stamp) = self.evict[(start + step) % len]; - let Some(slot) = self.lookup.get(&sequence) else { + let (key, stamp) = self.evict[(start + step) % len]; + let Some(slot) = self.lookup.get(&key) else { continue; }; if slot.stamp != stamp { @@ -2381,12 +2366,12 @@ impl TrackState { continue; } // Already aborted: the frames are gone, reclaim the slot so a - // later fetch can serve the sequence again. + // later fetch can serve the key again. if slot.is_spent() { - self.reclaim_aborted(sequence); + self.reclaim_aborted(key); continue; } - if self.protects(sequence) || !slot.is_expired(&scan) { + if self.protects(key) || !slot.is_expired(&scan) { // Writes keep their scan bounded. Cleanup visits the entire // queue to date pending accesses and find idle entries behind them. retained += 1; @@ -2397,13 +2382,13 @@ impl TrackState { } // An expired cut goes like the group it was, still forwarded as a drop. if slot.is_aborted() { - self.reclaim_aborted(sequence); + self.reclaim_aborted(key); continue; } // Take the group out of the cache and abort it, so any consumer // still reading surfaces `Error::Old` instead of blocking forever // on a frame that will never arrive. - self.evict_group(sequence, Error::Old); + self.evict_group(key, Error::Old); } } @@ -2411,8 +2396,8 @@ impl TrackState { // Trim dead leading arrival entries to advance the subscriber offset. An // entry is dead once its slot is gone or re-stamped by a newer incarnation. - while let Some((sequence, stamp)) = self.arrival.front() { - if self.lookup.get(sequence).is_some_and(|slot| slot.stamp == *stamp) { + while let Some((key, stamp)) = self.arrival.front() { + if self.lookup.get(key).is_some_and(|slot| slot.stamp == *stamp) { break; } self.arrival.pop_front(); @@ -2420,8 +2405,8 @@ impl TrackState { } // Drop dead leading eviction entries so scans stay over live candidates. - while let Some((sequence, stamp)) = self.evict.front() { - if self.lookup.get(sequence).is_some_and(|slot| slot.stamp == *stamp) { + while let Some((key, stamp)) = self.evict.front() { + if self.lookup.get(key).is_some_and(|slot| slot.stamp == *stamp) { break; } self.evict.pop_front(); @@ -2432,19 +2417,19 @@ impl TrackState { if self.evict.len() > 2 * self.lookup.len() + EVICT_SLACK { let lookup = &self.lookup; self.evict - .retain(|(sequence, stamp)| lookup.get(sequence).is_some_and(|slot| slot.stamp == *stamp)); + .retain(|(key, stamp)| lookup.get(key).is_some_and(|slot| slot.stamp == *stamp)); } } - /// Whether `(sequence, stamp)` names the currently cached incarnation. - fn is_current(&self, sequence: u64, stamp: u32) -> bool { - self.lookup.get(&sequence).is_some_and(|slot| slot.stamp == stamp) + /// Whether `(key, stamp)` names the currently cached incarnation. + fn is_current(&self, key: u64, stamp: u32) -> bool { + self.lookup.get(&key).is_some_and(|slot| slot.stamp == stamp) } - /// Whether `sequence` is the live edge, which eviction and expiry never take + /// Whether `key` is the live edge, which eviction and expiry never take /// while a producer remains. - fn protects(&self, sequence: u64) -> bool { - !self.outcome.is_released() && Some(sequence) == self.latest_group + fn protects(&self, key: u64) -> bool { + !self.outcome.is_released() && Some(key) == self.latest_group } /// Publishing ended: drop the ingress scope, whose session a reader still draining the @@ -2492,7 +2477,7 @@ impl TrackState { .lookup .iter() .filter(|(_, slot)| slot.visible && !slot.copies().any(group::Handle::is_finished)) - .filter_map(|(&sequence, _)| Some(sequence..sequence.checked_add(1)?)) + .filter_map(|(&key, _)| Some(key..key.checked_add(1)?)) .collect(); for groups in open { self.drop_groups(groups, Hole::drop(dropped.clone())); @@ -2500,41 +2485,33 @@ impl TrackState { self.lookup .retain(|_, slot| slot.copies().any(group::Handle::is_finished)); let lookup = &self.lookup; - self.visible.retain(|sequence| lookup.contains_key(sequence)); + self.visible.retain(|key| lookup.contains_key(key)); self.bounds .get_mut() .unwrap_or_else(PoisonError::into_inner) - .retain(|sequence, _| lookup.contains_key(sequence)); - self.starts - .get_mut() - .unwrap_or_else(PoisonError::into_inner) - .retain(|sequence, _| lookup.contains_key(sequence)); + .retain(|key, _| lookup.contains_key(key)); (resume, OpenGroups { groups, err }) } - /// Take `sequence` out of the cache, along with its checked start. - fn remove_slot(&mut self, sequence: u64) -> Option { - let slot = self.lookup.remove(&sequence)?; - self.visible.remove(&sequence); + /// Take `key` out of the cache, along with its checked start. + fn remove_slot(&mut self, key: u64) -> Option { + let slot = self.lookup.remove(&key)?; + self.visible.remove(&key); self.bounds .get_mut() .unwrap_or_else(PoisonError::into_inner) - .remove(&sequence); - self.starts - .get_mut() - .unwrap_or_else(PoisonError::into_inner) - .remove(&sequence); + .remove(&key); Some(slot) } - /// Record that the live feed delivered `sequence`, then fold the gaps that aged out. - fn record_arrival(&mut self, sequence: u64, now: crate::time::Instant) { - self.arrivals.arrive(sequence, now); + /// Record that the live feed delivered `key`, then fold the gaps that aged out. + fn record_arrival(&mut self, key: u64, now: crate::time::Instant) { + self.arrivals.arrive(key, now); if !self.anchors.received() - && self.appended.is_none_or(|appended| sequence >= appended) - && let Some(end) = sequence.checked_add(1) + && self.appended.is_none_or(|appended| key >= appended) + && let Some(end) = key.checked_add(1) { - self.produced.insert(sequence..end); + self.produced.insert(key..end); } // A feed still delivering keeps the gap above its last arrival open. self.refresh_tail(now); @@ -2635,21 +2612,21 @@ impl TrackState { state } - /// Record a written sequence. It raises the largest too, so a later group below it never + /// Record a written key. It raises the largest too, so a later group below it never /// changes it, even one below a reached end. /// /// `answers` when it is a group subscribers receive, which answers a pending refresh. /// /// The demand wakes it owes wait in [`Self::deferred`] for the caller to fire once its guard /// is released. - fn advance(&mut self, sequence: u64, answers: bool) { + fn advance(&mut self, key: u64, answers: bool) { let kept = self.kept_drops(); - self.max_sequence = Some(self.max_sequence.map_or(sequence, |max| max.max(sequence))); + self.max_key = Some(self.max_key.map_or(key, |max| max.max(key))); if self.kept_drops() > kept { self.rearm_drops(kept); } - self.cache.advance_largest(Position::group(sequence)); - let wakes = self.subscriptions.lock().wrote(sequence, answers); + self.cache.advance_largest(Position::group(key)); + let wakes = self.subscriptions.lock().wrote(key, answers); if !wakes.is_empty() { self.deferred.push(wakes); } @@ -2660,10 +2637,10 @@ impl TrackState { Deferred(std::mem::take(&mut self.deferred)) } - /// The sequence the next append takes: one past this track's own writes, short of the - /// reserved `u64::MAX`. - fn next_sequence(&self) -> Result { - match self.max_sequence { + /// One past every key this track holds, short of the reserved `u64::MAX`. + #[cfg(any(test, feature = "fuzz"))] + fn next_key(&self) -> Result { + match self.max_key { Some(max) => max .checked_add(1) .filter(|next| *next != u64::MAX) @@ -2672,30 +2649,30 @@ impl TrackState { } } - /// Reject a sequence that is still cached; a dead (aborted or evicted) - /// incarnation is removed so a fresh group can serve the sequence again. Subscribers may + /// Reject a key that is still cached; a dead (aborted or evicted) + /// incarnation is removed so a fresh group can serve the key again. Subscribers may /// have seen its start, so it stays as the eviction watermark. /// - /// Best effort: nothing remembers a sequence whose slot is already gone, so a - /// publisher re-sending a long-evicted sequence is accepted as new. + /// Best effort: nothing remembers a key whose slot is already gone, so a + /// publisher re-sending a long-evicted key is accepted as new. /// A group that starts above `frame_start` is also replaceable: it cannot answer a /// request from there, so the wider producer takes the slot. Readers already /// draining the old one keep their own handle. /// - /// A fetched backfill never holds the sequence against the feed: subscribers read only + /// A fetched backfill never holds the key against the feed: subscribers read only /// the feed's groups, so its copy takes the slot, and the fetch's readers keep theirs. - fn claim_sequence(&mut self, sequence: u64, frame_start: u64) -> Result<()> { - if self.holding_group(sequence, frame_start).is_some() { + fn claim_key(&mut self, key: u64, frame_start: u64) -> Result<()> { + if self.holding_group(key, frame_start).is_some() { return Err(Error::Duplicate); } - self.displace(sequence); + self.displace(key); Ok(()) } - /// The copy of the feed's `sequence` that still answers from `frame_start`, if any. - fn holding_group(&self, sequence: u64, frame_start: u64) -> Option<&group::Handle> { - self.lookup.get(&sequence).filter(|slot| slot.visible)?; - self.covering_group(sequence, frame_start) + /// The copy of the feed's `key` that still answers from `frame_start`, if any. + fn holding_group(&self, key: u64, frame_start: u64) -> Option<&group::Handle> { + self.lookup.get(&key).filter(|slot| slot.visible)?; + self.covering_group(key, frame_start) } /// Insert a freshly-created group into the cache. @@ -2703,61 +2680,61 @@ impl TrackState { /// Updates the live edge, demoting the previous latest into the eviction order; /// the current latest is never enqueued, which is what protects it from /// eviction. `visible` controls arrival-order delivery: publisher-produced - /// groups reach subscribers, fetched backfill is served by sequence only. A group + /// groups reach subscribers, fetched backfill is served by key only. A group /// counts as content only once [`Self::reveal`] reports its first frame or finish. fn insert_group(&mut self, group: &group::Handle, visible: bool) { self.latch_settled(); - let sequence = group.sequence; + let key = group.start; if visible { - self.contents.remove(sequence); + self.contents.remove(key); } group.register_content(&self.contents); self.next_stamp = self.next_stamp.wrapping_add(1); let stamp = self.next_stamp; - // The live edge is tracked separately from `max_sequence`, which can run past + // The live edge is tracked separately from `max_key`, which can run past // any cached group: demotion must still fire when the next group lands beyond it. - if self.latest_group.is_none_or(|latest| sequence >= latest) { + if self.latest_group.is_none_or(|latest| key >= latest) { // Demote the previous latest: it joins the eviction order (and the // pool's access average) like any other cached group. if let Some(latest) = self.latest_group - && sequence > latest + && key > latest && let Some(prev) = self.lookup.get(&latest) { prev.cache_demote(); self.evict.push_back((latest, prev.stamp)); } - self.latest_group = Some(sequence); + self.latest_group = Some(key); } if visible { if let Feed::Live { first, .. } = &mut self.feed { - first.get_or_insert(sequence); + first.get_or_insert(key); } - self.newest_produced = self.newest_produced.max(Some(sequence)); + self.newest_produced = self.newest_produced.max(Some(key)); } - if !self.protects(sequence) { + if !self.protects(key) { group.cache_demote(); - self.evict.push_back((sequence, stamp)); + self.evict.push_back((key, stamp)); } - self.advance(sequence, visible); + self.advance(key, visible); if !visible { self.fetched += 1; } - // A publisher may re-create a group it cut; the sequence is no longer given up. + // A publisher may re-create a group it cut; the key is no longer given up. if visible - && self.drops.contains_key(&sequence) - && let Some(end) = sequence.checked_add(1) + && self.drops.contains_key(&key) + && let Some(end) = key.checked_add(1) { - self.change_drops(sequence..end, None); + self.change_drops(key..end, None); } let now = self.cache.pool().now_wall(); if visible { - self.record_arrival(sequence, now); + self.record_arrival(key, now); } self.cached += 1; let replaced = self.lookup.insert( - sequence, + key, Slot { group: group.clone(), stamp, @@ -2770,26 +2747,24 @@ impl TrackState { }, ); match visible { - true => self.visible.insert(sequence), - false => self.visible.remove(&sequence), + true => self.visible.insert(key), + false => self.visible.remove(&key), }; if replaced.is_some() { self.bounds .get_mut() .unwrap_or_else(PoisonError::into_inner) - .remove(&sequence); + .remove(&key); } - // A group inserted with its first frame already written started before it was here. - self.index_start(sequence); - self.cache.wakes().arrived(sequence..=sequence); + self.cache.wakes().arrived(key..=key); if visible { // Readers and group demand reach it at once. Its arrival starts the wall clock of // the reads it succeeds, but it counts as content only once it starts. - self.arrival.push_back((sequence, stamp)); + self.arrival.push_back((key, stamp)); // Publishes the landing to reads judging expiry without the track lock, then // checks for one parked. Reveal only holds a read lock and cannot use the shortcut. - if self.cache.wakes().land(sequence) { - self.wake_landed(sequence); + if self.cache.wakes().land(key) { + self.wake_landed(key); } } } @@ -2801,30 +2776,30 @@ impl TrackState { /// Takes the read lock so a group starting wakes only the reads it can move, not every /// reader parked on the track. pub(super) fn reveal(&self, group: &group::Handle) -> bool { - let sequence = group.sequence; - match self.lookup.get(&sequence) { + let key = group.start; + match self.lookup.get(&key) { Some(slot) if slot.visible && slot.group.is_clone(group) && !slot.group.is_aborted() => {} _ => return false, } - self.wake_landed(sequence); - matches!(self.feed, Feed::Answered { from } if sequence >= from) + self.wake_landed(key); + matches!(self.feed, Feed::Answered { from } if key >= from) } - /// Wake the reads a group landing at `sequence` succeeds. - fn wake_landed(&self, sequence: u64) { + /// Wake the reads a group landing at `key` succeeds. + fn wake_landed(&self, key: u64) { let below = self - .live(..sequence) + .live(..key) .rev() .find(|(_, slot)| slot.is_shown()) .map(|(below, _)| *below); - let oldest = self.lookup.first_key_value().map_or(sequence, |(oldest, _)| *oldest); - self.cache.wakes().landed(below, sequence, oldest); + let oldest = self.lookup.first_key_value().map_or(key, |(oldest, _)| *oldest); + self.cache.wakes().landed(below, key, oldest); } - /// Content at `sequence` became readable: an answer waiting on it now shows. - pub(super) fn shown(&mut self, sequence: u64) { + /// Content at `key` became readable: an answer waiting on it now shows. + pub(super) fn shown(&mut self, key: u64) { if let Feed::Answered { from } = self.feed - && sequence >= from + && key >= from { self.go_live(Some(from)); } @@ -2882,10 +2857,10 @@ impl TrackState { let Some(average) = pool.average() else { return false; }; - let Some((sequence, stamp)) = self.evict.front() else { + let Some((key, stamp)) = self.evict.front() else { return false; }; - let Some(slot) = self.lookup.get(sequence) else { + let Some(slot) = self.lookup.get(key) else { return false; }; slot.stamp == *stamp && !slot.is_aborted() && slot.cache_accessed() <= average @@ -2916,10 +2891,10 @@ impl TrackState { swept = self.debt == 0 || paid >= cap; break; } - let Some((sequence, stamp)) = self.evict.pop_front() else { + let Some((key, stamp)) = self.evict.pop_front() else { break; }; - let Some(slot) = self.lookup.get(&sequence) else { + let Some(slot) = self.lookup.get(&key) else { // Evicted or expired; discard the dead entry. continue; }; @@ -2929,12 +2904,12 @@ impl TrackState { } if slot.is_aborted() { // Aborted upstream: the frames are already gone, reclaim the slot. - self.reclaim_aborted(sequence); + self.reclaim_aborted(key); continue; } - if self.protects(sequence) { + if self.protects(key) { // The live edge is never enqueued, but tolerate finding it anyway. - self.evict.push_back((sequence, stamp)); + self.evict.push_back((key, stamp)); continue; } @@ -2944,28 +2919,28 @@ impl TrackState { // being filled). Rotate to the back, but shed its front below if the debt // outlives the scan. if slot.cache_accessed() > average { - self.evict.push_back((sequence, stamp)); - fresh.push(sequence); + self.evict.push_back((key, stamp)); + fresh.push(key); continue; } // The full footprint including overhead, so even empty groups repay // their share of the budget when evicted. let size = slot.cache_size(); if size > self.debt { - self.evict.push_front((sequence, stamp)); + self.evict.push_front((key, stamp)); return paid; } self.debt -= size; paid = paid.saturating_add(size); - self.evict_group(sequence, Error::Evicted); + self.evict_group(key, Error::Evicted); } // Then the arrivals' oldest gaps, which costs a reader only the wait for a late group, as // the grace passing would. if self.debt > 0 && paid < cap { paid = paid.saturating_add(self.shed_arrivals(cap - paid)); } - // Then the drop set, oldest sequences first: losing a run costs a reader a judgment or + // Then the drop set, oldest keys first: losing a run costs a reader a judgment or // a fill, never content being read. if self.debt > 0 && paid < cap { paid = paid.saturating_add(self.shed_drops(cap - paid)); @@ -2973,11 +2948,11 @@ impl TrackState { // No cold group left to pay with: shed the front of the groups that could not go // whole, the ones being read and then the live edge, so neither grows without bound. fresh.extend(self.latest_group.filter(|latest| self.protects(*latest))); - for sequence in fresh { + for key in fresh { if self.debt == 0 || paid >= cap { break; } - let Some(slot) = self.lookup.get(&sequence) else { + let Some(slot) = self.lookup.get(&key) else { continue; }; for group in slot.copies() { @@ -3005,17 +2980,15 @@ impl TrackState { paid } - /// Group `sequence` once cached, `None` once the feed starts past it or ends before it, + /// Group `key` once cached, `None` once the feed starts past it or ends before it, /// and pending while it still may land. - fn landing(&self, sequence: u64) -> Poll> { - if let Some(slot) = self.lookup.get(&sequence) + fn landing(&self, key: u64) -> Poll> { + if let Some(slot) = self.lookup.get(&key) && slot.serves_end() { return Poll::Ready(Some(slot.serving().consume())); } - match self.start.sequence().is_some_and(|start| start > sequence) - || self.final_sequence().is_some_and(|fin| fin <= sequence) - { + match self.start.key().is_some_and(|start| start > key) || self.final_key().is_some_and(|fin| fin <= key) { true => Poll::Ready(None), false => Poll::Pending, } @@ -3025,26 +2998,24 @@ impl TrackState { /// the error it was given up with, or `None` once it is gone or never coming. fn poll_arrival( &self, - sequence: u64, + key: u64, now: crate::time::Instant, grace: Duration, waiter: &kio::Waiter, ) -> Poll>> { - if let Poll::Ready(Some(group)) = self.landing(sequence) { + if let Poll::Ready(Some(group)) = self.landing(key) { return Poll::Ready(Ok(Some(group))); } - if let Some(err) = self.given_up(sequence) { + if let Some(err) = self.given_up(key) { return Poll::Ready(Err(err)); } - if self.landing(sequence).is_ready() - || sequence - .checked_add(1) - .is_some_and(|end| self.arrivals.covers(sequence..end)) - || self.arrivals.folds_at(sequence, grace).is_some_and(|at| at <= now) + if self.landing(key).is_ready() + || key.checked_add(1).is_some_and(|end| self.arrivals.covers(key..end)) + || self.arrivals.folds_at(key, grace).is_some_and(|at| at <= now) { return Poll::Ready(Ok(None)); } - self.cache.wakes().watch_arrival(sequence, waiter); + self.cache.wakes().watch_arrival(key, waiter); Poll::Pending } @@ -3086,8 +3057,8 @@ impl TrackState { true => groups.start, false => groups.start.max(self.arrivals.first()?), }; - let start = start.max(self.start.sequence().unwrap_or(0)); - let end = groups.end.min(self.final_sequence().unwrap_or(u64::MAX)); + let start = start.max(self.start.key().unwrap_or(0)); + let end = groups.end.min(self.final_key().unwrap_or(u64::MAX)); let now = self.cache.pool().now_wall(); for (gap, opened) in self.arrivals.gaps(start..end) { let folds = opened.and_then(|opened| opened.checked_add(grace)); @@ -3101,7 +3072,7 @@ impl TrackState { None } - /// The lowest group in `groups` and `window` a sequence-order reader still waits for: + /// The lowest group in `groups` and `window` a key-order reader still waits for: /// one only a subscription owes, so a received copy's live feed may still land it. Not a /// group accounted for by its arrivals (since evicted or aborted), dropped, /// outside the feed, or in a gap its grace stopped waiting for. A local track owes none. @@ -3109,13 +3080,13 @@ impl TrackState { if !self.anchors.received() || self.outcome.is_released() || !self.readable() { return None; } - let asked = self.start.sequence().is_some(); + let asked = self.start.key().is_some(); self.find_landable(groups, asked, self.arrival_grace(), |open, _| { window.clip(open).map(|run| run.start) }) } - /// Record the declared first sequence of the live feed, replacing any earlier + /// Record the declared first key of the live feed, replacing any earlier /// declaration: the signal is scoped to the current subscription's demand, /// which may legitimately move in either direction. `None` clears it (the /// demand dropped to the live edge, whose floor is unknown until declared). @@ -3126,7 +3097,7 @@ impl TrackState { fn set_start(&mut self, start: Start) { self.latch_settled(); self.start = start; - if let Some(start) = start.sequence() { + if let Some(start) = start.key() { self.cache.wakes().arrived(..start); let now = self.cache.pool().now_wall(); self.arrivals.demand(start..u64::MAX, now); @@ -3160,8 +3131,8 @@ impl TrackState { tail: Option, settled: bool, ) -> Result<()> { - let fin = end.final_sequence().ok_or(coding::BoundsExceeded)?; - if let Some(max) = self.max_sequence + let fin = end.final_key().ok_or(coding::BoundsExceeded)?; + if let Some(max) = self.max_key && fin <= max { return Err(Error::ProtocolViolation); @@ -3203,23 +3174,23 @@ impl TrackState { } } - /// The final sequence and the error the track ended with, once a reader can reach it. + /// The final key and the error the track ended with, once a reader can reach it. fn end_error(&self) -> Option<(u64, Error)> { if !self.is_complete() { return None; } - Some((self.final_sequence()?, self.outcome.end()?.error.clone()?)) + Some((self.final_key()?, self.outcome.end()?.error.clone()?)) } /// The first group the end excludes whole: one past the last group that may exist. - fn final_sequence(&self) -> Option { - self.declared().and_then(Position::final_sequence) + fn final_key(&self) -> Option { + self.declared().and_then(Position::final_key) } /// Cap a new group the end cuts, so it never holds a frame past the end. fn cap(&self, group: &group::Handle) -> Result<()> { match self.declared() { - Some(end) if end.frame != 0 && end.group == group.sequence => group.limit(Some(end.frame)), + Some(end) if end.frame != 0 && end.group == group.start => group.limit(Some(end.frame)), _ => Ok(()), } } @@ -3237,7 +3208,7 @@ impl TrackState { match &self.outcome { Outcome::Ending(end) => end .at - .final_sequence() + .final_key() .is_some_and(|fin| self.reaches(fin) && !self.awaits_tail(fin)), // A declared end the last producer dropped short of. Outcome::Sealed(_) => true, @@ -3271,16 +3242,16 @@ impl TrackState { } /// Whether the largest can no longer advance while a producer is still held: the track - /// reached its declared end, and the largest group's content is final. Every sequence below + /// reached its declared end, and the largest group's content is final. Every key below /// the end raised it already, and a group name fixes its content, so no legal write moves /// it after this: not a late group below it, nor a recreated or fetched copy of it. fn largest_settled(&self) -> bool { self.is_complete() && !self.cache.largest_open() } - /// One past the highest sequence produced: where the live edge stands against the end. + /// One past the highest key produced: where the live edge stands against the end. fn edge(&self) -> u64 { - self.max_sequence.map_or(0, |max| max + 1) + self.max_key.map_or(0, |max| max + 1) } /// Whether a group below `fin` the wire subscription still owes has yet to arrive. @@ -3290,7 +3261,7 @@ impl TrackState { self.outcome.aborted().is_none() && self.owes(fin) } - /// Whether the tail misses a sequence from where the feed starts up to `fin` that is + /// Whether the tail misses a key from where the feed starts up to `fin` that is /// still worth waiting for; see [`Self::tail_deadline`]. fn owes(&self, fin: u64) -> bool { self.tail_deadline(fin) @@ -3298,11 +3269,11 @@ impl TrackState { } /// When a received copy's declared end stops waiting for its tail: the latest instant a - /// sequence below `fin` it still misses is given up, or `Some(None)` when one is never given + /// key below `fin` it still misses is given up, or `Some(None)` when one is never given /// up, as under an unbounded max delay, whose grace runs past the clock's range. `None` /// when nothing is missing. /// - /// A missing sequence is one no live group delivered and the publisher did + /// A missing key is one no live group delivered and the publisher did /// not drop. A gap below an arrival is given up a grace after that arrival opened it, as /// the session's tail does. The gap above the last arrival is given up a grace after /// the end or a start was declared, or a group last arrived, so a feed still delivering toward an @@ -3332,18 +3303,18 @@ impl TrackState { .reduce(|a, b| a.zip(b).map(|(a, b)| a.max(b))) } - /// The lowest sequence a received copy's tail owes: where the feed starts, or before a + /// The lowest key a received copy's tail owes: where the feed starts, or before a /// start is declared, its first arrival. fn tail_floor(&self) -> u64 { match self.start { Start::Pending(None) => 0, - start => start.sequence().or(self.arrivals.first()).unwrap_or(0), + start => start.key().or(self.arrivals.first()).unwrap_or(0), } } /// Wake `waiter` when a received copy's declared end stops waiting for its tail. fn watch_tail(&self, waiter: &kio::Waiter) { - if let Some(Some(at)) = self.final_sequence().and_then(|fin| self.tail_deadline(fin)) + if let Some(Some(at)) = self.final_key().and_then(|fin| self.tail_deadline(fin)) && self.cache.pool().now_wall() < at { self.cache.watch_wall(at, waiter); @@ -3354,7 +3325,7 @@ impl TrackState { /// every cached slot below it settled ([`Slot::is_settled`]), so nothing the end /// promised is still in flight. fn is_settled(&self) -> bool { - let Some(fin) = self.final_sequence() else { + let Some(fin) = self.final_key() else { return false; }; let reached = matches!(self.outcome, Outcome::Sealed(_)) || self.reaches(fin); @@ -3400,7 +3371,7 @@ impl TrackState { }), // A copy that wrote nothing has no frames to continue, and the reader // already holds its (empty) handle. Pointing a replacement at frame 0 would - // hand the same sequence out twice, so roll to the next group exactly as a + // hand the same key out twice, so roll to the next group exactly as a // finished one does. None => Some(Position::group(max.saturating_add(1))), } @@ -3415,7 +3386,7 @@ impl TrackState { .iter() .rev() .find(|(_, slot)| slot.is_shown()) - .map(|(sequence, _)| *sequence); + .map(|(key, _)| *key); // An open group stops because this copy dropped its feed, not because the publisher // gave it up: its stream resets as a cut would, so it leaves the cache first, and the // next feed sends it again. @@ -3423,10 +3394,10 @@ impl TrackState { .lookup .iter() .filter(|(_, slot)| slot.visible && !slot.is_aborted() && !slot.copies().any(group::Handle::is_finished)) - .map(|(&sequence, _)| sequence) + .map(|(&key, _)| key) .collect(); - for sequence in open { - self.displace(sequence); + for key in open { + self.displace(key); } // Largest names an object, so an empty group holds nothing to judge. let held = self @@ -3492,9 +3463,9 @@ impl TrackState { return Ok(()); } let err = Error::MalformedTrack; - let sequences: Vec = self.lookup.keys().copied().collect(); - for sequence in sequences { - self.evict_group(sequence, err.clone()); + let keys: Vec = self.lookup.keys().copied().collect(); + for key in keys { + self.evict_group(key, err.clone()); } Err(err) } @@ -3510,7 +3481,7 @@ impl TrackState { .range(..=newest?) .rev() .find(|(_, slot)| slot.is_shown()) - .map(|(sequence, _)| *sequence) + .map(|(key, _)| *key) } /// The route answered with `largest`. With `reaches`, its feed reaches that group, so @@ -3536,17 +3507,17 @@ impl TrackState { } } - /// Whether readers can already see content at or past `sequence`. + /// Whether readers can already see content at or past `key`. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn shows(&self, sequence: u64) -> bool { - self.lookup.range(sequence..).any(|(_, slot)| slot.is_shown()) + fn shows(&self, key: u64) -> bool { + self.lookup.range(key..).any(|(_, slot)| slot.is_shown()) } /// The newest live group the cache holds, and the number of frames it has so far. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] fn newest(&self) -> Option<(u64, u64)> { - let (sequence, slot) = self.live(..).rev().find(|(_, slot)| slot.group.is_started())?; - Some((*sequence, slot.group.frame_count() as u64)) + let (key, slot) = self.live(..).rev().find(|(_, slot)| slot.group.is_started())?; + Some((*key, slot.group.frame_count() as u64)) } /// The largest position the cache holds: the newest live group's last frame (its head, @@ -3600,7 +3571,7 @@ impl TrackState { // ([`Slot::is_settled`]). One whose last serving copy aborts leaves the outcome to // the track's own abort or drop. if self.outcome.aborted().is_none() - && let Some(fin) = self.final_sequence() + && let Some(fin) = self.final_key() { for (_, slot) in self.lookup.range(..fin).filter(|(_, slot)| !slot.is_settled()) { // Watch every copy, since a serving head that aborts hands the slot back to @@ -3642,11 +3613,11 @@ impl TrackState { Ok(()) } - fn content_for_copy(&self, sequence: u64) -> Result>> { - if let Some(content) = self.contents.get(sequence) { + fn content_for_copy(&self, key: u64) -> Result>> { + if let Some(content) = self.contents.get(key) { return Ok(Some(content)); } - if let Some(slot) = self.lookup.get(&sequence) { + if let Some(slot) = self.lookup.get(&key) { for copy in slot.copies() { if let Some(content) = copy.activate_content()? { return Ok(Some(content)); @@ -3663,20 +3634,20 @@ impl TrackState { /// [`Fetching`] resolves via [`Self::poll_fetch`]. pub(super) fn insert_group_request( &mut self, - sequence: u64, + key: u64, frames: group::Bounds, info: Option, ) -> Result { if let Some(err) = self.outcome.aborted() { return Err(err.clone()); } - if let Some(fin) = self.final_sequence() - && sequence >= fin + if let Some(fin) = self.final_key() + && key >= fin { return Err(Error::Closed); } // Never produced, so no fill holds it either. - if self.skipped(sequence) { + if self.skipped(key) { return Err(Error::Skipped); } @@ -3690,27 +3661,27 @@ impl TrackState { } let info = self.info.clone().unwrap(); - if self.covering_range(sequence, frames).is_some() { + if self.covering_range(key, frames).is_some() { return Err(Error::Duplicate); } - let content = self.content_for_copy(sequence)?; + let content = self.content_for_copy(key)?; // A copy cut short at the back never displaces a fetched one that reaches the end, // which whole-group readers still take. The fill hands it to its own fetch instead. let kept = frames.end.is_some() - && self.lookup.get(&sequence).is_some_and(|slot| match slot.is_live() { + && self.lookup.get(&key).is_some_and(|slot| match slot.is_live() { true => slot .head .as_deref() .is_some_and(|head| !head.is_aborted() && !head.back_cut()), false => slot.serves_end(), }); - if !kept && !self.lookup.get(&sequence).is_some_and(Slot::is_live) { - self.displace(sequence); + if !kept && !self.lookup.get(&key).is_some_and(Slot::is_live) { + self.displace(key); } let mut group = - group::Producer::new(group::Info { sequence }, info, self.cache.clone()).with_meter(self.ingress.meter()); + group::Producer::new(group::Info { start: key }, info, self.cache.clone()).with_meter(self.ingress.meter()); if let Some(content) = content { group.handle().share_content(content); } @@ -3726,8 +3697,8 @@ impl TrackState { // Settled first, like `commit_group`, which may evict the live slot itself. self.charge_debt(); - let protected = self.protects(sequence); - match self.lookup.get_mut(&sequence).filter(|slot| slot.is_live()) { + let protected = self.protects(key); + match self.lookup.get_mut(&key).filter(|slot| slot.is_live()) { _ if kept => self.detach(&group.handle()), Some(slot) => { // Joins the slot's standing: demoted already unless it is the live edge. @@ -3754,11 +3725,11 @@ impl TrackState { } impl TrackState { - /// Store a flight a [`Fill`] answered: a copy of group `sequence` of its own, from the + /// Store a flight a [`Fill`] answered: a copy of group `key` of its own, from the /// frame its riders start at, which the cache never serves in place of its cached copy. pub(super) fn insert_flight( &mut self, - sequence: u64, + key: u64, frames: group::Bounds, info: Option, ) -> Result { @@ -3769,12 +3740,12 @@ impl TrackState { self.install(info.unwrap_or_else(|| Info::default().with_max_age(None))); } let info = self.info.clone().unwrap(); - let mut group = group::Producer::new(group::Info { sequence }, info.clone(), self.cache.clone()) + let mut group = group::Producer::new(group::Info { start: key }, info.clone(), self.cache.clone()) .with_meter(self.ingress.meter()); // A compressed flight inflates from frame 0 with a decoder of its own, so it keeps // its own content rather than the cached copy's, whose inflated copy is further on. if !info.compressed() { - if let Some(content) = self.content_for_copy(sequence)? { + if let Some(content) = self.content_for_copy(key)? { group.handle().share_content(content); } group.handle().register_content(&self.contents); @@ -3788,7 +3759,7 @@ impl TrackState { flights.retain(|flight| flight.held().is_some()); !flights.is_empty() }); - self.flights.entry(sequence).or_default().push(group.handle().detach()); + self.flights.entry(key).or_default().push(group.handle().detach()); Ok(group) } @@ -3797,16 +3768,16 @@ impl TrackState { /// copy. When `ahead`, only one further along than the copy it reads, which it hands /// over from. pub(super) fn ride(&self, reader: &group::Consumer, ahead: bool) -> Option<(group::Handle, bool)> { - let sequence = reader.sequence; + let key = reader.start; let cached = self .lookup - .get(&sequence) + .get(&key) .into_iter() .flat_map(Slot::copies) .map(|copy| (copy.clone(), false)); let flights = self .flights - .get(&sequence) + .get(&key) .into_iter() .flatten() .filter_map(|flight| Some((flight.held()?, true))); @@ -3830,7 +3801,7 @@ pub(super) fn successor( index: usize, waiter: &kio::Waiter, ) -> Poll> { - let sequence = reader.sequence; + let key = reader.start; // A copy cut short at the back still continues a reader capped no later than the cut. let frames = reader.unread(index); let found = track.poll_ref(waiter, |state| { @@ -3839,7 +3810,7 @@ pub(super) fn successor( // arriving waits on that copy, which the reader takes instead. if state .holds - .get(&sequence) + .get(&key) .is_some_and(|held| held.iter().any(|copy| copy.is_read_by(reader) || copy.is_aborted())) { return Poll::Pending; @@ -3848,7 +3819,7 @@ pub(super) fn successor( Poll::Ready( state .lookup - .get(&sequence) + .get(&key) .into_iter() .flat_map(Slot::copies) .filter(|copy| copy.covers(frames) && !copy.is_read_by(reader)) @@ -3892,11 +3863,11 @@ pub(super) fn board(track: &kio::Producer, reader: &group::Consumer) max_delay: None, order: Order::Ascending, }; - let (sequence, frame) = (reader.sequence, if compress { 0 } else { reader.index() }); - let join = fills.lock().fly(&fills, sequence, frame, asking)?; + let (key, frame) = (reader.start, if compress { 0 } else { reader.index() }); + let join = fills.lock().fly(&fills, key, frame, asking)?; Some(Board::Wait(Boarding { join: Box::new(join), - sequence, + key, frame, })) } @@ -3915,7 +3886,7 @@ pub(super) fn hand_over( pub(super) struct Boarding { // Boxed: a reader holds one only while boarding. join: Box, - sequence: u64, + key: u64, frame: u64, } @@ -3933,17 +3904,12 @@ impl Boarding { byte: 0, end: None, }; - let sequence = self.sequence; - let answer = self - .join - .outcome() - .poll(waiter, |outcome| match outcome.refused(sequence) { - Some((err, _)) => Poll::Ready(Err(err.clone())), - None if outcome.is_accepted(sequence) => { - Poll::Ready(outcome.partial(sequence, frames).ok_or(Error::Cancel)) - } - None => Poll::Pending, - }); + let key = self.key; + let answer = self.join.outcome().poll(waiter, |outcome| match outcome.refused(key) { + Some((err, _)) => Poll::Ready(Err(err.clone())), + None if outcome.is_accepted(key) => Poll::Ready(outcome.partial(key, frames).ok_or(Error::Cancel)), + None => Poll::Pending, + }); Poll::Ready(match ready!(answer) { Ok(Ok(copy)) => copy.try_consume().ok_or(Error::Cancel), Ok(Err(err)) => Err(err), @@ -4019,9 +3985,9 @@ pub(super) struct Bounds { } impl Bounds { - /// Whether group `sequence` is within these bounds. - pub fn contains(&self, sequence: u64) -> bool { - sequence >= self.start && super::subscription::before_end(sequence, self.end) + /// Whether group `key` is within these bounds. + pub fn contains(&self, key: u64) -> bool { + key >= self.start && super::subscription::before_end(key, self.end) } } @@ -4049,6 +4015,25 @@ fn poll_largest( Poll::Ready(cache.take_largest(seen, bounds)) } +/// The lowest key of `runs`, which ascend, that no hole `hole_end` reports covers. +pub(super) fn first_open( + runs: impl Iterator>, + hole_end: impl Fn(u64) -> Option, +) -> Option { + let mut floor = 0; + for run in runs { + let mut key = (*run.start()).max(floor); + while key <= *run.end() { + match hole_end(key) { + Some(end) => key = end, + None => return Some(key), + } + } + floor = key; + } + None +} + /// Ready once the track's feed is live, or nothing more can move its largest. A declared /// end that is not reached yet does not count: an idle copy's largest is still only what /// its ended feed left. @@ -4077,19 +4062,19 @@ fn largest_ended(state: &TrackState, waiter: &kio::Waiter) -> Poll<()> { pub(crate) enum Received { /// A new group, for the copy to fill. Group(group::Producer), - /// Another copy of the sequence that is still arriving: whether this one is needed + /// Another copy of the key that is still arriving: whether this one is needed /// depends on how that one settles. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] Held(Hold), } -/// A received copy waiting on another copy of its sequence still arriving. While it waits, +/// A received copy waiting on another copy of its key still arriving. While it waits, /// a reader of that copy that fails waits for this one to take the group instead of failing. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) struct Hold { arriving: group::Handle, track: kio::Weak, - sequence: u64, + key: u64, } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] @@ -4108,7 +4093,7 @@ impl Drop for Hold { let Ok(mut state) = track.write() else { return; }; - if let btree_map::Entry::Occupied(mut holds) = state.holds.entry(self.sequence) { + if let btree_map::Entry::Occupied(mut holds) = state.holds.entry(self.key) { if let Some(at) = holds.get().iter().position(|held| held.is_copy(&self.arriving)) { holds.get_mut().swap_remove(at); } @@ -4224,13 +4209,14 @@ impl Producer { &self.info } - /// Create a new group with the given sequence number. + /// Create a new group starting at `group.start`, in the track's timescale. /// - /// It says nothing about the sequences around it: groups may be created out of order, and - /// an aborted or evicted sequence created again. A gap it leaves is skipped only once the + /// It says nothing about the starts around it: groups may be created out of order, and + /// an aborted or evicted start created again. A gap it leaves is skipped only once the /// producer says it will not fill it, with [`Self::skip_groups`] or [`Self::append_group`]. - /// A sequence already skipped is refused with [`Error::Skipped`], and the reserved - /// sequence `u64::MAX` with [`Error::BoundsExceeded`]. + /// A start already skipped is refused with [`Error::Skipped`], one held by another group + /// with [`Error::Duplicate`], and the reserved start `u64::MAX` with + /// [`Error::BoundsExceeded`]. /// /// Readers and group demand reach it at once, but the track counts it as content (its /// live edge, or the successor that ages the group before it) only once its first frame @@ -4244,7 +4230,7 @@ impl Producer { /// A received copy starts at its wire offset before any reader can see it. /// - /// A copy that already answers from `start` holds the sequence: a finished one refuses + /// A copy that already answers from `start` holds the key: a finished one refuses /// this copy with [`Error::Duplicate`], and one still arriving is returned to wait on. #[cfg(feature = "async")] pub(crate) fn create_group_at(&self, group: group::Info, start: u64) -> Result { @@ -4253,39 +4239,38 @@ impl Producer { fn create_group_inner(&self, info: group::Info, received: Option) -> Result { // Reserved, so every group fits an exclusive range. - if info.sequence == u64::MAX { + if info.start == u64::MAX { return Err(coding::BoundsExceeded.into()); } let mut state = self.modify()?; - if matches!(state.outcome, Outcome::Sealed(_)) || state.final_sequence().is_some_and(|fin| info.sequence >= fin) - { + if matches!(state.outcome, Outcome::Sealed(_)) || state.final_key().is_some_and(|fin| info.start >= fin) { return Err(Error::Closed); } - // A skip says the sequence will never be produced, so it is final. A drop is not: a + // A skip says the key will never be produced, so it is final. A drop is not: a // publisher may re-create a group it cut or aborted. - if state.skipped(info.sequence) { + if state.skipped(info.start) { return Err(Error::Skipped); } let start = received.unwrap_or(0); let content = if received.is_some() { - if let Some(holding) = state.holding_group(info.sequence, start) { + if let Some(holding) = state.holding_group(info.start, start) { if holding.is_settled() { return Err(Error::Duplicate); } let arriving = holding.clone(); - state.holds.entry(info.sequence).or_default().push(arriving.clone()); + state.holds.entry(info.start).or_default().push(arriving.clone()); return Ok(Received::Held(Hold { arriving, track: self.state.downgrade(), - sequence: info.sequence, + key: info.start, })); } - state.content_for_copy(info.sequence)? + state.content_for_copy(info.start)? } else { None }; let track = state.info.clone().unwrap(); - state.claim_sequence(info.sequence, start)?; + state.claim_key(info.start, start)?; let mut group = group::Producer::new(info, track, state.cache.clone()).with_meter(state.ingress.meter()); if start != 0 { group.bound(group::Bounds { @@ -4305,39 +4290,47 @@ impl Producer { Ok(Received::Group(group)) } - /// Create a new group with the next sequence number: one past this producer's own writes. + /// Create the next group, starting at `start` in the track's timescale: after every + /// group this track holds. /// - /// Appending says the producer will not go back, so it skips every sequence below the new + /// Appending says the producer will not go back, so it skips every start below the new /// group that it never produced, from just past its previous append (before the first, /// from the lowest group it created), as [`Self::skip_groups`] would, replacing any drop - /// there. Those sequences are then final: creating one is refused. Appending alone never - /// leaves such a gap; one left by out-of-order [`Self::create_group`] calls is what it names. + /// there: consecutive appends name the span between their starts. Those starts are then + /// final: creating one is refused. A `start` at or before a group this track holds is + /// [`Error::TimestampMismatch`], like a rewound frame. /// - /// The same broadcast, track, and group sequence always name the same content, and nothing - /// checks it: the broadcast remembers nothing of an earlier producer of the name, so a - /// replacement starts at 0. A recreated track, or a restarted publisher keeping its - /// instance, continues past its predecessor's groups with [`Self::create_group`], or uses - /// a new track name or instance. Reusing a sequence only corrupts this broadcast for its - /// readers. Datagram sequences name no cached content and are exempt. - pub fn append_group(&self) -> Result { + /// The same broadcast, track, and group start always name the same content, and nothing + /// checks it: the broadcast remembers nothing of an earlier producer of the name. A + /// recreated track, or a restarted publisher keeping its instance, keeps its starts moving + /// forward, as its media time does, or uses a new track name or instance. Reusing a start + /// only corrupts this broadcast for its readers. Datagram sequences name no cached content + /// and are exempt. + pub fn append_group(&self, start: u64) -> Result { + // Reserved, so every group fits an exclusive range. + if start == u64::MAX { + return Err(coding::BoundsExceeded.into()); + } let mut state = self.modify()?; - let sequence = state.next_sequence()?; - if let Some(fin) = state.final_sequence() - && sequence >= fin + if state.max_key.is_some_and(|max| start <= max) { + return Err(Error::TimestampMismatch); + } + if let Some(fin) = state.final_key() + && start >= fin { return Err(Error::Closed); } - if state.skipped(sequence) { + if state.skipped(start) { return Err(Error::Skipped); } let track = state.info.clone().unwrap(); - let group = group::Producer::new(group::Info { sequence }, track, state.cache.clone()) - .with_meter(state.ingress.meter()); + let group = + group::Producer::new(group::Info { start }, track, state.cache.clone()).with_meter(state.ingress.meter()); state.cap(&group.handle())?; state.commit_group(&group.handle()); - state.appended(sequence); + state.appended(start); let deferred = state.deferred(); drop(state); deferred.wake(); @@ -4345,28 +4338,33 @@ impl Producer { Ok(group) } - /// Create a group with a single frame, at the given presentation timestamp. + /// [`Self::append_group`] one past every group this track holds, or at 0 on an empty + /// track: for tests that count groups rather than time them. + #[cfg(any(test, feature = "fuzz"))] + pub(crate) fn append_next(&self) -> Result { + let start = self.modify()?.next_key()?; + self.append_group(start) + } + + /// Append a group holding one frame at `timestamp`, which is where the group starts. /// - /// The timestamp is converted into the track's timescale. Pass `None` on an untimed - /// track; a frame that doesn't match its track is [`Error::TimestampMismatch`]. - pub fn write_frame( - &mut self, - timestamp: impl Into>, - frame: B, - ) -> Result<()> { + /// The timestamp is converted into the track's timescale. On a scale-less track it is + /// [`Timestamp::unscaled`] at the group's key. The group closes after its frame, since + /// nothing says where it ends (see [`group::Producer::close`]). + pub fn write_frame(&mut self, timestamp: Timestamp, frame: B) -> Result<()> { let frame = crate::IntoBytes::into_bytes(frame); if frame.len() as u64 > group::MAX_CACHE_BYTES { return Err(Error::FrameTooLarge); } // Checked before the group exists, so a refused frame leaves no empty group behind. - let timestamp = self.modify()?.info.as_ref().unwrap().rescale(timestamp.into())?; - let mut group = self.append_group()?; + let timestamp = self.modify()?.info.as_ref().unwrap().rescale(timestamp)?; + let mut group = self.append_group(timestamp.value())?; if let Err(err) = group.write_frame(timestamp, frame) { // A rewind is only known once the group exists; end it rather than leave it open. let _ = group.abort(err.clone()); return Err(err); } - group.finish()?; + group.close()?; Ok(()) } @@ -4383,10 +4381,23 @@ impl Producer { /// [`Self::finish`] through a shared handle, which a session keeps to take the lower /// groups still in flight. + /// + /// On a received copy whose clean end was declared at its live edge, this ends the wait + /// for its tail instead: the session has every group stream it will get, so a key below + /// the end that never arrived is not coming, as with sparse starts it never existed. pub(crate) fn finish_shared(&self) -> Result<()> { let mut state = self.modify()?; - let final_sequence = state.max_sequence.map_or(0, |max| max + 1); - state.set_end(Position::group(final_sequence), None) + let end = Position::group(state.max_key.map_or(0, |max| max + 1)); + if let Outcome::Ending(declared) = &state.outcome + && declared.at == end + && declared.error.is_none() + && declared.tail.is_some() + { + let settled = declared.settled(); + state.outcome = Outcome::Open; + return state.declare(end, None, None, settled); + } + state.set_end(end, None) } /// Declare the track's exclusive end, possibly ahead of the live edge. @@ -4432,7 +4443,7 @@ impl Producer { Ok(()) } Outcome::Ending(declared) - if declared.at.frame == 0 && end.frame != 0 && end.final_sequence() == Some(declared.at.group) => + if declared.at.frame == 0 && end.frame != 0 && end.final_key() == Some(declared.at.group) => { let refined = state.declare(end, error, declared.tail.or(tail), declared.settled()); if refined.is_err() { @@ -4448,7 +4459,7 @@ impl Producer { } /// Declare the first group the live feed serves (the wire's SUBSCRIBE_START, - /// or the start the subscription itself requested): groups below `sequence` + /// or the start the subscription itself requested): groups below `key` /// are not waited for, so a reader waiting for one fails over instead of /// stalling. One may still arrive late, and a fetch can still retrieve them. /// @@ -4459,8 +4470,8 @@ impl Producer { /// its floor is unknown until the feed declares one. /// /// Resolves a start a subscriber session left pending. - pub fn start_at(&mut self, sequence: impl Into>) -> Result<()> { - let start = sequence.into().map_or(Start::Unknown, Start::Declared); + pub fn start_at(&mut self, key: impl Into>) -> Result<()> { + let start = key.into().map_or(Start::Unknown, Start::Declared); self.modify()?.set_start(start); Ok(()) } @@ -4469,8 +4480,8 @@ impl Producer { /// update can cross the response, so that budget is unknown: every older group stays /// fillable, and a fill that finds none ends empty. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn start_resolved(&mut self, sequence: u64) -> Result<()> { - self.modify()?.set_start(Start::Resolved(sequence)); + pub(crate) fn start_resolved(&mut self, key: u64) -> Result<()> { + self.modify()?.set_start(Start::Resolved(key)); Ok(()) } @@ -4498,21 +4509,34 @@ impl Producer { } } - /// Let readers use only time anchors this cache can already translate to group bounds. A - /// relative anchor counts back from the live edge the wire reports, once it does. + /// Have this received copy's readers ask no fills below the start its publisher resolved + /// from the budget, for a wire whose fills and timestamps cannot bound them by it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn anchored_cached(&self) { + pub(crate) fn fills_from_start(&self) { if let Ok(mut state) = self.modify() { - state.anchors = Anchors::Cached; + state.fills_from_start = true; } } - /// Have this received copy's readers ask no fills below the start its publisher resolved - /// from the budget, for a wire whose fills and timestamps cannot bound them by it. + /// Record where the upstream publisher started each of this lite-07 copy's live wire + /// ranges, as `(time, start)` pairs by the time each asked from, replacing the last set. + /// A reader's time start this copy cannot place among its own groups resolves to it. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn fills_from_start(&self) { + pub(crate) fn time_starts(&self, starts: impl IntoIterator) { + let starts: BTreeMap = starts.into_iter().collect(); + if let Ok(mut state) = self.modify() + && state.time_starts != starts + { + state.time_starts = starts; + } + } + + /// Have this received copy's readers start a time anchor at the group starting there, + /// held or not, for a wire whose publisher starts one at the first group at or after it. + #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] + pub(crate) fn starts_at_keys(&self) { if let Ok(mut state) = self.modify() { - state.fills_from_start = true; + state.starts_at_keys = true; } } @@ -4531,17 +4555,17 @@ impl Producer { /// /// This and [`Self::append_group`] are the only sources of a skip: a gap between created /// groups says nothing, since groups may be created out of order, so only the producer - /// knows a sequence will never exist. A skip is final: creating a group at a skipped - /// sequence is refused with [`Error::Skipped`], and a later drop of it is ignored. + /// knows a key will never exist. A skip is final: creating a group at a skipped + /// key is refused with [`Error::Skipped`], and a later drop of it is ignored. /// - /// A skip replaces any drop of the same sequences, whatever its error: a reader the drop + /// A skip replaces any drop of the same keys, whatever its error: a reader the drop /// already ended stays ended, and every later reader and relay sees the skip. A group /// already cached keeps what it is, since it was produced after all. pub fn skip_groups(&self, groups: ops::Range) { self.skip_received(groups); } - /// [`Self::skip_groups`], returning the cached sequences it left alone: a skip naming a + /// [`Self::skip_groups`], returning the cached keys it left alone: a skip naming a /// group that arrived contradicts it, which a subscriber reports. pub(crate) fn skip_received(&self, groups: ops::Range) -> RangeSet { let Ok(mut state) = self.modify() else { @@ -4573,19 +4597,19 @@ impl Producer { state.supersede_stale() } - /// Whether a stale drop under the current terms gave up `sequence`, which a group arriving + /// Whether a stale drop under the current terms gave up `key`, which a group arriving /// for it replaces and an update supersedes. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn holds_stale(&self, sequence: u64) -> bool { + pub(crate) fn holds_stale(&self, key: u64) -> bool { self.state .read() .drops - .get(&sequence) + .get(&key) .is_some_and(|hole| hole.stale_budget().is_some()) } /// Record `groups` given up as `hole` describes, returning the runs recorded: a drop - /// leaves a skipped sequence skipped, and a group a copy still serves whole, since a group + /// leaves a skipped key skipped, and a group a copy still serves whole, since a group /// stream wins over its drop: a stale drop leaves it still arriving or not, and a final one /// ends the live copy still arriving but leaves a group finished here or served by a /// fetched head. A head still filling holds that final drop until it finishes, which @@ -4613,7 +4637,7 @@ impl Producer { left.insert(groups.clone()); let mut open = Vec::new(); let mut carried = Vec::new(); - for (&sequence, slot) in state.lookup.range_mut(groups) { + for (&key, slot) in state.lookup.range_mut(groups) { if !slot.visible { continue; } @@ -4630,9 +4654,9 @@ impl Producer { let ended = !stale && arriving && std::ptr::eq(slot.serving(), &slot.group); if !ended && slot.serves_end() - && let Some(end) = sequence.checked_add(1) + && let Some(end) = key.checked_add(1) { - left.remove(sequence..end); + left.remove(key..end); // A fetched head still filling may yet die, leaving the live copy this // drop ended: the slot keeps the drop to surface as its cut then, and // the head announces its death to cut watchers. A copy that finished @@ -4643,7 +4667,7 @@ impl Producer { { slot.dropped = Some(Error::Stream(error.clone())); head.carry_drop(); - carried.push(sequence); + carried.push(key); } } } @@ -4672,11 +4696,11 @@ impl Producer { let state = self.state.read(); carried .into_iter() - .filter(|sequence| state.lookup.get(sequence).and_then(Slot::cut).is_some()) + .filter(|key| state.lookup.get(key).and_then(Slot::cut).is_some()) .collect() }; - for sequence in cut { - cache.aborted(sequence, true); + for key in cut { + cache.aborted(key, true); } } runs @@ -4780,35 +4804,42 @@ impl Producer { } } - /// Resolve cached time anchors before sending a wire that can name only groups. + /// Resolve relative anchors other than [`Range::live`](crate::track::Range::live)'s + /// against this copy before sending a wire that can name only groups: a start becomes + /// the group it resolved to, and an end the last group it includes. Fails with + /// [`Error::Unsupported`] while one waits on a latest group. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn resolve_subscription(&self, subscription: &Subscription) -> Result { - if !subscription.ranges.iter().any(|range| !range.is_wire()) { + use super::subscription::{Anchor, Range}; + + let relative: Vec = (0..subscription.ranges.len()) + .filter(|index| !subscription.ranges[*index].is_wire()) + .collect(); + if relative.is_empty() { return Ok(subscription.clone()); } + let ranges: Vec = relative.iter().map(|index| subscription.ranges[*index]).collect(); let state = self.state.read(); - let window = Window::new(&subscription.ranges, state.latest_live(), &*state)?; + let window = Window::new(&ranges, state.latest_live(), &*state)?; if !window.is_settled() { return Err(Error::Unsupported); } - let resolved = window.resolved(); let mut subscription = subscription.clone(); - for (range, resolved) in subscription.ranges.iter_mut().zip(resolved) { - if match range.start { - super::subscription::Anchor::Time(_) => true, - super::subscription::Anchor::Relative(back) => back != 0, - _ => false, - } { - range.start = super::subscription::Anchor::Absolute(resolved.ok_or(Error::Unsupported)?.start); - } - if matches!( - range.end, - Some(super::subscription::Anchor::Time(_) | super::subscription::Anchor::Relative(_)) - ) { - range.end = resolved - .ok_or(Error::Unsupported)? - .end - .map(super::subscription::Anchor::Absolute); + for (index, resolved) in relative.into_iter().zip(window.resolved()) { + let resolved = resolved.ok_or(Error::Unsupported)?; + let range = &mut subscription.ranges[index]; + if matches!(range.start, Anchor::Relative(back) if back != 0) { + range.start = Anchor::Time(resolved.start); + } + if matches!(range.end, Some(Anchor::Relative(_))) { + *range = match resolved.end.map(|end| end.checked_sub(1)) { + None => Range { end: None, ..*range }, + Some(Some(last)) => Range { + end: Some(Anchor::Time(last)), + ..*range + }, + Some(None) => Range::EMPTY, + }; } } Ok(subscription) @@ -4841,23 +4872,23 @@ impl Producer { /// [`Request::request_start`] for a copy already accepted, which a held copy's next /// subscription asks for again. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn request_start(&mut self, sequence: Option) -> Result<()> { - self.modify()?.set_start(Start::Pending(sequence)); + pub(crate) fn request_start(&mut self, key: Option) -> Result<()> { + self.modify()?.set_start(Start::Pending(key)); Ok(()) } - /// The declared first sequence of the live feed, once [`Self::start_at`] + /// The declared first key of the live feed, once [`Self::start_at`] /// declared one. `None` while nothing was declared. #[cfg(test)] #[cfg(feature = "async")] - pub(crate) fn start_sequence(&self) -> Option { - self.state.read().start.sequence() + pub(crate) fn start_key(&self) -> Option { + self.state.read().start.key() } - /// Evict group `sequence` from the cache, as expiry does. + /// Evict group `key` from the cache, as expiry does. #[cfg(all(test, feature = "async"))] - pub(crate) fn evict(&self, sequence: u64) { - self.modify().unwrap().evict_group(sequence, Error::Evicted); + pub(crate) fn evict(&self, key: u64) { + self.modify().unwrap().evict_group(key, Error::Evicted); } /// The exclusive end, once [`Self::finish`] or [`Self::finish_at`] declared one. @@ -4877,12 +4908,12 @@ impl Producer { /// [`cache::Config::with_expiry`]) reclaims the rest, the latest group included, so a /// stale [`Consumer`] can't pin them in memory forever. /// - /// If the declared end had settled (the final sequence from + /// If the declared end had settled (the final key from /// [`finish_at`](Self::finish_at) was reached and every group below it finished), /// consumers get that end instead of the error, as after [`finish`](Self::finish). /// /// [`finish`](Self::finish) is deliberately not terminal: it declares the final - /// sequence, and lower-numbered groups may still be written afterwards. + /// key, and lower-numbered groups may still be written afterwards. pub fn abort(self, err: Error) -> Result<()> { // Unlisted first, so the track leaves the listing before its name is free again. self.alive.unlist(false); @@ -4956,8 +4987,8 @@ impl Producer { } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn latest_sequence(&self) -> Option { - self.state.read().max_sequence + pub(crate) fn latest_key(&self) -> Option { + self.state.read().max_key } /// The track's largest location: its newest group at the newest frame (0 while empty), `None` before any. @@ -4965,12 +4996,12 @@ impl Producer { self.state.read().cache.largest() } - /// Ready once the track holds content past `sequence`, or is closed: a later group whose + /// Ready once the track holds content past `key`, or is closed: a later group whose /// first frame opened or that finished. A frameless group header /// does not count, since nothing says the copy reached it yet. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn poll_past(&self, sequence: u64, waiter: &kio::Waiter) -> Poll<()> { - let Some(next) = sequence.checked_add(1) else { + pub(crate) fn poll_past(&self, key: u64, waiter: &kio::Waiter) -> Poll<()> { + let Some(next) = key.checked_add(1) else { return Poll::Pending; }; let past = self.state.poll_ref(waiter, |state| { @@ -5138,7 +5169,7 @@ impl Producer { /// /// ```compile_fail,E0599 /// # fn cannot_append(track: moq_net::track::Finished) { -/// let _ = track.append_group(); +/// let _ = track.append_next(); /// # } /// ``` #[must_use = "dropping the last handle releases the track, which new readers can then no longer find"] @@ -5169,7 +5200,7 @@ impl Finished { /// Create an older group below the end, which arrived late; see [`Producer::create_group`]. /// - /// A sequence at or past the end is refused with [`Error::Closed`]. + /// A key at or past the end is refused with [`Error::Closed`]. pub fn create_group(&self, group: group::Info) -> Result { self.producer.create_group(group) } @@ -5641,7 +5672,7 @@ impl Demand { /// /// A publisher opens a group whenever `n` passes its largest. The first consumer of a /// track with no group asks for `Some(0)`, and a join judged by [`Info::refresh`] - /// asks for the next free sequence until a group lands there. `n` never goes back, so once a group reaches it nothing is + /// asks for the next free key until a group lands there. `n` never goes back, so once a group reaches it nothing is /// pending until another join asks. pub fn wanted(&self) -> Option { if !self.is_wanted() { @@ -5818,11 +5849,11 @@ impl ArrivalRead<'_> { // Legacy wires lack FETCH, so held requests need the live-feed verdict. pub(crate) fn poll_group( &self, - sequence: u64, + key: u64, now: crate::time::Instant, waiter: &kio::Waiter, ) -> Poll>> { - match self.state.poll_arrival(sequence, now, self.grace, waiter) { + match self.state.poll_arrival(key, now, self.grace, waiter) { Poll::Pending if self.state.is_closed() => Poll::Ready(Ok(None)), res => res, } @@ -5840,8 +5871,8 @@ impl ArrivalRead<'_> { } }; let closed = self.state.is_closed(); - let start = state.start.sequence().unwrap_or(0); - let fin = state.final_sequence().unwrap_or(u64::MAX); + let start = state.start.key().unwrap_or(0); + let fin = state.final_key().unwrap_or(u64::MAX); for hole in miss::holes(state, groups) { if closed { give_up(hole); @@ -6041,26 +6072,12 @@ impl Consumer { fn poll_state_start(state: &kio::Consumer, waiter: &kio::Waiter) -> Poll> { let res = state.poll(waiter, |state| match state.start { Start::Pending(_) if state.outcome.aborted().is_none() => Poll::Pending, - start => Poll::Ready(start.sequence()), + start => Poll::Ready(start.key()), }); match ready!(res) { Ok(start) => Poll::Ready(start), - Err(state) => Poll::Ready(state.start.sequence()), - } - } - - /// Refuse a new range before it changes the demand shared by this route's readers. - /// - /// Only an unresolved time anchor is refused, so callers skip ranges without one - /// before building a consumer or locking the track's state. - fn refuse_subscription(&self, subscription: &Subscription) -> Result<()> { - if let Some(serving) = self.serving() { - return serving.refuse_subscription(subscription); + Err(state) => Poll::Ready(state.start.key()), } - self.with_state(|state| match state { - Some(state) => refuse_cached_anchors(&state.read(), subscription), - None => Ok(()), - }) } /// The upstream publisher's Largest Object, independently of which groups are cached. @@ -6080,41 +6097,41 @@ impl Consumer { if let Some(serving) = self.serving() { return serving.peek_latest(); } - let sequence = self.with_state(|state| state?.read().max_sequence)?; - self.peek_group(sequence) + let key = self.with_state(|state| state?.read().max_key)?; + self.peek_group(key) } - /// The nearest cached group below `sequence`, under the same terms as + /// The nearest cached group below `key`, under the same terms as /// [`Self::peek_group`]. Walks the cache's own order, so gaps in the group numbering /// are crossed and aborted (evicted) entries are skipped. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn peek_before(&self, sequence: u64) -> Option { + pub(crate) fn peek_before(&self, key: u64) -> Option { if let Some(serving) = self.serving() { - return serving.peek_before(sequence); + return serving.peek_before(key); } self.with_state(|state| { state? .read() .lookup - .range(..sequence) + .range(..key) .rev() .find(|(_, slot)| slot.serves_end()) .map(|(_, slot)| slot.serving().consume()) }) } - /// A cached group by sequence, under the same terms as [`Self::peek_latest`]. Unlike a + /// A cached group by key, under the same terms as [`Self::peek_latest`]. Unlike a /// fetch, a peek does not refresh the group's cache standing, so it never keeps a /// group alive over one a subscriber actually read; an aborted (evicted) group, or a /// copy cut short at the back, is a miss. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn peek_group(&self, sequence: u64) -> Option { + pub(crate) fn peek_group(&self, key: u64) -> Option { if let Some(serving) = self.serving() { - return serving.peek_group(sequence); + return serving.peek_group(key); } self.with_state(|state| { let state = state?.read(); - let slot = state.lookup.get(&sequence).filter(|slot| slot.serves_end())?; + let slot = state.lookup.get(&key).filter(|slot| slot.serves_end())?; Some(slot.serving().consume()) }) } @@ -6128,7 +6145,7 @@ impl Consumer { /// Poll for a cached group, a terminal disposition, or a gap outliving its grace: the /// group, the error a dropped or cut group was given up with, or `None` once it is gone. - pub(crate) fn poll_group(&self, sequence: u64, waiter: &kio::Waiter) -> Poll>> { + pub(crate) fn poll_group(&self, key: u64, waiter: &kio::Waiter) -> Poll>> { self.with_state(|state| { let Some(state) = state else { return Poll::Ready(Ok(None)); @@ -6137,9 +6154,9 @@ impl Consumer { let max_delay = clamp_max_delay(state.subscriptions.lock().max_delay(), state.max_age_bound()); let grace = ledger::grace(max_delay); let now = state.cache.pool().now_wall(); - let res = state.poll_arrival(sequence, now, grace, waiter); + let res = state.poll_arrival(key, now, grace, waiter); if res.is_pending() { - let Some(at) = state.arrivals.folds_at(sequence, grace) else { + let Some(at) = state.arrivals.folds_at(key, grace) else { // No later arrival yet: only this wait uses the track's change channel. return Poll::Pending; }; @@ -6156,18 +6173,18 @@ impl Consumer { if res.is_pending() && state.poll_closed(waiter).is_ready() { // A cached arrival may have raced closure after the first lookup. let state = state.read(); - return Poll::Ready(match state.landing(sequence) { + return Poll::Ready(match state.landing(key) { Poll::Ready(Some(group)) => Ok(Some(group)), - _ => state.given_up(sequence).map_or(Ok(None), Err), + _ => state.given_up(key).map_or(Ok(None), Err), }); } res }) } - /// Lend `head` to the cached copies of group `sequence`; see + /// Lend `head` to the cached copies of group `key`; see /// [`group::Handle::adopt_head`]. - pub(super) fn adopt_head(&self, sequence: u64, head: &group::Head) { + pub(super) fn adopt_head(&self, key: u64, head: &group::Head) { let copies: Vec = self.with_state(|state| { let Some(state) = state else { return Vec::new(); @@ -6175,7 +6192,7 @@ impl Consumer { let state = state.read(); state .lookup - .get(&sequence) + .get(&key) .map_or_else(Vec::new, |slot| slot.copies().cloned().collect()) }); // With the track lock released (lock order is track then group): adopting settles @@ -6185,19 +6202,19 @@ impl Consumer { } } - /// Poll for group `sequence` falling a full `budget`, clamped to this track's max age, + /// Poll for group `key` falling a full `budget`, clamped to this track's max age, /// behind its live edge, as a reader would judge it, whether the track holds it or not; /// see `TrackState::drifted`. Parks in this track's expiry index, so only a change that /// can make the group stale, or land it here, wakes it. Pending for good once the track closes. - pub(super) fn poll_stale(&self, sequence: u64, budget: Duration, waiter: &kio::Waiter) -> Poll<()> { + pub(super) fn poll_stale(&self, key: u64, budget: Duration, waiter: &kio::Waiter) -> Poll<()> { self.with_state(|state| { let Some(state) = state else { return Poll::Pending; }; let state = state.read(); let budget = clamp_max_delay(budget, state.max_age_bound()); - state.cache.wakes().watch_held(sequence, waiter); - match state.poll_drifted(sequence, &Servable::ALL, budget, None, waiter) { + state.cache.wakes().watch_held(key, waiter); + match state.poll_drifted(key, &Servable::ALL, budget, None, waiter) { true => Poll::Ready(()), false => Poll::Pending, } @@ -6253,17 +6270,17 @@ impl Consumer { }) } - /// The group a front's parked copy would serve for `sequence` from `frame_start`, when it + /// The group a front's parked copy would serve for `key` from `frame_start`, when it /// is whole: finished from frame 0 on a copy still open. Anything less (open, headless, /// aborted, or on a failed copy) needs the route. The fetch joins the copy's demand, so /// a copy kept busy only by fetches stays cached at its session. - fn parked_whole(&self, sequence: u64, frame_start: u64) -> Option<(kio::Consumer, group::Consumer)> { + fn parked_whole(&self, key: u64, frame_start: u64) -> Option<(kio::Consumer, group::Consumer)> { let copy = self.with_state(|state| state?.read().routes.as_ref()?.parked())?; copy.with_state(|state| { let state = state?; let group = { let track = state.read(); - let group = track.covering_group(sequence, frame_start)?; + let group = track.covering_group(key, frame_start)?; let whole = !track.is_closed() && track.outcome.aborted().is_none() && group.live_first_frame() == Some(0) @@ -6322,13 +6339,13 @@ impl Consumer { /// or `group::Fetch::default()`. /// /// The returned future resolves to [`Error::NotFound`] when the group can never be served - /// (past the final sequence, or no [`Requests`] on the track), the handler's + /// (past the final key, or no [`Requests`] on the track), the handler's /// rejection (a relay's upstream miss is [`StreamError::NotFound`](StreamError::NotFound)), /// or the track's abort error if it's already closed. A miss queues a one-group [`Fill`], - /// which concurrent fetches of the same sequence join. - pub fn fetch_group(&self, sequence: u64, options: impl Into>) -> kio::Pending { + /// which concurrent fetches of the same key join. + pub fn fetch_group(&self, key: u64, options: impl Into>) -> kio::Pending { self.fetching( - sequence, + key, FetchOptions { options: options.into().unwrap_or_default(), end: None, @@ -6337,13 +6354,13 @@ impl Consumer { } /// Fill a wire request, widening a cold or compressed group to a reusable prefix. It asks - /// on a reader's behalf, so a sequence the serving copy gave up for good is answered from + /// on a reader's behalf, so a key the serving copy gave up for good is answered from /// that drop rather than asked upstream again, as the copy's own readers are. - pub(crate) fn fetch_range(&self, sequence: u64, mut request: FetchOptions) -> kio::Pending { + pub(crate) fn fetch_range(&self, key: u64, mut request: FetchOptions) -> kio::Pending { let state = self.resolve(None).0; { let state = state.read(); - if state.lookup.get(&sequence).is_none_or(Slot::is_aborted) + if state.lookup.get(&key).is_none_or(Slot::is_aborted) || state.info.as_ref().is_none_or(|info| info.compressed()) { request.options.frame_start = 0; @@ -6355,10 +6372,10 @@ impl Consumer { request.end = None; } } - self.fetching(sequence, request) + self.fetching(key, request) } - fn fetching(&self, sequence: u64, request: FetchOptions) -> kio::Pending { + fn fetching(&self, key: u64, request: FetchOptions) -> kio::Pending { let FetchOptions { options, end: frame_end, @@ -6371,12 +6388,12 @@ impl Consumer { // Nothing more feeds a whole group, so no route needs to stay up for it: the copy a // front parked serves it without reviving the logical track for one read. - if let Some((state, group)) = self.parked_whole(sequence, options.frame_start) { + if let Some((state, group)) = self.parked_whole(key, options.frame_start) { let fills = state.read().fills.clone(); return kio::Pending::new(Fetching { state, fills, - sequence, + key, frame_start: options.frame_start, byte_start: options.byte_start, frame_end, @@ -6398,7 +6415,7 @@ impl Consumer { return kio::Pending::new(Fetching { state: state.clone(), fills: state.read().fills.clone(), - sequence, + key, frame_start: options.frame_start, byte_start: options.byte_start, frame_end, @@ -6409,7 +6426,7 @@ impl Consumer { hit: None, stats: self.stats.clone(), resume: Some(Box::new(resume.fetch_group( - sequence, + key, FetchOptions { options, end: frame_end, @@ -6421,7 +6438,7 @@ impl Consumer { kio::Pending::new(Fetching::cached_or_queued( state, - sequence, + key, FetchOptions { options, end: frame_end, @@ -6430,7 +6447,27 @@ impl Consumer { )) } - /// The sequences of `span` the cache holds, lowest first. + /// The group `back` groups before `latest` among those the cache holds, `latest` itself + /// counting as the first: `None` once fewer are held. What a wire's relative start counts + /// back to, since keys are starts and no arithmetic on them counts groups. + #[cfg(feature = "async")] + pub(crate) fn group_back(&self, latest: u64, back: u64) -> Option { + if let Some(serving) = self.serving() { + return serving.group_back(latest, back); + } + self.with_state(|state| Holdings::back(&*state?.read(), latest, back)) + } + + /// The exclusive end of the run at `key` that no group will fill: one declared skipped, + /// or with `dropped`, also one given up for good. See [`Subscriber::next_in_range`]. + pub(super) fn hole_end(&self, key: u64, dropped: bool) -> Option { + if let Some(serving) = self.serving() { + return serving.hole_end(key, dropped); + } + self.with_state(|state| state?.read().hole_end(key, dropped)) + } + + /// The keys of `span` the cache holds, lowest first. #[cfg(feature = "async")] pub(crate) fn cached_groups(&self, span: ops::Range) -> Vec { if let Some(serving) = self.serving() { @@ -6445,7 +6482,7 @@ impl Consumer { .lookup .range(span) .filter(|(_, slot)| slot.serves_end()) - .map(|(sequence, _)| *sequence) + .map(|(key, _)| *key) .collect() }) } @@ -6506,14 +6543,14 @@ impl Consumer { }) } - pub(crate) fn latest_sequence(&self) -> Option { + pub(crate) fn latest_key(&self) -> Option { if let Some(serving) = self.serving() { - return serving.latest_sequence(); + return serving.latest_key(); } - self.with_state(|state| state?.read().max_sequence) + self.with_state(|state| state?.read().max_key) } - /// Ready once [`Self::latest_sequence`] is `Some`: the track holds a group. Stays pending + /// Ready once [`Self::latest_key`] is `Some`: the track holds a group. Stays pending /// on a track that closed without one. pub(super) fn poll_grouped(&self, waiter: &kio::Waiter) -> Poll<()> { self.with_state(|state| { @@ -6524,12 +6561,12 @@ impl Consumer { if let Some(routes) = routes { return routes.poll_grouped(waiter); } - match state.poll(waiter, |state| match state.max_sequence { + match state.poll(waiter, |state| match state.max_key { Some(_) => Poll::Ready(()), None => Poll::Pending, }) { Poll::Ready(Ok(())) => Poll::Ready(()), - Poll::Ready(Err(closed)) if closed.max_sequence.is_some() => Poll::Ready(()), + Poll::Ready(Err(closed)) if closed.max_key.is_some() => Poll::Ready(()), _ => Poll::Pending, } }) @@ -6537,11 +6574,11 @@ impl Consumer { /// The first group the declared end excludes whole, or `None` while the track is open ended. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn final_sequence(&self) -> Option { + pub(crate) fn final_key(&self) -> Option { if let Some(serving) = self.serving() { - return serving.final_sequence(); + return serving.final_key(); } - self.with_state(|state| state?.read().final_sequence()) + self.with_state(|state| state?.read().final_key()) } /// The frame-precise point a replacement route should resume from: one past the @@ -6618,10 +6655,6 @@ impl Subscribing { // Wait until the track info is available let info = ready!(self.state.poll(waiter, |state| state.poll_info())) .map_err(|e| e.outcome.aborted().cloned().unwrap_or(Error::Dropped))??; - refuse_untimed(&info, &self.subscription.read())?; - if self.subscription.read().has_time() { - Consumer::new(self.name.clone(), self.state.weak()).refuse_subscription(&self.subscription.read())?; - } let resume = self.state.read().routes.clone(); let inner = match resume { @@ -6651,9 +6684,6 @@ impl Subscribing { /// Returns [`Error::Closed`] if the track already ended; the update is /// meaningless at that point and can usually be ignored. pub fn update(&mut self, subscription: Subscription) -> Result<()> { - if subscription.has_time() && subscription.ranges != self.subscription.read().ranges { - Consumer::new(self.name.clone(), self.state.weak()).refuse_subscription(&subscription)?; - } self.subscription.update(subscription) } } @@ -6712,14 +6742,14 @@ impl kio::Task for Resolving { } } -/// Join the fill covering `sequence`, or queue a one-group fill; `None` when no handler -/// exists to take one, or for the reserved sequence `u64::MAX`, which no group holds. -fn queue(fills: &kio::Shared, sequence: u64, request: &FetchOptions) -> Option> { +/// Join the fill covering `key`, or queue a one-group fill; `None` when no handler +/// exists to take one, or for the reserved key `u64::MAX`, which no group holds. +fn queue(fills: &kio::Shared, key: u64, request: &FetchOptions) -> Option> { let options = &request.options; - let end = sequence.checked_add(1)?; + let end = key.checked_add(1)?; let ask = fill::Ask { start: Position { - group: sequence, + group: key, frame: options.frame_start, }, byte: options.byte_start, @@ -6754,7 +6784,7 @@ pub(crate) struct FetchOptions { pub struct Fetching { state: kio::Consumer, fills: kio::Shared, - sequence: u64, + key: u64, // This caller's own start, so a cached group that begins above it is a miss // rather than a short answer. frame_start: u64, @@ -6781,12 +6811,12 @@ pub struct Fetching { } impl Fetching { - /// A fetch of `sequence` from the track's own cache, queueing a fill for it only when + /// A fetch of `key` from the track's own cache, queueing a fill for it only when /// the cache can't resolve it (cached, aborted, or past-final all resolve through /// [`Self::poll`] without one). fn cached_or_queued( state: kio::Consumer, - sequence: u64, + key: u64, request: FetchOptions, stats: stats::Scope, ) -> Self { @@ -6797,10 +6827,10 @@ impl Fetching { }; let (fills, cached) = { let state = state.read(); - (state.fills.clone(), state.poll_fetch_cached(sequence, frames)) + (state.fills.clone(), state.poll_fetch_cached(key, frames)) }; let result = match cached.is_pending() { - true => queue(&fills, sequence, &request), + true => queue(&fills, key, &request), false => None, }; // Hold a hit until the caller polls: the held consumer keeps the group wanted, @@ -6812,7 +6842,7 @@ impl Fetching { Self { state, fills, - sequence, + key, frame_start: request.options.frame_start, byte_start: request.options.byte_start, frame_end: request.end, @@ -6844,7 +6874,7 @@ impl kio::Task for Fetching { // caller keeps the fetch and its join. if let Some(join) = &self.result { let mut picked = RangeSet::new(); - picked.insert(self.sequence..self.sequence.saturating_add(1)); + picked.insert(self.key..self.key.saturating_add(1)); join.saw(&picked); } Poll::Ready(res) @@ -6865,7 +6895,7 @@ impl Fetching { byte_start: self.byte_start, }; self.resume = Some(Box::new(routes.fetch_group( - self.sequence, + self.key, FetchOptions { options, end: self.frame_end, @@ -6877,18 +6907,18 @@ impl Fetching { let group = ready!(kio::Task::poll(&mut **resume, waiter))?; return Poll::Ready(Ok(group.with_meter(self.stats.meter()))); } - let (state, fills, sequence, frame_start, result) = ( + let (state, fills, key, frame_start, result) = ( &self.state, &self.fills, - self.sequence, + self.key, self.frame_start, self.result.as_ref(), ); let cache = { let state = state.read(); let budget = clamp_max_delay(self.max_delay.unwrap_or(Duration::MAX), state.max_age_bound()); - let _ = state.cache.wakes().watch_landing(sequence, waiter); - if self.judges_drift && state.poll_drifted(sequence, &Servable::ALL, budget, None, waiter) { + let _ = state.cache.wakes().watch_landing(key, waiter); + if self.judges_drift && state.poll_drifted(key, &Servable::ALL, budget, None, waiter) { return Poll::Ready(Err(Error::Old)); } state.cache.clone() @@ -6900,7 +6930,7 @@ impl Fetching { state: state.weak(), cache, reading: Reading::new(state.clone()), - sequence, + key, max_delay: self.max_delay, superseded: Superseded::default(), }); @@ -6922,8 +6952,7 @@ impl Fetching { byte: self.byte_start, end: self.frame_end, }; - let cached = |waiter: &kio::Waiter| match state.poll(waiter, |state| state.poll_fetch_cached(sequence, frames)) - { + let cached = |waiter: &kio::Waiter| match state.poll(waiter, |state| state.poll_fetch_cached(key, frames)) { Poll::Ready(Ok(res)) => Poll::Ready(res.map(resolve)), Poll::Ready(Err(closed)) => Poll::Ready(Err(closed.outcome.aborted().cloned().unwrap_or(Error::Dropped))), Poll::Pending => Poll::Pending, @@ -6934,8 +6963,8 @@ impl Fetching { // Handler side. let Some(result) = result else { - // No group is ever created at the reserved sequence, so no handler is asked for it. - if sequence == u64::MAX { + // No group is ever created at the reserved key, so no handler is asked for it. + if key == u64::MAX { return Poll::Ready(Err(Error::NotFound)); } // Never queued: no handler existed when the fetch was made. Fail fast while @@ -6951,15 +6980,11 @@ impl Fetching { // An answered fill fails a joined fetch the cache cannot cover. The channel // closing without an answer means its handlers went away. - let (err, asked, partial) = match result - .outcome() - .poll(waiter, |outcome| match outcome.refused(sequence) { - Some((err, asked)) => Poll::Ready((err.clone(), asked, None)), - None if outcome.is_accepted(sequence) => { - Poll::Ready((Error::NotFound, None, outcome.partial(sequence, frames))) - } - None => Poll::Pending, - }) { + let (err, asked, partial) = match result.outcome().poll(waiter, |outcome| match outcome.refused(key) { + Some((err, asked)) => Poll::Ready((err.clone(), asked, None)), + None if outcome.is_accepted(key) => Poll::Ready((Error::NotFound, None, outcome.partial(key, frames))), + None => Poll::Pending, + }) { Poll::Ready(Ok(res)) => res, Poll::Ready(Err(_closed)) => (Error::NotFound, None, None), Poll::Pending => return Poll::Pending, @@ -6990,7 +7015,7 @@ impl Fetching { }; self.result = queue( &self.fills, - self.sequence, + self.key, &FetchOptions { options, end: self.frame_end, @@ -7099,14 +7124,14 @@ pub(super) enum Reserved { } impl Reserved { - pub(super) fn sequence(&self) -> u64 { + pub(super) fn key(&self) -> u64 { match self { - Self::Plain(peeked) => peeked.group.sequence, - Self::Resume(candidate) => candidate.sequence(), + Self::Plain(peeked) => peeked.group.start, + Self::Resume(candidate) => candidate.key(), } } - /// Whether it was found in sequence order rather than arrival order. + /// Whether it was found in key order rather than arrival order. pub(super) fn ordered(&self) -> bool { match self { Self::Plain(peeked) => peeked.arrival.is_none(), @@ -7187,7 +7212,7 @@ struct GroupExpiry { subscription: kio::Consumer, /// What the cursor can be served; see [`Cursor::published`]. servable: kio::Consumer, - sequence: u64, + key: u64, superseded: Superseded, } @@ -7197,7 +7222,7 @@ struct FetchExpiry { /// See [`GroupExpiry::cache`]. cache: Arc, reading: Reading, - sequence: u64, + key: u64, max_delay: Option, superseded: Superseded, } @@ -7232,7 +7257,7 @@ struct Superseded(Mutex>); /// A successor, when it arrived, and whether it has since failed. struct Successor { - sequence: u64, + key: u64, arrived: crate::time::Instant, fate: group::Fate, } @@ -7252,7 +7277,7 @@ impl group::Expiry for FetchExpiry { reading: self.reading.moved(&state), state, cache, - sequence: self.sequence, + key: self.key, max_delay: self.max_delay, superseded: Superseded::default(), }) @@ -7264,7 +7289,7 @@ impl group::Expiry for FetchExpiry { fn is_expired(&self, recover: Option<&super::resume::Recover>, waiter: &kio::Waiter) -> bool { // Parked first, so a group landing above while this judges wakes it. - if !self.cache.wakes().watch_landing(self.sequence, waiter) { + if !self.cache.wakes().watch_landing(self.key, waiter) { return false; } let max_delay = recover.and_then(|recover| recover.poll_budget(waiter)); @@ -7276,7 +7301,7 @@ impl group::Expiry for FetchExpiry { (None, None) => Duration::MAX, }; let budget = clamp_max_delay(budget, state.max_age_bound()); - state.poll_drifted(self.sequence, &Servable::ALL, budget, Some(&self.superseded), waiter) + state.poll_drifted(self.key, &Servable::ALL, budget, Some(&self.superseded), waiter) } } @@ -7297,7 +7322,7 @@ impl group::Expiry for GroupExpiry { cache, subscription: self.subscription.clone(), servable: self.servable.clone(), - sequence: self.sequence, + key: self.key, superseded: Superseded::default(), }) } @@ -7313,7 +7338,7 @@ impl group::Expiry for GroupExpiry { /// without this poll, since every update mirrors them at once. fn is_expired(&self, recover: Option<&super::resume::Recover>, waiter: &kio::Waiter) -> bool { // Parked first, so a group landing above while this judges wakes it. - if !self.cache.wakes().watch_landing(self.sequence, waiter) { + if !self.cache.wakes().watch_landing(self.key, waiter) { return false; } let max_delay = recover.and_then(|recover| recover.poll_budget(waiter)); @@ -7334,12 +7359,12 @@ impl group::Expiry for GroupExpiry { let state = self.state.read(); let budget = clamp_max_delay(max_delay, state.max_age_bound()); - state.poll_drifted(self.sequence, &servable, budget, Some(&self.superseded), waiter) + state.poll_drifted(self.key, &servable, budget, Some(&self.superseded), waiter) } } /// The group a poll's drift is measured against, identified well enough to tell it apart -/// from whatever may occupy its sequence by the time a candidate is judged. +/// from whatever may occupy its key by the time a candidate is judged. #[derive(Clone)] struct Edge { /// This track's own edge, revalidated before it convicts anything. @@ -7477,7 +7502,7 @@ impl Published { enum LiveWalk { /// The cache, which holds only the live feed. Cached(C), - /// The live index, each sequence looked up in the cache. + /// The live index, each key looked up in the cache. Indexed(I), } @@ -7526,11 +7551,8 @@ fn leap<'a, R>( if !run.contains(&first) { return Leap::Past(first); } - let mut within = std::iter::once(slot).chain( - cached - .take_while(|(sequence, _)| run.contains(*sequence)) - .map(|(_, slot)| slot), - ); + let mut within = + std::iter::once(slot).chain(cached.take_while(|(key, _)| run.contains(*key)).map(|(_, slot)| slot)); match within.find_map(f) { Some(found) => Leap::Found(found), None => Leap::Exhausted, @@ -7587,7 +7609,7 @@ struct Indexed { track: Weak, /// How far into the arrival order this has taken in, as an absolute index. taken: usize, - /// The stamp of each live group inside the fixed runs, by sequence, as it arrived. + /// The stamp of each live group inside the fixed runs, by key, as it arrived. groups: BTreeMap, } @@ -7604,26 +7626,25 @@ impl Indexed { /// Take in what arrived since the last call. fn catch_up(&mut self, state: &TrackState, fixed: &Runs) { let skip = self.taken.saturating_sub(state.offset).min(state.arrival.len()); - for (sequence, stamp) in state.arrival.range(skip..) { - if fixed.contains(*sequence) { - self.groups.insert(*sequence, *stamp); + for (key, stamp) in state.arrival.range(skip..) { + if fixed.contains(*key) { + self.groups.insert(*key, *stamp); } } self.taken = state.offset + state.arrival.len(); // A live entry is a visible slot, so past twice their count most entries are stale, // and a prune costs each entry it drops one lookup. if self.groups.len() > 2 * state.visible.len().max(LIVE_BATCH) { - self.groups - .retain(|sequence, stamp| current(state, *sequence, *stamp).is_some()); + self.groups.retain(|key, stamp| current(state, *key, *stamp).is_some()); } } } -/// The live slot at `sequence`, if it is still the incarnation stamped `stamp`. -fn current(state: &TrackState, sequence: u64, stamp: u32) -> Option<&Slot> { +/// The live slot at `key`, if it is still the incarnation stamped `stamp`. +fn current(state: &TrackState, key: u64, stamp: u32) -> Option<&Slot> { state .lookup - .get(&sequence) + .get(&key) .filter(|slot| slot.stamp == stamp && slot.visible) } @@ -7647,7 +7668,7 @@ impl Live { } /// The first live group inside `fixed`, within `lo..=hi`, for which `f` answers, lowest - /// first or, with `rev`, highest first, with its sequence. + /// first or, with `rev`, highest first, with its key. fn find<'a, R>( &self, state: &'a TrackState, @@ -7670,10 +7691,7 @@ impl Live { indexed.catch_up(state, fixed); caught_up = true; } - let entries = indexed - .groups - .range(lo..=hi) - .map(|(&sequence, &stamp)| (sequence, stamp)); + let entries = indexed.groups.range(lo..=hi).map(|(&key, &stamp)| (key, stamp)); let mut taken = 0; let mut put = |entry| { batch[taken] = entry; @@ -7688,22 +7706,22 @@ impl Live { let batch = &batch[..taken]; let mut stale = Vec::new(); let mut found = None; - for &(sequence, stamp) in batch { - match current(state, sequence, stamp) { + for &(key, stamp) in batch { + match current(state, key, stamp) { Some(slot) => { if let Some(answer) = f(slot) { - found = Some((sequence, answer)); + found = Some((key, answer)); break; } } - None => stale.push(sequence), + None => stale.push(key), } } if !stale.is_empty() { let mut indexes = self.0.lock().unwrap_or_else(PoisonError::into_inner); let indexed = Self::index(&mut indexes, state); - for sequence in stale { - indexed.groups.remove(&sequence); + for key in stale { + indexed.groups.remove(&key); } } if found.is_some() || taken < LIVE_BATCH { @@ -7725,11 +7743,11 @@ impl Servable { /// No group. const NONE: Self = Self::Run { start: 0, end: Some(0) }; - /// Whether group `sequence` is among these. - fn contains(&self, sequence: u64) -> bool { + /// Whether group `key` is among these. + fn contains(&self, key: u64) -> bool { match self { - Self::Run { start, end } => sequence >= *start && end.is_none_or(|end| sequence < end), - Self::Runs(several) => several.cap.is_none_or(|cap| sequence < cap) && several.groups.contains(sequence), + Self::Run { start, end } => key >= *start && end.is_none_or(|end| key < end), + Self::Runs(several) => several.cap.is_none_or(|cap| key < cap) && several.groups.contains(key), } } @@ -7799,9 +7817,9 @@ impl Servable { /// The newest servable group that has presented at least one frame. #[derive(Clone, Copy)] struct PresentationEdge { - sequence: u64, + key: u64, /// The slot incarnation this timestamp was read from, so an eviction or a re-served - /// sequence between resolving the edge and using it is detectable. + /// key between resolving the edge and using it is detectable. stamp: u32, /// The newest frame this group has presented, not its first. The candidate side of the /// comparison is an upper bound on what a group could still reach, so the edge side has @@ -7813,7 +7831,19 @@ struct PresentationEdge { impl PresentationEdge { /// Whether a group that can present until `reach` is a full `budget` behind this edge. fn behind(&self, reach: Option, budget: Duration) -> bool { - reach.is_some_and(|reach| matches!(self.timestamp.checked_sub(reach), Ok(age) if Duration::from(age) >= budget)) + reach + .and_then(|reach| self.timestamp.checked_sub(reach).ok()) + .and_then(age) + .is_some_and(|age| age >= budget) + } +} + +/// A span of presentation time as a duration: a scale-less track's bare units count as +/// milliseconds, over the whole key range. +fn age(span: Timestamp) -> Option { + match span.scale() { + Some(_) => Duration::try_from(span).ok(), + None => Some(Duration::from_millis(span.value())), } } @@ -7827,19 +7857,19 @@ struct Cursor { subscription: Preferences, /// Arrival-order cursor used by `recv_group`. index: usize, - /// Minimum sequence to return from any `recv` method. Set by `set_range`. - min_sequence: u64, + /// Minimum key to return from any `recv` method. Set by `set_range`. + min_key: u64, /// This subscriber's own ranges, resolved. Boxed: every reader embeds a cursor. filter: Box, - /// One past the highest sequence returned by `next_group`. + /// One past the highest key returned by `next_group`. /// Used only by that method to skip late arrivals; does not affect `recv_group`. - next_sequence: u64, - /// Exclusive upper sequence bound for `next_group` and `recv_group`. `None` means no + next_key: u64, + /// Exclusive upper key bound for `next_group` and `recv_group`. `None` means no /// cap. Set by `set_range`; /// can be raised, lowered, or unset at any time. Groups at or past the cap stay in /// the producer's cache and become eligible again when the cap rises (or is removed). /// `Some(0)` is the empty range. - end_sequence: Option, + end_key: Option, /// What this cursor can be served, the only groups its drift is measured within: its /// window below its cap, so a range behind the live edge is judged against its own /// newest group, and a group in a gap between ranges against nothing. @@ -7852,8 +7882,8 @@ struct Cursor { /// here rather than metered in place because the stats scope lives on the owning /// [`Subscriber`], which drains this after every read. stale: stats::Content, - /// Groups the seek path has convicted but whose sequences no caller has committed - /// past yet, keyed by sequence; see [`Self::commit_seek_stale`]. + /// Groups the seek path has convicted but whose keys no caller has committed + /// past yet, keyed by key; see [`Self::commit_seek_stale`]. seek_pending: BTreeMap, /// What this cursor handed out, skipped, gave up, and reported. record: Record, @@ -7873,7 +7903,7 @@ struct Cursor { /// A time start a [`Cursor`]'s window looks up through a fill. struct Lookup { - pts: Timestamp, + time: u64, /// The fill looking it up, `None` once its answer was taken in. join: Option, } @@ -7882,7 +7912,7 @@ struct Lookup { pub(super) struct Peeked { group: group::Consumer, /// How taking it moves the cursor: an arrival read past where it came from, or a - /// sequence read past its sequence. + /// key read past its key. arrival: Option, } @@ -7903,13 +7933,13 @@ enum Taken { /// fill answered from their first frame: a whole-group read never starts short. #[derive(Default)] struct Heads { - /// Asked, by sequence. + /// Asked, by key. fetching: BTreeMap, /// Answered and not handed out yet. filled: BTreeMap, } -/// The sequences a [`DropCursor`] reports from: a cursor's span, narrowed to its window, and +/// The keys a [`DropCursor`] reports from: a cursor's span, narrowed to its window, and /// the drift that says which received stale runs settle it. struct Within<'a> { span: ops::Range, @@ -8001,18 +8031,18 @@ impl DropCursor { } } - /// Record `slot`, cached at `sequence`, if it holds a cut no fetched copy covers, or a + /// Record `slot`, cached at `key`, if it holds a cut no fetched copy covers, or a /// final drop a fetched head died before winning over: a copy still serving the group's /// end delivers it, so it is never also dropped. - fn cut(record: &mut Record, sequence: u64, slot: &Slot) { + fn cut(record: &mut Record, key: u64, slot: &Slot) { if slot.visible && !slot.serves_end() && let Some(err) = slot.verdict() - && let Some(end) = sequence.checked_add(1) + && let Some(end) = key.checked_add(1) { record .pending - .insert(sequence..end, Pending::own(Hole::drop(StreamError::from(err)))); + .insert(key..end, Pending::own(Hole::drop(StreamError::from(err)))); } } @@ -8026,9 +8056,9 @@ impl DropCursor { ) -> Option<(ops::Range, Hole)> { let (span, window) = (&within.span, within.window); // A cut already reclaimed from the cache is in the track's drops instead. - for sequence in cut { - if let Some(slot) = state.lookup.get(&sequence) { - Self::cut(record, sequence, slot); + for key in cut { + if let Some(slot) = state.lookup.get(&key) { + Self::cut(record, key, slot); } } // Rescan the span only after the window moves or what settles a stale run does, since @@ -8089,8 +8119,8 @@ impl DropCursor { } } if full { - for (&sequence, slot) in state.lookup.range(span.clone()) { - Self::cut(record, sequence, slot); + for (&key, slot) in state.lookup.range(span.clone()) { + Self::cut(record, key, slot); } } while let Some((range, pending)) = record @@ -8191,11 +8221,11 @@ impl Cursor { state, cache, subscription, - min_sequence: 0, + min_key: 0, filter: Box::new(filter), index: 0, - next_sequence: 0, - end_sequence: None, + next_key: 0, + end_key: None, // Uncapped, the servable groups are the window's. servable: published.servable.clone(), published: kio::Producer::new(published), @@ -8228,14 +8258,14 @@ impl Cursor { && let Poll::Ready(Ok(drift)) = self.poll_drift(&self.servable, waiter) { // From where the reader reads next: an ordered one asks nothing below what it took. - let from = self.next_sequence.max(self.floor()); + let from = self.next_key.max(self.floor()); self.probe(from, &drift, waiter); judged = Some(drift); } // A cap at or below the floor leaves nothing to report, not a reversed span. let floor = self.floor(); let within = Within { - span: floor..self.end_sequence.unwrap_or(u64::MAX).max(floor), + span: floor..self.end_key.unwrap_or(u64::MAX).max(floor), window: &self.filter.window, revision: self.filter.revision, drift: judged.as_ref(), @@ -8267,7 +8297,7 @@ impl Cursor { // A moved window or cap notifies nothing this cursor parked on. idle::Idle::disarm(&mut self.idle); let runs = Servable::of(&self.filter.window, &self.published.read().runs); - let (revision, cap) = (self.filter.resolved, self.end_sequence); + let (revision, cap) = (self.filter.resolved, self.end_key); self.servable = runs.below(cap); let next = |current: &Published| { // Ranges a `Control` republished past this window stay until this cursor syncs @@ -8299,28 +8329,17 @@ impl Cursor { let revision = self.subscription.ranges_revision(); (revision != self.filter.resolved).then(|| (revision, preferences.ranges.clone())) }; - // Taken even when nothing walks, so it never piles up. - let newly_gone = self - .misses - .as_deref_mut() - .map(miss::Misses::take_gone) - .unwrap_or_default(); // A settled window has nothing to read from the track, whose lock every subscriber // of it takes. if replaced.is_none() && self.filter.window.is_settled() { return false; } let state = self.state.read(); - // Every walk reads the reader's own gone runs, so none parks on a gap they fill. - let holdings = ReaderHoldings { - state: &state, - misses: self.misses.as_deref(), - newly_gone, - }; + let holdings = &*state; let moved = match replaced { Some((revision, ranges)) => { self.filter.resolved = revision; - match Window::new(&ranges, state.latest_live(), &holdings) { + match Window::new(&ranges, state.latest_live(), holdings) { Ok(window) => { self.filter.window = window; self.filter.refused = None; @@ -8334,8 +8353,8 @@ impl Cursor { // The live feed had no group when the ranges resolved, so the newest one then was // the first to arrive, or the newest once it went live, however many have arrived // or left since. - None if self.filter.window.is_pending() => self.filter.window.settle(state.first_live(), &holdings), - None => self.filter.window.settle(state.latest_live(), &holdings), + None if self.filter.window.is_pending() => self.filter.window.settle(state.first_live(), holdings), + None => self.filter.window.settle(state.latest_live(), holdings), }; drop(state); if moved { @@ -8345,39 +8364,28 @@ impl Cursor { moved } - /// Park a read the cap of an unsettled time end holds back on what raises it: the - /// speculative group starting or ending, or any group it waits on starting. Returns - /// whether the cap already moved, so the read goes again rather than parking. - fn watch_time(&mut self, waiter: &kio::Waiter) -> bool { + /// Settle a parked read's window on the groups that arrived since: a window waiting on a + /// group parks on the track, which every arrival wakes. Returns whether the window + /// moved, so the read goes again rather than parking. + fn watch_time(&mut self) -> bool { if self.filter.window.is_settled() || self.filter.refused.is_some() { return false; } - { - let state = self.state.read(); - if let Some(from) = self.filter.window.watch_starts() { - state.cache.wakes().watch_start(from, waiter); - } - for sequence in self.filter.window.speculative() { - if let Some(slot) = state.lookup.get(&sequence) { - let _ = slot.serving().poll_started(waiter); - } - } - } - // Registered first, so a start landing since the last sync is either seen here or - // wakes the read. self.sync() } /// The lowest group this cursor reads: its own floor, raised to its window's. fn floor(&self) -> u64 { - self.min_sequence.max(self.filter.window.floor().unwrap_or(0)) + self.min_key.max(self.filter.window.floor().unwrap_or(0)) } /// Whether a bounded window has nothing left to deliver: every group in its ranges was /// handed out (in any of `taken`), dropped, skipped as stale, or given up as unfillable. /// An open window, or one with a fill still out, always has more, and so does one its - /// cap keeps a group out of, since a reader may raise a cap again. - fn is_exhausted(&self, logical: &[&Delivered]) -> bool { + /// cap keeps a group out of, since a reader may raise a cap again. Groups under `from` are + /// not judged: a time start resolves to the group containing it, which may lie under a + /// copy's resume point. + fn is_exhausted(&self, logical: &[&Delivered], from: u64) -> bool { let taken: Vec<&Delivered> = std::iter::once(&self.record.delivered) .chain(logical.iter().copied()) .collect(); @@ -8410,14 +8418,14 @@ impl Cursor { }) { left.remove(run.clone()); } - for sequence in self.seek_pending.range(gap).map(|(sequence, _)| *sequence) { - left.remove(sequence..sequence + 1); + for key in self.seek_pending.range(gap).map(|(key, _)| *key) { + left.remove(key..key + 1); } left.is_empty() }; self.filter .window - .runs(self.filter.window.floor().unwrap_or(0)) + .runs(self.filter.window.floor().unwrap_or(0).max(from)) .take_while(|run| *run.start() < end) .all(|run| { let run = *run.start()..run.end().checked_add(1).map_or(end, |next| next.min(end)); @@ -8425,10 +8433,10 @@ impl Cursor { }) } - /// Whether group `sequence`, at or above [`Self::floor`], is one this cursor reads now: + /// Whether group `key`, at or above [`Self::floor`], is one this cursor reads now: /// in its window and below its cap. One that isn't may be later. - fn admits(&self, sequence: u64) -> bool { - super::subscription::before_end(sequence, self.end_sequence) && self.filter.window.contains(sequence) + fn admits(&self, key: u64) -> bool { + super::subscription::before_end(key, self.end_key) && self.filter.window.contains(key) } /// [`Filter::owed`] as of `admitted`: what was owed and what the walk newly reads since @@ -8458,19 +8466,16 @@ impl Cursor { } /// Take in a move of the floor, the ranges, or the cap: whatever it newly admits is - /// owed. Costs the ranges, never the cache, and only when one moved. - /// - /// A time end whose cap moves on each group moves only the walks' runs layered over the - /// window's fixed ones, so that costs those runs: what they gain is owed only where the - /// cache holds a group now, the only place a scan finds one, and what they lose is cut - /// only from what is owed. + /// owed. Costs the ranges, never the cache, and only when one moved: what they gain is + /// owed only where the cache holds a group now, the only place a scan finds one, and + /// what they lose is cut only from what is owed. fn admit(&mut self) { - let at = (self.floor(), self.filter.revision, self.end_sequence); + let at = (self.floor(), self.filter.revision, self.end_key); if self.filter.admitted_at == Some(at) { return; } // The arrival walk reads the window from the floor. - let admitted = Admitted::new(&self.filter.window, self.floor(), self.end_sequence); + let admitted = Admitted::new(&self.filter.window, self.floor(), self.end_key); let moved = self.filter.admitted_at.and_then(|_| { let cached = { let state = self.state.read(); @@ -8565,8 +8570,8 @@ impl Cursor { continue; }; let scanned = match ascending { - true => group.sequence.checked_sub(1).map(|below| *run.start()..=below), - false => group.sequence.checked_add(1).map(|above| above..=*run.end()), + true => group.start.checked_sub(1).map(|below| *run.start()..=below), + false => group.start.checked_add(1).map(|above| above..=*run.end()), }; if let Some(scanned) = scanned.filter(|scanned| !scanned.is_empty()) { self.filter.owed.remove(scanned); @@ -8575,9 +8580,9 @@ impl Cursor { } } - /// Record `sequence` as skipped stale under `budget`, so a wider one owes it again. - fn skip(&mut self, sequence: u64, budget: Duration) { - self.record.skipped.insert(sequence, budget); + /// Record `key` as skipped stale under `budget`, so a wider one owes it again. + fn skip(&mut self, key: u64, budget: Duration) { + self.record.skipped.insert(key, budget); self.prune_skipped(); } @@ -8715,7 +8720,7 @@ impl Cursor { /// The groups this reader reads, and its cap; see [`Coverage`]. fn coverage(&self) -> Coverage { - Some((self.filter.window.groups()?.clone(), self.end_sequence)) + Some((self.filter.window.groups()?.clone(), self.end_key)) } /// This subscriber's clamped drift budget and the live edge to measure against, @@ -8747,7 +8752,7 @@ impl Cursor { } /// Pending while a group in `groups` that the live feed still owes is within the drift - /// budget, so a read in sequence order waits for it; see `TrackState::owed`. A budget + /// budget, so a read in key order waits for it; see `TrackState::owed`. A budget /// with no deadline waits until the hole resolves: the group lands, or the track learns /// it won't. fn poll_owed(&self, groups: ops::Range, drift: &Drift, waiter: &kio::Waiter) -> Poll> { @@ -8781,21 +8786,21 @@ impl Cursor { /// for this poll. fn poll_stale(&self, group: &group::Consumer, drift: &Drift, waiter: &kio::Waiter) -> Poll> { self.poll(waiter, |state| { - Poll::Ready(Ok(state.is_stale(group.sequence, &drift.edge, drift.budget))) + Poll::Ready(Ok(state.is_stale(group.start, &drift.edge, drift.budget))) }) } - /// Record `sequence` as handed out. An ordered read never goes back, and any reader + /// Record `key` as handed out. An ordered read never goes back, and any reader /// stops expecting what is past its budget, so only the gaps within it cost memory. - fn hand_out(&mut self, sequence: u64, ordered: bool) { + fn hand_out(&mut self, key: u64, ordered: bool) { if ordered { - self.record.delivered.forget_below(sequence); + self.record.delivered.forget_below(key); } self.prune_skipped(); - self.record.delivered.insert(sequence); + self.record.delivered.insert(key); // The group won over any stale report on it. if self.drops.is_some() { - self.record.retire(sequence); + self.record.retire(key); } let mut delivered = std::mem::take(&mut self.record.delivered); self.forget_past_budget(&mut delivered); @@ -8826,12 +8831,12 @@ impl Cursor { break; }; let shared = if now.saturating_duration_since(arrival.arrived) >= budget { - arrival.group.sequence + arrival.group.start } else if let Some(successor) = state.successor(oldest, &edge.servable) && let Some(live) = state.live_above(oldest, &edge) && live.behind(successor.group.timestamp(), budget) { - successor.group.sequence.min(live.sequence) + successor.group.start.min(live.key) } else { break; }; @@ -8855,14 +8860,14 @@ impl Cursor { /// A group missing its head reads it as [`Error::Lagged`], which a route copy recovers /// from another. fn deliver(&self, group: group::Consumer) -> group::Consumer { - let sequence = group.sequence; + let key = group.start; group.with_expiry(Arc::new(GroupExpiry { state: self.state.weak(), cache: self.cache.clone(), reading: Reading::new(self.state.clone()), subscription: self.subscription.consume(), servable: self.published.consume(), - sequence, + key, superseded: Superseded::default(), })) } @@ -8870,14 +8875,14 @@ impl Cursor { /// Look up each time start the window can't resolve from the cache, once, and take in /// the answers, parking `waiter` on the lookups still out. fn look_up(&mut self, waiter: &kio::Waiter) { - let lookups: Vec<(Timestamp, u64)> = self.filter.window.lookups().collect(); + let lookups: Vec<(u64, u64)> = self.filter.window.lookups().collect(); if lookups.is_empty() && self.filter.lookups.iter().all(|lookup| lookup.join.is_none()) { return; } let asking = &asking(&self.subscription); let fills = self.state.read().fills.clone(); - for (pts, reach) in lookups { - if self.filter.lookups.iter().any(|lookup| lookup.pts == pts) { + for (time, reach) in lookups { + if self.filter.lookups.iter().any(|lookup| lookup.time == time) { continue; } let ask = fill::Ask { @@ -8886,21 +8891,25 @@ impl Cursor { end: reach, frame_end: None, asking: *asking, - time: Some(pts), + time: Some(time), }; let (joins, _) = fills.lock().join(&fills, &ask); self.filter.lookups.push(Lookup { - pts, + time, join: joins.into_iter().next(), }); } let mut moved = false; for lookup in &mut self.filter.lookups { // The window stops looking once replaced; with no join, nothing can answer. - if !self.filter.window.lookups().any(|(pts, _)| pts == lookup.pts) { + if !self.filter.window.lookups().any(|(time, _)| time == lookup.time) { lookup.join = None; continue; } + // What the reader asks with still reaches the lookups it waits on. + if let Some(join) = &mut lookup.join { + join.update(asking); + } let answer = match &lookup.join { None => None, Some(join) => match join.outcome().poll(waiter, |outcome| { @@ -8913,22 +8922,8 @@ impl Cursor { }, }; let state = self.state.read(); - // The answer's first frame says whether it starts at or before the time. - if let Some(group) = answer - && let Some(slot) = state.lookup.get(&group) - && slot.may_start(state.looks_up()) - { - let _ = slot.serving().poll_started(waiter); - continue; - } lookup.join = None; - // What the reader's fills gave up since its last sync wakes its walks at the next. - let holdings = ReaderHoldings { - state: &state, - misses: self.misses.as_deref(), - newly_gone: RangeSet::new(), - }; - moved |= self.filter.window.found(lookup.pts, answer, &holdings); + moved |= self.filter.window.found(lookup.time, answer, &*state); } if moved { self.filter.revision += 1; @@ -8946,7 +8941,7 @@ impl Cursor { fn probe(&mut self, from: u64, drift: &Drift, waiter: &kio::Waiter) { self.look_up(waiter); let taken = &self.record.delivered; - let (window, cap, revision) = (&self.filter.window, self.end_sequence, self.filter.revision); + let (window, cap, revision) = (&self.filter.window, self.end_key, self.filter.revision); let misses = self.misses.get_or_insert_with(Box::default); let mut given_up = Vec::new(); misses.retain((from, revision, cap), window.missable_groups(), || { @@ -9042,10 +9037,10 @@ impl Cursor { fn take(&mut self, peeked: Peeked) -> group::Consumer { let Peeked { group, arrival } = peeked; let Some(taken) = arrival else { - return self.advance_sequence(group); + return self.advance_key(group); }; - self.advance_arrival(group.sequence, taken); - self.record.skipped.remove(group.sequence); + self.advance_arrival(group.start, taken); + self.record.skipped.remove(group.start); // Taking is the cache access a peek is not. group.cache_refresh(); self.deliver(group) @@ -9093,7 +9088,7 @@ impl Cursor { state.poll_recv_group(self.index, floor, &self.filter.window) }); // A moved window moved the servable edge `drift` was judged against. - if polled.is_pending() && self.watch_time(waiter) { + if polled.is_pending() && self.watch_time() { self.admit(); drift = ready!(self.poll_drift(&self.servable, waiter))?; self.expire_heads(&drift, waiter); @@ -9117,7 +9112,7 @@ impl Cursor { if !self.filter.window.is_pending() { drift = ready!(self.poll_drift(&self.servable, waiter))?; } - if consumer.sequence < self.floor() { + if consumer.start < self.floor() { self.index = found_index + 1; continue; } @@ -9126,7 +9121,7 @@ impl Cursor { // Pass a group beyond the cap or outside the ranges, holding nothing: a // widening finds it in the cache again. Keep scanning so an in-range group // that arrived behind it still flows. - if !self.admits(consumer.sequence) { + if !self.admits(consumer.start) { self.index = found_index + 1; continue; } @@ -9137,13 +9132,13 @@ impl Cursor { // Drop a group the drift budget has given up on and keep scanning, so one // poll walks a whole backlog off rather than handing it out group by group. if ready!(self.poll_stale(&consumer, &drift, waiter))? { - self.advance_arrival(consumer.sequence, taken); - self.write_off_stale(consumer.sequence, consumer.content(), &drift); + self.advance_arrival(consumer.start, taken); + self.write_off_stale(consumer.start, consumer.content(), &drift); continue; } if self.lacks_head(&consumer) { - self.advance_arrival(consumer.sequence, taken); - self.ask_head(consumer.sequence, waiter); + self.advance_arrival(consumer.start, taken); + self.ask_head(consumer.start, waiter); continue; } return Poll::Ready(Ok(Some(Peeked { @@ -9153,43 +9148,43 @@ impl Cursor { } } - /// Write off group `sequence`, which the drift budget gave up on, as dropped `Old`: + /// Write off group `key`, which the drift budget gave up on, as dropped `Old`: /// skipped for a wider budget to offer again, unless the budget is at its bound. - fn write_off_stale(&mut self, sequence: u64, content: stats::Content, drift: &Drift) { + fn write_off_stale(&mut self, key: u64, content: stats::Content, drift: &Drift) { self.stale.add(content); if self.drops.is_some() - && let Some(end) = sequence.checked_add(1) + && let Some(end) = key.checked_add(1) { self.record .pending - .insert(sequence..end, Pending::own(Hole::stale(drift.budget))); + .insert(key..end, Pending::own(Hole::stale(drift.budget))); } match drift.bounded { // A re-offered group stale at the bound is final. - true => self.record.skipped.remove(sequence), - false => self.skip(sequence, drift.budget), + true => self.record.skipped.remove(key), + false => self.skip(key, drift.budget), } // Whatever the window is now: narrowed to a bounded one later, it accounts for this // group by it, even once the group left the cache. - if let Some(end) = sequence.checked_add(1) { - self.record.written_off.insert(sequence..end); + if let Some(end) = key.checked_add(1) { + self.record.written_off.insert(key..end); } } /// Move the arrival cursor past a group it read: from the walk, to `index`; from what /// a widening owes, out of the owed spans. - fn advance_arrival(&mut self, sequence: u64, taken: Taken) { + fn advance_arrival(&mut self, key: u64, taken: Taken) { match taken { Taken::Walk(index) => self.index = index, - Taken::Owed => self.filter.owed.remove(sequence..=sequence), + Taken::Owed => self.filter.owed.remove(key..=key), Taken::Filled => { if let Some(misses) = &mut self.misses { - misses.take_filled(sequence); + misses.take_filled(key); } } Taken::Head => { if let Some(heads) = &mut self.heads { - heads.filled.remove(&sequence); + heads.filled.remove(&key); } } } @@ -9201,9 +9196,9 @@ impl Cursor { self.filter.window.end().is_some() && group.first_frame() > 0 } - /// Ask a fill for the head of `sequence`, as a fetch from frame 0 would, taking in an + /// Ask a fill for the head of `key`, as a fetch from frame 0 would, taking in an /// answer already at hand. - fn ask_head(&mut self, sequence: u64, waiter: &kio::Waiter) { + fn ask_head(&mut self, key: u64, waiter: &kio::Waiter) { let asking = self.subscription.read().clone(); let request = FetchOptions { options: group::Fetch { @@ -9216,16 +9211,16 @@ impl Cursor { }; // The cursor judges the answer against its window when it hands it out. let mut fetching = - Fetching::cached_or_queued(self.state.clone(), sequence, request, stats::Scope::default()).unjudged(); + Fetching::cached_or_queued(self.state.clone(), key, request, stats::Scope::default()).unjudged(); let heads = self.heads.get_or_insert_default(); match kio::Task::poll(&mut fetching, waiter) { Poll::Pending => { - heads.fetching.insert(sequence, fetching); + heads.fetching.insert(key, fetching); } Poll::Ready(Ok(group)) => { - heads.filled.insert(sequence, group); + heads.filled.insert(key, group); } - Poll::Ready(Err(_)) => self.give_up_heads(std::iter::once(sequence..sequence.saturating_add(1))), + Poll::Ready(Err(_)) => self.give_up_heads(std::iter::once(key..key.saturating_add(1))), } } @@ -9234,20 +9229,20 @@ impl Cursor { fn poll_heads(&mut self, waiter: &kio::Waiter) { let Some(mut heads) = self.heads.take() else { return }; let floor = self.floor(); - let admitted = |sequence: &u64| *sequence >= floor && self.admits(*sequence); - heads.fetching.retain(|sequence, _| admitted(sequence)); - heads.filled.retain(|sequence, _| admitted(sequence)); + let admitted = |key: &u64| *key >= floor && self.admits(*key); + heads.fetching.retain(|key, _| admitted(key)); + heads.filled.retain(|key, _| admitted(key)); let mut given_up = Vec::new(); heads .fetching - .retain(|sequence, fetching| match kio::Task::poll(fetching, waiter) { + .retain(|key, fetching| match kio::Task::poll(fetching, waiter) { Poll::Pending => true, Poll::Ready(Ok(group)) => { - heads.filled.insert(*sequence, group); + heads.filled.insert(*key, group); false } Poll::Ready(Err(_)) => { - given_up.push(*sequence..sequence.saturating_add(1)); + given_up.push(*key..key.saturating_add(1)); false } }); @@ -9268,22 +9263,22 @@ impl Cursor { let Poll::Ready(Ok(stale)) = self.poll(waiter, |state| { let stale: Vec<(u64, stats::Content)> = asked .iter() - .filter_map(|&sequence| { + .filter_map(|&key| { // Parked first, so a group landing above while this judges wakes it. - let _ = state.cache.wakes().watch_landing(sequence, waiter); + let _ = state.cache.wakes().watch_landing(key, waiter); // Judged whether or not the cache still holds the headless copy: an evicted // one goes stale all the same, and counts as a group of unknown size. - (sequence < state.cache_floor(drift.budget) - || state.poll_drifted(sequence, &servable, drift.budget, None, waiter)) + (key < state.cache_floor(drift.budget) + || state.poll_drifted(key, &servable, drift.budget, None, waiter)) .then(|| { - let content = match state.lookup.get(&sequence) { + let content = match state.lookup.get(&key) { Some(slot) => slot.group.consume().content(), None => stats::Content { groups: 1, ..Default::default() }, }; - (sequence, content) + (key, content) }) }) .collect(); @@ -9291,11 +9286,11 @@ impl Cursor { }) else { return; }; - for (sequence, content) in stale { + for (key, content) in stale { if let Some(heads) = &mut self.heads { - heads.fetching.remove(&sequence); + heads.fetching.remove(&key); } - self.write_off_stale(sequence, content, drift); + self.write_off_stale(key, content, drift); } } @@ -9331,11 +9326,11 @@ impl Cursor { /// Pass groups another copy already handed out, without delivering or metering them. fn skip_ordered(&mut self, floor: u64) { - if floor <= self.next_sequence { + if floor <= self.next_key { return; } idle::Idle::disarm(&mut self.idle); - self.next_sequence = floor; + self.next_key = floor; // A group delivered on another copy is not a stale skip on this one. if !self.seek_pending.is_empty() { self.seek_pending = self.seek_pending.split_off(&floor); @@ -9355,7 +9350,7 @@ impl Cursor { /// The lowest servable group past the last one returned, walking off everything the /// drift budget has convicted on the way. fn poll_peek_next(&mut self, waiter: &kio::Waiter) -> Poll>> { - let next = self.next_sequence; + let next = self.next_key; self.sync(); if let Some(err) = &self.filter.refused { return Poll::Ready(Err(err.clone())); @@ -9377,7 +9372,7 @@ impl Cursor { mut drift: Drift, waiter: &kio::Waiter, ) -> Poll>> { - let end = self.end_sequence; + let end = self.end_key; self.poll_heads(waiter); // Where the gap below the candidate starts: groups stepped over as stale are cached. let mut from = floor; @@ -9385,7 +9380,7 @@ impl Cursor { let window = &self.filter.window; let polled = self.poll(waiter, |state| state.poll_next_in_range(floor, end, window)); // A moved window moved the servable edge `drift` was judged against. - if polled.is_pending() && self.watch_time(waiter) { + if polled.is_pending() && self.watch_time() { drift = ready!(self.poll_drift(&self.servable, waiter))?; continue; } @@ -9406,7 +9401,7 @@ impl Cursor { .misses .as_ref() .and_then(|misses| misses.lowest_wanted()) - .is_some_and(|wanted| wanted < producer.sequence) + .is_some_and(|wanted| wanted < producer.start) { return Poll::Pending; } @@ -9421,8 +9416,8 @@ impl Cursor { drift = ready!(self.poll_drift(&self.servable, waiter))?; } from = from.max(self.floor()); - if group.sequence < self.floor() || !self.filter.window.contains(group.sequence) { - floor = floor.max(self.floor()).max(group.sequence.saturating_add(1)); + if group.start < self.floor() || !self.filter.window.contains(group.start) { + floor = floor.max(self.floor()).max(group.start.saturating_add(1)); continue; } } @@ -9434,38 +9429,33 @@ impl Cursor { // the edge can be evicted), so the group may still be delivered. // Re-snapshot on every re-examination so the eventual count reflects the // group's latest observed content. - self.seek_pending.insert(group.sequence, group.content()); - floor = group.sequence.saturating_add(1); + self.seek_pending.insert(group.start, group.content()); + floor = group.start.saturating_add(1); continue; } // In order means a group the live feed still owes below this one comes first, // until the budget gives up on it. Its landing, its drop, or that verdict wakes us. - ready!(self.poll_owed(from..group.sequence, &drift, waiter))?; + ready!(self.poll_owed(from..group.start, &drift, waiter))?; // A group missing its head comes from the fill asked for it, in its turn. let group = match self.lacks_head(&group) { false => group, true => { - let sequence = group.sequence; - if self.heads.as_ref().is_none_or(|heads| { - !heads.fetching.contains_key(&sequence) && !heads.filled.contains_key(&sequence) - }) && self - .misses + let key = group.start; + if self + .heads .as_ref() - .is_none_or(|misses| misses.gone(sequence).is_none()) + .is_none_or(|heads| !heads.fetching.contains_key(&key) && !heads.filled.contains_key(&key)) + && self.misses.as_ref().is_none_or(|misses| misses.gone(key).is_none()) { - self.ask_head(sequence, waiter); + self.ask_head(key, waiter); } - match self.heads.as_ref().and_then(|heads| heads.filled.get(&sequence)) { + match self.heads.as_ref().and_then(|heads| heads.filled.get(&key)) { Some(head) => head.clone(), // Given up: past it, as past any group no fill can serve. - None if self - .misses - .as_ref() - .is_some_and(|misses| misses.gone(sequence).is_some()) => - { - floor = sequence.saturating_add(1); + None if self.misses.as_ref().is_some_and(|misses| misses.gone(key).is_some()) => { + floor = key.saturating_add(1); continue; } None => return Poll::Pending, @@ -9475,32 +9465,32 @@ impl Cursor { // A conviction the budget walked back: the group is handed over after all, // so it must never reach the stale count. - self.seek_pending.remove(&group.sequence); + self.seek_pending.remove(&group.start); break group; }; Poll::Ready(Ok(Some(Peeked { group, arrival: None }))) } - fn advance_sequence(&mut self, group: group::Consumer) -> group::Consumer { - self.next_sequence = group.sequence + 1; + fn advance_key(&mut self, group: group::Consumer) -> group::Consumer { + self.next_key = group.start + 1; // The reader holds the group itself now, as on the arrival path. if let Some(misses) = &mut self.misses { - misses.take_filled(group.sequence); + misses.take_filled(group.start); } - self.forget_heads_below(self.next_sequence); + self.forget_heads_below(self.next_key); // The delivery commits everything the seek stepped over to reach this group. - self.commit_seek_stale(self.next_sequence); + self.commit_seek_stale(self.next_key); group.cache_refresh(); self.deliver(group) } - /// Count the sequence path's convictions below `committed` into [`Self::stale`], + /// Count the key path's convictions below `committed` into [`Self::stale`], /// exactly once each. /// /// A conviction only becomes a real skip once a delivery commits past it. Until then /// the entry waits in [`Self::seek_pending`], where a delivered group removes itself - /// (see [`Self::poll_next_with`]). `committed` must be the sequence watermark (one + /// (see [`Self::poll_next_with`]). `committed` must be the key watermark (one /// past the last delivery), which only rises. The seek's floor is NOT that: it /// includes the local floor, which can be lowered again, and a conviction it flushed could /// then be delivered after all. @@ -9520,15 +9510,12 @@ impl Cursor { #[derive(Clone)] pub struct Control { subscription: Preferences, - track: Consumer, /// A plain cursor's servable window, republished when the ranges are replaced, so a /// group it handed out is judged against them before the cursor next reads. `None` /// for a reader that follows a route, which [`Self::mirror`] serves instead. republish: Option, /// A reader that follows a route, whose copies take the new preferences at once. mirror: Option, - // Whether the track is timed, so a time anchor on an untimed one is refused. - timed: bool, } /// What a [`Control`] needs to republish a plain cursor's [`Published`] window. @@ -9576,15 +9563,8 @@ impl Control { /// Replace this subscriber's preferences, updating the producer's aggregate. /// /// Returns [`Error::Closed`] if the track already ended; the update is - /// meaningless at that point and can usually be ignored. Refuses a time anchor on - /// an untimed track with [`Error::TimestampMismatch`], leaving the preferences as they were. + /// meaningless at that point and can usually be ignored. pub fn update(&self, subscription: Subscription) -> Result<()> { - if !self.timed && subscription.has_time() { - return Err(Error::TimestampMismatch); - } - if subscription.has_time() && subscription.ranges != self.subscription.read().ranges { - self.track.refuse_subscription(&subscription)?; - } self.subscription.update(subscription)?; if let Some(republish) = &self.republish { republish.publish(&self.subscription); @@ -9596,25 +9576,6 @@ impl Control { } } -/// A group-only wire refuses an unresolved time anchor before changing shared demand. -fn refuse_cached_anchors(state: &TrackState, subscription: &Subscription) -> Result<()> { - if state.anchors == Anchors::Cached && subscription.has_time() { - let window = Window::new(&subscription.ranges, state.latest_live(), state)?; - if !window.is_settled() { - return Err(Error::Unsupported); - } - } - Ok(()) -} - -/// Refuse `subscription` when it asks an untimed track for a presentation time. -fn refuse_untimed(info: &Info, subscription: &Subscription) -> Result<()> { - match info.timescale.is_none() && subscription.has_time() { - true => Err(Error::TimestampMismatch), - false => Ok(()), - } -} - impl Subscriber { /// Count the payload of the groups taken from now on only as the caller delivers it, /// through the returned [`stats::Bill`], rather than as it is read. A reader that drops @@ -9672,24 +9633,19 @@ impl Subscriber { /// Create a handle for updating this subscriber's delivery preferences. pub fn control(&self) -> Control { - let timed = self.info.timescale.is_some(); match &self.inner { Inner::Plain(cursor) => Control { - track: self.consume(), subscription: cursor.subscription.clone(), republish: Some(Republish { state: cursor.state.weak(), published: cursor.published.weak(), }), mirror: None, - timed, }, Inner::Resume(resume, _) => Control { - track: self.consume(), subscription: resume.subscription().clone(), republish: None, mirror: Some(resume.mirror()), - timed, }, } } @@ -9697,11 +9653,11 @@ impl Subscriber { /// The next group in arrival order. /// /// Returns each group it delivers exactly once, in the order it landed on the wire, - /// which may be out of sequence due to network reordering or loss. Use - /// [`Self::ordered`] if you only want groups whose sequence number is higher than any + /// which may be out of key due to network reordering or loss. Use + /// [`Self::ordered`] if you only want groups whose key is higher than any /// previously returned. /// - /// Once means once per sequence: a group the publisher aborts and re-creates, or that + /// Once means once per key: a group the publisher aborts and re-creates, or that /// a second route delivers too, is not handed out again. Once a group is past the /// budget below, the subscriber stops expecting every group before it, so a gap that /// old is never filled, which holds because a timed track refuses a group whose start @@ -9748,7 +9704,7 @@ impl Subscriber { self.poll_group(false, waiter) } - /// [`Self::poll_recv_group`], handing out the next group only if its sequence lies in + /// [`Self::poll_recv_group`], handing out the next group only if its key lies in /// `admit`: one outside stays next, and the read is `Pending` without parking on it. #[cfg(feature = "async")] pub(crate) fn poll_recv_group_in( @@ -9759,13 +9715,13 @@ impl Subscriber { self.poll_group_in(false, admit, waiter) } - /// The next group in arrival order, or in sequence order for `ordered`, skipping any + /// The next group in arrival order, or in key order for `ordered`, skipping any /// already handed out. fn poll_group(&mut self, ordered: bool, waiter: &kio::Waiter) -> Poll>> { self.poll_group_in(ordered, (ops::Bound::Unbounded, ops::Bound::Unbounded), waiter) } - /// [`Self::poll_group`], refusing a next group whose sequence lies outside `admit`. + /// [`Self::poll_group`], refusing a next group whose key lies outside `admit`. fn poll_group_in( &mut self, ordered: bool, @@ -9776,7 +9732,7 @@ impl Subscriber { // What the read settled is billed now, and pays what it can of its readers' debt. The // group it hands out stays above the fold until its reader accounts for it. let handed = match &res { - Poll::Ready(Ok(Some(group))) => Some(group.sequence), + Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; match self.settle_record(handed) { @@ -9808,7 +9764,7 @@ impl Subscriber { _ => {} } match self.poll_peek_group(ordered, waiter) { - Poll::Ready(Ok(Some(reserved))) if admit.contains(&reserved.sequence()) => { + Poll::Ready(Ok(Some(reserved))) if admit.contains(&reserved.key()) => { Poll::Ready(self.take(reserved).map(Some)) } res => { @@ -9843,7 +9799,7 @@ impl Subscriber { Poll::Pending => { // A read that only skipped since its last hand-out settles its write-offs too. cursor.retire_written_off(); - break match !self.parks && cursor.is_exhausted(&[]) { + break match !self.parks && cursor.is_exhausted(&[], 0) { true => Poll::Ready(Ok(None)), false => Poll::Pending, }; @@ -9852,7 +9808,7 @@ impl Subscriber { Poll::Ready(Ok(None)) => break Poll::Ready(Ok(None)), Poll::Ready(Ok(Some(peeked))) => peeked, }; - if !cursor.record.delivered.contains(peeked.group.sequence) { + if !cursor.record.delivered.contains(peeked.group.start) { break Poll::Ready(Ok(Some(Reserved::Plain(peeked)))); } // A replay advances only this copy's cursor, as an ordinary read would. @@ -9869,9 +9825,10 @@ impl Subscriber { /// Whether this route copy's bounded window has nothing left for the logical readers that /// handed out `logical`: what this copy took itself, wrote off, or gave up counts too. - pub(super) fn exhausted_for(&mut self, logical: &[&Delivered], waiter: &kio::Waiter) -> bool { + /// It judges from its resume point `from`: the logical reader judges what lies under it. + pub(super) fn exhausted_for(&mut self, logical: &[&Delivered], from: Option, waiter: &kio::Waiter) -> bool { match &mut self.inner { - Inner::Plain(cursor) => cursor.is_exhausted(logical), + Inner::Plain(cursor) => cursor.is_exhausted(logical, from.unwrap_or(0)), // A copy that is another front's logical track judges across its own copies. Inner::Resume(resume, own) => { let mut taken = Vec::with_capacity(logical.len() + 1); @@ -9892,7 +9849,7 @@ impl Subscriber { _ => unreachable!("a reservation is taken from the subscriber that made it"), }; if let Ok(group) = &res { - self.hand_out(group.sequence, ordered); + self.hand_out(group.start, ordered); } self.count_stale(&meter); res.map(|group| group.with_meter(meter)) @@ -9926,7 +9883,7 @@ impl Subscriber { } } - /// Whether this reader's range reaches an end at the exclusive final sequence `fin`: a + /// Whether this reader's range reaches an end at the exclusive final key `fin`: a /// range that stops short of it, past the cap or past every range as resolved, whatever /// its anchors, never reaches the end. pub(crate) fn reaches(&self, fin: u64) -> bool { @@ -9950,20 +9907,20 @@ impl Subscriber { } } - /// Record `sequence` as handed out. An ordered read never goes back, and any reader + /// Record `key` as handed out. An ordered read never goes back, and any reader /// stops expecting what is past its budget, so only the gaps within it cost memory. - fn hand_out(&mut self, sequence: u64, ordered: bool) { + fn hand_out(&mut self, key: u64, ordered: bool) { match &mut self.inner { - Inner::Plain(cursor) => cursor.hand_out(sequence, ordered), + Inner::Plain(cursor) => cursor.hand_out(key, ordered), Inner::Resume(resume, logical) => { let delivered = &mut logical.record.delivered; if ordered { - delivered.forget_below(sequence); + delivered.forget_below(key); } - delivered.insert(sequence); + delivered.insert(key); resume.forget_past_budget(delivered); // The group won over any stale report its publisher made on it. - logical.record.retire(sequence); + logical.record.retire(key); } } } @@ -10044,11 +10001,11 @@ impl Subscriber { // A finite budget skips the cache below its floor at once, so it is not unread. let budget = clamp_max_delay(budget, state.max_age_bound()); let floor = cursor.floor().max(state.cache_floor(budget)).max(unread.start); - let eligible = |sequence: u64| { - sequence >= floor - && super::subscription::before_end(sequence, unread.end) - && cursor.filter.window.contains(sequence) - && !unread.delivered.contains(sequence) + let eligible = |key: u64| { + key >= floor + && super::subscription::before_end(key, unread.end) + && cursor.filter.window.contains(key) + && !unread.delivered.contains(key) }; // What a widening owes, the walk has yet to take in, or lifting the local cap would: // a group past it is unread, not gone. @@ -10065,7 +10022,7 @@ impl Subscriber { && state .lookup .range(*run.start()..=end) - .any(|(sequence, slot)| eligible(*sequence) && slot.is_read()) + .any(|(key, slot)| eligible(*key) && slot.is_read()) }) }; state.readable() @@ -10073,24 +10030,24 @@ impl Subscriber { .arrival .iter() .skip(cursor.index.saturating_sub(state.offset)) - .any(|(sequence, stamp)| { - eligible(*sequence) + .any(|(key, stamp)| { + eligible(*key) && state .lookup - .get(sequence) + .get(key) .is_some_and(|slot| slot.stamp == *stamp && slot.is_read()) }) || owed()) } - /// The sequence cursor behind [`Ordered`], which owns the only public door to it. + /// The key cursor behind [`Ordered`], which owns the only public door to it. pub(super) fn poll_next_group(&mut self, waiter: &kio::Waiter) -> Poll>> { self.poll_group(true, waiter) } - /// Read this track's groups in sequence order instead of arrival order. + /// Read this track's groups in key order instead of arrival order. /// /// Consumes the subscriber, so one handle carries exactly one cursor: an - /// arrival-order [`Subscriber`] or a sequence-order [`Ordered`], never both at once. + /// arrival-order [`Subscriber`] or a key-order [`Ordered`], never both at once. /// The two advance independently, and interleaving them produces a group stream that /// is neither, which is why the choice is a handle rather than a method. pub fn ordered(self) -> Ordered { @@ -10111,8 +10068,11 @@ impl Subscriber { /// below the floor serves the floor's group too, and a fill this cursor waits on starts it /// lower still. `None` when the source declares none and no fill is out. /// - /// Where a reader that waits on each group in sequence starts when [`Self::floor`] names + /// Where a reader that waits on each group in key starts when [`Self::floor`] names /// none: groups arrive newest first, so the first to arrive is not the oldest served. + /// + /// Pending too while a time start waits on the groups held to place it, since where its + /// range starts is unknown until then. #[cfg_attr(feature = "async", kio::wrapper)] pub fn poll_start(&mut self, waiter: &kio::Waiter) -> Poll> { let cursor = match &mut self.inner { @@ -10121,6 +10081,15 @@ impl Subscriber { }; let start = ready!(Consumer::poll_state_start(&cursor.state, waiter)); cursor.sync(); + if cursor.filter.window.is_waiting() { + // Registered before syncing again, so a change placing it after this wakes it. A + // closed track places it at once, since nothing more can arrive. + let closed = cursor.state.poll(waiter, |_| Poll::<()>::Pending).is_ready(); + cursor.sync(); + if !closed && cursor.filter.window.is_waiting() { + return Poll::Pending; + } + } let start = start.map(|start| start.max(cursor.floor())); let filling = cursor.misses.as_ref().and_then(|misses| misses.lowest()); Poll::Ready(match (start, filling) { @@ -10134,7 +10103,7 @@ impl Subscriber { /// oldest group still around, or a relative start still waits on the first group from /// the live feed. /// - /// Where a reader that waits on each group in sequence starts, rather than at whatever + /// Where a reader that waits on each group in key starts, rather than at whatever /// arrives first, which newest-first delivery makes the latest. pub fn floor(&self) -> Option { let start = match &self.inner { @@ -10177,18 +10146,42 @@ impl Subscriber { } /// The lowest group at or above `from` this subscriber's ranges and - /// [`Self::with_range`] floor cover, `None` when none does: the groups between are - /// never read, so a reader waiting on each group in sequence need not wait for them. - /// `Some` from there while a relative anchor waits on the first group, which can't - /// say yet. + /// [`Self::with_range`] floor cover, past the starts the publisher declared skipped + /// (see [`Producer::skip_groups`]) and those given up for good, `None` when none does: + /// the groups between are never read, so a reader waiting on each group in key need not + /// wait for them. A wire before lite-07 forwards a skip as such a drop. `Some` from there + /// while a relative anchor waits on the first group, which can't say yet. pub fn next_in_range(&self, from: u64) -> Option { let from = from.max(self.bounds().start); match &self.inner { - Inner::Plain(cursor) => cursor.filter.window.runs(from).next().map(|run| *run.start()), + Inner::Plain(cursor) => { + let state = cursor.state.read(); + first_open(cursor.filter.window.runs(from), |key| state.hole_end(key, true)) + } Inner::Resume(resume, _) => resume.next_in_range(from), } } + /// Whether the publisher declared every start in `groups` skipped, never to be produced + /// (see [`Producer::skip_groups`], which appending does between groups), so no group can + /// arrive there: true for an empty range. Unlike [`Self::next_in_range`], this ignores the + /// subscriber's ranges. A copy knows only the skips its wire forwarded. + /// + /// What tells a reader that nothing lies between two groups, or before a track's first. + pub fn is_skipped(&self, groups: ops::Range) -> bool { + let Some(last) = groups.end.checked_sub(1).filter(|last| *last >= groups.start) else { + return true; + }; + let run = std::iter::once(groups.start..=last); + match &self.inner { + Inner::Plain(cursor) => { + let state = cursor.state.read(); + first_open(run, |key| state.hole_end(key, false)).is_none() + } + Inner::Resume(resume, _) => resume.is_skipped(run), + } + } + /// The groups from `from` up to the exclusive `end` that this subscriber's ranges and /// [`Self::with_range`] bounds name, lowest first: the groups it is owed, without the /// ones between disjoint ranges. @@ -10267,7 +10260,7 @@ impl Subscriber { } } - /// Poll for a run of sequences this subscription gave up without serving, with the + /// Poll for a run of keys this subscription gave up without serving, with the /// error it was given up with, so a publisher can tell its peer not to wait for them. /// /// Stale skips are recorded only once this has been polled, so a reader that never @@ -10357,7 +10350,7 @@ impl Subscriber { } } - /// Limit subsequent reads to these group sequences, returning `self` for chaining. + /// Limit subsequent reads to these group keys, returning `self` for chaining. /// /// `2..=5` includes groups 2 through 5; `2..5` excludes group 5, and `..0` is the /// empty range. Both bounds are assigned, in either direction: an omitted start is @@ -10417,11 +10410,11 @@ impl Subscriber { } pub(crate) fn set_range(&mut self, groups: impl RangeBounds) { - let (start, end) = super::subscription::sequence_bounds(groups); + let (start, end) = super::subscription::key_bounds(groups); match &mut self.inner { Inner::Plain(cursor) => { - cursor.min_sequence = start; - cursor.end_sequence = end; + cursor.min_key = start; + cursor.end_key = end; cursor.update_servable(); } Inner::Resume(resume, logical) => resume.set_range(start, end, &logical.record.delivered), @@ -10433,8 +10426,8 @@ impl Subscriber { fn bounds(&self) -> Bounds { match &self.inner { Inner::Plain(cursor) => Bounds { - start: cursor.min_sequence, - end: cursor.end_sequence, + start: cursor.min_key, + end: cursor.end_key, }, Inner::Resume(resume, _) => resume.bounds(), } @@ -10453,17 +10446,12 @@ impl Subscriber { /// the cached groups they newly cover, even ones that arrived before the groups /// already read. Returns [`Error::Closed`] /// if the track already ended; the update is meaningless at that point and can usually - /// be ignored. Refuses a time anchor on an untimed track with - /// [`Error::TimestampMismatch`], leaving the preferences as they were. + /// be ignored. pub fn update(&mut self, subscription: Subscription) -> Result<()> { - refuse_untimed(&self.info, &subscription)?; let channel = match &self.inner { Inner::Plain(cursor) => &cursor.subscription, Inner::Resume(resume, _) => resume.subscription(), }; - if subscription.has_time() && subscription.ranges != channel.read().ranges { - self.consume().refuse_subscription(&subscription)?; - } channel.update(subscription)?; match &mut self.inner { Inner::Plain(cursor) => { @@ -10559,16 +10547,14 @@ impl Subscriber { state .lookup .range(run) - .filter(|&(&sequence, slot)| { - slot.serves_end() && cursor.admits(sequence) && !state.is_stale(sequence, &edge, budget) - }) - .map(|(&sequence, slot)| (sequence, slot.serving().consume())) + .filter(|&(&key, slot)| slot.serves_end() && cursor.admits(key) && !state.is_stale(key, &edge, budget)) + .map(|(&key, slot)| (key, slot.serving().consume())) .collect() }; let mut groups = Vec::new(); - for (sequence, group) in fresh { - if let Some(end) = sequence.checked_add(1) { - given_up.remove(sequence..end); + for (key, group) in fresh { + if let Some(end) = key.checked_add(1) { + given_up.remove(key..end); group.cache_refresh(); groups.push(cursor.deliver(group)); } @@ -10606,9 +10592,9 @@ impl Subscriber { } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn latest_sequence(&self) -> Option { + pub(crate) fn latest_key(&self) -> Option { match &self.inner { - Inner::Plain(cursor) => cursor.state.read().max_sequence, + Inner::Plain(cursor) => cursor.state.read().max_key, Inner::Resume(resume, _) => resume.latest(), } } @@ -10629,10 +10615,10 @@ impl Subscriber { } } -/// A [`Subscriber`] that reads groups in sequence order. +/// A [`Subscriber`] that reads groups in key order. /// /// Created by [`Subscriber::ordered`], which consumes the subscriber, so a track is read -/// one way or the other and never both. Every group it returns has a higher sequence +/// one way or the other and never both. Every group it returns has a higher key /// than the last, so a late arrival (network reordering, or a gap the cache filled after /// the fact) is skipped rather than delivered out of turn. /// @@ -10645,7 +10631,7 @@ impl Subscriber { /// budget: a group's own timestamps say where it starts presenting, not where it stops, /// and an unstamped successor leaves it unbounded and therefore kept. A backlog inside /// the budget is still delivered whole, as a burst in order, which is what a decoder -/// reading a gap-free sequence needs. +/// reading a gap-free key needs. /// /// The same budget follows a group already handed out: once newer content pulls that far /// ahead, a read still waiting on it ends with [`Error::Old`] and the cursor moves on. @@ -10688,14 +10674,14 @@ impl Ordered { self.inner.broadcast() } - /// The next group with a higher sequence number than any previously returned. + /// The next group with a higher key than any previously returned. /// /// Honors the group range set by [`Self::with_range`]: /// a group past the cap stays in the producer's cache and becomes eligible again if /// the cap rises or is removed. Ranges that all end end the read once they are /// accounted for, as for [`Subscriber::poll_recv_group`]. /// Later reads return `Ok(None)` immediately too; widening the ranges with - /// [`Self::update`] resumes reading above the last sequence returned. + /// [`Self::update`] resumes reading above the last key returned. /// /// Returns `Ok(Some(group))` when a group is available, `Ok(None)` when the track is /// finished (including an abort after its declared end settled, see [`Producer::abort`]) @@ -10705,7 +10691,7 @@ impl Ordered { self.inner.poll_next_group(waiter) } - /// Limit subsequent reads to these group sequences, returning `self` for chaining. + /// Limit subsequent reads to these group keys, returning `self` for chaining. /// /// See [`Subscriber::with_range`] for how the bounds apply. An ordered read never goes /// back, so lowering the floor below a group it already returned offers nothing. @@ -11156,7 +11142,7 @@ mod test { producer.drop_groups(15..25, StreamError::Cancel); indexed(); // Still arriving, so the next drop ends it and covers it again. - let _recreated = producer.create_group(group::Info { sequence: 17 }).unwrap(); + let _recreated = producer.create_group(group::Info { start: 17 }).unwrap(); indexed(); producer.drop_groups(14..26, StreamError::Old); indexed(); @@ -11172,18 +11158,18 @@ mod test { fn a_final_drop_leaves_a_finished_group() { let producer = track_producer("test", Info::default()); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); - let open = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let open = producer.create_group(group::Info { start: 1 }).unwrap(); producer.drop_groups(0..2, StreamError::Cancel); let drops: Vec<_> = producer.state.read().drops.iter().map(|(run, _)| run.clone()).collect(); assert_eq!(drops, vec![1..2]); assert!(open.is_aborted(), "the group still arriving ends"); let mut consumer = producer.subscribe(None); - assert_eq!(consumer.assert_group().sequence, 0); + assert_eq!(consumer.assert_group().start, 0); } /// A drop leaves a group a fetched head serves whole, whatever its live copy is doing: a @@ -11195,19 +11181,19 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); let mut lives = Vec::new(); - for sequence in 0..2 { - let mut live = producer.create_group(group::Info { sequence }).unwrap(); + for key in 0..2 { + let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); - let fetch = consumer.fetch_group(sequence, None); + let fetch = consumer.fetch_group(key, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap(); for payload in [&b"snapshot"[..], b"delta"] { - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(payload)) + head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } - head.finish().unwrap(); + head.close().unwrap(); fetch.await.unwrap(); lives.push(live); } @@ -11220,8 +11206,8 @@ mod test { assert!(lives[0].is_aborted(), "the live copy still arriving ends"); let mut sub = producer.subscribe(None); assert!(sub.poll_drop(&kio::Waiter::noop()).is_pending()); - for sequence in 0..2 { - assert_eq!(sub.assert_group().sequence, sequence); + for key in 0..2 { + assert_eq!(sub.assert_group().start, key); } } @@ -11231,14 +11217,14 @@ mod test { async fn arriving_head(producer: &Producer) -> (group::Producer, group::Producer) { let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let _fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"snapshot")) + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); (live, head) } @@ -11295,7 +11281,7 @@ mod test { async fn a_finished_live_copy_wins_over_a_final_drop_a_head_waits_on() { let producer = track_producer("test", Info::timed()); let (live, head) = arriving_head(&producer).await; - let live = live.finish().unwrap(); + let live = live.close().unwrap(); producer.drop_groups(0..1, StreamError::Old); head.abort(Error::Cancel).unwrap(); @@ -11314,9 +11300,9 @@ mod test { let (_live, mut head) = arriving_head(&producer).await; producer.drop_groups(0..1, StreamError::Old); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); - let head = head.finish().unwrap(); + let head = head.close().unwrap(); { let state = producer.state.read(); assert!(state.drops.iter().next().is_none()); @@ -11324,7 +11310,7 @@ mod test { } let mut sub = producer.subscribe(None); assert!(sub.poll_drop(&kio::Waiter::noop()).is_pending()); - assert_eq!(sub.assert_group().sequence, 0); + assert_eq!(sub.assert_group().start, 0); head.abort(Error::Evicted).unwrap(); assert!( @@ -11346,9 +11332,9 @@ mod test { let feed = producer.consume(); let waiter = kio::Waiter::noop(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); assert!(feed.poll_group(0, &waiter).is_pending()); assert!(feed.poll_group(u64::MAX, &waiter).is_pending()); @@ -11382,13 +11368,13 @@ mod test { state.lookup.len() } - /// Helper: get the sequence number of the first live group in arrival order. - fn first_live_sequence(state: &TrackState) -> u64 { + /// Helper: get the key of the first live group in arrival order. + fn first_live_key(state: &TrackState) -> u64 { state .arrival .iter() - .find(|(sequence, stamp)| state.lookup.get(sequence).is_some_and(|slot| slot.stamp == *stamp)) - .map(|(sequence, _)| *sequence) + .find(|(key, stamp)| state.lookup.get(key).is_some_and(|slot| slot.stamp == *stamp)) + .map(|(key, _)| *key) .unwrap() } @@ -11400,44 +11386,52 @@ mod test { let mut subscriber = producer.subscribe(replay()); assert_eq!(producer.largest(), None); - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); assert_eq!( subscriber.largest().await, Some(Position::group(0)), "an empty group is the largest at its head" ); - group.write_frame(None, &b"a"[..]).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"a"[..]) + .unwrap(); assert_eq!(producer.largest(), Some(Position { group: 0, frame: 0 })); assert!( subscriber.largest().now_or_never().is_none(), "the first frame is the head already named" ); - group.write_frame(None, &b"b"[..]).unwrap(); - group.write_frame(None, &b"c"[..]).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"b"[..]) + .unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"c"[..]) + .unwrap(); assert_eq!(subscriber.largest().await, Some(Position { group: 0, frame: 2 })); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let producer = producer.finish().unwrap(); drop(group); drop(producer); assert_eq!(subscriber.largest().await, None); } - /// Every sequence below a reached end already raised the largest, so a late group below + /// Every key below a reached end already raised the largest, so a late group below /// it changes nothing, and the ended stream stays ended. #[test] fn a_late_group_below_the_largest_changes_nothing() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); - let mut framed = producer.create_group(group::Info { sequence: 2 }).unwrap(); - framed.write_frame(None, &b"a"[..]).unwrap(); - framed.finish().unwrap(); + let mut framed = producer.create_group(group::Info { start: 2 }).unwrap(); + framed + .write_frame(crate::Timestamp::unscaled(framed.start), &b"a"[..]) + .unwrap(); + framed.close().unwrap(); producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); producer.finish_at(Position::group(6), None).unwrap(); @@ -11446,8 +11440,9 @@ mod test { assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); // Late content below the end is still admitted; it just isn't the largest. - let mut late = producer.create_group(group::Info { sequence: 4 }).unwrap(); - late.write_frame(None, &b"b"[..]).unwrap(); + let mut late = producer.create_group(group::Info { start: 4 }).unwrap(); + late.write_frame(crate::Timestamp::unscaled(late.start), &b"b"[..]) + .unwrap(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); let mut fresh = producer.consume(); assert_eq!(fresh.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); @@ -11459,8 +11454,10 @@ mod test { fn a_recreated_largest_group_is_reported_before_the_end() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); - let mut first = producer.create_group(group::Info { sequence: 0 }).unwrap(); - first.write_frame(None, &b"a"[..]).unwrap(); + let mut first = producer.create_group(group::Info { start: 0 }).unwrap(); + first + .write_frame(crate::Timestamp::unscaled(first.start), &b"a"[..]) + .unwrap(); first.abort(Error::Cancel).unwrap(); producer.finish_at(Position::group(1), None).unwrap(); @@ -11471,15 +11468,17 @@ mod test { "the aborted group can be recreated" ); - let mut again = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut again = producer.create_group(group::Info { start: 0 }).unwrap(); for payload in [&b"a"[..], b"b"] { - again.write_frame(None, payload).unwrap(); + again + .write_frame(crate::Timestamp::unscaled(again.start), payload) + .unwrap(); } assert_eq!( reader.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 1 })) ); - again.finish().unwrap(); + again.close().unwrap(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); } @@ -11491,8 +11490,8 @@ mod test { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let noop = kio::Waiter::noop(); - let mut first = producer.create_group(group::Info { sequence: 0 }).unwrap(); - first.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); + let mut first = producer.create_group(group::Info { start: 0 }).unwrap(); + first.write_frame(first.key_time(), &b"a"[..]).unwrap(); first.abort(Error::Cancel).unwrap(); producer.finish_at(Position::group(1), None).unwrap(); @@ -11511,8 +11510,8 @@ mod test { .unwrap() .accept(Info::timed()) .unwrap(); - bounded.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); - bounded.finish().unwrap(); + bounded.write_frame(bounded.key_time(), &b"a"[..]).unwrap(); + bounded.close().unwrap(); fetch.await.unwrap(); assert!( reader.poll_largest(&noop).is_pending(), @@ -11526,10 +11525,12 @@ mod test { fn a_live_edge_reader_starts_on_an_empty_largest_group() { let producer = track_producer("test", None); let noop = kio::Waiter::noop(); - let mut first = producer.append_group().unwrap(); - first.write_frame(None, &b"a"[..]).unwrap(); - first.finish().unwrap(); - let empty = producer.append_group().unwrap(); + let mut first = producer.append_next().unwrap(); + first + .write_frame(crate::Timestamp::unscaled(first.start), &b"a"[..]) + .unwrap(); + first.close().unwrap(); + let empty = producer.append_next().unwrap(); let mut reader = producer.subscribe(replay()); let Poll::Ready(Some(edge)) = reader.poll_largest(&noop) else { @@ -11538,18 +11539,19 @@ mod test { assert_eq!(edge, Position::group(1)); reader.set_range(edge.group..); - empty.finish().unwrap(); - let mut next = producer.append_group().unwrap(); - next.write_frame(None, &b"b"[..]).unwrap(); + empty.close().unwrap(); + let mut next = producer.append_next().unwrap(); + next.write_frame(crate::Timestamp::unscaled(next.start), &b"b"[..]) + .unwrap(); let mut group = reader.assert_group(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); assert!( matches!(group.poll_read_frame(&noop), Poll::Ready(Ok(None))), "it ends with nothing" ); let mut group = reader.assert_group(); - assert_eq!(group.sequence, 2); + assert_eq!(group.start, 2); assert!(matches!(group.poll_read_frame(&noop), Poll::Ready(Ok(Some(_))))); } @@ -11568,8 +11570,10 @@ mod test { let producer = track_producer("test", Info::default()); let mut subscriber = producer.subscribe(replay()); - let mut group = producer.append_group().unwrap(); - group.write_frame(None, &b"a"[..]).unwrap(); + let mut group = producer.append_next().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"a"[..]) + .unwrap(); let noop = kio::Waiter::noop(); assert_eq!( subscriber.poll_largest(&noop), @@ -11581,7 +11585,9 @@ mod test { subscriber.poll_largest(&noop).is_pending(), "the open group can still grow" ); - group.write_frame(None, &b"b"[..]).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"b"[..]) + .unwrap(); assert_eq!( subscriber.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 1 })) @@ -11590,7 +11596,7 @@ mod test { let flag = Arc::new(Flag(AtomicBool::new(false))); let waiter = kio::Waiter::new(futures::task::waker(flag.clone())); assert!(subscriber.poll_largest(&waiter).is_pending()); - group.finish().unwrap(); + group.close().unwrap(); assert!( flag.0.load(Ordering::SeqCst), "finishing the last group woke the reader" @@ -11612,8 +11618,10 @@ mod test { // An older report or frame does not move it back. producer.report_largest(Position { group: 7, frame: 1 }); - let mut _open = producer.create_group(group::Info { sequence: 2 }).unwrap(); - _open.write_frame(None, &b"a"[..]).unwrap(); + let mut _open = producer.create_group(group::Info { start: 2 }).unwrap(); + _open + .write_frame(crate::Timestamp::unscaled(_open.start), &b"a"[..]) + .unwrap(); assert!(subscriber.largest().now_or_never().is_none()); producer.report_largest(Position { group: 8, frame: 0 }); @@ -11648,7 +11656,7 @@ mod test { for ordered in [false, true] { let producer = track_producer("test", Info::default()); let first = framed(&producer, 0, 1); - first.finish().unwrap(); + first.close().unwrap(); let mut reader = producer.subscribe(replay()); let mut read = |waiter: &kio::Waiter| match ordered { true => reader.poll_group(true, waiter), @@ -11663,17 +11671,19 @@ mod test { let (other, moved) = Flag::waiter(); assert!(read(&moved).is_pending()); let next = framed(&producer, 1, 1); - next.finish().unwrap(); + next.close().unwrap(); assert!(other.take(), "the next group wakes the task that polled last"); assert!(matches!(read(&moved), Poll::Ready(Ok(Some(_))))); } } - /// Write group `sequence` with `frames` frames, left open. - fn framed(producer: &Producer, sequence: u64, frames: usize) -> group::Producer { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + /// Write group `key` with `frames` frames, left open. + fn framed(producer: &Producer, key: u64, frames: usize) -> group::Producer { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); for _ in 0..frames { - group.write_frame(None, &b"x"[..]).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); } group } @@ -11692,27 +11702,25 @@ mod test { match group.poll_read_frame(&noop) { Poll::Ready(Ok(Some(_))) => frames.push(index), Poll::Ready(Ok(None)) => break, - res => panic!("group {} read {res:?}", group.sequence), + res => panic!("group {} read {res:?}", group.start), } } - groups.push((group.sequence, frames)); + groups.push((group.start, frames)); } groups } /// A range starting and ending inside cached groups reads only the frames it covers, - /// in arrival and sequence order alike, however wide another subscriber made the + /// in arrival and key order alike, however wide another subscriber made the /// aggregate. #[test] fn a_range_reads_only_its_groups() { let producer = track_producer("test", None); - let _groups: Vec<_> = (3..8) - .map(|sequence| framed(&producer, sequence, 8).finish().unwrap()) - .collect(); + let _groups: Vec<_> = (3..8).map(|key| framed(&producer, key, 8).close().unwrap()).collect(); let _wide = producer.subscribe(replay()); let range = Range { - start: Anchor::Absolute(4), - end: Some(Anchor::Absolute(6)), + start: Anchor::Time(4), + end: Some(Anchor::Time(5)), }; let bounded = || producer.subscribe(replay().with_ranges([range])); @@ -11728,15 +11736,17 @@ mod test { #[test] fn a_group_missing_its_head_lags() { let producer = track_producer("test", Info::default()); - let mut headless = producer.create_group(group::Info { sequence: 4 }).unwrap(); + let mut headless = producer.create_group(group::Info { start: 4 }).unwrap(); headless.start_at(2).unwrap(); for _ in 0..3 { - headless.write_frame(None, &b"x"[..]).unwrap(); + headless + .write_frame(crate::Timestamp::unscaled(headless.start), &b"x"[..]) + .unwrap(); } - headless.finish().unwrap(); + headless.close().unwrap(); let noop = kio::Waiter::noop(); - let start = Anchor::Absolute(4); + let start = Anchor::Time(4); let mut sub = producer.subscribe(replay().with_ranges([Range { start, end: None }])); let Poll::Ready(Ok(Some(mut group))) = sub.poll_recv_group(&noop) else { panic!("no group"); @@ -11750,7 +11760,7 @@ mod test { #[test] fn floor_honors_with_range() { let producer = track_producer("test", Info::default()); - let _groups: Vec<_> = (0..6).map(|sequence| framed(&producer, sequence, 2)).collect(); + let _groups: Vec<_> = (0..6).map(|key| framed(&producer, key, 2)).collect(); let reader = producer.subscribe(replay().with_range(2..)); assert_eq!((reader.floor(), reader.next_in_range(0)), (Some(2), Some(2))); @@ -11773,7 +11783,7 @@ mod test { fn largest_reports_only_within_the_bounds() { let producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); - let _groups: Vec<_> = (0..6).map(|sequence| framed(&producer, sequence, 2)).collect(); + let _groups: Vec<_> = (0..6).map(|key| framed(&producer, key, 2)).collect(); let mut capped = producer.subscribe(replay()); capped.set_range(..3); @@ -11803,7 +11813,7 @@ mod test { fn largest_follows_the_bounds_as_they_move() { let producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); - let _groups: Vec<_> = (0..6).map(|sequence| framed(&producer, sequence, 1)).collect(); + let _groups: Vec<_> = (0..6).map(|key| framed(&producer, key, 1)).collect(); let mut reader = producer.subscribe(replay()).ordered(); reader.set_range(..3); @@ -11838,7 +11848,7 @@ mod test { #[test] fn largest_stays_ended_when_the_bounds_widen() { let producer = track_producer("test", Info::default()); - framed(&producer, 5, 1).finish().unwrap(); + framed(&producer, 5, 1).close().unwrap(); let producer = producer.finish().unwrap(); let noop = kio::Waiter::noop(); @@ -11887,7 +11897,7 @@ mod test { fn largest_keeps_its_last_report_across_an_idle_move() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); - let _groups: Vec<_> = (0..3).map(|sequence| framed(&producer, sequence, 1)).collect(); + let _groups: Vec<_> = (0..3).map(|key| framed(&producer, key, 1)).collect(); let mut reader = producer.subscribe(replay()); assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(2)))); @@ -11910,7 +11920,7 @@ mod test { fn largest_waits_for_an_idle_copy_with_a_future_end() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); - let _groups: Vec<_> = (0..3).map(|sequence| framed(&producer, sequence, 1)).collect(); + let _groups: Vec<_> = (0..3).map(|key| framed(&producer, key, 1)).collect(); producer.finish_at(Position::group(10), None).unwrap(); producer.set_idle(); @@ -11921,7 +11931,7 @@ mod test { assert_eq!(reader.poll_largest(&noop), Poll::Ready(Some(Position::group(5)))); let mut finished = track_producer("test", Info::default()); - framed(&finished, 0, 1).finish().unwrap(); + framed(&finished, 0, 1).close().unwrap(); finished.set_idle(); let finished = finished.finish().unwrap(); let mut reader = finished.subscribe(replay()); @@ -11934,7 +11944,7 @@ mod test { fn largest_waits_for_an_idle_copy_to_go_live() { let mut producer = track_producer("test", Info::default()); let noop = kio::Waiter::noop(); - let _groups: Vec<_> = (0..3).map(|sequence| framed(&producer, sequence, 1)).collect(); + let _groups: Vec<_> = (0..3).map(|key| framed(&producer, key, 1)).collect(); producer.set_idle(); let mut reader = producer.subscribe(replay()); @@ -11957,8 +11967,8 @@ mod test { /// A relay copy holding groups 0 to 3 from an ended feed, idle until its route answers. fn idle_copy() -> Producer { let mut producer = track_producer("test", None); - for sequence in 0..4 { - framed(&producer, sequence, 1).finish().unwrap(); + for key in 0..4 { + framed(&producer, key, 1).close().unwrap(); } producer.set_idle(); producer @@ -11967,11 +11977,11 @@ mod test { /// Every group `reader` can take right now, in delivery order. fn ready_groups(reader: &mut Subscriber) -> Vec { let noop = kio::Waiter::noop(); - let mut sequences = Vec::new(); + let mut keys = Vec::new(); while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&noop) { - sequences.push(group.sequence); + keys.push(group.start); } - sequences + keys } /// A relative start taken while the copy is idle counts back from the live feed's @@ -11989,7 +11999,7 @@ mod test { assert_eq!(reader.floor(), None, "the start waits for the live feed"); producer.set_live(Some(Position::group(4)), None).unwrap(); - framed(&producer, 4, 1).finish().unwrap(); + framed(&producer, 4, 1).close().unwrap(); assert_eq!(ready_groups(&mut reader), expected, "{ranges:?}"); } } @@ -12001,10 +12011,10 @@ mod test { let mut producer = idle_copy(); let mut reader = producer.subscribe(replay().with_ranges([Range::live()])); producer.set_live(Some(Position::group(5)), None).unwrap(); - framed(&producer, 4, 1).finish().unwrap(); + framed(&producer, 4, 1).close().unwrap(); assert_eq!(ready_groups(&mut reader), Vec::::new(), "the answer has not shown"); - framed(&producer, 5, 1).finish().unwrap(); + framed(&producer, 5, 1).close().unwrap(); assert_eq!(ready_groups(&mut reader), vec![5]); } @@ -12020,8 +12030,8 @@ mod test { .chain(current.map(|largest| (largest, true))) { let mut producer = track_producer("test", None); - framed(&producer, 2, 1).finish().unwrap(); - framed(&producer, 3, 2).finish().unwrap(); + framed(&producer, 2, 1).close().unwrap(); + framed(&producer, 3, 2).close().unwrap(); producer.set_idle(); let mut reader = producer.subscribe(replay()); @@ -12048,8 +12058,8 @@ mod test { // Largest names the newest object, which an empty newest group does not hold. for (largest, ok) in [(Some(Position::group(2)), true), (None, false)] { let mut producer = track_producer("test", None); - framed(&producer, 2, 1).finish().unwrap(); - framed(&producer, 3, 0).finish().unwrap(); + framed(&producer, 2, 1).close().unwrap(); + framed(&producer, 3, 0).close().unwrap(); producer.set_idle(); assert_eq!(producer.set_live(largest, None).is_ok(), ok, "{largest:?}"); } @@ -12060,7 +12070,7 @@ mod test { #[test] fn an_idle_copy_judges_its_answer_against_the_ended_feed() { let mut producer = track_producer("test", None); - framed(&producer, 2, 1).finish().unwrap(); + framed(&producer, 2, 1).close().unwrap(); let _open = framed(&producer, 3, 1); producer.set_idle(); @@ -12085,14 +12095,14 @@ mod test { }; for (object, ok) in [(50, false), (100, true)] { let mut producer = track_producer("test", None); - let mut group = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 2 }).unwrap(); group.set_wire(wire.clone()).unwrap(); for id in [0, 100] { let mut frame = group .create_object(frame::Header { info: frame::Info { size: 1, - timestamp: None, + timestamp: Timestamp::unscaled(2), }, fields: Some(Arc::new(super::super::object::Fields { id, @@ -12104,7 +12114,7 @@ mod test { frame.write(&b"x"[..]).unwrap(); group = frame.finish().unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); producer.set_idle(); let answered = producer.set_live( Some(Position { @@ -12140,7 +12150,7 @@ mod test { assert_eq!(ready_groups(&mut oldest), Vec::::new(), "the copy is idle"); producer.set_live(Some(Position::group(4)), None).unwrap(); - framed(&producer, 4, 1).finish().unwrap(); + framed(&producer, 4, 1).close().unwrap(); assert_eq!(ready_groups(&mut live), vec![4]); assert_eq!(ready_groups(&mut oldest), vec![0, 1, 2, 3, 4]); } @@ -12158,22 +12168,27 @@ mod test { assert_eq!(ready_groups(&mut reader), vec![3]); } - /// A live join's answer anchors a relative start where it says the feed is, both for the - /// copy's readers and for the bounds sent upstream, even once a newer group lands before - /// the one it named. + /// A live join's answer anchors a relative start where it says the feed is, counting back + /// over the groups held below it, both for the copy's readers and for the bounds sent + /// upstream, even once a newer group lands before the one it named. #[test] fn a_relative_start_counts_from_the_answered_edge() { let producer = track_producer("test", Info::default()); + // Counting back counts groups held: two below the edge the answer names. + for key in [2, 3] { + framed(&producer, key, 1).close().unwrap(); + } let back = Range { start: super::super::subscription::Anchor::Relative(2), end: None, }; - let mut reader = producer.subscribe(replay().with_ranges([back])); producer.report_edge(4); + let mut reader = producer.subscribe(replay().with_ranges([back])); let upstream = producer.resolve_subscription(&replay().with_ranges([back])).unwrap(); assert_eq!(upstream.start(), Some(2), "the answer resolves the upstream bounds"); - framed(&producer, 5, 1).finish().unwrap(); + framed(&producer, 5, 1).close().unwrap(); + // The default max age serves only the live edge, so the groups held below it are stale. assert_eq!(ready_groups(&mut reader), vec![5]); assert_eq!(reader.floor(), Some(2), "the answer anchors the start"); } @@ -12183,9 +12198,9 @@ mod test { async fn dropped_group_writer_aborts_and_releases_its_cached_copy() { let pool = cache::Pool::unbounded(); let producer = track_producer_pooled("test", pool.clone()); - let mut group = producer.create_group(group::Info { sequence: 3 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); let mut reader = group.consume(); assert!(pool.used() > 0); @@ -12203,11 +12218,11 @@ mod test { } #[kio::test] - async fn dropped_group_writer_is_absent_and_its_sequence_can_be_recreated() { + async fn dropped_group_writer_is_absent_and_its_key_can_be_recreated() { let producer = track_producer("test", Info::timed()); - let mut group = producer.create_group(group::Info { sequence: 3 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); drop(group); @@ -12221,12 +12236,12 @@ mod test { Poll::Ready(Err(Error::Dropped)) )); let mut replacement = producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .expect("the dead copy is replaceable"); replacement - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .write_frame(replacement.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); - replacement.finish().unwrap(); + replacement.close().unwrap(); let mut later = producer.subscribe(Subscription::default().with_range(0..)); assert!(matches!( later.poll_recv_group(&kio::Waiter::noop()), @@ -12238,15 +12253,15 @@ mod test { async fn dropped_group_writer_preserves_the_committed_resume_position() { let producer = track_producer("test", Info::timed()); let consumer = producer.consume(); - let mut group = producer.create_group(group::Info { sequence: 3 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); let handle = group.handle(); let _partial = group .create_frame(frame::Info { size: 2, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); assert_eq!(handle.frame_count(), 2); @@ -12255,11 +12270,11 @@ mod test { assert_eq!(consumer.resume_position(), Some(Position { group: 3, frame: 1 })); } - /// No group holds the reserved sequence, so its fetch fails at once rather than parking + /// No group holds the reserved key, so its fetch fails at once rather than parking /// until every handler goes. #[test] #[cfg(feature = "async")] - fn the_reserved_sequence_misses_with_a_handler() { + fn the_reserved_key_misses_with_a_handler() { let producer = track_producer("test", None); let dynamic = producer.requests(); let consumer = producer.consume(); @@ -12273,9 +12288,9 @@ mod test { let producer = track_producer("test", Info::timed()); // Create 3 groups at time 0. - producer.append_group().unwrap().finish().unwrap(); // seq 0 - producer.append_group().unwrap().finish().unwrap(); // seq 1 - producer.append_group().unwrap().finish().unwrap(); // seq 2 + producer.append_next().unwrap().close().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 1 + producer.append_next().unwrap().close().unwrap(); // seq 2 { let state = producer.state.read(); @@ -12287,7 +12302,7 @@ mod test { elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // Append a new group to trigger eviction. - producer.append_group().unwrap().finish().unwrap(); // seq 3 + producer.append_next().unwrap().close().unwrap(); // seq 3 // Groups 0, 1, 2 are expired but seq 3 (the live edge) is kept. Their arrival // entries no longer resolve, so the leading ones are trimmed and the offset @@ -12295,7 +12310,7 @@ mod test { { let state = producer.state.read(); assert_eq!(live_groups(&state), 1); - assert_eq!(first_live_sequence(&state), 3); + assert_eq!(first_live_key(&state), 3); assert_eq!(state.offset, 3); assert!(!state.lookup.contains_key(&0)); assert!(!state.lookup.contains_key(&1)); @@ -12311,7 +12326,7 @@ mod test { #[cfg(feature = "async")] async fn aging_out_a_finished_group_keeps_the_clean_end() { let producer = track_producer("test", Info::timed()); - let mut group = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); let mut consumer = group.consume(); group @@ -12321,8 +12336,8 @@ mod test { // The group stays open well past the LRU window, then the next period starts. elapse(&producer, cache::DEFAULT_EXPIRY * 2); - group.finish().unwrap(); - let _next = producer.create_group(group::Info { sequence: 1 }).unwrap(); + group.close().unwrap(); + let _next = producer.create_group(group::Info { start: 1 }).unwrap(); assert!(consumer.next_frame().await.unwrap().is_none()); } @@ -12339,13 +12354,13 @@ mod test { // A finished group with one frame per step of the read loop below. let mut group = producer.create_group(0u64.into()).unwrap(); for _ in 0..10 { - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let mut reading = subscriber.assert_group(); // A sibling written at the same time that nobody ever reads. - producer.create_group(1u64.into()).unwrap().finish().unwrap(); + producer.create_group(1u64.into()).unwrap().close().unwrap(); // Each step stays well inside the retention window, but the whole read // spans several windows. New groups keep the expiry scan running. @@ -12356,7 +12371,7 @@ mod test { matches!(frame, Ok(Some(_))), "an actively-read group must not expire mid-read (step {seq})" ); - producer.create_group(seq.into()).unwrap().finish().unwrap(); + producer.create_group(seq.into()).unwrap().close().unwrap(); } let state = producer.state.read(); @@ -12376,9 +12391,9 @@ mod test { let mut group = producer.create_group(0u64.into()).unwrap(); for _ in 0..20 { - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let mut reading = subscriber.assert_group(); // One whole-frame read per half-window: most are served straight from the @@ -12390,7 +12405,7 @@ mod test { matches!(frame, Ok(Some(_))), "a slow prefetch reader must not expire mid-read (step {seq})" ); - producer.create_group(seq.into()).unwrap().finish().unwrap(); + producer.create_group(seq.into()).unwrap().close().unwrap(); } } @@ -12404,10 +12419,10 @@ mod test { let mut subscriber = producer.subscribe(replay()); let mut group = producer.create_group(0u64.into()).unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); + group.close().unwrap(); // A second group so seq 0 leaves the protected live edge. - producer.create_group(1u64.into()).unwrap().finish().unwrap(); + producer.create_group(1u64.into()).unwrap().close().unwrap(); // Deliver just inside the window: the delivery stamps the group. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); @@ -12416,7 +12431,7 @@ mod test { // Almost another full window passes: far beyond the write, inside the // delivery stamp. The new group runs the expiry scan. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); - producer.create_group(2u64.into()).unwrap().finish().unwrap(); + producer.create_group(2u64.into()).unwrap().close().unwrap(); let frame = reading.read_frame().await.unwrap(); assert!(frame.is_some(), "a just-delivered group must not expire unread"); @@ -12430,12 +12445,12 @@ mod test { let producer = track_producer("test", Info::timed()); let straggler = producer.create_group(0u64.into()).unwrap(); // The live edge moves on, so the straggler is demoted and expirable. - producer.create_group(1u64.into()).unwrap().finish().unwrap(); + producer.create_group(1u64.into()).unwrap().close().unwrap(); let mut frame = straggler .create_frame(frame::Info { size: 10, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); // One chunk per half-window; the whole frame spans several windows. New @@ -12443,10 +12458,10 @@ mod test { for seq in 2..12u64 { elapse(&producer, cache::DEFAULT_EXPIRY / 2); frame.write(bytes::Bytes::from_static(b"x")).unwrap(); - producer.create_group(seq.into()).unwrap().finish().unwrap(); + producer.create_group(seq.into()).unwrap().close().unwrap(); } let straggler = frame.finish().unwrap(); - straggler.finish().unwrap(); + straggler.close().unwrap(); let state = producer.state.read(); assert!( @@ -12464,12 +12479,12 @@ mod test { let producer = track_producer("test", Info::timed()); let straggler = producer.create_group(0u64.into()).unwrap(); // The live edge moves on, so the straggler is demoted and expirable. - producer.create_group(1u64.into()).unwrap().finish().unwrap(); + producer.create_group(1u64.into()).unwrap().close().unwrap(); let mut frame = straggler .create_frame(frame::Info { size: 3, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); @@ -12479,10 +12494,10 @@ mod test { elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); frame.write_deferred(bytes::Bytes::from_static(b"abc")).unwrap(); let straggler = frame.finish().unwrap(); - straggler.finish().unwrap(); + straggler.close().unwrap(); // A new group runs the expiry scan. - producer.create_group(2u64.into()).unwrap().finish().unwrap(); + producer.create_group(2u64.into()).unwrap().close().unwrap(); let state = producer.state.read(); assert!( @@ -12502,44 +12517,44 @@ mod test { for seq in 0..2u64 { let mut group = producer.create_group(seq.into()).unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); + group.close().unwrap(); } // Group 0 is in range; group 1 is beyond the cap and parks. - assert_eq!(subscriber.assert_group().sequence, 0); + assert_eq!(subscriber.assert_group().start, 0); subscriber.assert_no_group(); // Just inside the window, the cap rises and the re-offer stamps group 1. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); subscriber.set_range(..2); let mut reading = subscriber.assert_group(); - assert_eq!(reading.sequence, 1); + assert_eq!(reading.start, 1); // Almost another full window passes: far beyond the write, inside the // re-offer stamp. The new group runs the expiry scan. elapse(&producer, cache::DEFAULT_EXPIRY - Duration::from_secs(1)); - producer.create_group(2u64.into()).unwrap().finish().unwrap(); + producer.create_group(2u64.into()).unwrap().close().unwrap(); let frame = reading.read_frame().await.unwrap(); assert!(frame.is_some(), "a just-re-offered group must not expire unread"); } #[test] - fn evict_keeps_max_sequence() { + fn evict_keeps_max_key() { let producer = track_producer("test", Info::timed()); - let _open = producer.append_group().unwrap(); // seq 0 + let _open = producer.append_next().unwrap(); // seq 0 // Advance time past the LRU window. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // Append another group; seq 0 is expired and evicted. - let _open = producer.append_group().unwrap(); // seq 1 + let _open = producer.append_next().unwrap(); // seq 1 { let state = producer.state.read(); assert_eq!(live_groups(&state), 1); - assert_eq!(first_live_sequence(&state), 1); + assert_eq!(first_live_key(&state), 1); assert_eq!(state.offset, 1); } } @@ -12547,9 +12562,9 @@ mod test { #[test] fn no_eviction_when_fresh() { let producer = track_producer("test", Info::timed()); - producer.append_group().unwrap().finish().unwrap(); // seq 0 - producer.append_group().unwrap().finish().unwrap(); // seq 1 - producer.append_group().unwrap().finish().unwrap(); // seq 2 + producer.append_next().unwrap().close().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 1 + producer.append_next().unwrap().close().unwrap(); // seq 2 { let state = producer.state.read(); @@ -12561,16 +12576,16 @@ mod test { #[test] fn consumer_skips_evicted_groups() { let producer = track_producer("test", Info::timed()); - producer.append_group().unwrap().finish().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 0 let mut consumer = producer.subscribe(None); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); - producer.append_group().unwrap().finish().unwrap(); // seq 1 + producer.append_next().unwrap().close().unwrap(); // seq 1 // Group 0 was evicted. Consumer should get group 1. let group = consumer.assert_group(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); } /// Mint a track under an origin whose pool has the given wall-clock LRU window. @@ -12597,9 +12612,9 @@ mod test { fn pool_sweep_expires_without_a_write() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let producer = track_producer_pooled("test", pool.clone()); - let mut stalled = producer.append_group().unwrap(); // seq 0, left open - stalled.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); - let _open = producer.append_group().unwrap(); // seq 1, the live edge + let mut stalled = producer.append_next().unwrap(); // seq 0, left open + stalled.write_frame(stalled.key_time(), b"x".as_slice()).unwrap(); + let _open = producer.append_next().unwrap(); // seq 1, the live edge elapse(&producer, Duration::from_secs(2)); pool.sweep(); @@ -12619,9 +12634,9 @@ mod test { Some(now.saturating_add(Duration::from_millis(500))) ); let producer = track_producer_pooled("test", pool.clone()); - let mut group = producer.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"first".as_slice()).unwrap(); - let _open = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); + group.write_frame(group.key_time(), b"first".as_slice()).unwrap(); + let _open = producer.append_next().unwrap(); // Activity written while cleanup was idle gets the new supplied time. let later = now.saturating_add(Duration::from_secs(60)); pool.poll(later, &kio::Waiter::noop()); @@ -12636,8 +12651,8 @@ mod test { let pool = cache::Pool::new(cache::Config::default().with_expiry(expiry)); let producer = track_producer_pooled("test", pool.clone()); let now = crate::model::clock::now(); - let groups: Vec<_> = (0..EVICT_SCAN * 3).map(|_| producer.append_group().unwrap()).collect(); - let _open = producer.append_group().unwrap(); + let groups: Vec<_> = (0..EVICT_SCAN * 3).map(|_| producer.append_next().unwrap()).collect(); + let _open = producer.append_next().unwrap(); pool.poll(now, &kio::Waiter::noop()); // Keep more than a write scan's worth of leading entries fresh. for group in &groups[..EVICT_SCAN * 2] { @@ -12645,11 +12660,11 @@ mod test { } pool.poll(now.saturating_add(expiry * 2), &kio::Waiter::noop()); let state = producer.state.read(); - for sequence in 0..EVICT_SCAN * 2 { - assert!(state.lookup.contains_key(&(sequence as u64)), "fresh front survives"); + for key in 0..EVICT_SCAN * 2 { + assert!(state.lookup.contains_key(&(key as u64)), "fresh front survives"); } - for sequence in EVICT_SCAN * 2..EVICT_SCAN * 3 { - assert!(!state.lookup.contains_key(&(sequence as u64)), "old tail is reclaimed"); + for key in EVICT_SCAN * 2..EVICT_SCAN * 3 { + assert!(!state.lookup.contains_key(&(key as u64)), "old tail is reclaimed"); } } @@ -12665,11 +12680,11 @@ mod test { let backlog = 4 * EVICT_SCAN; let mut _open = Vec::new(); for _ in 0..backlog { - let mut group = producer.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + let mut group = producer.append_next().unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); _open.push(group); } - let _open = producer.append_group().unwrap(); // the live edge, always protected + let _open = producer.append_next().unwrap(); // the live edge, always protected elapse(&producer, Duration::from_secs(2)); pool.sweep(); @@ -12683,26 +12698,26 @@ mod test { fn pool_expiry_controls_eviction() { // A shorter LRU window on the pool evicts sooner than the default. let producer = track_producer_expiring("test", Duration::from_secs(1)); - let _open = producer.append_group().unwrap(); // seq 0 + let _open = producer.append_next().unwrap(); // seq 0 // Past the pool's window but well within cache::DEFAULT_EXPIRY. elapse(&producer, Duration::from_secs(2)); - let _open = producer.append_group().unwrap(); // seq 1 + let _open = producer.append_next().unwrap(); // seq 1 // Seq 0 is gone because the pool only keeps idle groups for 1s. let state = producer.state.read(); assert_eq!(live_groups(&state), 1); - assert_eq!(first_live_sequence(&state), 1); + assert_eq!(first_live_key(&state), 1); } #[test] fn small_frame_write_expires_idle_siblings() { let producer = track_producer_expiring("test", Duration::from_secs(1)); - producer.append_group().unwrap().finish().unwrap(); // seq 0 - let mut live = producer.append_group().unwrap(); // seq 1 + producer.append_next().unwrap().close().unwrap(); // seq 0 + let mut live = producer.append_next().unwrap(); // seq 1 elapse(&producer, Duration::from_secs(2)); - live.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"x".as_slice()).unwrap(); let expired = !producer.state.read().lookup.contains_key(&0); assert!(expired, "a small frame write runs expiry"); @@ -12713,11 +12728,11 @@ mod test { use std::sync::atomic::{AtomicBool, Ordering}; let producer = track_producer_expiring("test", cache::DEFAULT_EXPIRY); - producer.append_group().unwrap().finish().unwrap(); - let mut live = producer.append_group().unwrap(); + producer.append_next().unwrap().close().unwrap(); + let mut live = producer.append_next().unwrap(); let mut consumer = producer.subscribe(replay()); - assert_eq!(consumer.assert_group().sequence, 0); - assert_eq!(consumer.assert_group().sequence, 1); + assert_eq!(consumer.assert_group().start, 0); + assert_eq!(consumer.assert_group().start, 1); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); @@ -12728,12 +12743,12 @@ mod test { let mut follower = producer.subscribe(replay()); let noop = kio::Waiter::noop(); assert_eq!(follower.poll_largest(&noop), Poll::Ready(Some(Position::group(1)))); - live.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"x".as_slice()).unwrap(); let advanced = Arc::new(AtomicBool::new(false)); let largest = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(advanced.clone())))); assert!(follower.poll_largest(&largest).is_pending()); - live.write_frame(Timestamp::ZERO, b"y".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"y".as_slice()).unwrap(); assert!( !woken.load(Ordering::SeqCst), "a no-op expiry scan must not wake track consumers" @@ -12744,12 +12759,12 @@ mod test { #[test] fn streaming_frame_write_expires_idle_siblings() { let producer = track_producer_expiring("test", Duration::from_secs(1)); - producer.append_group().unwrap().finish().unwrap(); // seq 0 - let live = producer.append_group().unwrap(); // seq 1 + producer.append_next().unwrap().close().unwrap(); // seq 0 + let live = producer.append_next().unwrap(); // seq 1 let mut frame = live .create_frame(frame::Info { size: 1, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); @@ -12786,18 +12801,18 @@ mod test { ); } - /// A peer whose timestamps stop advancing never ages a group out on media time. The wall + /// A peer whose timestamps barely advance never ages a group out on media time. The wall /// clock still does. #[test] #[cfg(feature = "async")] fn frozen_timestamps_still_age_out_by_wall_clock() { let mut producer = track_producer("test", Info::timed()); let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1))); - append_at(&mut producer, 0); + append_frozen(&mut producer); elapse(&producer, Duration::from_millis(500)); - append_at(&mut producer, 0); + append_frozen(&mut producer); elapse(&producer, Duration::from_millis(800)); - append_at(&mut producer, 0); + append_frozen(&mut producer); // Group 0's successor arrived 800ms ago, group 1's just now. assert_eq!(drain(&mut producer.subscribe(budget.clone())), vec![0, 1, 2]); @@ -12835,19 +12850,20 @@ mod test { .flat_map(|info| [(info.clone(), true), (info, false)]) { let producer = track_producer("test", info.clone()); - let at = info.timescale.map(|_| Timestamp::ZERO); + // The first group, at key 0. + let at = info.timescale.map_or(Timestamp::unscaled(0), |_| Timestamp::ZERO); if primed { elapse(&producer, Duration::ZERO); } let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); - let mut held = producer.append_group().unwrap(); + let mut held = producer.append_next().unwrap(); held.write_frame(at, bytes::Bytes::from_static(b"x")).unwrap(); let mut reading = subscriber.assert_group(); assert!(reading.read_frame().now_or_never().unwrap().unwrap().is_some()); // The successor never presents a frame, so media time never bounds the held group. - let _successor = producer.append_group().unwrap(); + let _successor = producer.append_next().unwrap(); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(reading.poll_read_frame(&waiter).is_pending()); @@ -12875,6 +12891,57 @@ mod test { } } + /// The next group in range passes over the starts the publisher declared skipped, + /// whether by appending past them or outright, and those given up for good, so a reader + /// waiting on each group in key need not wait for them. + #[test] + fn the_next_group_in_range_passes_over_skips() { + let producer = track_producer("test", Info::default().with_max_age(None)); + let subscriber = producer.subscribe(Subscription::default()); + producer.append_group(0).unwrap().close().unwrap(); + assert_eq!(subscriber.next_in_range(1), Some(1), "nothing skipped yet"); + assert!(!subscriber.is_skipped(1..2)); + producer.append_group(10).unwrap().close().unwrap(); + assert_eq!(subscriber.next_in_range(1), Some(10)); + assert_eq!(subscriber.next_in_range(11), Some(11)); + assert!(subscriber.is_skipped(1..10)); + assert!(!subscriber.is_skipped(0..10), "group 0 is held"); + assert!(!subscriber.is_skipped(1..11), "group 10 is held"); + assert!(subscriber.is_skipped(5..5), "an empty range"); + producer.skip_groups(11..20); + assert_eq!(subscriber.next_in_range(11), Some(20)); + assert!(subscriber.is_skipped(11..20)); + assert!(!subscriber.is_skipped(11..21)); + + // A drop for good is passed over too, as a wire before lite-07 forwards a skip, but + // it was produced, so it is no skip. + producer.drop_groups(20..30, StreamError::NotFound); + assert_eq!(subscriber.next_in_range(20), Some(30)); + assert!(!subscriber.is_skipped(20..30)); + } + + /// A group finished whole ages from its own end, which a zero budget waits for too: a + /// fetch before the end still gets it, and one after is told it is old. + #[kio::test] + #[cfg(feature = "async")] + async fn a_whole_group_ages_only_once_it_ends() { + let producer = track_producer("test", Info::timed().with_max_age(Duration::ZERO)); + elapse(&producer, Duration::ZERO); + let mut group = producer.append_group(0).unwrap(); + group + .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) + .unwrap(); + group.finish(40).unwrap(); + + let consumer = producer.consume(); + elapse(&producer, Duration::from_millis(39)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + assert!(fetched.is_ok(), "before its end: {:?}", fetched.err()); + elapse(&producer, Duration::from_millis(1)); + let fetched = consumer.fetch_group(0, None).now_or_never().unwrap(); + assert!(matches!(fetched, Err(Error::Old)), "{:?}", fetched.err()); + } + /// A successor whose header landed but whose first frame never does still starts the /// wall clock, so a relay ages the group before it as the origin does. #[kio::test] @@ -12882,11 +12949,12 @@ mod test { async fn a_frameless_successor_ages_its_predecessor() { for info in [Info::default().with_max_age(None), Info::timed()] { let producer = track_producer("test", info.clone()); - let at = info.timescale.map(|_| Timestamp::ZERO); + // The first group, at key 0. + let at = info.timescale.map_or(Timestamp::unscaled(0), |_| Timestamp::ZERO); elapse(&producer, Duration::ZERO); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); - let mut held = producer.append_group().unwrap(); + let mut held = producer.append_next().unwrap(); held.write_frame(at, bytes::Bytes::from_static(b"x")).unwrap(); let mut reading = subscriber.assert_group(); assert!(reading.read_frame().now_or_never().unwrap().unwrap().is_some()); @@ -12894,7 +12962,7 @@ mod test { let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(reading.poll_read_frame(&waiter).is_pending()); - let _successor = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let _successor = producer.create_group(group::Info { start: 1 }).unwrap(); assert!( woken.swap(false, Ordering::SeqCst), "{info:?}: the header never woke the read" @@ -12924,7 +12992,7 @@ mod test { let expired = |held: &mut [group::Consumer; 2]| held.each_mut().map(|group| group.poll_expired(&waiter)); // Frozen timestamps, so the wall clock alone convicts. - append_at(&mut producer, 0); + append_frozen(&mut producer); let fetch = group::Fetch::default().with_max_delay(budget); let mut held = [ subscriber.assert_group(), @@ -12942,13 +13010,13 @@ mod test { assert_eq!(expired(&mut held), [false; 2], "the newest group"); elapse(&producer, Duration::from_secs(1)); - let mut last = append_at(&mut producer, 0); // the successor + let mut last = append_frozen(&mut producer); // the successor assert_eq!(expired(&mut held), [false; 2]); // The track runs on, a group a second, until a second short of the deadline. for _ in 0..budget.as_secs() - 1 { elapse(&producer, Duration::from_secs(1)); - last = append_at(&mut producer, 0); + last = append_frozen(&mut producer); } producer.broadcast.pool.sweep(); let state = producer.state.read(); @@ -12964,7 +13032,7 @@ mod test { // The newest group outlives any budget. let mut newest = subscriber.assert_group(); - while newest.sequence < last { + while newest.start < last { newest = subscriber.assert_group(); } elapse(&producer, budget * 2); @@ -12982,7 +13050,7 @@ mod test { let waiter = kio::Waiter::noop(); let expired = |held: &mut [group::Consumer; 2]| held.each_mut().map(|group| group.poll_expired(&waiter)); - append_at(&mut producer, 0); + append_frozen(&mut producer); let fetch = group::Fetch::default().with_max_delay(budget); let mut held = [ subscriber.assert_group(), @@ -12998,12 +13066,12 @@ mod test { assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); } - let failed = producer.append_group().unwrap(); + let failed = producer.append_next().unwrap(); assert_eq!(expired(&mut held), [false; 2], "the successor is recorded"); failed.abort(Error::Cancel).unwrap(); let replacement = Duration::from_secs(10); elapse(&producer, replacement); - append_at(&mut producer, 0); + append_frozen(&mut producer); producer.broadcast.pool.sweep(); assert!(!producer.state.read().lookup.contains_key(&1), "reclaimed"); @@ -13029,13 +13097,13 @@ mod test { let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); let waiter = kio::Waiter::noop(); - append_at(&mut producer, 0); + append_frozen(&mut producer); let mut held = subscriber.assert_group(); assert!(matches!(held.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); assert!(matches!(held.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); pool.step(Duration::from_secs(1)); - append_at(&mut producer, 0); // the successor + append_frozen(&mut producer); // the successor assert!(!held.poll_expired(&waiter), "the successor is recorded"); let state = producer.state.downgrade(); let producer = producer.finish().unwrap(); @@ -13069,11 +13137,11 @@ mod test { let mut subscriber = producer.subscribe(budget.clone()); append_at(&mut producer, 0); let mut open = Vec::new(); - for sequence in 1..=RUN { - let group = producer.append_group().unwrap(); - match sequence % 2 { + for key in 1..=RUN { + let group = producer.append_next().unwrap(); + match key % 2 { 0 => { - group.finish().unwrap(); + group.close().unwrap(); } _ => open.push(group), } @@ -13090,10 +13158,10 @@ mod test { #[test] fn disabled_pool_expiry_never_reclaims() { let producer = track_producer_expiring("test", None); - producer.append_group().unwrap().finish().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 0 elapse(&producer, Duration::from_secs(3600)); - producer.append_group().unwrap().finish().unwrap(); // seq 1 + producer.append_next().unwrap().close().unwrap(); // seq 1 let state = producer.state.read(); assert_eq!(live_groups(&state), 2); @@ -13167,7 +13235,7 @@ mod test { guard.reject(Error::NotFound, None); assert!(!guard.protects(0)); assert!( - guard.evict.iter().any(|(sequence, _)| *sequence == 0), + guard.evict.iter().any(|(key, _)| *key == 0), "the released live edge is reclaimable" ); } @@ -13216,11 +13284,11 @@ mod test { Info::timed().with_max_age(Duration::from_secs(60)), Duration::from_secs(1), ); - producer.append_group().unwrap().finish().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 0 // Far past the ceiling in wall time, but inside the pool's LRU window. elapse(&producer, Duration::from_secs(2)); - producer.append_group().unwrap().finish().unwrap(); // seq 1 + producer.append_next().unwrap().close().unwrap(); // seq 1 let state = producer.state.read(); assert_eq!(live_groups(&state), 2); @@ -13351,22 +13419,33 @@ mod test { /// Append a finished group presenting at `millis`, so the track carries a media /// timeline for the drift budget to measure against. fn append_at(producer: &mut Producer, millis: u64) -> u64 { - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - let group = group.finish().unwrap(); - group.sequence + let group = group.close().unwrap(); + group.start + } + + /// Append a finished group presenting at its key in milliseconds: a timeline that barely + /// advances, since starts strictly increase and can't stop. + fn append_frozen(producer: &mut Producer) -> u64 { + let mut group = producer.append_next().unwrap(); + group + .write_frame(group.key_time(), bytes::Bytes::from_static(b"x")) + .unwrap(); + let group = group.close().unwrap(); + group.start } /// Every group the subscriber can read right now, in delivery order. #[cfg(feature = "async")] fn drain(subscriber: &mut Subscriber) -> Vec { - let mut sequences = Vec::new(); + let mut keys = Vec::new(); while let Some(Ok(Some(group))) = subscriber.recv_group().now_or_never() { - sequences.push(group.sequence); + keys.push(group.start); } - sequences + keys } /// A rejoined copy judges the route's answer by the groups that survived the leave. @@ -13381,14 +13460,14 @@ mod test { fn an_answer_past_a_reset_group_skips_the_cache() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); - let mut open = producer.append_group().unwrap(); + let mut open = producer.append_next().unwrap(); open.write_frame(Timestamp::from_millis(500).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); producer.set_idle(); open.abort(Error::Cancel).unwrap(); - let mut answer = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let mut answer = producer.create_group(group::Info { start: 2 }).unwrap(); producer.set_live(Some(Position::group(2)), None).unwrap(); answer .write_frame(Timestamp::from_millis(533).unwrap(), bytes::Bytes::from_static(b"")) @@ -13415,8 +13494,8 @@ mod test { fn an_unjudgeable_gap_does_not_hide_a_fresh_ordered_gap() { let mut producer = track_producer("test", None); producer.received(); - for sequence in [0, 4] { - framed(&producer, sequence, 1).finish().unwrap(); + for key in [0, 4] { + framed(&producer, key, 1).close().unwrap(); } producer.set_idle(); let mut reader = producer.subscribe( @@ -13425,11 +13504,11 @@ mod test { .with_range(0..), ); producer.set_live(Some(Position::group(8)), None).unwrap(); - for sequence in [6, 8] { - framed(&producer, sequence, 1).finish().unwrap(); + for key in [6, 8] { + framed(&producer, key, 1).close().unwrap(); } assert_eq!(read_now(&mut reader, true), None, "group 5 is still owed"); - framed(&producer, 5, 1).finish().unwrap(); + framed(&producer, 5, 1).close().unwrap(); assert_eq!(read_now(&mut reader, true), Some(Some(5))); assert_eq!(read_now(&mut reader, true), Some(Some(6))); } @@ -13444,16 +13523,16 @@ mod test { let mut producer = track_producer("test", Info::default()); let requests = producer.requests(); producer.received(); - for sequence in 0..3 { - framed(&producer, sequence, 1).finish().unwrap(); + for key in 0..3 { + framed(&producer, key, 1).close().unwrap(); } producer.set_idle(); producer.start_at(6).unwrap(); producer.set_live(Some(Position::group(6)), None).unwrap(); - framed(&producer, 6, 1).finish().unwrap(); + framed(&producer, 6, 1).close().unwrap(); if !cached { - for sequence in 0..3 { - evict(&producer, sequence); + for key in 0..3 { + evict(&producer, key); } } let mut reader = producer.subscribe( @@ -13473,15 +13552,17 @@ mod test { ); assert_eq!(read_now(&mut reader, false), None, "the fill is still owed"); assert_eq!(fill.wanted(), Some(5), "the fresh suffix stays wanted"); - for sequence in 3..6 { - let mut group = fill.accept(sequence, None).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + for key in 3..6 { + let mut group = fill.accept(key, None).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); } let mut got = Vec::new(); loop { match read_now(&mut reader, false) { - Some(Some(sequence)) => got.push(sequence), + Some(Some(key)) => got.push(key), Some(None) => break, None => panic!("cached: {cached}: the bounded read still owes history"), } @@ -13500,15 +13581,15 @@ mod test { let mut producer = track_producer("test", Info::default().with_max_age(None)); let requests = producer.requests(); producer.received(); - for sequence in 0..3 { - framed(&producer, sequence, 1).finish().unwrap(); + for key in 0..3 { + framed(&producer, key, 1).close().unwrap(); } producer.set_idle(); producer.start_at(6).unwrap(); producer.set_live(Some(Position::group(6)), None).unwrap(); - framed(&producer, 6, 1).finish().unwrap(); - for sequence in 0..3 { - evict(&producer, sequence); + framed(&producer, 6, 1).close().unwrap(); + for key in 0..3 { + evict(&producer, key); } let unlimited = Subscription::default() .with_delay(Delay::default().with_max(None)) @@ -13536,13 +13617,15 @@ mod test { "the unjudgeable prefix is given up" ); assert!(fill.wanted().is_some(), "the fresh suffix stays wanted"); - for sequence in 3..6 { - let mut group = fill.accept(sequence, None).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + for key in 3..6 { + let mut group = fill.accept(key, None).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); } - for sequence in 3..6 { - assert_eq!(read_now(&mut reader, true), Some(Some(sequence))); + for key in 3..6 { + assert_eq!(read_now(&mut reader, true), Some(Some(key))); } assert_eq!(read_now(&mut reader, true), Some(None)); } @@ -13554,12 +13637,12 @@ mod test { fn the_cache_below_the_floor_is_unread_only_to_unlimited_reads() { let mut producer = track_producer("test", Info::default().with_max_age(None)); producer.received(); - for sequence in 0..3 { - framed(&producer, sequence, 1).finish().unwrap(); + for key in 0..3 { + framed(&producer, key, 1).close().unwrap(); } producer.set_idle(); producer.set_live(Some(Position::group(6)), None).unwrap(); - framed(&producer, 6, 1).finish().unwrap(); + framed(&producer, 6, 1).close().unwrap(); let delivered = Delivered::default(); let unread = Unread { start: 0, @@ -13583,10 +13666,10 @@ mod test { fn an_evicted_group_below_the_floor_stays_stale() { let mut producer = track_producer("test", Info::default()); producer.received(); - framed(&producer, 0, 1).finish().unwrap(); + framed(&producer, 0, 1).close().unwrap(); producer.set_idle(); producer.set_live(Some(Position::group(6)), None).unwrap(); - framed(&producer, 6, 1).finish().unwrap(); + framed(&producer, 6, 1).close().unwrap(); evict(&producer, 0); let state = producer.state.read(); let edge = state.drift_edge(&Servable::ALL); @@ -13598,7 +13681,7 @@ mod test { } /// An idle cache's old groups and the holes after them stay owed by unlimited reads. - /// A refused run reports every missing sequence and releases a bounded reader, while + /// A refused run reports every missing key and releases a bounded reader, while /// an open reader keeps receiving the live feed. Both cursor orders use the same rule. #[test] #[cfg(feature = "async")] @@ -13608,7 +13691,7 @@ mod test { let mut producer = track_producer("test", Info::default().with_max_age(None)); let requests = producer.requests(); producer.received(); - framed(&producer, 0, 1).finish().unwrap(); + framed(&producer, 0, 1).close().unwrap(); producer.set_idle(); let subscription = Subscription::default() .with_delay(Delay::default().with_max(None)) @@ -13623,7 +13706,7 @@ mod test { producer.start_at(3).unwrap(); producer.set_live(Some(Position::group(3)), None).unwrap(); - framed(&producer, 3, 1).finish().unwrap(); + framed(&producer, 3, 1).close().unwrap(); assert_eq!(read_now(&mut reader, ordered), Some(Some(0)), "cached history"); // An arrival-order reader may take the live group before its history. if !bounded && !ordered { @@ -13651,7 +13734,7 @@ mod test { if ordered { assert_eq!(read_now(&mut reader, ordered), Some(Some(3))); } - framed(&producer, 4, 1).finish().unwrap(); + framed(&producer, 4, 1).close().unwrap(); assert_eq!(read_now(&mut reader, ordered), Some(Some(4)), "the feed survives"); } } @@ -13665,7 +13748,7 @@ mod test { fn a_reset_group_is_not_dropped_and_comes_again() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); - let mut open = producer.append_group().unwrap(); + let mut open = producer.append_next().unwrap(); open.write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); @@ -13676,7 +13759,7 @@ mod test { let waiter = kio::Waiter::noop(); assert!(subscriber.poll_drop(&waiter).is_pending(), "nothing was given up"); - let mut again = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let mut again = producer.create_group(group::Info { start: 1 }).unwrap(); producer.set_live(Some(Position::group(1)), None).unwrap(); again .write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) @@ -13730,13 +13813,7 @@ mod test { let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); assert_eq!( - subscriber - .recv_group() - .now_or_never() - .unwrap() - .unwrap() - .unwrap() - .sequence, + subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 3 ); @@ -13753,20 +13830,14 @@ mod test { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); append_at(&mut producer, 1000); - let mut edge = producer.append_group().unwrap(); + let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"a")) .unwrap(); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(2)))); assert_eq!( - subscriber - .recv_group() - .now_or_never() - .unwrap() - .unwrap() - .unwrap() - .sequence, + subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0 ); @@ -13809,7 +13880,7 @@ mod test { let budget = Duration::from_millis(1_500) + Duration::from_micros(i); let fetch = group::Fetch::default().with_max_delay(budget); let group = consumer.fetch_group(2, fetch).now_or_never().unwrap().unwrap(); - assert_eq!(group.sequence, 2); + assert_eq!(group.start, 2); } let cache = producer.state.read().cache.clone(); let (media, wall) = cache.wakes().deadlines_indexed(); @@ -13871,39 +13942,92 @@ mod test { assert_eq!(drain(&mut subscriber), vec![2, 3, 4]); } + /// A start past the live edge is a time, which the newest group contains until a later + /// one starts: the cursor reads that group, then each one after it. #[test] #[cfg(feature = "async")] - fn a_floor_above_the_live_edge_waits_there() { - let mut producer = track_producer("test", Info::timed()); - for second in 0..3 { - append_at(&mut producer, second * 1000); + fn a_start_past_the_live_edge_reads_the_group_containing_it() { + let producer = track_producer("test", Info::timed()); + for key in [0, 1000, 2000] { + write_group(&producer, key); } - // A resumed subscription names where it left off, which may not exist yet. The - // cursor sits at the floor rather than sliding back to what is cached. let resumed = Subscription::default() - .with_start(7) + .with_start(7000) .with_delay(Delay::default().with_max(Duration::from_secs(10))); let mut subscriber = producer.subscribe(resumed); - assert_eq!(drain(&mut subscriber), Vec::::new()); - append_at(&mut producer, 3000); // sequence 3: still below the floor - assert_eq!(drain(&mut subscriber), Vec::::new()); - for second in 4..8 { - append_at(&mut producer, second * 1000); + assert_eq!(drain(&mut subscriber), vec![2000]); + for key in [3000, 4000] { + write_group(&producer, key); + } + assert_eq!(drain(&mut subscriber), vec![3000, 4000]); + } + + /// A lite-07 copy holds only part of the track, so it places a time start only once it + /// knows which group contains it: one starting there, one before it once the keys between + /// are skipped or a later group arrived, or where its publisher reported starting the + /// range asked from that time. Meanwhile the reader reads the groups from the time. + #[test] + #[cfg(feature = "async")] + fn a_received_copy_places_a_time_start_once_it_knows() { + let from = |time| { + Subscription::default() + .with_ranges([Range { + start: Anchor::Time(time), + end: None, + }]) + .with_delay(Delay::default().with_max(None)) + }; + let placed = + |subscriber: &mut Subscriber| subscriber.resolved()[0].map(|resolved| (resolved.start, resolved.floored)); + let producer = track_producer("test", Info::timed()); + producer.received(); + producer.anchored(); + write_group(&producer, 1000); + + // A group between 1000 and the time may still arrive. + let mut later = producer.subscribe(from(1500)); + let mut answered = producer.subscribe(from(1700)); + let mut skipped = producer.subscribe(from(1800)); + let mut before = producer.subscribe(from(500)); + let mut at = producer.subscribe(from(1000)); + assert_eq!(placed(&mut at), Some((1000, true)), "a group starts there"); + for subscriber in [&mut later, &mut answered, &mut skipped, &mut before] { + assert_eq!(placed(subscriber), None); } - assert_eq!(drain(&mut subscriber), vec![7]); + assert_eq!(drain(&mut before), vec![1000], "reads from the time meanwhile"); + assert_eq!(drain(&mut later), Vec::::new()); + + // The publisher started the range asked from 1700 at 1000, and the one from 500 at 1000 + // too, past it: nothing at or before 500 is served. + producer.time_starts([(1700, 1000), (500, 1000)]); + assert_eq!(placed(&mut answered), Some((1000, true))); + assert_eq!(placed(&mut before), Some((0, false))); + assert_eq!(placed(&mut later), None); + + // Skipped keys between say nothing arrives there. + producer.skip_groups(1001..1801); + assert_eq!(placed(&mut skipped), Some((1000, true))); + assert_eq!(placed(&mut later), Some((1000, true))); + + // A group past the time says the feed went by it. + let mut past = producer.subscribe(from(2500)); + assert_eq!(placed(&mut past), None); + write_group(&producer, 3000); + assert_eq!(placed(&mut past), Some((1000, true))); + assert_eq!(drain(&mut past), vec![1000, 3000]); } #[test] #[cfg(feature = "async")] fn a_late_lower_group_within_the_budget_is_delivered() { let producer = track_producer("test", Info::timed()); - for (sequence, millis) in [(5, 1000), (6, 2000), (7, 3000)] { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + for (key, millis) in [(5, 1000), (6, 2000), (7, 3000)] { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let mut subscriber = @@ -13912,11 +14036,11 @@ mod test { // Arriving below everything already delivered is not what makes content stale: // the budget is the only gate, and this straggler's timestamp is within it. A - // consumer that needs sequence order reorders (or drops) it itself. - let mut late = producer.create_group(group::Info { sequence: 4 }).unwrap(); + // consumer that needs key order reorders (or drops) it itself. + let mut late = producer.create_group(group::Info { start: 4 }).unwrap(); late.write_frame(Timestamp::from_millis(500).unwrap(), bytes::Bytes::from_static(b"late")) .unwrap(); - late.finish().unwrap(); + late.close().unwrap(); assert_eq!(drain(&mut subscriber), vec![4]); } @@ -13941,7 +14065,7 @@ mod test { fn a_stamped_successor_expires_an_unstamped_group() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); - let _open = producer.append_group().unwrap(); // seq 0 stalls before its first frame + let _open = producer.append_next().unwrap(); // seq 0 stalls before its first frame append_at(&mut producer, 1000); // seq 1 proves the live feed moved on @@ -13955,7 +14079,7 @@ mod test { async fn a_handed_out_group_expires_while_its_first_frame_is_stalled() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); - let _writer_1 = producer.append_group().unwrap(); + let _writer_1 = producer.append_next().unwrap(); let mut stalled = subscriber.recv_group().await.unwrap().expect("stalled group"); let pending = kio::clock::sim::spawn(async move { stalled.read_frame().await }); @@ -13983,8 +14107,8 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); - let mut old = producer.append_group().unwrap(); - old.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"old")) + let mut old = producer.append_next().unwrap(); + old.write_frame(old.key_time(), bytes::Bytes::from_static(b"old")) .unwrap(); // Group 1 bounds group 0's reach at 1s. @@ -13993,7 +14117,7 @@ mod test { let mut held = subscriber.recv_group().await.unwrap().expect("old group"); assert!(held.read_frame().await.unwrap().is_some()); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); let pending = kio::clock::sim::spawn(async move { held.read_frame().await }); kio::clock::sim::yield_now().await; assert!(!pending.is_finished(), "the newer group has no timestamp yet"); @@ -14035,7 +14159,7 @@ mod test { // Left open, so every read parks instead of ending. let _open: Vec<_> = (0..PARKED) .map(|i| { - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); group .write_frame( Timestamp::from_micros(i * FRAME).unwrap(), @@ -14072,7 +14196,7 @@ mod test { // Read `i` can reach its successor's start, `(i + 1) * FRAME`, so an edge at 210ms // puts reads 0..=3 a full 200ms behind and leaves every later one in budget. - let mut next = producer.append_group().unwrap(); + let mut next = producer.append_next().unwrap(); next.write_frame( Timestamp::from_micros(210_000).unwrap(), bytes::Bytes::from_static(b"x"), @@ -14110,11 +14234,11 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); - let mut head = producer.append_group().unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + let mut head = producer.append_next().unwrap(); + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach - let mut between = producer.append_group().unwrap(); + let mut between = producer.append_next().unwrap(); append_at(&mut producer, 3000); // the edge let mut held = subscriber @@ -14123,7 +14247,7 @@ mod test { .unwrap() .unwrap() .expect("head group"); - assert_eq!(held.sequence, 0); + assert_eq!(held.start, 0); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -14140,7 +14264,7 @@ mod test { "a first frame below the edge moves neither the reach nor the edge" ); - let mut beyond = producer.append_group().unwrap(); + let mut beyond = producer.append_next().unwrap(); beyond .write_frame(Timestamp::from_millis(4000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); @@ -14168,11 +14292,11 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); - let mut head = producer.append_group().unwrap(); + let mut head = producer.append_next().unwrap(); head.write_frame(Timestamp::from_millis(0).unwrap(), bytes::Bytes::new()) .unwrap(); append_at(&mut producer, 1000); - let mut edge = producer.append_group().unwrap(); + let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::new()) .unwrap(); let mut held = subscriber.recv_group().now_or_never().unwrap().unwrap().unwrap(); @@ -14197,8 +14321,8 @@ mod test { "the batch's highest PTS reaches the deadline" ); assert!(matches!(next.as_mut().poll(&mut cx), Poll::Ready(Err(Error::Old)))); - head.finish().unwrap(); - edge.finish().unwrap(); + head.close().unwrap(); + edge.close().unwrap(); } #[test] @@ -14207,11 +14331,11 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(10)))); - let mut old = producer.append_group().unwrap(); + let mut old = producer.append_next().unwrap(); old.write_frame(Timestamp::from_millis(0).unwrap(), bytes::Bytes::new()) .unwrap(); append_at(&mut producer, 1000); - let mut edge = producer.append_group().unwrap(); + let mut edge = producer.append_next().unwrap(); edge.start_at(2).unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::new()) .unwrap(); @@ -14222,7 +14346,7 @@ mod test { let mut cx = std::task::Context::from_waker(&waker); let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); - let mut prefix = group::Info { sequence: 2 }.produce(); + let mut prefix = group::Info { start: 2 }.produce(); for ms in [1500, 11_000] { prefix .write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::new()) @@ -14236,9 +14360,9 @@ mod test { "the adopted prefix reaches the drift deadline" ); assert!(matches!(next.as_mut().poll(&mut cx), Poll::Ready(Err(Error::Old)))); - old.finish().unwrap(); - edge.finish().unwrap(); - prefix.finish().unwrap(); + old.close().unwrap(); + edge.close().unwrap(); + prefix.close().unwrap(); } /// An aborted edge hands the edge to a group between it and the successor, and neither @@ -14249,15 +14373,15 @@ mod test { let mut producer = track_producer("test", crate::track::Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(5)))); - let mut head = producer.append_group().unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + let mut head = producer.append_next().unwrap(); + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach - let mut between = producer.append_group().unwrap(); - let mut edge = producer.append_group().unwrap(); + let mut between = producer.append_next().unwrap(); + let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - let edge = edge.finish().unwrap(); + let edge = edge.close().unwrap(); let mut held = subscriber .recv_group() @@ -14265,7 +14389,7 @@ mod test { .unwrap() .unwrap() .expect("head group"); - assert_eq!(held.sequence, 0); + assert_eq!(held.start, 0); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -14300,11 +14424,11 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); - let mut head = producer.append_group().unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + let mut head = producer.append_next().unwrap(); + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); - let successor = producer.append_group().unwrap(); - let mut replacement = producer.append_group().unwrap(); + let successor = producer.append_next().unwrap(); + let mut replacement = producer.append_next().unwrap(); append_at(&mut producer, 20_000); // the edge let mut held = subscriber @@ -14313,7 +14437,7 @@ mod test { .unwrap() .unwrap() .expect("head group"); - assert_eq!(held.sequence, 0, "an unstamped successor leaves the reach unbounded"); + assert_eq!(held.start, 0, "an unstamped successor leaves the reach unbounded"); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -14356,21 +14480,21 @@ mod test { let mut producer = track_producer("test", crate::track::Info::timed()); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(5)))); - let mut head = producer.append_group().unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + let mut head = producer.append_next().unwrap(); + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach // Starts below the edge but presents past the deadline. - let mut long = producer.append_group().unwrap(); + let mut long = producer.append_next().unwrap(); for millis in [1500, 10_000] { long.write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } - long.finish().unwrap(); - let mut edge = producer.append_group().unwrap(); + long.close().unwrap(); + let mut edge = producer.append_next().unwrap(); edge.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - let edge = edge.finish().unwrap(); + let edge = edge.close().unwrap(); let mut held = subscriber .recv_group() @@ -14378,7 +14502,7 @@ mod test { .unwrap() .unwrap() .expect("head group"); - assert_eq!(held.sequence, 0, "the edge keeps the head within budget"); + assert_eq!(held.start, 0, "the edge keeps the head within budget"); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -14405,8 +14529,8 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); - let mut head = producer.append_group().unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + let mut head = producer.append_next().unwrap(); + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); append_at(&mut producer, 1000); // the successor bounds the head's reach @@ -14416,7 +14540,7 @@ mod test { .unwrap() .unwrap() .expect("head group"); - assert_eq!(held.sequence, 0); + assert_eq!(held.start, 0); assert!(held.read_frame().now_or_never().unwrap().unwrap().is_some()); let woken = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -14425,7 +14549,7 @@ mod test { let mut next = std::pin::pin!(held.read_frame()); assert!(next.as_mut().poll(&mut cx).is_pending()); - let mut received = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let mut received = producer.create_group(group::Info { start: 2 }).unwrap(); woken.store(false, Ordering::SeqCst); assert!( next.as_mut().poll(&mut cx).is_pending(), @@ -14454,11 +14578,11 @@ mod test { producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(60)))); producer.finish_at(Position::group(2), None).unwrap(); let mut received: Vec<_> = (0..2) - .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) + .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); let mut first = subscriber.assert_group(); let mut second = subscriber.assert_group(); - assert_eq!((first.sequence, second.sequence), (0, 1)); + assert_eq!((first.start, second.start), (0, 1)); assert!(subscriber.recv_group().now_or_never().unwrap().unwrap().is_none()); let waiter = kio::Waiter::noop(); @@ -14468,7 +14592,7 @@ mod test { ); for group in &mut received { group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); } assert!( @@ -14489,14 +14613,14 @@ mod test { let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(producer.poll_past(0, &waiter).is_pending()); - let mut group = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 1 }).unwrap(); woken.store(false, Ordering::SeqCst); assert!( producer.poll_past(0, &waiter).is_pending(), "a frameless header is not content" ); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); assert!(woken.load(Ordering::SeqCst), "its first frame wakes the parked poll"); assert!(producer.poll_past(0, &waiter).is_ready()); @@ -14512,9 +14636,9 @@ mod test { let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(60)))); producer.finish_at(Position::group(1), None).unwrap(); - let group = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let group = producer.create_group(group::Info { start: 0 }).unwrap(); let mut reading = subscriber.assert_group(); - assert_eq!(reading.sequence, 0); + assert_eq!(reading.start, 0); let ended = { let mut next = std::pin::pin!(reading.read_frame()); @@ -14522,7 +14646,7 @@ mod test { match abort { true => group.abort(Error::Cancel).unwrap(), false => { - group.finish().unwrap(); + group.close().unwrap(); } } kio::clock::timeout(Duration::from_secs(1), next) @@ -14562,22 +14686,22 @@ mod test { group.write_frame(timestamp, bytes::Bytes::from_static(b"x")).unwrap(); }; - let mut open = producer.append_group().unwrap(); + let mut open = producer.append_next().unwrap(); write(&mut open, gop(0)[0]); write(&mut open, gop(0)[1]); let mut reading = subscriber.recv_group().await.unwrap().expect("the live group"); let mut read = Vec::new(); for n in 1..5u64 { - let sequence = reading.sequence; + let key = reading.start; let mut frames = 0; while let Some(res) = reading.read_frame().now_or_never() { match res.expect("no truncation while draining") { Some(_) => frames += 1, - None => panic!("group {sequence} ended early"), + None => panic!("group {key} ended early"), } } - read.push((sequence, frames)); + read.push((key, frames)); let next = { // Parked at the end of the current group: every frame is read and no FIN @@ -14587,18 +14711,18 @@ mod test { // The next keyframe opens its group. The verdict is taken here, in the // window before the previous group's FIN arrives. - let mut opened = producer.append_group().unwrap(); + let mut opened = producer.append_next().unwrap(); write(&mut opened, gop(n)[0]); let verdict = futures::poll!(end.as_mut()); - open.finish().unwrap(); + open.close().unwrap(); let res = match verdict { Poll::Ready(res) => res, Poll::Pending => end.await, }; assert!( matches!(res, Err(Error::Old)), - "group {sequence} gives up at the boundary, after its last frame: {res:?}" + "group {key} gives up at the boundary, after its last frame: {res:?}" ); opened }; @@ -14621,13 +14745,13 @@ mod test { let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(80)))); for ms in [0, 100] { - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); group .write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::new()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - let mut edge = producer.append_group().unwrap(); + let mut edge = producer.append_next().unwrap(); for ms in [150, 200] { edge.write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::new()) .unwrap(); @@ -14636,11 +14760,11 @@ mod test { edge.write_frame(Timestamp::from_millis(160).unwrap(), bytes::Bytes::new()) .unwrap(); } - edge.finish().unwrap(); + edge.close().unwrap(); // Group 0 can reach 100ms: 100ms behind the 200ms edge, past its 80ms budget. // Group 1 can reach 150ms and stays within the same budget. assert_eq!( - subscriber.recv_group().await.unwrap().unwrap().sequence, + subscriber.recv_group().await.unwrap().unwrap().start, 1, "reordered {reordered}" ); @@ -14663,8 +14787,8 @@ mod test { let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); - let mut open = producer.append_group().unwrap(); - open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"key")) + let mut open = producer.append_next().unwrap(); + open.write_frame(open.key_time(), bytes::Bytes::from_static(b"key")) .unwrap(); open.write_frame( Timestamp::from_millis(1900).unwrap(), @@ -14680,7 +14804,7 @@ mod test { assert!(futures::poll!(end.as_mut()).is_pending(), "parked at 1900ms"); // The next GOP opens. The reader is 100ms behind it, inside its 1s budget. - let mut next = producer.append_group().unwrap(); + let mut next = producer.append_next().unwrap(); next.write_frame(Timestamp::from_millis(2000).unwrap(), bytes::Bytes::from_static(b"key")) .unwrap(); assert!( @@ -14697,7 +14821,7 @@ mod test { let late = end.await.expect("the straggler is not truncated"); assert_eq!( late.map(|frame| frame.timestamp), - Some(Some(Timestamp::from_millis(1950).unwrap())) + Some(Timestamp::from_millis(1950).unwrap()) ); } @@ -14708,8 +14832,8 @@ mod test { async fn an_expired_group_stays_old_when_probed_again() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); - let mut open = producer.append_group().unwrap(); - open.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")) + let mut open = producer.append_next().unwrap(); + open.write_frame(open.key_time(), bytes::Bytes::from_static(b"a")) .unwrap(); let mut group = subscriber.recv_group().await.unwrap().expect("group"); @@ -14751,11 +14875,11 @@ mod test { async fn a_handed_out_partial_frame_expires_while_its_payload_is_stalled() { let mut producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); - let source = producer.append_group().unwrap(); + let source = producer.append_next().unwrap(); let mut writing = source .create_frame(frame::Info { size: 6, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); writing.write(bytes::Bytes::from_static(b"old")).unwrap(); @@ -14810,8 +14934,8 @@ mod test { // Timed: a successor at the same instant already supersedes. let mut timed = track_producer("timed", Info::default().with_timescale(Timescale::MILLI)); - append_at(&mut timed, 0); - append_at(&mut timed, 0); + append_frozen(&mut timed); + append_frozen(&mut timed); assert_eq!(drain(&mut timed.subscribe(history())), vec![1]); // Untimed: the wall clock agrees, and an hour on the unchanged newest group still @@ -14872,12 +14996,12 @@ mod test { let mut producer = track_producer("test", Info::timed()); // Group 0 spans 0..2000ms; group 1 starts at 2000ms, where group 0 ends. - let mut long = producer.append_group().unwrap(); + let mut long = producer.append_next().unwrap(); for ms in [0u64, 500, 1000, 1500, 2000] { long.write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } - long.finish().unwrap(); + long.close().unwrap(); append_at(&mut producer, 2000); // A budget far shorter than the group's own span still keeps it. @@ -14896,12 +15020,12 @@ mod test { fn a_long_group_is_stale_once_its_successor_falls_behind() { let mut producer = track_producer("test", Info::timed()); - let mut long = producer.append_group().unwrap(); + let mut long = producer.append_next().unwrap(); for ms in [0u64, 500, 1000] { long.write_frame(Timestamp::from_millis(ms).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } - long.finish().unwrap(); + long.close().unwrap(); // Group 0 reaches at most 3s (where group 1 starts), a full second behind the 4s // edge and so past the budget. Group 1 reaches the edge itself and is kept. append_at(&mut producer, 3000); @@ -14924,7 +15048,7 @@ mod test { .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); append_at(&mut producer, 0); // seq 0 - let _writer_1 = producer.append_group().unwrap(); // seq 1 stalls before its first frame + let _writer_1 = producer.append_next().unwrap(); // seq 1 stalls before its first frame append_at(&mut producer, 10_000); // seq 2 append_at(&mut producer, 11_000); // seq 3 @@ -14950,17 +15074,22 @@ mod test { track.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")), Err(Error::TimestampMismatch) )); - track.write_frame(None, bytes::Bytes::from_static(b"x")).unwrap(); + let mut group = track.append_group(0).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), bytes::Bytes::from_static(b"x")) + .unwrap(); } } /// A group of untimed frames. #[cfg(feature = "async")] fn append_untimed(producer: &mut Producer) -> u64 { - let mut group = producer.append_group().unwrap(); - group.write_frame(None, bytes::Bytes::from_static(b"x")).unwrap(); - let group = group.finish().unwrap(); - group.sequence + let mut group = producer.append_next().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), bytes::Bytes::from_static(b"x")) + .unwrap(); + let group = group.close().unwrap(); + group.start } /// The wall clock judges an untimed track, so a new subscriber starts at the oldest @@ -15026,53 +15155,22 @@ mod test { /// subscriber waiting for a frame that never comes. #[test] fn a_mismatched_frame_appends_no_group() { - let mut producer = track_producer("test", Info::timed()); + let mut producer = untimed_producer(); assert!(matches!( - producer.write_frame(None, bytes::Bytes::from_static(b"x")), + producer.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")), Err(Error::TimestampMismatch) )); - assert_eq!(producer.append_group().unwrap().sequence, 0); + assert_eq!(producer.append_next().unwrap().start, 0); } - /// A subscriber read by either cursor, so one test covers both. - #[cfg(feature = "async")] - enum Reader { - Arrival(Subscriber), - Ordered(Ordered), - } - - #[cfg(feature = "async")] - impl Reader { - fn new(subscriber: Subscriber, ordered: bool) -> Self { - match ordered { - true => Self::Ordered(subscriber.ordered()), - false => Self::Arrival(subscriber), - } - } - - fn drain(&mut self) -> Vec { - match self { - Self::Arrival(subscriber) => drain(subscriber), - Self::Ordered(ordered) => drain_ordered(ordered), - } - } - - fn resolved(&mut self) -> Vec> { - match self { - Self::Arrival(subscriber) => subscriber.resolved_ranges(), - Self::Ordered(ordered) => ordered.inner.resolved_ranges(), - } - } - } - - /// Every group the ordered cursor can read right now, in sequence order. + /// Every group the ordered cursor can read right now, in key order. #[cfg(feature = "async")] fn drain_ordered(subscriber: &mut Ordered) -> Vec { - let mut sequences = Vec::new(); + let mut keys = Vec::new(); while let Some(Ok(Some(group))) = subscriber.next_group().now_or_never() { - sequences.push(group.sequence); + keys.push(group.start); } - sequences + keys } /// A cached backlog is not free to deliver: a consumer that stalls and resumes would @@ -15123,7 +15221,7 @@ mod test { fn next_group_judges_past_a_frameless_successor() { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 0); // seq 0 - let _writer_1 = producer.append_group().unwrap(); // seq 1 stalls before its first frame + let _writer_1 = producer.append_next().unwrap(); // seq 1 stalls before its first frame append_at(&mut producer, 10_000); // seq 2 append_at(&mut producer, 11_000); // seq 3 @@ -15136,10 +15234,10 @@ mod test { #[test] #[cfg(feature = "async")] - fn real_time_skips_older_sequences_with_equal_ages() { + fn real_time_skips_older_keys_with_equal_ages() { let mut producer = track_producer("test", Info::timed()); - append_at(&mut producer, 0); - append_at(&mut producer, 0); + append_frozen(&mut producer); + append_frozen(&mut producer); let mut subscriber = producer.subscribe(None); assert_eq!(drain(&mut subscriber), vec![1]); @@ -15155,7 +15253,7 @@ mod test { // Omission adds no reader bound when the publisher declared none. let consumer = producer.consume(); let group = consumer.fetch_group(0, None).now_or_never().unwrap().unwrap(); - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); } #[test] @@ -15178,10 +15276,7 @@ mod test { // A reader cannot relax max_age, but can tighten it independently. let loose = group::Fetch::default().with_max_delay(Duration::from_secs(60)); - assert_eq!( - consumer.fetch_group(2, loose).now_or_never().unwrap().unwrap().sequence, - 2 - ); + assert_eq!(consumer.fetch_group(2, loose).now_or_never().unwrap().unwrap().start, 2); let tight = group::Fetch::default().with_max_delay(Duration::from_millis(500)); assert!(matches!( consumer.fetch_group(1, tight).now_or_never().unwrap(), @@ -15193,11 +15288,11 @@ mod test { fn cold_fetch_expires_without_waiting_for_the_upstream_group() { let mut producer = track_producer("test", Info::timed()); let _dynamic = producer.requests(); - let mut successor = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let mut successor = producer.create_group(group::Info { start: 1 }).unwrap(); successor .write_frame(Timestamp::from_millis(1000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - successor.finish().unwrap(); + successor.close().unwrap(); append_at(&mut producer, 2000); let options = group::Fetch::default().with_max_delay(Duration::ZERO); @@ -15211,8 +15306,8 @@ mod test { #[cfg(feature = "async")] async fn fetched_group_keeps_its_budget_while_reading() { let mut producer = track_producer("test", Info::timed()); - let mut old = producer.append_group().unwrap(); - old.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"first")) + let mut old = producer.append_next().unwrap(); + old.write_frame(old.key_time(), bytes::Bytes::from_static(b"first")) .unwrap(); let consumer = producer.consume(); @@ -15251,11 +15346,11 @@ mod test { // Nothing below the floor is waited for, and a group above it reads at once. producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(subscriber.assert_group().sequence, 3); + assert_eq!(subscriber.assert_group().start, 3); assert!(subscriber.poll_start(&waiter).is_pending()); producer.start_at(3).unwrap(); @@ -15285,12 +15380,14 @@ mod test { panic!("fetch request should be ready"); }; let mut fetched = req.accept(None).unwrap(); - fetched.write_frame(None, b"2-0".as_ref()).unwrap(); - fetched.finish().unwrap(); + fetched + .write_frame(crate::Timestamp::unscaled(fetched.start), b"2-0".as_ref()) + .unwrap(); + fetched.close().unwrap(); let Poll::Ready(Ok(mut group)) = kio::Task::poll(&mut *pending, &waiter) else { panic!("fetched group should be cached"); }; - assert_eq!(group.sequence, 2); + assert_eq!(group.start, 2); let Poll::Ready(Ok(Some(frame))) = group.poll_read_frame(&waiter) else { panic!("fetched frame should be cached"); }; @@ -15333,11 +15430,11 @@ mod test { bytes::Bytes::from_static(b"fetched"), ) .unwrap(); - fetched.finish().unwrap(); + fetched.close().unwrap(); pending.await.unwrap(); let mut groups = producer.subscribe(None); - assert_eq!(groups.assert_group().sequence, 0); + assert_eq!(groups.assert_group().start, 0); groups.assert_no_group(); } @@ -15415,9 +15512,9 @@ mod test { #[test] #[cfg(feature = "async")] - fn a_lower_sequence_is_never_the_live_edge() { + fn a_lower_key_is_never_the_live_edge() { let producer = track_producer("test", Info::timed()); - // A high timestamp on a lower sequence is not a live edge: a group whose frames run + // A high timestamp on a lower key is not a live edge: a group whose frames run // past where its successor starts sits there. Only groups above the candidate // anchor the measure, so it cannot convict the group that follows it. let mut straggler = producer.create_group(0u64.into()).unwrap(); @@ -15426,12 +15523,12 @@ mod test { .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); } - straggler.finish().unwrap(); + straggler.close().unwrap(); let mut next = producer.create_group(1u64.into()).unwrap(); - next.write_frame(Timestamp::from_millis(0).unwrap(), bytes::Bytes::from_static(b"x")) + next.write_frame(next.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); - next.finish().unwrap(); + next.close().unwrap(); let mut subscriber = producer.subscribe(replay()); assert_eq!(drain(&mut subscriber), vec![0, 1]); @@ -15451,7 +15548,7 @@ mod test { assert_eq!(drain(&mut subscriber), vec![1]); let mut subscriber = producer.subscribe(Subscription::default().with_range(0..2)).ordered(); let next = subscriber.next_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(next.sequence, 1); + assert_eq!(next.start, 1); // A budget that covers it delivers the range, and nothing above it. let mut subscriber = producer.subscribe(replay().with_end(2)); @@ -15460,26 +15557,27 @@ mod test { /// Drift is measured only against groups the ranges cover: a group in the gap between /// two spans is never read, so it neither sets the edge nor replaces the newest group - /// below it. A group in the later span is read, so it does. + /// below it. A group in the later span is read, so it does: here group 9, which contains + /// its start, while group 8 below it is in the gap. #[test] #[cfg(feature = "async")] fn a_gap_between_ranges_is_not_a_drift_edge() { let ranges = || Subscription::default().with_ranges([Range::groups(0..2), Range::groups(10..12)]); let producer = track_producer("test", Info::timed()); - for (sequence, millis) in [(0, 0), (1, 1000), (9, 9000)] { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + for (key, millis) in [(0, 0), (1, 1000), (8, 8000), (9, 9000)] { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } // With no budget, the newest group the ranges cover is the edge. - assert_eq!(drain(&mut producer.subscribe(ranges())), vec![1]); + assert_eq!(drain(&mut producer.subscribe(ranges())), vec![9]); - let mut ten = producer.create_group(group::Info { sequence: 10 }).unwrap(); + let mut ten = producer.create_group(group::Info { start: 10 }).unwrap(); ten.write_frame(Timestamp::from_millis(10_000).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - ten.finish().unwrap(); + ten.close().unwrap(); assert_eq!(drain(&mut producer.subscribe(ranges())), vec![10]); } @@ -15493,16 +15591,20 @@ mod test { let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(15))); let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(budget.clone()); - let mut zero = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut zero = producer.create_group(group::Info { start: 0 }).unwrap(); zero.write_frame(at(0), bytes::Bytes::from_static(b"x")).unwrap(); let mut held = subscriber.recv_group().await.unwrap().expect("group 0"); assert!(held.read_frame().await.unwrap().is_some()); + // The run's own group 10 is held, so its time start names it. + let mut ten = producer.create_group(group::Info { start: 10 }).unwrap(); + ten.write_frame(at(10_000), bytes::Bytes::from_static(b"x")).unwrap(); + ten.close().unwrap(); subscriber.update(budget.with_range(10..20)).unwrap(); - for (sequence, millis) in [(1, 1000), (10, 10_000), (19, 19_000)] { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + for (key, millis) in [(1, 1000), (19, 19_000)] { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group.write_frame(at(millis), bytes::Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let pending = kio::clock::sim::spawn(async move { held.read_frame().await }); @@ -15520,7 +15622,7 @@ mod test { let subscription = Subscription::default() .with_delay(Delay::default().with_max(Duration::ZERO)) .with_ranges([Range { - start: Anchor::Absolute(0), + start: Anchor::Time(0), end: Some(Anchor::Relative(0)), }]); let mut subscriber = producer.subscribe(subscription); @@ -15543,7 +15645,7 @@ mod test { let Poll::Ready(Ok(Some(peeked))) = res else { panic!("group 0 is the newest group its range reads (ordered: {ordered})"); }; - assert_eq!(peeked.group.sequence, 0); + assert_eq!(peeked.group.start, 0); } } @@ -15564,6 +15666,8 @@ mod test { let expected = (Servable::NONE, Some(5)); for order in ["ranges first", "cap first", "cap between"] { let producer = track_producer("test", Info::timed()); + // Group 10 is held, so the time start names it. + producer.append_group(10).unwrap().close().unwrap(); let mut subscriber = producer.subscribe(Subscription::default()); let control = subscriber.control(); match order { @@ -15604,7 +15708,7 @@ mod test { seed.set(x); x % bound }; - const SEQUENCES: u64 = 40; + const KEYS: u64 = 40; let at = |millis| Timestamp::from_millis(millis).unwrap(); for round in 0..300 { let producers = [ @@ -15613,7 +15717,7 @@ mod test { ]; let runs = |count: u64| -> Runs { Runs::new((0..next(count + 1)).map(|_| { - let start = next(SEQUENCES); + let start = next(KEYS); start..=start + next(5) })) }; @@ -15621,42 +15725,42 @@ mod test { let mut groups = [Vec::new(), Vec::new()]; let mut servable = Servable::NONE; for step in 0..30 { - let sequence = next(SEQUENCES); + let key = next(KEYS); let track = next(2) as usize; let (producer, groups) = (&producers[track], &mut groups[track]); match next(6) { 0 | 1 => { - if let Ok(mut group) = producer.create_group(group::Info { sequence }) { + if let Ok(mut group) = producer.create_group(group::Info { start: key }) { if next(3) > 0 { group - .write_frame(at(sequence * 10), bytes::Bytes::from_static(b"x")) + .write_frame(at(key * 10), bytes::Bytes::from_static(b"x")) .unwrap(); } groups.push(group); } } 2 => { - if let Some(index) = groups.iter().position(|group| group.sequence == sequence) { + if let Some(index) = groups.iter().position(|group| group.start == key) { let _ = groups.swap_remove(index).abort(Error::Cancel); } } 3 => { - let _ = producer.modify().unwrap().uncache(sequence); + let _ = producer.modify().unwrap().uncache(key); } _ => {} } // Now and then the window's walks move, keeping the fixed runs and their index. if step == 0 || next(4) == 0 { - let cap = (next(4) == 0).then(|| next(SEQUENCES + 2)); + let cap = (next(4) == 0).then(|| next(KEYS + 2)); let layered = Layered::new(fixed.clone(), runs(2)); servable = Servable::layered(layered, cap, &servable); } for (track, producer) in producers.iter().enumerate() { let state = producer.state.read(); - let shown = |slot: &Slot| slot.is_shown().then_some(slot.group.sequence); - for from in [0, next(SEQUENCES), next(SEQUENCES)] { + let shown = |slot: &Slot| slot.is_shown().then_some(slot.group.start); + for from in [0, next(KEYS), next(KEYS)] { for rev in [false, true] { - let scan = state.live(from..).filter(|(sequence, _)| servable.contains(**sequence)); + let scan = state.live(from..).filter(|(key, _)| servable.contains(**key)); let expected = match rev { true => scan.map(|(_, slot)| slot).rev().find_map(shown), false => scan.map(|(_, slot)| slot).find_map(shown), @@ -15671,10 +15775,7 @@ mod test { several.cap ), }; - let cached: Vec<_> = state - .live(..) - .map(|(sequence, slot)| (*sequence, slot.is_shown())) - .collect(); + let cached: Vec<_> = state.live(..).map(|(key, slot)| (*key, slot.is_shown())).collect(); assert_eq!( found, expected, "round {round} step {step} track {track} from {from}, rev {rev}: {shape}, cached {cached:?}" @@ -15693,7 +15794,7 @@ mod test { fn the_live_index_shrinks_with_the_cache() { let producer = track_producer("test", Info::timed()); let groups: Vec<_> = (0..100) - .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) + .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); let fixed = Arc::new(Runs::new([0..=999, 1001..=u64::MAX])); let servable = Servable::layered(Layered::new(fixed, Runs::default()), None, &Servable::NONE); @@ -15704,7 +15805,7 @@ mod test { let indexes = several.live.0.lock().unwrap(); indexes.iter().map(|indexed| indexed.groups.len()).sum() }; - let newest = |slot: &Slot| Some(slot.group.sequence); + let newest = |slot: &Slot| Some(slot.group.start); assert_eq!( producer.state.read().find_servable(&servable, 0, true, newest), Some(99) @@ -15712,8 +15813,8 @@ mod test { assert_eq!(entries(), 100); // Most of the cache leaves, so a prune keeps only the groups still cached. - for sequence in 0..60 { - producer.modify().unwrap().uncache(sequence).unwrap(); + for key in 0..60 { + producer.modify().unwrap().uncache(key).unwrap(); } assert_eq!( producer.state.read().find_servable(&servable, 0, true, newest), @@ -15722,8 +15823,8 @@ mod test { assert_eq!(entries(), 40); // Then all but the newest leaves. - for sequence in 60..99 { - producer.modify().unwrap().uncache(sequence).unwrap(); + for key in 60..99 { + producer.modify().unwrap().uncache(key).unwrap(); } assert_eq!( producer.state.read().find_servable(&servable, 0, true, newest), @@ -15815,16 +15916,20 @@ mod test { let budget = Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(15))); let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(budget.clone()); - let mut zero = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut zero = producer.create_group(group::Info { start: 0 }).unwrap(); zero.write_frame(at(0), bytes::Bytes::from_static(b"x")).unwrap(); let mut held = subscriber.recv_group().await.unwrap().expect("group 0"); assert!(held.read_frame().await.unwrap().is_some()); + // The run's own group 10 is held, so its time start names it. + let mut ten = producer.create_group(group::Info { start: 10 }).unwrap(); + ten.write_frame(at(10_000), bytes::Bytes::from_static(b"x")).unwrap(); + ten.close().unwrap(); subscriber.control().update(budget.with_range(10..20)).unwrap(); - for (sequence, millis) in [(1, 1000), (10, 10_000), (19, 19_000)] { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + for (key, millis) in [(1, 1000), (19, 19_000)] { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group.write_frame(at(millis), bytes::Bytes::from_static(b"x")).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let pending = kio::clock::sim::spawn(async move { held.read_frame().await }); @@ -15978,32 +16083,32 @@ mod test { } #[test] - fn out_of_order_max_sequence_at_front() { + fn out_of_order_max_key_at_front() { let producer = track_producer("test", Info::timed()); // Arrive out of order: seq 5 first, then 3, then 4. - let _open = producer.create_group(group::Info { sequence: 5 }).unwrap(); - let _open = producer.create_group(group::Info { sequence: 3 }).unwrap(); - let _open = producer.create_group(group::Info { sequence: 4 }).unwrap(); + let _open = producer.create_group(group::Info { start: 5 }).unwrap(); + let _open = producer.create_group(group::Info { start: 3 }).unwrap(); + let _open = producer.create_group(group::Info { start: 4 }).unwrap(); - // max_sequence = 5, which is at the front of the VecDeque. + // max_key = 5, which is at the front of the VecDeque. { let state = producer.state.read(); - assert_eq!(state.max_sequence, Some(5)); + assert_eq!(state.max_key, Some(5)); } // Expire all three groups. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); - // Append seq 6 (becomes new max_sequence). - let _open = producer.append_group().unwrap(); // seq 6 + // Append seq 6 (becomes new max_key). + let _open = producer.append_next().unwrap(); // seq 6 - // Seq 3, 4, 5 are all expired. Seq 5 was the old max_sequence but now 6 is. + // Seq 3, 4, 5 are all expired. Seq 5 was the old max_key but now 6 is. // All old groups are evicted. { let state = producer.state.read(); assert_eq!(live_groups(&state), 1); - assert_eq!(first_live_sequence(&state), 6); + assert_eq!(first_live_key(&state), 6); assert!(!state.lookup.contains_key(&3)); assert!(!state.lookup.contains_key(&4)); assert!(!state.lookup.contains_key(&5)); @@ -16012,26 +16117,26 @@ mod test { } #[test] - fn max_sequence_at_front_blocks_trim() { + fn max_key_at_front_blocks_trim() { let producer = track_producer("test", Info::timed()); // Arrive: seq 5, then seq 3. producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); - // Seq 3 arrives late; max_sequence is still 5 (at front). + // Seq 3 arrives late; max_key is still 5 (at front). producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); - // Seq 5 is max_sequence (protected). Seq 3 is not expired (just created). + // Seq 5 is max_key (protected). Seq 3 is not expired (just created). // Nothing should be evicted. { let state = producer.state.read(); @@ -16044,9 +16149,9 @@ mod test { // Seq 2 arrives late, triggering eviction. producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); // Seq 5 is the live edge (protected) and still resolves at the front of @@ -16066,15 +16171,15 @@ mod test { let mut consumer = producer.subscribe(None); let group = consumer.assert_group(); // consume() starts at index 0; the first arrival entry that still resolves is seq 5. - assert_eq!(group.sequence, 5); + assert_eq!(group.start, 5); } #[test] #[cfg(feature = "async")] fn abort_drops_open_groups() { let producer = track_producer("test", Info::timed()); - let _writer_1 = producer.append_group().unwrap(); - let _writer_2 = producer.append_group().unwrap(); + let _writer_1 = producer.append_next().unwrap(); + let _writer_2 = producer.append_next().unwrap(); let mut consumer = producer.subscribe(None); assert_eq!(live_groups(&producer.state.read()), 2); @@ -16095,7 +16200,7 @@ mod test { fn drop_unfinished_clears_cached_groups() { let producer = track_producer("test", Info::timed()); let writer = producer.clone(); - let _open = writer.append_group().unwrap(); + let _open = writer.append_next().unwrap(); // A stale consumer keeps the channel (and thus the cache) alive. let mut consumer = producer.subscribe(None); @@ -16116,9 +16221,9 @@ mod test { fn abort_ends_a_pulled_open_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); - let mut writer = producer.append_group().unwrap(); + let mut writer = producer.append_next().unwrap(); writer - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) + .write_frame(writer.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); let mut group = consumer.assert_group(); assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); @@ -16126,7 +16231,7 @@ mod test { producer.clone().abort(Error::Cancel).unwrap(); let read = group.read_frame().now_or_never().expect("should not block"); assert!(matches!(read, Err(Error::Cancel)), "{read:?}"); - let write = writer.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"y")); + let write = writer.write_frame(writer.key_time(), bytes::Bytes::from_static(b"y")); assert!(matches!(write, Err(Error::Cancel)), "{write:?}"); } @@ -16137,9 +16242,9 @@ mod test { fn abort_unwanted_waits_for_a_pulled_open_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); - let mut writer = producer.append_group().unwrap(); + let mut writer = producer.append_next().unwrap(); writer - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) + .write_frame(writer.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); let mut group = consumer.assert_group(); assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); @@ -16150,7 +16255,7 @@ mod test { .expect_err("the group's reader keeps the track"); drop(group); assert!(producer.abort_unwanted(Error::Cancel).is_ok()); - let write = writer.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"y")); + let write = writer.write_frame(writer.key_time(), bytes::Bytes::from_static(b"y")); assert!(matches!(write, Err(Error::Cancel)), "{write:?}"); } @@ -16161,9 +16266,9 @@ mod test { fn drop_unfinished_ends_a_pulled_open_group() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); - let mut writer = producer.append_group().unwrap(); + let mut writer = producer.append_next().unwrap(); writer - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"x")) + .write_frame(writer.key_time(), bytes::Bytes::from_static(b"x")) .unwrap(); let mut group = consumer.assert_group(); assert!(group.read_frame().now_or_never().unwrap().unwrap().is_some()); @@ -16171,7 +16276,7 @@ mod test { drop(producer); let read = group.read_frame().now_or_never().expect("should not block"); assert!(matches!(read, Err(Error::Dropped)), "{read:?}"); - let write = writer.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"y")); + let write = writer.write_frame(writer.key_time(), bytes::Bytes::from_static(b"y")); assert!(matches!(write, Err(Error::Dropped)), "{write:?}"); } @@ -16183,15 +16288,15 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let _fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"snapshot")) + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); let mut reading = consumer.peek_group(0).expect("the head serves group 0"); assert!(reading.read_frame().now_or_never().unwrap().unwrap().is_some()); @@ -16199,7 +16304,7 @@ mod test { producer.abort(Error::App(7)).unwrap(); let read = reading.read_frame().now_or_never().expect("the abort settles the head"); assert!(matches!(read, Err(Error::App(7))), "{read:?}"); - let write = head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")); + let write = head.write_frame(head.key_time(), bytes::Bytes::from_static(b"delta")); assert!(matches!(write, Err(Error::App(7))), "{write:?}"); } @@ -16211,8 +16316,8 @@ mod test { let producer = track_producer("test", Info::timed()); let keep = producer.clone(); let writer = producer.clone(); - let group = writer.append_group().unwrap(); - group.finish().unwrap(); + let group = writer.append_next().unwrap(); + group.close().unwrap(); let _consumer = producer.subscribe(None); writer.abort(Error::Cancel).unwrap(); drop(keep); @@ -16225,7 +16330,7 @@ mod test { let warns = count_drop_warnings("track::Producer dropped without finish", || { let producer = track_producer("test", Info::timed()); let writer = producer.clone(); - let _open = writer.append_group().unwrap(); + let _open = writer.append_next().unwrap(); let _consumer = producer.subscribe(None); drop(writer); drop(producer); @@ -16237,14 +16342,14 @@ mod test { #[cfg(feature = "async")] fn drop_finished_keeps_cached_groups() { let producer = track_producer("test", Info::timed()); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); let producer = producer.finish().unwrap(); let mut consumer = producer.subscribe(None); drop(producer); // A cleanly finished track keeps its cache so the consumer can still drain. - assert_eq!(consumer.assert_group().sequence, 0); + assert_eq!(consumer.assert_group().start, 0); let done = consumer.recv_group().now_or_never().expect("should not block").unwrap(); assert!(done.is_none(), "consumer should drain then see clean finish"); } @@ -16256,11 +16361,11 @@ mod test { #[cfg(feature = "async")] fn drop_short_of_the_boundary_ends_cleanly() { let mut producer = track_producer("test", Info::timed()); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); producer.finish_at(Position::group(3), None).unwrap(); let mut consumer = producer.subscribe(None); - assert_eq!(consumer.assert_group().sequence, 0); + assert_eq!(consumer.assert_group().start, 0); assert!( consumer.recv_group().now_or_never().is_none(), "groups 1 and 2 are owed" @@ -16274,21 +16379,21 @@ mod test { ); } - /// The sequence cursor ends the same way. A gap below the boundary is skipped, + /// The key cursor ends the same way. A gap below the boundary is skipped, /// a later cached group is still delivered, and the read finishes cleanly. #[test] #[cfg(feature = "async")] fn ordered_ends_cleanly_when_sealed_short_of_the_boundary() { let mut producer = track_producer("test", Info::timed()); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); producer.finish_at(Position::group(4), None).unwrap(); @@ -16296,9 +16401,9 @@ mod test { drop(producer); let first = ordered.next_group().now_or_never().expect("group 0").unwrap().unwrap(); - assert_eq!(first.sequence, 0); + assert_eq!(first.start, 0); let second = ordered.next_group().now_or_never().expect("group 2").unwrap().unwrap(); - assert_eq!(second.sequence, 2); + assert_eq!(second.start, 2); let done = ordered.next_group().now_or_never().expect("end").unwrap(); assert!(done.is_none(), "missing groups below the boundary are not an error"); } @@ -16315,19 +16420,19 @@ mod test { if given_up { producer.drop_groups(3..4, StreamError::DeliveryTimeout); } - for sequence in [0, 1, 2, 4, 5] { - write_group(&producer, sequence); + for key in [0, 1, 2, 4, 5] { + write_group(&producer, key); } let _producer = producer.finish().unwrap(); - for sequence in 0..3 { + for key in 0..3 { let Poll::Ready(Ok(Some(group))) = ordered.poll_next_group(&kio::Waiter::noop()) else { - panic!("group {sequence}, given_up={given_up}") + panic!("group {key}, given_up={given_up}") }; - assert_eq!(group.sequence, sequence); + assert_eq!(group.start, key); let Poll::Ready(Ok(Some(group))) = arrival.poll_recv_group(&kio::Waiter::noop()) else { - panic!("arrival group {sequence}, given_up={given_up}") + panic!("arrival group {key}, given_up={given_up}") }; - assert_eq!(group.sequence, sequence); + assert_eq!(group.start, key); } assert!( matches!(ordered.poll_next_group(&kio::Waiter::noop()), Poll::Ready(Ok(None))), @@ -16348,17 +16453,17 @@ mod test { // neither finish again nor append. assert!(producer.clone().finish().is_ok()); assert!(producer.clone().finish().is_err()); - assert!(producer.append_group().is_err()); + assert!(producer.append_next().is_err()); } #[test] fn finish_after_groups() { let producer = track_producer("test", Info::timed()); - let _open = producer.append_group().unwrap(); + let _open = producer.append_next().unwrap(); assert!(producer.clone().finish().is_ok()); assert!(producer.clone().finish().is_err()); - assert!(producer.append_group().is_err()); + assert!(producer.append_next().is_err()); } /// The handle `finish` returns still fills an older group below the end, refuses one at or @@ -16369,24 +16474,24 @@ mod test { let producer = track_producer("test", Info::timed()); let mut ordered = producer.subscribe(replay()).ordered(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); let finished = producer.finish().unwrap(); assert!(matches!( - finished.create_group(group::Info { sequence: 2 }), + finished.create_group(group::Info { start: 2 }), Err(Error::Closed) )); - let mut late = finished.create_group(group::Info { sequence: 0 }).unwrap(); - late.write_frame(Timestamp::ZERO, &b"late"[..]).unwrap(); - let late = late.finish().unwrap(); + let mut late = finished.create_group(group::Info { start: 0 }).unwrap(); + late.write_frame(late.key_time(), &b"late"[..]).unwrap(); + let late = late.close().unwrap(); assert_eq!(late.frame_count(), 1); let first = ordered.next_group().now_or_never().expect("group 0").unwrap().unwrap(); - assert_eq!(first.sequence, 0); + assert_eq!(first.start, 0); let second = ordered.next_group().now_or_never().expect("group 1").unwrap().unwrap(); - assert_eq!(second.sequence, 1); + assert_eq!(second.start, 1); let done = ordered.next_group().now_or_never().expect("end").unwrap(); assert!(done.is_none(), "the finished track ends after its late group"); assert!(finished.closed().now_or_never().expect("settled").is_ok()); @@ -16395,7 +16500,7 @@ mod test { #[test] fn finish_at_rejects_a_boundary_at_or_below_the_live_edge() { let mut producer = track_producer("test", Info::timed()); - let _open = producer.create_group(group::Info { sequence: 5 }).unwrap(); + let _open = producer.create_group(group::Info { start: 5 }).unwrap(); // The boundary is exclusive, so it must be strictly above the highest produced // group. 5 or below would orphan groups that already exist. @@ -16409,8 +16514,8 @@ mod test { // boundary can be created. assert!(producer.finish_at(Position::group(6), None).is_ok()); assert!(producer.finish_at(Position::group(7), None).is_err()); - assert!(producer.create_group(group::Info { sequence: 4 }).is_ok()); - assert!(producer.create_group(group::Info { sequence: 6 }).is_err()); + assert!(producer.create_group(group::Info { start: 4 }).is_ok()); + assert!(producer.create_group(group::Info { start: 6 }).is_err()); } #[test] @@ -16419,11 +16524,11 @@ mod test { let mut producer = track_producer_pooled("test", pool.clone()); let end = Position { group: 5, frame: 5 }; producer.finish_at(end, None).unwrap(); - let mut group = producer.create_group(group::Info { sequence: 5 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 5 }).unwrap(); for _ in 0..5 { - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let consumer = producer.consume(); drop(producer); // Date the last activity before advancing the cache clock past expiry. @@ -16457,9 +16562,9 @@ mod test { #[test] fn a_mid_group_end_refuses_to_cut_existing_frames() { let mut producer = track_producer("test", Info::timed()); - let mut group = producer.create_group(group::Info { sequence: 5 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 5 }).unwrap(); for _ in 0..6 { - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); } assert!(matches!( producer.finish_at(Position { group: 5, frame: 5 }, None), @@ -16478,14 +16583,14 @@ mod test { if early { producer.finish_at(end, None).unwrap(); } - let mut group = producer.create_group(group::Info { sequence: 5 }).unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + let mut group = producer.create_group(group::Info { start: 5 }).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); if !early { producer.finish_at(end, None).unwrap(); } - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"x".as_slice()).unwrap(); assert!(matches!( - group.write_frame(Timestamp::ZERO, b"x".as_slice()), + group.write_frame(group.key_time(), b"x".as_slice()), Err(Error::Closed) )); let mut frames = frame::Buffer::<1>::new(); @@ -16500,11 +16605,11 @@ mod test { assert!(matches!( group.create_frame(frame::Info { size: 1, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }), Err(Error::Closed) )); - assert!(producer.create_group(group::Info { sequence: 6 }).is_err()); + assert!(producer.create_group(group::Info { start: 6 }).is_err()); } } @@ -16513,7 +16618,7 @@ mod test { let mut producer = track_producer("test", Info::timed()); assert_eq!(producer.finished(), None); - let _open = producer.create_group(group::Info { sequence: 5 }).unwrap(); + let _open = producer.create_group(group::Info { start: 5 }).unwrap(); assert_eq!(producer.finished(), None, "a group does not declare an end"); producer.finish_at(Position::group(9), None).unwrap(); @@ -16526,7 +16631,7 @@ mod test { #[test] fn end_reports_the_live_edge_after_finish() { let producer = track_producer("test", Info::timed()); - let _open = producer.create_group(group::Info { sequence: 5 }).unwrap(); + let _open = producer.create_group(group::Info { start: 5 }).unwrap(); producer.finish_shared().unwrap(); assert_eq!(producer.finished(), Some(Position::group(6))); } @@ -16536,16 +16641,16 @@ mod test { fn finish_at_declares_a_future_boundary() { let mut producer = track_producer("test", Info::timed()); producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); // Learn the track ends at group 6 (exclusive 7) while the live edge is still 5. producer.finish_at(Position::group(7), None).unwrap(); let mut consumer = producer.subscribe(None); - assert_eq!(consumer.assert_group().sequence, 5); + assert_eq!(consumer.assert_group().start, 5); // The boundary is known immediately, but the track isn't done: group 6 is still // outstanding, so the consumer parks rather than seeing end-of-stream. @@ -16562,11 +16667,11 @@ mod test { // The trailing group arrives (below the boundary), then the track completes. producer - .create_group(group::Info { sequence: 6 }) + .create_group(group::Info { start: 6 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(consumer.assert_group().sequence, 6); + assert_eq!(consumer.assert_group().start, 6); let done = consumer .recv_group() .now_or_never() @@ -16582,11 +16687,11 @@ mod test { #[cfg(feature = "async")] async fn an_errored_end_keeps_its_history() { let mut producer = track_producer("test", Info::timed()); - for sequence in 0..2 { + for key in 0..2 { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } producer.finish_at(Position::group(2), Some(Error::App(7))).unwrap(); @@ -16599,30 +16704,30 @@ mod test { let mut arrival = producer.subscribe(replay().with_range(0..)); let mut ordered = producer.subscribe(replay().with_range(0..)).ordered(); - for sequence in 0..2 { - assert_eq!(arrival.assert_group().sequence, sequence); - assert_eq!(ordered.next_group().await.unwrap().unwrap().sequence, sequence); + for key in 0..2 { + assert_eq!(arrival.assert_group().start, key); + assert_eq!(ordered.next_group().await.unwrap().unwrap().start, key); } assert!(matches!(arrival.recv_group().await, Err(Error::App(7)))); assert!(matches!(ordered.next_group().await, Err(Error::App(7)))); assert!(matches!(producer.closed().await, Err(Error::App(7)))); let mut capped = producer.subscribe(replay().with_range(0..)).with_range(..1); - assert_eq!(capped.assert_group().sequence, 0); + assert_eq!(capped.assert_group().start, 0); assert!(capped.recv_group().await.unwrap().is_none()); // So does a range whose relative end excludes the last group. let mut relative = producer.subscribe(replay().with_ranges([Range { - start: Anchor::Absolute(0), + start: Anchor::Time(0), end: Some(Anchor::Relative(1)), }])); - assert_eq!(relative.assert_group().sequence, 0); + assert_eq!(relative.assert_group().start, 0); assert!(relative.recv_group().await.unwrap().is_none()); // Unlike an abort, the history stays: a later reader and a fetch still get it. producer.clone().abort(Error::Timeout).unwrap(); let mut late = producer.subscribe(replay().with_range(0..)); - assert_eq!(late.assert_group().sequence, 0); - assert_eq!(late.assert_group().sequence, 1); + assert_eq!(late.assert_group().start, 0); + assert_eq!(late.assert_group().start, 1); assert!(matches!(late.recv_group().await, Err(Error::App(7)))); producer.consume().fetch_group(1, None).await.unwrap(); } @@ -16634,12 +16739,12 @@ mod test { let mut arrival = producer.subscribe(None); let mut ordered = producer.subscribe(None).ordered(); producer.finish_at(Position::group(3), None).unwrap(); - let _open = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let _open = producer.create_group(group::Info { start: 2 }).unwrap(); // Missing lower groups do not hold either reader for the wire's grace. // The last group's own stream remains open independently of the track end. - assert_eq!(arrival.assert_group().sequence, 2); - assert_eq!(ordered.next_group().await.unwrap().unwrap().sequence, 2); + assert_eq!(arrival.assert_group().start, 2); + assert_eq!(ordered.next_group().await.unwrap().unwrap().start, 2); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); assert!(ordered.next_group().now_or_never().unwrap().unwrap().is_none()); @@ -16667,16 +16772,16 @@ mod test { // The wire declares where the feed starts. producer.start_at(0).unwrap(); producer.finish_at(Position::group(3), None).unwrap(); - let _high = producer.create_group(group::Info { sequence: 2 }).unwrap(); - assert_eq!(arrival.assert_group().sequence, 2); + let _high = producer.create_group(group::Info { start: 2 }).unwrap(); + assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); // The late lower groups are delivered, then the full tail ends the reader. - let _low = producer.create_group(group::Info { sequence: 0 }).unwrap(); - assert_eq!(arrival.assert_group().sequence, 0); + let _low = producer.create_group(group::Info { start: 0 }).unwrap(); + assert_eq!(arrival.assert_group().start, 0); assert!(arrival.recv_group().now_or_never().is_none(), "group 1 is still owed"); - let _mid = producer.create_group(group::Info { sequence: 1 }).unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + let _mid = producer.create_group(group::Info { start: 1 }).unwrap(); + assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); // The grace passing ends readers at the boundary, hole and all. @@ -16685,8 +16790,8 @@ mod test { let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); - let _high = producer.create_group(group::Info { sequence: 1 }).unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + let _high = producer.create_group(group::Info { start: 1 }).unwrap(); + assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().is_none()); // The subscription's max delay is the grace. elapse(&producer, replay().budget()); @@ -16698,8 +16803,8 @@ mod test { let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); - let _high = producer.create_group(group::Info { sequence: 1 }).unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + let _high = producer.create_group(group::Info { start: 1 }).unwrap(); + assert_eq!(arrival.assert_group().start, 1); drop(producer); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } @@ -16719,11 +16824,11 @@ mod test { let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + assert_eq!(arrival.assert_group().start, 1); if arrived_late { // The end comes later than the arrival, so its gap runs from the end. elapse(&producer, grace / 2); @@ -16770,11 +16875,11 @@ mod test { producer.start_at(0).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); producer - .create_group(group::Info { sequence: arrived }) + .create_group(group::Info { start: arrived }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, arrived); + assert_eq!(arrival.assert_group().start, arrived); elapse(&producer, Duration::from_secs(3600)); assert!( @@ -16783,11 +16888,11 @@ mod test { ); producer - .create_group(group::Info { sequence: missing }) + .create_group(group::Info { start: missing }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, missing); + assert_eq!(arrival.assert_group().start, missing); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } } @@ -16803,14 +16908,14 @@ mod test { let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(5), None).unwrap(); - for sequence in 0..3 { + for key in 0..3 { elapse(&producer, grace / 2); producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, sequence); + assert_eq!(arrival.assert_group().start, key); } elapse(&producer, grace - Duration::from_millis(1)); assert!(arrival.recv_group().now_or_never().is_none(), "still within the grace"); @@ -16846,11 +16951,11 @@ mod test { producer.start_at(0).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } @@ -16864,11 +16969,11 @@ mod test { producer.start_at(0).unwrap(); producer.finish_at(Position::group(3), None).unwrap(); producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, 2); + assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); producer.drop_groups(0..2, StreamError::NotFound); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); @@ -16893,11 +16998,11 @@ mod test { producer.request_start(None).unwrap(); let mut arrival = producer.subscribe(replay()); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, 0); + assert_eq!(arrival.assert_group().start, 0); producer.drop_groups(1..2, StreamError::NotFound); producer.finish_at(Position::group(2), None).unwrap(); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); @@ -16908,11 +17013,11 @@ mod test { producer.request_start(None).unwrap(); let mut arrival = producer.subscribe(replay()); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + assert_eq!(arrival.assert_group().start, 1); producer.finish_at(Position::group(2), None).unwrap(); assert!(arrival.recv_group().now_or_never().is_none(), "group 0 is owed"); elapse(&producer, replay().budget()); @@ -16931,12 +17036,12 @@ mod test { let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); - let _high = producer.create_group(group::Info { sequence: 1 }).unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + let _high = producer.create_group(group::Info { start: 1 }).unwrap(); + assert_eq!(arrival.assert_group().start, 1); let _pending = consumer.fetch_group(0, None); let req = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - req.accept(None).unwrap().finish().unwrap(); + req.accept(None).unwrap().close().unwrap(); assert!( arrival.recv_group().now_or_never().is_none(), "the live group 0 is still owed" @@ -16953,9 +17058,9 @@ mod test { let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); - let high = producer.create_group(group::Info { sequence: 1 }).unwrap(); - high.finish().unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); + let high = producer.create_group(group::Info { start: 1 }).unwrap(); + high.close().unwrap(); + assert_eq!(arrival.assert_group().start, 1); assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); producer.abort(Error::Dropped).unwrap(); @@ -16975,16 +17080,16 @@ mod test { let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); - for sequence in [1, 0] { + for key in [1, 0] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } producer.clone().abort(Error::Dropped).unwrap(); - assert_eq!(arrival.assert_group().sequence, 1); - assert_eq!(arrival.assert_group().sequence, 0); + assert_eq!(arrival.assert_group().start, 1); + assert_eq!(arrival.assert_group().start, 0); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } @@ -16999,13 +17104,13 @@ mod test { let mut arrival = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(3), None).unwrap(); - for sequence in [0, 2] { + for key in [0, 2] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, sequence); + assert_eq!(arrival.assert_group().start, key); } assert!(arrival.recv_group().now_or_never().is_none(), "held at the hole"); @@ -17026,22 +17131,22 @@ mod test { let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); producer.finish_at(Position::group(3), None).unwrap(); - for sequence in [0, 1] { + for key in [0, 1] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(arrival.assert_group().sequence, sequence); + assert_eq!(arrival.assert_group().start, key); } elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); assert!(!producer.state.read().lookup.contains_key(&0), "group 0 was evicted"); - assert_eq!(arrival.assert_group().sequence, 2); + assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } @@ -17055,8 +17160,8 @@ mod test { producer.received(); let mut arrival = producer.subscribe(replay()); producer.start_at(0).unwrap(); - let _high = producer.create_group(group::Info { sequence: 2 }).unwrap(); - assert_eq!(arrival.assert_group().sequence, 2); + let _high = producer.create_group(group::Info { start: 2 }).unwrap(); + assert_eq!(arrival.assert_group().start, 2); assert!(arrival.recv_group().now_or_never().is_none()); producer.finish_at(Position::group(3), None).unwrap(); @@ -17071,9 +17176,9 @@ mod test { assert_eq!(producer.finished(), Some(Position::group(3)), "the first end stands"); let mut _low = Vec::new(); - for sequence in [0, 1] { - _low.push(producer.create_group(group::Info { sequence }).unwrap()); - assert_eq!(arrival.assert_group().sequence, sequence); + for key in [0, 1] { + _low.push(producer.create_group(group::Info { start: key }).unwrap()); + assert_eq!(arrival.assert_group().start, key); } assert!(arrival.recv_group().now_or_never().unwrap().unwrap().is_none()); } @@ -17086,11 +17191,11 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); - let head = producer.create_group(group::Info { sequence: 0 }).unwrap(); - head.finish().unwrap(); - let _tail = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let head = producer.create_group(group::Info { start: 0 }).unwrap(); + head.close().unwrap(); + let _tail = producer.create_group(group::Info { start: 1 }).unwrap(); producer.finish_at(Position::group(2), None).unwrap(); - assert_eq!(consumer.assert_group().sequence, 0); + assert_eq!(consumer.assert_group().start, 0); producer.abort(Error::Timeout).unwrap(); let res = consumer.recv_group().now_or_never().expect("should not block"); @@ -17103,13 +17208,13 @@ mod test { fn session_close_is_not_completion_and_keeps_the_resume_position() { let producer = track_producer("local", Info::timed()); let consumer = producer.consume(); - producer.append_group().unwrap().finish().unwrap(); - let mut group = producer.append_group().unwrap(); + producer.append_next().unwrap().close().unwrap(); + let mut group = producer.append_next().unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"a")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"a")) .unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"b")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"b")) .unwrap(); producer .abort(Error::Stream(StreamError::Session(crate::SessionError::Cancel))) @@ -17129,12 +17234,12 @@ mod test { let waiter = kio::Waiter::noop(); let mut producer = track_producer("test", Info::timed()); let demand = producer.demand(); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); producer.finish_at(Position::group(2), None).unwrap(); assert!(producer.poll_closed(&waiter).is_pending(), "group 1 is still to come"); assert!(demand.closed().now_or_never().is_none()); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); assert!(matches!(demand.closed().now_or_never(), Some(Ok(())))); assert!( @@ -17155,17 +17260,17 @@ mod test { use std::sync::atomic::{AtomicBool, Ordering}; let mut producer = track_producer("test", Info::timed()); - let open = producer.append_group().unwrap(); + let open = producer.append_next().unwrap(); producer.finish_at(Position::group(1), None).unwrap(); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); assert!(producer.poll_closed(&waiter).is_pending(), "group 0 may still abort"); - open.finish().unwrap(); + open.close().unwrap(); assert!(woken.load(Ordering::SeqCst), "the group finishing wakes the wait"); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); let mut cut = track_producer("cut", Info::timed()); - let _open = cut.append_group().unwrap(); + let _open = cut.append_next().unwrap(); cut.finish_at(Position::group(1), None).unwrap(); assert!(cut.poll_closed(&kio::Waiter::noop()).is_pending()); cut.clone().abort(Error::Cancel).unwrap(); @@ -17185,11 +17290,11 @@ mod test { let _short = producer.subscribe(None); producer.start_at(0).unwrap(); producer.finish_at(Position::group(3), None).unwrap(); - for sequence in [0, 2] { + for key in [0, 2] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } assert!(producer.poll_closed(&waiter).is_pending(), "held at the hole"); @@ -17212,14 +17317,14 @@ mod test { let mut producer = track_producer("test", Info::timed()); let demand = producer.demand(); producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); producer.finish_at(Position::group(6), None).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); - let _late = producer.create_group(group::Info { sequence: 4 }).unwrap(); + let _late = producer.create_group(group::Info { start: 4 }).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); producer.clone().abort(Error::Cancel).unwrap(); assert!(matches!(producer.poll_closed(&waiter), Poll::Ready(Ok(())))); @@ -17251,18 +17356,18 @@ mod test { let mut arrival = producer.subscribe(replay()); let mut ordered = producer.subscribe(replay()).ordered(); - for sequence in 0..2 { + for key in 0..2 { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } - let _open = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let _open = producer.create_group(group::Info { start: 2 }).unwrap(); producer.abort(Error::Timeout).unwrap(); - assert_eq!(arrival.assert_group().sequence, 0); - assert_eq!(arrival.assert_group().sequence, 1); + assert_eq!(arrival.assert_group().start, 0); + assert_eq!(arrival.assert_group().start, 1); let res = arrival.recv_group().now_or_never().expect("should not block"); assert!(matches!(res, Err(Error::Timeout)), "expected the abort"); @@ -17271,7 +17376,7 @@ mod test { .next_group() .now_or_never() .expect("should not block") - .map(|group| group.map(|group| group.sequence)); + .map(|group| group.map(|group| group.start)); assert!(matches!(res, Err(Error::Timeout)), "expected the abort, got {res:?}"); } @@ -17282,13 +17387,13 @@ mod test { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); drop(producer); - assert_eq!(consumer.assert_group().sequence, 0); + assert_eq!(consumer.assert_group().start, 0); let res = consumer.recv_group().now_or_never().expect("should not block"); assert!(matches!(res, Err(Error::Dropped)), "expected the drop"); } @@ -17300,15 +17405,15 @@ mod test { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let live = track_producer_pooled("live", pool.clone()); - live.append_group().unwrap().finish().unwrap(); + live.append_next().unwrap().close().unwrap(); let aborted = track_producer_pooled("aborted", pool.clone()); - aborted.append_group().unwrap().finish().unwrap(); + aborted.append_next().unwrap().close().unwrap(); let stale_aborted = aborted.consume(); aborted.abort(Error::Timeout).unwrap(); let finished = track_producer_pooled("finished", pool.clone()); - finished.append_group().unwrap().finish().unwrap(); + finished.append_next().unwrap().close().unwrap(); let finished = finished.finish().unwrap(); let stale_finished = finished.consume(); drop(finished); @@ -17337,8 +17442,8 @@ mod test { fn closed_track_retires_an_expired_cut() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(1))); let producer = track_producer_pooled("cut", pool.clone()); - let mut group = producer.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); + let mut group = producer.append_next().unwrap(); + group.write_frame(group.key_time(), &b"a"[..]).unwrap(); group.abort(Error::Stream(StreamError::App(5))).unwrap(); let producer = producer.finish().unwrap(); let stale = producer.consume(); @@ -17346,7 +17451,7 @@ mod test { let mut reader = stale.subscribe(replay()).now_or_never().unwrap().unwrap(); assert_eq!( - reader.assert_group().sequence, + reader.assert_group().start, 0, "the cut is handed out within the window" ); @@ -17369,13 +17474,13 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); - for sequence in 0..2 { - let group = producer.create_group(group::Info { sequence }).unwrap(); - group.finish().unwrap(); + for key in 0..2 { + let group = producer.create_group(group::Info { start: key }).unwrap(); + group.close().unwrap(); } producer.finish_at(Position::group(2), None).unwrap(); - assert_eq!(consumer.assert_group().sequence, 0); - assert_eq!(consumer.assert_group().sequence, 1); + assert_eq!(consumer.assert_group().start, 0); + assert_eq!(consumer.assert_group().start, 1); producer.abort(Error::Timeout).unwrap(); let res = consumer.recv_group().now_or_never().expect("should not block"); @@ -17390,9 +17495,9 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); - for sequence in 0..2 { - let group = producer.create_group(group::Info { sequence }).unwrap(); - group.finish().unwrap(); + for key in 0..2 { + let group = producer.create_group(group::Info { start: key }).unwrap(); + group.close().unwrap(); } producer.finish_at(Position::group(2), None).unwrap(); producer.abort(Error::Timeout).unwrap(); @@ -17402,7 +17507,7 @@ mod test { .next_group() .now_or_never() .expect("should not block") - .map(|group| group.map(|group| group.sequence)); + .map(|group| group.map(|group| group.start)); assert!(matches!(end, Ok(None)), "expected the clean end, got {end:?}"); } @@ -17414,11 +17519,11 @@ mod test { fn abort_after_the_end_settles_ends_clean_below_the_cap() { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None).ordered(); - for sequence in 0..2 { + for key in 0..2 { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } producer.finish_at(Position::group(2), None).unwrap(); @@ -17430,7 +17535,7 @@ mod test { .next_group() .now_or_never() .expect("should not block") - .map(|group| group.map(|group| group.sequence)); + .map(|group| group.map(|group| group.start)); assert!( matches!(end, Ok(None)), "capped at the end: expected the clean end, got {end:?}" @@ -17438,11 +17543,11 @@ mod test { let mut producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(None).ordered(); - for sequence in [0, 2] { + for key in [0, 2] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } producer.finish_at(Position::group(3), None).unwrap(); @@ -17454,7 +17559,7 @@ mod test { .next_group() .now_or_never() .expect("should not block") - .map(|group| group.map(|group| group.sequence)); + .map(|group| group.map(|group| group.start)); assert!( matches!(end, Ok(None)), "gap below the cap: expected the clean end, got {end:?}" @@ -17466,14 +17571,14 @@ mod test { fn recv_group_finishes_without_waiting_for_gaps() { let producer = track_producer("test", Info::timed()); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); let producer = producer.finish().unwrap(); let mut consumer = producer.subscribe(None); - assert_eq!(consumer.assert_group().sequence, 1); + assert_eq!(consumer.assert_group().start, 1); let done = consumer .recv_group() @@ -17502,17 +17607,17 @@ mod test { for case in ["reoffer", "released", "past"] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); let mut reader = first.consume(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a", "{case}"); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; - let Ok(Received::Held(second)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(second)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("a second duplicate waits too"); }; first.abort(Error::Cancel).unwrap(); @@ -17528,7 +17633,7 @@ mod test { continue; } let start = if case == "past" { 2 } else { 0 }; - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, start) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, start) else { panic!("the re-offer takes the group"); }; drop(hold); @@ -17538,7 +17643,7 @@ mod test { ); // The second duplicate now waits on the re-offer, which holds nobody reading the // failed copy. - let Ok(Received::Held(third)) = producer.create_group_at(group::Info { sequence: 3 }, start) else { + let Ok(Received::Held(third)) = producer.create_group_at(group::Info { start: 3 }, start) else { panic!("the second duplicate waits on the re-offer still arriving"); }; drop(second); @@ -17551,7 +17656,7 @@ mod test { again.write_frame(at(300), &b"a"[..]).unwrap(); again.write_frame(at(301), &b"b"[..]).unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); - again.finish().unwrap(); + again.close().unwrap(); drop(third); assert!(reader.read_frame().await.unwrap().is_none()); } @@ -17564,28 +17669,28 @@ mod test { async fn a_failed_copy_finishes_on_its_held_reoffer() { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); let mut reader = first.consume(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let waiter = kio::Waiter::noop(); assert!(reader.poll_finished(&waiter).is_pending(), "waits on the held re-offer"); - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; drop(hold); again.write_frame(at(300), &b"a"[..]).unwrap(); again.write_frame(at(301), &b"b"[..]).unwrap(); assert!(reader.poll_finished(&waiter).is_pending(), "the re-offer is still open"); - again.finish().unwrap(); + again.close().unwrap(); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); } @@ -17599,21 +17704,21 @@ mod test { for torn in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); let mut reader = first.consume(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a", "{torn}"); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; - let Ok(Received::Held(next)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(next)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("another re-offer waits on the first"); }; drop(hold); @@ -17625,7 +17730,7 @@ mod test { let mut frame = again .create_frame(frame::Info { size: 2, - timestamp: Some(at(301)), + timestamp: at(301), }) .unwrap(); frame.write(&b"b"[..]).unwrap(); @@ -17637,7 +17742,7 @@ mod test { let mut frame = again .create_frame(frame::Info { size: 2, - timestamp: Some(at(300)), + timestamp: at(300), }) .unwrap(); frame.write(&b"a"[..]).unwrap(); @@ -17655,13 +17760,13 @@ mod test { reader.poll_read_frame(&waiter).is_pending(), "waits on the next re-offer" ); - let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the next re-offer takes the group"); }; drop(next); last.write_frame(at(300), &b"a"[..]).unwrap(); last.write_frame(at(301), &b"b"[..]).unwrap(); - last.finish().unwrap(); + last.close().unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert!(reader.read_frame().await.unwrap().is_none()); } @@ -17676,7 +17781,7 @@ mod test { for again_fails in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); @@ -17685,14 +17790,14 @@ mod test { // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 2) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 2) else { panic!("the re-offer takes the group"); }; - let Ok(Received::Held(next)) = producer.create_group_at(group::Info { sequence: 3 }, 2) else { + let Ok(Received::Held(next)) = producer.create_group_at(group::Info { start: 3 }, 2) else { panic!("another re-offer waits on the first"); }; drop(hold); @@ -17704,7 +17809,7 @@ mod test { if !again_fails { drop(next); again.write_frame(at(302), &b"c"[..]).unwrap(); - again.finish().unwrap(); + again.close().unwrap(); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); @@ -17714,14 +17819,14 @@ mod test { again.abort(Error::Cancel).unwrap(); let res = reader.poll_finished(&waiter); assert!(res.is_pending(), "waits on the next re-offer: {res:?}"); - let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut last)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the next re-offer takes the group"); }; drop(next); for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { last.write_frame(at(millis), &payload[..]).unwrap(); } - last.finish().unwrap(); + last.close().unwrap(); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); @@ -17738,7 +17843,7 @@ mod test { for start in [2, 3] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); @@ -17747,13 +17852,13 @@ mod test { // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); let waiter = kio::Waiter::noop(); assert!(reader.poll_finished(&waiter).is_pending(), "{start}: waits on the hold"); - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, start) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, start) else { panic!("the re-offer takes the group"); }; drop(hold); @@ -17771,7 +17876,7 @@ mod test { again.handle().poll_unwanted(&waiter).is_pending(), "the re-offer stays wanted" ); - again.finish().unwrap(); + again.close().unwrap(); // From where the reader stands, with frame 1 still to read. assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"b"); @@ -17788,7 +17893,7 @@ mod test { let producer = received_producer(Info::timed()); let mut subscriber = producer.subscribe(None); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); @@ -17798,16 +17903,16 @@ mod test { // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 2) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 2) else { panic!("the re-offer takes the group"); }; drop(hold); again.write_frame(at(302), &b"c"[..]).unwrap(); - again.finish().unwrap(); + again.close().unwrap(); assert_eq!(producer.demand().wanted(), Some(0), "not read to its end yet"); assert_eq!(reader.finished().await.unwrap(), 1); assert_eq!(producer.demand().wanted(), None, "the re-offer answered the end"); @@ -17821,7 +17926,7 @@ mod test { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); // Received from frame 1, so a fetch of the group's head can sit beside it. - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 1) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 1) else { panic!("the first copy opens"); }; first.write_frame(at(301), &b"b"[..]).unwrap(); @@ -17846,7 +17951,7 @@ mod test { for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { head.write_frame(at(millis), &payload[..]).unwrap(); } - head.finish().unwrap(); + head.close().unwrap(); first.abort(Error::Cancel).unwrap(); // An uncapped reader needs frames past the head, which no copy holds. assert!(reader.clone().with_end(None).read_frame().await.is_err()); @@ -17881,7 +17986,7 @@ mod test { for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { head.write_frame(at(millis), &payload[..]).unwrap(); } - head.finish().unwrap(); + head.close().unwrap(); // Takes frames 1 and 2 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); let mut reader = reader.with_end(None); @@ -17904,7 +18009,7 @@ mod test { async fn a_retained_end_check_follows_a_lifted_cap() { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(300), &b"a"[..]).unwrap(); @@ -17912,7 +18017,7 @@ mod test { let mut reader = first.consume().with_range(..3); // Takes frame 1 into its prefetch with frame 0. assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); @@ -17936,7 +18041,7 @@ mod test { let waiter = kio::Waiter::noop(); assert!(reader.poll_finished(&waiter).is_pending(), "the cut copy is still open"); let mut reader = reader.with_range(..); - cut.finish().unwrap(); + cut.close().unwrap(); let res = reader.finished().await; assert!(res.is_err(), "frame 3 is in no copy: {res:?}"); } @@ -17949,7 +18054,7 @@ mod test { for chunked in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; let mut reader = first.consume(); @@ -17958,7 +18063,7 @@ mod test { let mut frame = first .create_frame(frame::Info { size: 1, - timestamp: Some(at(300)), + timestamp: at(300), }) .unwrap(); let mut part = reader.next_frame().await.unwrap().expect("frame 0 streams"); @@ -17971,7 +18076,7 @@ mod test { } reader.skip_to(3); - let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); @@ -17980,14 +18085,14 @@ mod test { reader.poll_read_frame(&waiter).is_pending(), "waits on the held re-offer" ); - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; drop(hold); for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c"), (303, b"d")] { again.write_frame(at(millis), &payload[..]).unwrap(); } - again.finish().unwrap(); + again.close().unwrap(); assert_eq!( &reader.read_frame().await.unwrap().unwrap().payload[..], b"d", @@ -18006,7 +18111,7 @@ mod test { for reoffer in [false, true] { let producer = received_producer(Info::timed()); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 1) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 1) else { panic!("the first copy opens"); }; first.write_frame(at(301), &b"b"[..]).unwrap(); @@ -18031,7 +18136,7 @@ mod test { for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c")] { head.write_frame(at(millis), &payload[..]).unwrap(); } - head.finish().unwrap(); + head.close().unwrap(); first.abort(Error::Cancel).unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"c"); @@ -18044,13 +18149,13 @@ mod test { assert!(res.is_err(), "nor is the group's end: {res:?}"); continue; } - let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut again)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer takes the group"); }; for (millis, payload) in [(300, b"a"), (301, b"b"), (302, b"c"), (303, b"d")] { again.write_frame(at(millis), &payload[..]).unwrap(); } - again.finish().unwrap(); + again.close().unwrap(); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"d"); assert!(reader.read_frame().await.unwrap().is_none()); // Its end, from where that reader stood, on the copy holding the rest. @@ -18067,14 +18172,14 @@ mod test { let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(500)))); let at = |millis| Timestamp::from_millis(millis).unwrap(); - let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Group(mut first)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the first copy opens"); }; first.write_frame(at(0), &b"a"[..]).unwrap(); let mut reader = subscriber.recv_group().await.unwrap().expect("group 3"); assert_eq!(&reader.read_frame().await.unwrap().unwrap().payload[..], b"a"); - let Ok(Received::Held(_hold)) = producer.create_group_at(group::Info { sequence: 3 }, 0) else { + let Ok(Received::Held(_hold)) = producer.create_group_at(group::Info { start: 3 }, 0) else { panic!("the re-offer waits on the copy still arriving"); }; first.abort(Error::Cancel).unwrap(); @@ -18083,9 +18188,9 @@ mod test { // Group 4 bounds group 3's reach at 1s, and group 5 puts the edge a second past it. let mut later = Vec::new(); - for (sequence, millis) in [(4, 1000), (5, 2000)] { - let Ok(Received::Group(mut next)) = producer.create_group_at(group::Info { sequence }, 0) else { - panic!("group {sequence} opens"); + for (start, millis) in [(4, 1000), (5, 2000)] { + let Ok(Received::Group(mut next)) = producer.create_group_at(group::Info { start }, 0) else { + panic!("group {start} opens"); }; next.write_frame(at(millis), &b"x"[..]).unwrap(); later.push(next); @@ -18098,16 +18203,16 @@ mod test { } } - /// Create and finish group `sequence`, with one frame at `millis` on a timed track. + /// Create and finish group `key`, with one frame at `millis` on a timed track. #[cfg(feature = "async")] - fn put(producer: &Producer, sequence: u64, millis: Option) { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + fn put(producer: &Producer, key: u64, millis: Option) { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); if let Some(millis) = millis { group .write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } /// A received group reordered behind its successor is still delivered in order: the @@ -18163,9 +18268,9 @@ mod test { elapse(&producer, Duration::from_millis(1)); assert!(woken.load(Ordering::SeqCst), "the budget never woke the reader"); assert_eq!(drain_ordered(&mut consumer), [2]); - for sequence in 3..10 { - put(&producer, sequence, None); - assert_eq!(drain_ordered(&mut consumer), [sequence]); + for key in 3..10 { + put(&producer, key, None); + assert_eq!(drain_ordered(&mut consumer), [key]); } } @@ -18181,7 +18286,7 @@ mod test { put(&producer, 5, Some(5000)); assert_eq!(drain_ordered(&mut consumer), [5]); - let mut later = producer.create_group(group::Info { sequence: 7 }).unwrap(); + let mut later = producer.create_group(group::Info { start: 7 }).unwrap(); later .write_frame(Timestamp::from_millis(7000).unwrap(), &b"start"[..]) .unwrap(); @@ -18212,7 +18317,7 @@ mod test { .ordered(); put(&producer, 0, Some(0)); assert_eq!(drain_ordered(&mut consumer), [0]); - let mut edge = producer.create_group(group::Info { sequence: 4 }).unwrap(); + let mut edge = producer.create_group(group::Info { start: 4 }).unwrap(); edge.write_frame(Timestamp::from_millis(4000).unwrap(), &b"edge"[..]) .unwrap(); @@ -18237,7 +18342,7 @@ mod test { .ordered(); put(&producer, 0, None); assert_eq!(drain_ordered(&mut consumer), [0]); - let opener = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let opener = producer.create_group(group::Info { start: 2 }).unwrap(); elapse(&producer, Duration::from_millis(40)); put(&producer, 3, None); opener.abort(Error::Cancel).unwrap(); @@ -18268,7 +18373,7 @@ mod test { put(&producer, 4, None); assert_eq!(drain_ordered(&mut consumer), [4], "groups 1 to 3 were dropped"); - let aborted = producer.create_group(group::Info { sequence: 5 }).unwrap(); + let aborted = producer.create_group(group::Info { start: 5 }).unwrap(); aborted.abort(Error::Cancel).unwrap(); producer.modify().unwrap().reclaim_aborted(5); assert!(!producer.state.read().lookup.contains_key(&5), "the slot was reclaimed"); @@ -18335,9 +18440,9 @@ mod test { // Seq 5 arrives first. producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); let group = consumer .next_group() @@ -18345,25 +18450,25 @@ mod test { .expect("should not block") .expect("would have errored") .expect("track should not be closed"); - assert_eq!(group.sequence, 5); + assert_eq!(group.start, 5); // Seq 3 arrives late, skipped because 3 <= 5. producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); // Seq 4 arrives late and is also skipped. producer - .create_group(group::Info { sequence: 4 }) + .create_group(group::Info { start: 4 }) .unwrap() - .finish() + .close() .unwrap(); // Seq 7 arrives and is returned. producer - .create_group(group::Info { sequence: 7 }) + .create_group(group::Info { start: 7 }) .unwrap() - .finish() + .close() .unwrap(); let group = consumer @@ -18372,12 +18477,12 @@ mod test { .expect("should not block") .expect("would have errored") .expect("track should not be closed"); - assert_eq!(group.sequence, 7); + assert_eq!(group.start, 7); // No more groups. This would block. assert!( consumer.next_group().now_or_never().is_none(), - "should block waiting for a higher sequence" + "should block waiting for a higher key" ); } @@ -18389,14 +18494,14 @@ mod test { // Seq 3 arrives first, then seq 5. Both should be returned in arrival order. producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); let group = consumer @@ -18405,7 +18510,7 @@ mod test { .expect("should not block") .expect("would have errored") .expect("track should not be closed"); - assert_eq!(group.sequence, 3); + assert_eq!(group.start, 3); let group = consumer .next_group() @@ -18413,25 +18518,25 @@ mod test { .expect("should not block") .expect("would have errored") .expect("track should not be closed"); - assert_eq!(group.sequence, 5); + assert_eq!(group.start, 5); } - /// The sequence a peek reserves, without taking it. + /// The key a peek reserves, without taking it. fn peek(subscriber: &mut Subscriber, ordered: bool) -> Poll> { subscriber .poll_peek_group(ordered, &kio::Waiter::noop()) - .map(|res| res.unwrap().map(|reserved| reserved.sequence())) + .map(|res| res.unwrap().map(|reserved| reserved.key())) } #[test] - fn skipping_another_copys_groups_preserves_arrival_bounds_and_the_last_sequence() { + fn skipping_another_copys_groups_preserves_arrival_bounds_and_the_last_key() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); - for sequence in [u64::MAX - 2, u64::MAX - 1] { + for key in [u64::MAX - 2, u64::MAX - 1] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let _finished = producer.finish().unwrap(); @@ -18446,8 +18551,8 @@ mod test { let Inner::Plain(cursor) = &subscriber.inner else { unreachable!() }; - assert_eq!(cursor.min_sequence, 0); - assert_eq!(cursor.end_sequence, None); + assert_eq!(cursor.min_key, 0); + assert_eq!(cursor.end_key, None); assert_eq!(cursor.index, 0); assert_eq!(peek(&mut subscriber, false), Poll::Ready(Some(u64::MAX - 2))); } @@ -18462,9 +18567,9 @@ mod test { let Poll::Ready(Ok(Some(reserved))) = subscriber.poll_peek_group(true, &kio::Waiter::noop()) else { panic!("group 3 is servable"); }; - assert_eq!(reserved.sequence(), 3); + assert_eq!(reserved.key(), 3); subscriber.skip_ordered(2); - assert_eq!(subscriber.take(reserved).unwrap().sequence, 3); + assert_eq!(subscriber.take(reserved).unwrap().start, 3); assert_eq!(subscriber.take_stale().groups, 1, "only group 2 was skipped for age"); assert_eq!(subscriber.take_stale().groups, 0, "counted once"); } @@ -18476,11 +18581,11 @@ mod test { for raise_floor in [false, true] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); - for sequence in [1, 2] { + for key in [1, 2] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let waiter = kio::Waiter::noop(); @@ -18490,7 +18595,7 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { panic!("group 2 is inside the new floor"); }; - assert_eq!(group.sequence, 2); + assert_eq!(group.start, 2); } else { subscriber.set_range(..0); assert!(subscriber.poll_group(ordered, &waiter).is_pending()); @@ -18498,7 +18603,7 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { panic!("group 1 was parked, not consumed"); }; - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); } } } @@ -18509,11 +18614,11 @@ mod test { for ordered in [false, true] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); - for sequence in [1, 3] { + for key in [1, 3] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let Poll::Ready(Ok(Some(reserved))) = subscriber.poll_peek_group(ordered, &kio::Waiter::noop()) else { @@ -18521,7 +18626,7 @@ mod test { }; evict(&producer, 1); assert_eq!( - subscriber.take(reserved).unwrap().sequence, + subscriber.take(reserved).unwrap().start, 1, "ordered={ordered}: eviction must not substitute group 3" ); @@ -18534,11 +18639,11 @@ mod test { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); subscriber.set_range(..0); - for sequence in [1, 2] { + for key in [1, 2] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let waiter = kio::Waiter::noop(); @@ -18549,7 +18654,7 @@ mod test { } let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(false, &waiter) { - read.push(group.sequence); + read.push(group.start); } assert_eq!(read, [1, 2]); } @@ -18559,11 +18664,11 @@ mod test { for ordered in [false, true] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); - for sequence in [2, 1] { + for key in [2, 1] { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let waiter = kio::Waiter::noop(); @@ -18574,7 +18679,7 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_group(ordered, &waiter) else { panic!("the peeked group is still available"); }; - assert_eq!(group.sequence, first); + assert_eq!(group.start, first); let second = if ordered { 2 } else { 1 }; subscriber.set_range(..0); assert!(subscriber.poll_peek_group(ordered, &waiter).is_pending()); @@ -18592,17 +18697,17 @@ mod test { // Out-of-order arrivals: seq 5 first, then seq 3. producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); - // The ordered handle returns the smallest sequence first, regardless of + // The ordered handle returns the smallest key first, regardless of // arrival order. let group = ordered .next_group() @@ -18610,10 +18715,10 @@ mod test { .expect("should not block") .expect("would have errored") .expect("track should not be closed"); - assert_eq!(group.sequence, 3); + assert_eq!(group.start, 3); // The plain handle walks arrivals, so it still starts at seq 5. - assert_eq!(arrival.assert_group().sequence, 5); + assert_eq!(arrival.assert_group().start, 5); } #[test] @@ -18624,27 +18729,18 @@ mod test { for s in 0..6 { producer - .create_group(group::Info { sequence: s }) + .create_group(group::Info { start: s }) .unwrap() - .finish() + .close() .unwrap(); } consumer.set_range(..3); // Groups 0, 1, 2 are within the cap. - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 1 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 2 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 0); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 1); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 2); // Group 3 is beyond the cap: next_group parks even though cached groups exist. assert!( @@ -18661,45 +18757,27 @@ mod test { for s in 0..6 { producer - .create_group(group::Info { sequence: s }) + .create_group(group::Info { start: s }) .unwrap() - .finish() + .close() .unwrap(); } consumer.set_range(..2); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 1 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 0); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 1); assert!(consumer.next_group().now_or_never().is_none(), "capped at 2"); // Raise the cap; previously-blocked cached groups become available again. consumer.set_range(..5); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 2 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 3 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 4 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 2); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 3); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 4); assert!(consumer.next_group().now_or_never().is_none(), "capped at 5"); // Remove the cap; everything remaining flows. consumer.set_range(..); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 5 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 5); assert!(consumer.next_group().now_or_never().is_none(), "no more groups"); } @@ -18711,37 +18789,28 @@ mod test { for s in 0..3 { producer - .create_group(group::Info { sequence: s }) + .create_group(group::Info { start: s }) .unwrap() - .finish() + .close() .unwrap(); } // Drain everything with no cap. - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 1 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 2 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 0); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 1); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 2); // Lower the cap below the cursor. New groups beyond the cap are blocked. consumer.set_range(..2); producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 4 }) + .create_group(group::Info { start: 4 }) .unwrap() - .finish() + .close() .unwrap(); assert!( consumer.next_group().now_or_never().is_none(), @@ -18750,14 +18819,8 @@ mod test { // Restoring the cap to no-limit (or any value >= cursor) releases them. consumer.set_range(..); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 3 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 4 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 3); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 4); } #[test] @@ -18770,58 +18833,43 @@ mod test { // Out-of-order arrivals all within the cap. producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 5 }) + .create_group(group::Info { start: 5 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); // One beyond the cap; should be held even though it arrived in the middle. producer - .create_group(group::Info { sequence: 8 }) + .create_group(group::Info { start: 8 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 4 }) + .create_group(group::Info { start: 4 }) .unwrap() - .finish() + .close() .unwrap(); - // next_group walks in sequence order through everything <= cap. - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 2 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 3 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 4 - ); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 5 - ); + // next_group walks in key order through everything <= cap. + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 2); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 3); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 4); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 5); // Now blocked: 8 is still beyond the cap. assert!(consumer.next_group().now_or_never().is_none()); // Raise the cap; cached seq 8 is finally served. consumer.set_range(..11); - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 8 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 8); } /// `recv_group` (arrival order) honors the `end_at` cap like `next_group`: a group @@ -18835,31 +18883,22 @@ mod test { for s in 0..3 { producer - .create_group(group::Info { sequence: s }) + .create_group(group::Info { start: s }) .unwrap() - .finish() + .close() .unwrap(); } consumer.set_range(..2); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 1 - ); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 1); assert!(consumer.recv_group().now_or_never().is_none(), "capped at 2"); consumer.set_range(..); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 2 - ); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2); consumer.set_range(..3); - let _open = producer.create_group(group::Info { sequence: 3 }).unwrap(); + let _open = producer.create_group(group::Info { start: 3 }).unwrap(); assert!(consumer.recv_group().now_or_never().is_none(), "capped at 3"); let _producer = producer.finish().unwrap(); assert!( @@ -18880,36 +18919,27 @@ mod test { // Reordered burst: the beyond-cap group arrives first. producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 1 - ); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 1); assert!(consumer.recv_group().now_or_never().is_none(), "capped at 2"); consumer.set_range(..3); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 2 - ); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2); } #[kio::test] @@ -18919,35 +18949,35 @@ mod test { let mut consumer = producer.subscribe(None); consumer.set_range(2..=2); producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(consumer.recv_group().await.unwrap().unwrap().sequence, 2); + assert_eq!(consumer.recv_group().await.unwrap().unwrap().start, 2); consumer.set_range(..4); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 3 }) + .create_group(group::Info { start: 3 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(consumer.recv_group().await.unwrap().unwrap().sequence, 3); + assert_eq!(consumer.recv_group().await.unwrap().unwrap().start, 3); consumer.set_range(0..=4); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 4 }) + .create_group(group::Info { start: 4 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(consumer.recv_group().await.unwrap().unwrap().sequence, 4); + assert_eq!(consumer.recv_group().await.unwrap().unwrap().start, 4); } /// A raised `start_at` drops parked groups it overtook instead of re-offering @@ -18960,9 +18990,9 @@ mod test { consumer.set_range(..1); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); assert!( consumer.recv_group().now_or_never().is_none(), @@ -18971,12 +19001,12 @@ mod test { consumer.set_range(2..); producer - .create_group(group::Info { sequence: 2 }) + .create_group(group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, + consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2, "the overtaken parked group is dropped, not re-offered" ); @@ -18992,17 +19022,14 @@ mod test { let mut consumer = producer.subscribe(None); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); consumer.set_range(..1); - let straggler = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let straggler = producer.create_group(group::Info { start: 1 }).unwrap(); assert!( consumer.recv_group().now_or_never().is_none(), "group 1 parked at the cap" @@ -19026,14 +19053,14 @@ mod test { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); consumer.set_range(..0); @@ -19043,10 +19070,7 @@ mod test { ); consumer.set_range(..1); - assert_eq!( - consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); + assert_eq!(consumer.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); assert!(consumer.recv_group().now_or_never().is_none(), "group 1 stays parked"); } @@ -19062,20 +19086,14 @@ mod test { for s in 0..3 { producer - .create_group(group::Info { sequence: s }) + .create_group(group::Info { start: s }) .unwrap() - .finish() + .close() .unwrap(); } assert_eq!( - everything - .recv_group() - .now_or_never() - .unwrap() - .unwrap() - .unwrap() - .sequence, + everything.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0 ); assert!( @@ -19084,12 +19102,12 @@ mod test { ); empty.set_range(..2); - assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, 0); - assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, 1); + assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 0); + assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 1); assert!(empty.recv_group().now_or_never().is_none(), "still capped at 2"); empty.set_range(..); - assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().sequence, 2); + assert_eq!(empty.recv_group().now_or_never().unwrap().unwrap().unwrap().start, 2); } /// A frame-limited exclusive end includes the last group and stops before that frame. @@ -19102,19 +19120,16 @@ mod test { consumer.set_range((Bound::Unbounded, end.group_end())); for s in 0..3u64 { - let mut group = producer.create_group(group::Info { sequence: s }).unwrap(); + let mut group = producer.create_group(group::Info { start: s }).unwrap(); for i in 0..3u8 { - group.write_frame(Timestamp::ZERO, vec![i]).unwrap(); + group.write_frame(group.key_time(), vec![i]).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } - assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, - 0 - ); + assert_eq!(consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, 0); let mut last = consumer.next_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(last.sequence, 1); + assert_eq!(last.start, 1); last = last.with_range(..end.frame); assert_eq!( last.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[0], @@ -19138,26 +19153,26 @@ mod test { /// nor an append reaches it, and an inclusive bound there withholds nothing. #[test] #[cfg(feature = "async")] - fn the_reserved_sequence_is_refused() { + fn the_reserved_key_is_refused() { let producer = track_producer("test", Info::timed()); let mut consumer = producer.subscribe(replay()).ordered(); consumer.set_range(..=u64::MAX); assert!(matches!( - producer.create_group(group::Info { sequence: u64::MAX }), + producer.create_group(group::Info { start: u64::MAX }), Err(Error::BoundsExceeded(_)) )); producer - .create_group(group::Info { sequence: u64::MAX - 1 }) + .create_group(group::Info { start: u64::MAX - 1 }) .unwrap() - .finish() + .close() .unwrap(); - assert!(matches!(producer.append_group(), Err(Error::BoundsExceeded(_)))); + assert!(matches!(producer.append_next(), Err(Error::BoundsExceeded(_)))); assert_eq!( - consumer.next_group().now_or_never().unwrap().unwrap().unwrap().sequence, + consumer.next_group().now_or_never().unwrap().unwrap().unwrap().start, u64::MAX - 1 ); - // The track's end lands on the reserved sequence. + // The track's end lands on the reserved key. let _finished = producer.finish().unwrap(); assert!(consumer.next_group().now_or_never().unwrap().unwrap().is_none()); } @@ -19166,19 +19181,22 @@ mod test { fn rescale_matches_the_track_timeline() { let timed = Info::timed(); let ms = Timestamp::from_micros(2_400).unwrap(); - let rescaled = timed.rescale(Some(ms)).unwrap().unwrap(); + let rescaled = timed.rescale(ms).unwrap(); assert_eq!(rescaled, Timestamp::from_millis(2).unwrap()); // Halfway rounds down too, as `@moq/net`'s `Track.rescale` does. - let half = timed.rescale(Timestamp::from_micros(1_500).ok()).unwrap().unwrap(); + let half = timed.rescale(Timestamp::from_micros(1_500).unwrap()).unwrap(); assert_eq!(half, Timestamp::from_millis(1).unwrap()); - assert!(matches!(timed.rescale(None), Err(Error::TimestampMismatch))); + assert!(matches!( + timed.rescale(Timestamp::unscaled(2)), + Err(Error::TimestampMismatch) + )); // (2^61-1) seconds overflows conversion to the milli-scale track. let huge = Timestamp::from_secs((1 << 61) - 1).unwrap(); - assert!(matches!(timed.rescale(Some(huge)), Err(Error::TimestampMismatch))); + assert!(matches!(timed.rescale(huge), Err(Error::TimestampMismatch))); let untimed = Info::default(); - assert_eq!(untimed.rescale(None).unwrap(), None); - assert!(matches!(untimed.rescale(Some(ms)), Err(Error::TimestampMismatch))); + assert_eq!(untimed.rescale(Timestamp::unscaled(2)).unwrap(), Timestamp::unscaled(2)); + assert!(matches!(untimed.rescale(ms), Err(Error::TimestampMismatch))); } #[test] @@ -19191,7 +19209,7 @@ mod test { Err(Error::FrameTooLarge) )); assert_eq!( - producer.latest_sequence(), + producer.latest_key(), None, "the rejected frame did not publish a group" ); @@ -19203,11 +19221,11 @@ mod test { let producer = track_producer("test", Info::timed()); // Produce a cached group. - let mut group = producer.append_group().unwrap(); // seq 0 + let mut group = producer.append_next().unwrap(); // seq 0 group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"hello")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"hello")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); // A cached group resolves immediately and never queues a request. `peek_group` // also returns it synchronously. @@ -19215,7 +19233,7 @@ mod test { let consumer = producer.consume(); assert!(consumer.peek_group(0).is_some()); let mut g = consumer.fetch_group(0, None).await.unwrap(); - assert_eq!(g.sequence, 0); + assert_eq!(g.start, 0); assert_eq!(&g.read_frame().await.unwrap().unwrap().payload[..], b"hello"); // Nothing was queued for the dynamic handler to serve. @@ -19241,18 +19259,18 @@ mod test { .now_or_never() .expect("should not block") .unwrap(); - assert_eq!(req.sequence(), 5); + assert_eq!(req.key(), 5); assert_eq!(req.priority(), 7); // Serve it by accepting the request; the fetch then resolves. let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"hi")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut g = pending.await.unwrap(); - assert_eq!(g.sequence, 5); + assert_eq!(g.start, 5); assert_eq!(&g.read_frame().await.unwrap().unwrap().payload[..], b"hi"); } @@ -19324,8 +19342,8 @@ mod test { .unwrap(); drop(req); assert!(fresh.poll_unwanted(&kio::Waiter::noop()).is_pending()); - fresh.accept(Info::timed()).unwrap().finish().unwrap(); - assert_eq!(retry.await.unwrap().sequence, 5); + fresh.accept(Info::timed()).unwrap().close().unwrap(); + assert_eq!(retry.await.unwrap().start, 5); } /// A fetch abandoned before any handler took it is withdrawn from the queue too, so @@ -19369,8 +19387,8 @@ mod test { .now_or_never() .expect("the retry queues a fresh request") .unwrap(); - fresh.accept(Info::timed()).unwrap().finish().unwrap(); - assert_eq!(retry.await.unwrap().sequence, 5); + fresh.accept(Info::timed()).unwrap().close().unwrap(); + assert_eq!(retry.await.unwrap().start, 5); } /// A handler that accepts after every caller left still caches the group, so a fetch @@ -19399,9 +19417,9 @@ mod test { let mut group = req.accept(Info::timed()).expect("a withdrawn request still accepts"); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"hi")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut fetched = retry.await.unwrap(); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"hi"); @@ -19435,7 +19453,7 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); drop(first.await.unwrap()); @@ -19467,9 +19485,9 @@ mod test { assert_eq!((fill.frame_start(), fill.frame_end()), (0, Some(4))); let mut prefix = fill.accept(Info::timed()).unwrap(); for _ in 0..4 { - prefix.write_frame(Timestamp::ZERO, b"prefix".as_slice()).unwrap(); + prefix.write_frame(prefix.key_time(), b"prefix".as_slice()).unwrap(); } - prefix.finish().unwrap(); + prefix.close().unwrap(); let mut bounded = bounded.await.unwrap().with_range(2..).with_end(Some(4)); for expected in 2..4 { assert_eq!(bounded.index(), expected); @@ -19494,9 +19512,9 @@ mod test { assert_eq!((fill.frame_start(), fill.frame_end()), (0, None)); let mut complete = fill.accept(Info::timed()).unwrap(); for _ in 0..6 { - complete.write_frame(Timestamp::ZERO, b"prefix".as_slice()).unwrap(); + complete.write_frame(complete.key_time(), b"prefix".as_slice()).unwrap(); } - complete.finish().unwrap(); + complete.close().unwrap(); let mut whole = whole.await.unwrap(); for _ in 0..6 { assert!(whole.read_frame().await.unwrap().is_some()); @@ -19512,9 +19530,9 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(200).unwrap(); - live.write_frame(Timestamp::ZERO, b"live".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); for start in (1..100).rev() { let fetched = consumer.fetch_range( 0, @@ -19529,8 +19547,8 @@ mod test { let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (start, Some(start + 1))); let mut tail = fill.accept(Info::timed()).unwrap(); - tail.write_frame(Timestamp::ZERO, b"tail".as_slice()).unwrap(); - tail.finish().unwrap(); + tail.write_frame(tail.key_time(), b"tail".as_slice()).unwrap(); + tail.close().unwrap(); assert_eq!(fetched.await.unwrap().index(), start); assert_eq!(producer.state.read().lookup[&0].copies().count(), 2); } @@ -19554,9 +19572,9 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(200).unwrap(); - live.write_frame(Timestamp::ZERO, b"live".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); let range = |start, end| FetchOptions { options: group::Fetch { frame_start: start, @@ -19569,9 +19587,9 @@ mod test { assert_eq!((fill.frame_start(), fill.frame_end()), (frames.start, Some(frames.end))); let mut group = fill.accept(Info::timed()).unwrap(); for _ in frames { - group.write_frame(Timestamp::ZERO, b"tail".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"tail".as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); }; let later = consumer.fetch_range(0, range(4, 10)); fill(4..10).await; @@ -19589,14 +19607,14 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(200).unwrap(); - live.write_frame(Timestamp::ZERO, b"live".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); let fill = async |start: u64, end: Option| { let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!((fill.frame_start(), fill.frame_end()), (start, end)); let mut group = fill.accept(Info::timed()).unwrap(); - group.write_frame(Timestamp::ZERO, b"tail".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"tail".as_slice()).unwrap(); group }; let tail = consumer.fetch_group( @@ -19614,7 +19632,7 @@ mod test { end: Some(1), }, ); - fill(0, Some(1)).await.finish().unwrap(); + fill(0, Some(1)).await.close().unwrap(); assert_eq!(head.await.unwrap().index(), 0); assert_eq!(tail.await.unwrap().index(), 5); let mut sub = producer.subscribe(None); @@ -19643,19 +19661,19 @@ mod test { (frames.options.frame_start, frames.end) ); let mut group = fill.accept(Info::timed()).unwrap(); - group.write_frame(Timestamp::ZERO, b"copy".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"copy".as_slice()).unwrap(); group }; - let fetch = async |sequence, frames: FetchOptions| { - let pending = consumer.fetch_range(sequence, frames.clone()); + let fetch = async |key, frames: FetchOptions| { + let pending = consumer.fetch_range(key, frames.clone()); let held = fill(frames).await; (held, pending.await.unwrap()) }; let mut lives = Vec::new(); - for sequence in 0..2 { - let mut live = producer.create_group(group::Info { sequence }).unwrap(); + for key in 0..2 { + let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(200).unwrap(); - live.write_frame(Timestamp::ZERO, b"live".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); lives.push(live); } // A bounded copy beside a cached whole tail never enters the cache. @@ -19674,7 +19692,7 @@ mod test { producer.abort(Error::Cancel).unwrap(); for (mut held, mut fetched) in detached { assert!(matches!(fetched.read_frame().now_or_never(), Some(Err(Error::Cancel)))); - assert!(held.write_frame(Timestamp::ZERO, b"late".as_slice()).is_err()); + assert!(held.write_frame(held.key_time(), b"late".as_slice()).is_err()); } } @@ -19700,9 +19718,9 @@ mod test { .accept(Info::timed()) .unwrap(); for _ in 0..2 { - group.write_frame(Timestamp::ZERO, b"copy".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"copy".as_slice()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let mut fetched = fetch.await.unwrap(); producer.abort(Error::Cancel).unwrap(); @@ -19729,16 +19747,16 @@ mod test { let fill = dynamic.requested_fill_group().await.unwrap(); assert_eq!(fill.frame_end(), Some(1)); let mut group = fill.accept(Info::timed()).unwrap(); - group.write_frame(Timestamp::ZERO, b"copy".as_slice()).unwrap(); + group.write_frame(group.key_time(), b"copy".as_slice()).unwrap(); // The end is reached while the copy is still being written, so only the abort // can find it settled. producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); producer.finish_at(Position::group(2), None).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); fetch.await.unwrap(); producer.abort(Error::Timeout).unwrap(); @@ -19757,9 +19775,9 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let consumer = producer.consume(); - let bounded = async |sequence: u64| { + let bounded = async |key: u64| { let fetch = consumer.fetch_range( - sequence, + key, FetchOptions { options: group::Fetch::default(), end: Some(1), @@ -19771,28 +19789,28 @@ mod test { .unwrap() .accept(Info::timed()) .unwrap(); - group.write_frame(Timestamp::ZERO, b"head".as_slice()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.key_time(), b"head".as_slice()).unwrap(); + group.close().unwrap(); fetch.await.unwrap(); }; let mut arrival = producer.subscribe(replay()); let mut ordered = producer.subscribe(replay()).ordered(); - for (sequence, err) in [(0, Error::App(5)), (1, Error::Cancel)] { - let mut live = producer.create_group(group::Info { sequence }).unwrap(); + for (key, err) in [(0, Error::App(5)), (1, Error::Cancel)] { + let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(200).unwrap(); - live.write_frame(Timestamp::ZERO, b"live".as_slice()).unwrap(); - bounded(sequence).await; + live.write_frame(live.key_time(), b"live".as_slice()).unwrap(); + bounded(key).await; live.abort(err).unwrap(); } // Fetched only, so it reaches the ordered reader alone. bounded(2).await; let cut = arrival.assert_group(); - assert_eq!(cut.sequence, 0); + assert_eq!(cut.start, 0); assert!(matches!(cut.with_range(200..).read_frame().await, Err(Error::App(5)))); arrival.assert_no_group(); let cut = ordered.next_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(cut.sequence, 0); + assert_eq!(cut.start, 0); assert!(matches!(cut.with_range(200..).read_frame().await, Err(Error::App(5)))); assert!( ordered.next_group().now_or_never().is_none(), @@ -19879,7 +19897,7 @@ mod test { .now_or_never() .expect("the retry queues a fresh request") .unwrap(); - req.accept(Info::timed()).unwrap().finish().unwrap(); + req.accept(Info::timed()).unwrap().close().unwrap(); assert!(retry.await.unwrap().read_frame().await.unwrap().is_none()); } @@ -19907,9 +19925,9 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"retry")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"retry")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut group = retry.await.unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"retry"); @@ -19926,12 +19944,12 @@ mod test { let consumer = producer.consume(); // The live subscription resumed mid-group, so only the tail is cached. - let mut group = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); group.start_at(3).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"tail")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"tail")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); // A fetch for the tail is covered and resolves from the cache. let fetch = consumer.fetch_group( @@ -19953,14 +19971,14 @@ mod test { ); let request = dynamic.requested_fill_group().await.unwrap(); - assert_eq!((request.sequence(), request.frame_start()), (0, 0)); + assert_eq!((request.key(), request.frame_start()), (0, 0)); // Serving it replaces the too-narrow entry rather than colliding with it. let mut whole = request.accept(Info::timed()).unwrap(); whole - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .write_frame(whole.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); - whole.finish().unwrap(); + whole.close().unwrap(); let mut served = fetch.await.unwrap(); assert_eq!(served.index(), 0); @@ -19980,16 +19998,16 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap(); for payload in [&b"snapshot"[..], b"delta"] { - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(payload)) + head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } assert_eq!(fetch.await.unwrap().index(), 0); @@ -20004,7 +20022,7 @@ mod test { // An abandoned fetch leaves the live group in its slot, still written by the feed. head.abort(Error::Cancel).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta2")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta2")) .unwrap(); let mut sub = producer.subscribe(None); let group = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); @@ -20025,11 +20043,11 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); - live.finish().unwrap(); + live.close().unwrap(); let mut sub = producer.subscribe(replay().with_range(0..1)); let reading = kio::clock::sim::spawn(async move { @@ -20047,10 +20065,10 @@ mod test { assert_eq!(request.frame_start(), 0); let mut head = request.accept(Info::timed()).unwrap(); for payload in [&b"snapshot"[..], b"delta"] { - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(payload)) + head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } - head.finish().unwrap(); + head.close().unwrap(); let (frames, ended) = reading.await.unwrap(); assert_eq!(frames, [&b"snapshot"[..], b"delta"]); assert!(ended, "the read ends with its range"); @@ -20064,20 +20082,20 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); - live.finish().unwrap(); - let mut newer = producer.create_group(group::Info { sequence: 1 }).unwrap(); + live.close().unwrap(); + let mut newer = producer.create_group(group::Info { start: 1 }).unwrap(); newer .write_frame(Timestamp::from_secs(1).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - newer.finish().unwrap(); + newer.close().unwrap(); let mut sub = producer.subscribe(Subscription::default().with_range(0..1)); let reading = kio::clock::sim::spawn(async move { - let group = sub.recv_group().await.unwrap().map(|group| group.sequence); + let group = sub.recv_group().await.unwrap().map(|group| group.start); (group, sub.recv_group().await.unwrap().is_none()) }); let request = kio::clock::timeout(std::time::Duration::from_secs(1), dynamic.requested_fill_group()) @@ -20086,10 +20104,10 @@ mod test { .unwrap(); let mut head = request.accept(Info::timed()).unwrap(); for payload in [&b"snapshot"[..], b"delta"] { - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(payload)) + head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } - head.finish().unwrap(); + head.close().unwrap(); assert_eq!(reading.await.unwrap(), (Some(0), true)); } @@ -20103,22 +20121,22 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); - live.finish().unwrap(); + live.close().unwrap(); let mut sub = producer.subscribe(Subscription::default().with_range(0..2)); assert!(sub.recv_group().now_or_never().is_none(), "the head is still asked"); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - assert_eq!(request.sequence(), 0); + assert_eq!(request.key(), 0); - let mut newer = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let mut newer = producer.create_group(group::Info { start: 1 }).unwrap(); newer .write_frame(Timestamp::from_millis(1).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - newer.finish().unwrap(); + newer.close().unwrap(); if evict { producer.modify().unwrap().evict_group(0, Error::Evicted); } @@ -20127,7 +20145,7 @@ mod test { .expect("group 1 timed out") .unwrap() .expect("group 1"); - assert_eq!(group.sequence, 1, "evict={evict}"); + assert_eq!(group.start, 1, "evict={evict}"); let end = kio::clock::timeout(std::time::Duration::from_secs(1), sub.recv_group()) .await .unwrap_or_else(|_| panic!("evict={evict}: the read never ended")) @@ -20144,11 +20162,15 @@ mod test { for (order, expected) in [(Order::Ascending, [0, 1]), (Order::Descending, [1, 0])] { let producer = track_producer("test", None); let dynamic = producer.requests(); - for sequence in 0..2 { - let mut live = producer.create_group(group::Info { sequence }).unwrap(); + for key in 0..2 { + let mut live = producer.create_group(group::Info { start: key }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(None, bytes::Bytes::from_static(b"delta")).unwrap(); - live.finish().unwrap(); + live.write_frame( + crate::Timestamp::unscaled(live.start), + bytes::Bytes::from_static(b"delta"), + ) + .unwrap(); + live.close().unwrap(); } let mut sub = producer.subscribe(replay().with_order(order).with_range(0..2)); @@ -20163,14 +20185,18 @@ mod test { assert_eq!(requests.len(), 2, "{order:?}: both heads are asked"); for request in requests { let mut head = request.accept(Info::default()).unwrap(); - head.write_frame(None, bytes::Bytes::from_static(b"snapshot")).unwrap(); - head.finish().unwrap(); + head.write_frame( + crate::Timestamp::unscaled(head.start), + bytes::Bytes::from_static(b"snapshot"), + ) + .unwrap(); + head.close().unwrap(); } let mut got = Vec::new(); for _ in 0..2 { let group = sub.recv_group().now_or_never().unwrap().unwrap().expect("a head"); - got.push(group.sequence); + got.push(group.start); } assert_eq!(got, expected, "{order:?}"); assert!( @@ -20187,22 +20213,30 @@ mod test { let producer = track_producer("test", None); let dynamic = producer.requests(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(None, bytes::Bytes::from_static(b"delta")).unwrap(); - live.finish().unwrap(); - framed(&producer, 1, 1).finish().unwrap(); + live.write_frame( + crate::Timestamp::unscaled(live.start), + bytes::Bytes::from_static(b"delta"), + ) + .unwrap(); + live.close().unwrap(); + framed(&producer, 1, 1).close().unwrap(); let mut sub = producer.subscribe(replay().with_range(0..2)); let group = sub.recv_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); let request = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - assert_eq!(request.sequence(), 0); + assert_eq!(request.key(), 0); sub.update(replay().with_range(1..2)).unwrap(); let mut head = request.accept(Info::default()).unwrap(); - head.write_frame(None, bytes::Bytes::from_static(b"snapshot")).unwrap(); - head.finish().unwrap(); + head.write_frame( + crate::Timestamp::unscaled(head.start), + bytes::Bytes::from_static(b"snapshot"), + ) + .unwrap(); + head.close().unwrap(); assert!( matches!(sub.recv_group().now_or_never(), Some(Ok(None))), "the narrowed read ends without group 0" @@ -20217,17 +20251,21 @@ mod test { let producer = track_producer("test", None); let dynamic = producer.requests(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(None, bytes::Bytes::from_static(b"delta")).unwrap(); - live.finish().unwrap(); - framed(&producer, 1, 1).finish().unwrap(); + live.write_frame( + crate::Timestamp::unscaled(live.start), + bytes::Bytes::from_static(b"delta"), + ) + .unwrap(); + live.close().unwrap(); + framed(&producer, 1, 1).close().unwrap(); let mut sub = producer.subscribe(replay().with_range(0..2)).ordered(); let reading = kio::clock::sim::spawn(async move { let mut read = Vec::new(); while let Some(group) = sub.next_group().await.unwrap() { - read.push(group.sequence); + read.push(group.start); } read }); @@ -20248,9 +20286,9 @@ mod test { let consumer = producer.consume(); // The live copy holds frames 0 and 1, the fetched head frames 0 to 2. - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, b"delta".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"delta".as_slice()).unwrap(); let fetch = consumer.fetch_group(0, None); let mut head = dynamic .requested_fill_group() @@ -20264,7 +20302,7 @@ mod test { } else { b"delta".as_slice() }; - head.write_frame(Timestamp::ZERO, payload).unwrap(); + head.write_frame(head.key_time(), payload).unwrap(); } fetch.await.unwrap(); @@ -20273,7 +20311,7 @@ mod test { Err(Error::ProtocolViolation) )); assert_eq!(producer.finished(), None); - live.write_frame(Timestamp::ZERO, b"delta".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"delta".as_slice()).unwrap(); } /// A live group with a fetched head: group 0, fed from frame 1, plus a two-frame head. @@ -20282,19 +20320,19 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let fetch = consumer.fetch_group(0, None); let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap(); for payload in [&b"snapshot"[..], b"delta"] { - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(payload)) + head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } - let head = head.finish().unwrap(); + let head = head.close().unwrap(); assert_eq!(fetch.await.unwrap().index(), 0); (live, head) } @@ -20307,10 +20345,10 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - let mut group = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); group.start_at(1).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); let _fetch = consumer.fetch_group(0, None); let mut head = dynamic @@ -20320,14 +20358,14 @@ mod test { .accept(Info::timed()) .unwrap(); for payload in [&b"snapshot"[..], b"delta"] { - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(payload)) + head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } match live { Some(err) => group.abort(err).unwrap(), None => { - group.finish().unwrap(); + group.close().unwrap(); } } producer.finish_at(Position::group(1), None).unwrap(); @@ -20353,7 +20391,7 @@ mod test { "{live:?}: the head still writes" ); } - head.finish().unwrap(); + head.close().unwrap(); if polled { assert!( woken.load(Ordering::SeqCst), @@ -20442,9 +20480,9 @@ mod test { let noop = kio::Waiter::noop(); // The feed delivered group 1 and never group 0, which a fetch asks of the source. producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); let fetch = consumer.fetch_group(0, None); @@ -20455,7 +20493,7 @@ mod test { .accept(Info::timed()) .unwrap(); fetched - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"backfill")) + .write_frame(fetched.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); let woken = Arc::new(AtomicBool::new(false)); let waiter = kio::Waiter::new(futures::task::waker(Arc::new(FlagWake(woken.clone())))); @@ -20497,9 +20535,9 @@ mod test { let mut reader = consumer.clone(); let noop = kio::Waiter::noop(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); assert_eq!( reader.poll_largest(&noop), @@ -20510,7 +20548,7 @@ mod test { let request = dynamic.requested_fill_group().await.unwrap(); let mut head = request.accept(Info::timed()).unwrap(); for payload in [&b"snapshot"[..], b"delta"] { - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(payload)) + head.write_frame(head.key_time(), bytes::Bytes::from_static(payload)) .unwrap(); } @@ -20518,14 +20556,14 @@ mod test { let _producer = producer.finish().unwrap(); assert!(reader.poll_largest(&noop).is_pending(), "the head still writes"); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"more")) + head.write_frame(head.key_time(), bytes::Bytes::from_static(b"more")) .unwrap(); assert_eq!( reader.poll_largest(&noop), Poll::Ready(Some(Position { group: 0, frame: 2 })) ); - head.finish().unwrap(); + head.close().unwrap(); assert_eq!(reader.poll_largest(&noop), Poll::Ready(None)); } @@ -20557,7 +20595,7 @@ mod test { live.abort(Error::Cancel).unwrap(); // The next group demotes 0 into the eviction order and runs an expiry scan. - let _open = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let _open = producer.create_group(group::Info { start: 1 }).unwrap(); let mut group = producer.consume().peek_group(0).expect("the head still serves group 0"); assert_eq!( @@ -20587,13 +20625,13 @@ mod test { async fn reading_the_head_keeps_the_slot_from_expiring() { let producer = track_producer("test", Info::timed()); let (live, _head) = live_group_with_head(&producer).await; - live.finish().unwrap(); - let _open = producer.create_group(group::Info { sequence: 1 }).unwrap(); + live.close().unwrap(); + let _open = producer.create_group(group::Info { start: 1 }).unwrap(); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); // A fetch hit on the head, past the window the live copy was last written in. producer.consume().fetch_group(0, None).await.unwrap(); - let _open = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let _open = producer.create_group(group::Info { start: 2 }).unwrap(); assert!(producer.consume().peek_group(0).is_some(), "a read head keeps its slot"); } @@ -20607,11 +20645,11 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"delta")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"delta")) .unwrap(); - let live = live.finish().unwrap(); + let live = live.close().unwrap(); let fetch = consumer.fetch_group(0, None); let mut head = dynamic .requested_fill_group() @@ -20619,17 +20657,17 @@ mod test { .unwrap() .accept(Info::timed()) .unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 10_000])) + head.write_frame(head.key_time(), bytes::Bytes::from(vec![0u8; 10_000])) .unwrap(); - head.finish().unwrap(); + head.close().unwrap(); drop(fetch.await.unwrap()); if abort_live { live.abort(Error::Cancel).unwrap(); } - let mut next = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let mut next = producer.create_group(group::Info { start: 1 }).unwrap(); for _ in 0..30 { - next.write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 10_000])) + next.write_frame(next.key_time(), bytes::Bytes::from(vec![0u8; 10_000])) .unwrap(); } consumer.peek_group(0).is_none() @@ -20681,7 +20719,7 @@ mod test { assert_eq!(request.frame_start(), 2, "the in-flight range is already on the wire"); let whole = dynamic.requested_fill_group().await.unwrap(); assert_eq!( - (whole.sequence(), whole.frame_start()), + (whole.key(), whole.frame_start()), (0, 0), "the earlier frames are asked" ); @@ -20694,7 +20732,7 @@ mod test { // does; that offset is what makes the cached group too narrow for `widest`. group.start_at(2).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"from2")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"from2")) .unwrap(); // The two callers it covers resolve; the one it doesn't waits on its own fill @@ -20702,14 +20740,14 @@ mod test { assert_eq!(narrow.await.unwrap().index(), 5); assert_eq!(wider.await.unwrap().index(), 2); assert!(futures::poll!(widest.as_mut()).is_pending()); - group.finish().unwrap(); + group.close().unwrap(); let mut group = whole.accept(None).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"from0")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"from0")) .unwrap(); assert_eq!(widest.await.unwrap().index(), 0); - group.finish().unwrap(); + group.close().unwrap(); } #[kio::test] @@ -20740,7 +20778,7 @@ mod test { .now_or_never() .expect("should not block") .unwrap(); - assert_eq!(req.sequence(), 5); + assert_eq!(req.key(), 5); assert_eq!(req.priority(), 7); assert_eq!(req.max_delay(), Some(Duration::from_secs(10))); assert!( @@ -20754,13 +20792,13 @@ mod test { // One accept resolves all of them. let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"hi")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); - assert_eq!(first.await.unwrap().sequence, 5); - assert_eq!(second.await.unwrap().sequence, 5); - assert_eq!(third.await.unwrap().sequence, 5); + assert_eq!(first.await.unwrap().start, 5); + assert_eq!(second.await.unwrap().start, 5); + assert_eq!(third.await.unwrap().start, 5); } /// A looser caller that joined after a tighter budget went on the wire is not failed by @@ -20786,10 +20824,10 @@ mod test { assert_eq!(req.max_delay(), None); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"hi")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"hi")) .unwrap(); - group.finish().unwrap(); - assert_eq!(loose.await.unwrap().sequence, 5); + group.close().unwrap(); + assert_eq!(loose.await.unwrap().start, 5); } #[kio::test] @@ -20819,7 +20857,7 @@ mod test { .now_or_never() .expect("should not block") .unwrap(); - assert_eq!(req.sequence(), 5); + assert_eq!(req.key(), 5); } #[kio::test] @@ -20846,7 +20884,7 @@ mod test { // A track with no `Requests` can't serve old content, so a cache miss // resolves to NotFound instead of blocking forever. let producer = track_producer("test", Info::timed()); - let _open = producer.append_group().unwrap(); // seq 0, but we miss on seq 5 + let _open = producer.append_next().unwrap(); // seq 0, but we miss on seq 5 let consumer = producer.consume(); assert!(matches!(consumer.fetch_group(5, None).await, Err(Error::NotFound))); } @@ -20855,10 +20893,10 @@ mod test { #[cfg(feature = "async")] async fn fetch_past_final_not_found() { let producer = track_producer("test", Info::timed()); - let _open = producer.append_group().unwrap(); // seq 0 - producer.finish_shared().unwrap(); // final_sequence = 1 + let _open = producer.append_next().unwrap(); // seq 0 + producer.finish_shared().unwrap(); // final_key = 1 - // A group at or past the final sequence can never exist, even with a handler, + // A group at or past the final key can never exist, even with a handler, // so it resolves to NotFound. let dynamic = producer.requests(); let consumer = producer.consume(); @@ -20883,12 +20921,12 @@ mod test { } fn finished_group(producer: &mut Producer, size: usize) -> u64 { - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; size])) + .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; size])) .unwrap(); - let group = group.finish().unwrap(); - group.sequence + let group = group.close().unwrap(); + group.start } /// While the pool is over capacity, every append accrues debt and pays it by @@ -20915,7 +20953,7 @@ mod test { // A fresh subscriber skips the evicted groups entirely. let mut subscriber = producer.subscribe(replay()); - assert!(subscriber.assert_group().sequence > 0, "evicted group is not delivered"); + assert!(subscriber.assert_group().start > 0, "evicted group is not delivered"); } /// A plain reader inflating a finished compressed track pays the debt its inflated copies @@ -20933,21 +20971,21 @@ mod test { let frame = bytes::Bytes::from(vec![0u8; 256 * 1024]); let groups = 16; for _ in 0..groups { - let mut group = producer.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, frame.clone()).unwrap(); - group.finish().unwrap(); + let mut group = producer.append_next().unwrap(); + group.write_frame(group.key_time(), frame.clone()).unwrap(); + group.close().unwrap(); } assert!(pool.used() < capacity, "the slices fit: {}", pool.used()); let consumer = producer.consume(); - for sequence in 0..groups { - let Some(mut group) = consumer.peek_group(sequence) else { + for key in 0..groups { + let Some(mut group) = consumer.peek_group(key) else { continue; }; match group.read_frame().now_or_never().expect("cached") { Ok(Some(read)) => assert_eq!(read.payload.len(), frame.len()), - Ok(None) => panic!("group {sequence} ended without its frame"), - Err(err) => assert!(matches!(err, Error::Evicted), "group {sequence}: {err}"), + Ok(None) => panic!("group {key} ended without its frame"), + Err(err) => assert!(matches!(err, Error::Evicted), "group {key}: {err}"), } } // The live edge and the group being read stay, whatever the budget. @@ -21036,12 +21074,12 @@ mod test { let (mut producer, _) = pooled_producer(22 * unit); finished_group(&mut producer, 20 * unit as usize); - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); let mut reader = group.consume(); for _ in 0..3 { - group.write_frame(Timestamp::ZERO, vec![0u8; unit as usize]).unwrap(); + group.write_frame(group.key_time(), vec![0u8; unit as usize]).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); for _ in 0..3 { assert_eq!(reader.read_frame().await.unwrap().unwrap().payload.len(), unit as usize); } @@ -21091,9 +21129,9 @@ mod test { ..Default::default() }); let failed = Producer::new(broadcast.clone(), "failed", Info::timed()); - let mut group = failed.append_group().unwrap(); + let mut group = failed.append_next().unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 1_000])) + .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 1_000])) .unwrap(); let handoff = group.consume().handoff(); group.abort(crate::Error::Cancel).unwrap(); @@ -21165,14 +21203,14 @@ mod test { let mut backfill = req.accept(Info::timed()).unwrap(); pending.await.unwrap(); backfill - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 30_000])) + .write_frame(backfill.key_time(), bytes::Bytes::from(vec![0u8; 30_000])) .unwrap(); // Accept with a fresh Info: the pre-accept group's writes must still be // drained by this track's future charges. let producer = request.accept(Info::timed()); - producer.append_group().unwrap().finish().unwrap(); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert!( producer.consume().peek_group(0).is_none(), @@ -21181,12 +21219,12 @@ mod test { assert!(pool.used() <= 13_000, "usage converges: {}", pool.used()); } - /// Re-serving a sequence many times must not accumulate eviction hints: stale + /// Re-serving a key many times must not accumulate eviction hints: stale /// hints die on stamp mismatch and compaction reclaims them. #[test] - fn recreated_sequence_bounds_eviction_hints() { + fn recreated_key_bounds_eviction_hints() { let (producer, _pool) = pooled_producer(1 << 40); - producer.create_group(5u64.into()).unwrap().finish().unwrap(); + producer.create_group(5u64.into()).unwrap().close().unwrap(); for _ in 0..200 { let group = producer.create_group(1u64.into()).unwrap(); @@ -21212,7 +21250,7 @@ mod test { // No time advances: every stamp lands in the same tick. let (mut producer, _pool) = pooled_producer(10 * unit); - producer.append_group().unwrap().finish().unwrap(); // seq 0: empty + producer.append_next().unwrap().close().unwrap(); // seq 0: empty finished_group(&mut producer, 3 * unit as usize); // seq 1: written finished_group(&mut producer, 3 * unit as usize); // seq 2 finished_group(&mut producer, 3 * unit as usize); // seq 3 @@ -21228,13 +21266,13 @@ mod test { #[test] fn frame_only_writer_pays() { let (producer, pool) = pooled_producer(2_000); - let mut demoted = producer.append_group().unwrap(); // seq 0 - producer.append_group().unwrap().finish().unwrap(); // seq 1 demotes seq 0 + let mut demoted = producer.append_next().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 1 demotes seq 0 // One large frame crosses the charge threshold: the write itself pays, // with no further group insert on this track. demoted - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 300_000])) + .write_frame(demoted.key_time(), bytes::Bytes::from(vec![0u8; 300_000])) .unwrap(); assert!( @@ -21242,7 +21280,7 @@ mod test { "the frame write settled the debt: {}", pool.used() ); - assert!(matches!(demoted.finish(), Err(Error::Evicted))); + assert!(matches!(demoted.close(), Err(Error::Evicted))); } /// One `Info` describing several tracks must not join their eviction accounting: @@ -21298,9 +21336,9 @@ mod test { fn teardown_ignores_a_settling_group() { let (producer, pool) = pooled_producer(1 << 40); // Open, so only the abrupt teardown releases it. - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 100])) + .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); drop(group); @@ -21324,8 +21362,8 @@ mod test { #[test] fn cached_group_outlives_its_track() { let (mut producer, pool) = pooled_producer(1 << 40); - let sequence = finished_group(&mut producer, 100); - let group = producer.consume().peek_group(sequence).expect("cached"); + let key = finished_group(&mut producer, 100); + let group = producer.consume().peek_group(key).expect("cached"); let state = producer.state.downgrade(); let producer = producer.finish().unwrap(); drop(producer); @@ -21366,12 +21404,12 @@ mod test { // Accept, then demote the backfill with a live group. let producer = request.accept(Info::timed()); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); // No further insert: the late write into the demoted backfill is the only // thing that can pay the debt it just took on. backfill - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 300_000])) + .write_frame(backfill.key_time(), bytes::Bytes::from(vec![0u8; 300_000])) .unwrap(); assert!( @@ -21387,22 +21425,22 @@ mod test { #[test] fn write_restarts_retention_clock() { let (producer, _pool) = pooled_producer(1 << 40); - let mut straggler = producer.append_group().unwrap(); // seq 0 - producer.append_group().unwrap().finish().unwrap(); // seq 1 demotes seq 0 + let mut straggler = producer.append_next().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 1 demotes seq 0 // Idle past the window, then the straggler receives a late frame. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); straggler - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 100])) + .write_frame(straggler.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); - producer.append_group().unwrap().finish().unwrap(); // seq 2 runs expiry + producer.append_next().unwrap().close().unwrap(); // seq 2 runs expiry let consumer = producer.consume(); assert!(consumer.peek_group(0).is_some(), "the write restarted the clock"); // Once the writes stop, the group ages out normally. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); - producer.append_group().unwrap().finish().unwrap(); // seq 3 runs expiry + producer.append_next().unwrap().close().unwrap(); // seq 3 runs expiry assert!(consumer.peek_group(0).is_none(), "idle content still expires"); } @@ -21415,9 +21453,9 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - producer.create_group(10u64.into()).unwrap().finish().unwrap(); - for sequence in 1..=5u64 { - let pending = consumer.fetch_group(sequence, None); + producer.create_group(10u64.into()).unwrap().close().unwrap(); + for key in 1..=5u64 { + let pending = consumer.fetch_group(key, None); let req = dynamic .requested_fill_group() .now_or_never() @@ -21425,44 +21463,44 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 100])) + .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pending.await.unwrap(); } // Age everything out, then refresh the first four backfills so they sit // fresh at the front of the eviction order, hiding the expired fifth. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); - for sequence in 1..=4u64 { - consumer.fetch_group(sequence, None).await.unwrap(); + for key in 1..=4u64 { + consumer.fetch_group(key, None).await.unwrap(); } // The rotating cursor reaches the fifth entry within a few writes. for _ in 0..3 { - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); } assert!(consumer.peek_group(5).is_none(), "expired backfill is reclaimed"); assert!(consumer.peek_group(1).is_some(), "refreshed backfill survives"); } - /// A publisher re-creating an aborted sequence is delivered exactly once, at + /// A publisher re-creating an aborted key is delivered exactly once, at /// its actual arrival position: the historical arrival entry is dead. #[test] - fn recreated_sequence_delivered_once() { + fn recreated_key_delivered_once() { let (producer, _pool) = pooled_producer(1 << 40); - producer.create_group(0u64.into()).unwrap().finish().unwrap(); + producer.create_group(0u64.into()).unwrap().close().unwrap(); let aborted = producer.create_group(1u64.into()).unwrap(); aborted.abort(Error::Cancel).unwrap(); - producer.create_group(2u64.into()).unwrap().finish().unwrap(); - producer.create_group(1u64.into()).unwrap().finish().unwrap(); + producer.create_group(2u64.into()).unwrap().close().unwrap(); + producer.create_group(1u64.into()).unwrap().close().unwrap(); let mut subscriber = producer.subscribe(replay()); - assert_eq!(subscriber.assert_group().sequence, 0); - assert_eq!(subscriber.assert_group().sequence, 2); + assert_eq!(subscriber.assert_group().start, 0); + assert_eq!(subscriber.assert_group().start, 2); assert_eq!( - subscriber.assert_group().sequence, + subscriber.assert_group().start, 1, "replacement arrives at its own position" ); @@ -21475,8 +21513,8 @@ mod test { #[test] fn aborted_group_leaves_no_ghost_sample() { let (producer, pool) = pooled_producer(1 << 40); - let group0 = producer.append_group().unwrap(); - let _open = producer.append_group().unwrap(); // demotes seq 0 into the mean + let group0 = producer.append_next().unwrap(); + let _open = producer.append_next().unwrap(); // demotes seq 0 into the mean assert!(pool.average().is_some(), "demoted group is sampled"); group0.abort(Error::Cancel).unwrap(); @@ -21489,8 +21527,8 @@ mod test { fn empty_groups_repay_overhead() { let (producer, pool) = pooled_producer(1_000); for _ in 0..100 { - let group = producer.append_group().unwrap(); - group.finish().unwrap(); + let group = producer.append_next().unwrap(); + group.close().unwrap(); } assert!( @@ -21505,35 +21543,35 @@ mod test { #[test] fn growth_on_demoted_group_is_billed() { let (producer, pool) = pooled_producer(2_000); - let mut straggler = producer.append_group().unwrap(); // seq 0 - producer.append_group().unwrap().finish().unwrap(); // seq 1 demotes seq 0 + let mut straggler = producer.append_next().unwrap(); // seq 0 + producer.append_next().unwrap().close().unwrap(); // seq 1 demotes seq 0 // The demoted group balloons: no eviction yet (nothing ran), but billed. straggler - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 10_000])) + .write_frame(straggler.key_time(), bytes::Bytes::from(vec![0u8; 10_000])) .unwrap(); // The next append observes the growth and evicts the straggler. - producer.append_group().unwrap().finish().unwrap(); // seq 2 + producer.append_next().unwrap().close().unwrap(); // seq 2 let consumer = producer.consume(); assert!(consumer.peek_group(0).is_none(), "the ballooned group is evicted"); assert!(pool.used() <= 3_000, "growth is reclaimed: {}", pool.used()); } - /// A stale arrival entry whose sequence was later re-served by fetched backfill must + /// A stale arrival entry whose key was later re-served by fetched backfill must /// not put the replacement in the arrival walk, which a subscriber reads as backfill. #[kio::test] #[cfg(feature = "async")] - async fn a_refilled_sequence_stays_out_of_the_arrival_walk() { + async fn a_refilled_key_stays_out_of_the_arrival_walk() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); - producer.create_group(0u64.into()).unwrap().finish().unwrap(); + producer.create_group(0u64.into()).unwrap().close().unwrap(); let aborted = producer.create_group(1u64.into()).unwrap(); aborted.abort(Error::Cancel).unwrap(); - producer.create_group(2u64.into()).unwrap().finish().unwrap(); + producer.create_group(2u64.into()).unwrap().close().unwrap(); // Re-serve seq 1 as backfill; its slot replaces the aborted one, and the // old arrival entry for seq 1 now resolves to it. @@ -21545,12 +21583,12 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"backfill")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pending.await.unwrap(); - // The backfill serves by sequence, but never in arrival order. + // The backfill serves by key, but never in arrival order. assert!(consumer.peek_group(1).is_some()); let mut walked = Vec::new(); let mut index = 0; @@ -21559,14 +21597,14 @@ mod test { Window::new(&replay().ranges, None, &*state).unwrap() }; while let Poll::Ready(Ok(Some((group, next)))) = producer.state.read().poll_recv_group(index, 0, &window) { - walked.push(group.sequence); + walked.push(group.start); index = next + 1; } assert_eq!(walked, [0, 2]); // A reader takes it as the fill answer for the group the feed gave up, as it would // have asked a fill for it, and the stale arrival entry adds nothing. let mut subscriber = producer.subscribe(replay()); - let mut read: Vec<_> = (0..3).map(|_| subscriber.assert_group().sequence).collect(); + let mut read: Vec<_> = (0..3).map(|_| subscriber.assert_group().start).collect(); read.sort(); assert_eq!(read, [0, 1, 2]); subscriber.assert_no_group(); @@ -21581,9 +21619,9 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - producer.create_group(5u64.into()).unwrap().finish().unwrap(); - for sequence in [2u64, 3u64] { - let pending = consumer.fetch_group(sequence, None); + producer.create_group(5u64.into()).unwrap().close().unwrap(); + for key in [2u64, 3u64] { + let pending = consumer.fetch_group(key, None); let req = dynamic .requested_fill_group() .now_or_never() @@ -21591,9 +21629,9 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 100])) + .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 100])) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pending.await.unwrap(); } @@ -21601,7 +21639,7 @@ mod test { elapse(&producer, cache::DEFAULT_EXPIRY / 2 + Duration::from_secs(1)); consumer.fetch_group(2, None).await.unwrap(); elapse(&producer, cache::DEFAULT_EXPIRY / 2 + Duration::from_secs(1)); - producer.create_group(6u64.into()).unwrap().finish().unwrap(); + producer.create_group(6u64.into()).unwrap().close().unwrap(); let consumer = producer.consume(); assert!(consumer.peek_group(2).is_some(), "refreshed backfill survives"); @@ -21630,7 +21668,7 @@ mod test { assert!(consumer.peek_group(1).is_none(), "the unread group dies instead"); } - /// A refetched group that reclaims max_sequence is the live edge again: it must + /// A refetched group that reclaims max_key is the live edge again: it must /// not re-enter the eviction order, or memory pressure could evict the newest /// content. #[kio::test] @@ -21640,13 +21678,13 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - let straggler = producer.append_group().unwrap(); // seq 0 + let straggler = producer.append_next().unwrap(); // seq 0 - // The publisher aborts its own latest group; the sequence stays at the live edge. - let latest = producer.append_group().unwrap(); // seq 1 + // The publisher aborts its own latest group; the key stays at the live edge. + let latest = producer.append_next().unwrap(); // seq 1 latest.abort(Error::Cancel).unwrap(); - // Re-fetch it: the replacement takes over max_sequence. + // Re-fetch it: the replacement takes over max_key. let pending = consumer.fetch_group(1, None); let req = dynamic .requested_fill_group() @@ -21655,9 +21693,9 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 1000])) + .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 1000])) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pending.await.unwrap(); // The refetched latest is protected by omission: it has no entry in the @@ -21666,7 +21704,7 @@ mod test { let state = producer.state.read(); assert!(state.lookup.contains_key(&1), "refetched group is cached"); assert!( - state.evict.iter().all(|(sequence, _)| *sequence != 1), + state.evict.iter().all(|(key, _)| *key != 1), "the live edge must not be an eviction candidate" ); } @@ -21674,7 +21712,7 @@ mod test { } /// An evicted group is a cache miss, so a fetch re-fetches it and the accepted - /// replacement serves the sequence again (not `Error::Duplicate`). + /// replacement serves the key again (not `Error::Duplicate`). #[kio::test] #[cfg(feature = "async")] async fn eviction_allows_refetch() { @@ -21694,19 +21732,19 @@ mod test { .now_or_never() .expect("should not block") .unwrap(); - assert_eq!(req.sequence(), 0); + assert_eq!(req.key(), 0); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"refetched")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"refetched")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut group = pending.await.unwrap(); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"refetched"); } - /// An aborted group is dead whatever frame it starts at: the sequence is claimable + /// An aborted group is dead whatever frame it starts at: the key is claimable /// again, and a cache lookup misses rather than handing back a slot that can no /// longer serve it. /// @@ -21716,18 +21754,18 @@ mod test { /// because it died. That interleaving is what the single guard rules out; this /// pins the committed semantics it has to preserve. #[test] - fn an_aborted_group_releases_its_sequence() { + fn an_aborted_group_releases_its_key() { let producer = track_producer("test", Info::timed()); let consumer = producer.consume(); - let mut group = producer.create_group(group::Info { sequence: 3 }).unwrap(); + let mut group = producer.create_group(group::Info { start: 3 }).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"head")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"head")) .unwrap(); - // While it lives the slot answers, so the sequence is taken. + // While it lives the slot answers, so the key is taken. assert!(matches!( - producer.create_group(group::Info { sequence: 3 }), + producer.create_group(group::Info { start: 3 }), Err(Error::Duplicate) )); assert!(consumer.peek_group(3).is_some()); @@ -21736,9 +21774,9 @@ mod test { assert!(consumer.peek_group(3).is_none(), "an aborted slot is a cache miss"); producer - .create_group(group::Info { sequence: 3 }) - .expect("an aborted slot releases its sequence") - .finish() + .create_group(group::Info { start: 3 }) + .expect("an aborted slot releases its key") + .close() .unwrap(); } @@ -21751,9 +21789,9 @@ mod test { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); let consumer = producer.consume(); - producer.create_group(5u64.into()).unwrap().finish().unwrap(); + producer.create_group(5u64.into()).unwrap().close().unwrap(); let mut subscriber = producer.subscribe(replay().with_range(5..)); - assert_eq!(subscriber.assert_group().sequence, 5); + assert_eq!(subscriber.assert_group().start, 5); let pending = consumer.fetch_group(2, None); let req = dynamic @@ -21763,21 +21801,21 @@ mod test { .unwrap(); let mut fetched = req.accept(Info::timed()).unwrap(); fetched - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"backfill")) + .write_frame(fetched.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); - fetched.finish().unwrap(); + fetched.close().unwrap(); pending.await.unwrap(); - let _frameless = producer.create_group(group::Info { sequence: 3 }).unwrap(); + let _frameless = producer.create_group(group::Info { start: 3 }).unwrap(); subscriber.update(replay().with_range(0..)).unwrap(); - assert_eq!(subscriber.assert_group().sequence, 3, "offered before its first frame"); - assert_eq!(subscriber.assert_group().sequence, 2, "cached backfill is owed"); + assert_eq!(subscriber.assert_group().start, 3, "offered before its first frame"); + assert_eq!(subscriber.assert_group().start, 2, "cached backfill is owed"); subscriber.assert_no_group(); // A cursor floor lowered before the next read counts only walked groups as unread. let mut lowered = producer.subscribe(replay()); lowered.set_range(5..); - assert_eq!(lowered.assert_group().sequence, 5); + assert_eq!(lowered.assert_group().start, 5); lowered.set_range(0..); let delivered = super::super::delivered::Delivered::default(); let unread = Unread { @@ -21799,8 +21837,8 @@ mod test { let consumer = producer.consume(); // The publisher starts at seq 5; earlier groups exist only upstream. - producer.create_group(5u64.into()).unwrap().finish().unwrap(); - producer.create_group(6u64.into()).unwrap().finish().unwrap(); + producer.create_group(5u64.into()).unwrap().close().unwrap(); + producer.create_group(6u64.into()).unwrap().close().unwrap(); // Fetch the gap: it lands in the cache and resolves the fetch... let pending = consumer.fetch_group(2, None); @@ -21811,16 +21849,16 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"backfill")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"backfill")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut fetched = pending.await.unwrap(); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"backfill"); assert!(consumer.peek_group(2).is_some(), "backfill is cached for later fetches"); // ...and a subscriber reads it beside the live groups, asking no fill of its own. let mut subscriber = producer.subscribe(replay()); - let mut read: Vec<_> = (0..3).map(|_| subscriber.assert_group().sequence).collect(); + let mut read: Vec<_> = (0..3).map(|_| subscriber.assert_group().start).collect(); read.sort_unstable(); assert_eq!(read, [2, 5, 6]); subscriber.assert_no_group(); @@ -21853,7 +21891,7 @@ mod test { .unwrap(); let mut fetching = req.accept(Info::timed()).unwrap(); fetching - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"snapshot")) + .write_frame(fetching.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); let mut fetched = pending.await.unwrap(); subscriber.assert_no_group(); @@ -21861,17 +21899,17 @@ mod test { let Ok(Received::Group(mut live)) = producer.create_group_at(0u64.into(), 0) else { panic!("the feed takes the slot"); }; - live.write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"snapshot")) + live.write_frame(live.key_time(), bytes::Bytes::from_static(b"snapshot")) .unwrap(); let mut group = subscriber.assert_group(); - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"snapshot"); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"snapshot"); // The fetch giving up never cuts the group short for the subscriber. drop(fetched); fetching.abort(Error::Cancel).unwrap(); - live.finish().unwrap(); + live.close().unwrap(); assert!(group.read_frame().await.unwrap().is_none()); assert!(matches!( producer.create_group_at(0u64.into(), 0), @@ -21888,9 +21926,9 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - producer.create_group(5u64.into()).unwrap().finish().unwrap(); + producer.create_group(5u64.into()).unwrap().close().unwrap(); - // Serve a backfill fetch for an old sequence. + // Serve a backfill fetch for an old key. let pending = consumer.fetch_group(2, None); let req = dynamic .requested_fill_group() @@ -21899,15 +21937,15 @@ mod test { .unwrap(); let mut group = req.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![0u8; 1000])) + .write_frame(group.key_time(), bytes::Bytes::from(vec![0u8; 1000])) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pending.await.unwrap(); let used = pool.used(); // Age past the pool's LRU window; the next write reclaims the backfill. elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); - producer.create_group(6u64.into()).unwrap().finish().unwrap(); + producer.create_group(6u64.into()).unwrap().close().unwrap(); assert!(consumer.peek_group(2).is_none(), "expired backfill is reclaimed"); assert!(pool.used() < used, "its bytes are released"); @@ -21915,12 +21953,16 @@ mod test { /// Append a finished group of a 3 byte first frame and a `delta` byte delta. fn refresh_group(producer: &Producer, delta: usize) { - let mut group = producer.append_group().unwrap(); - group.write_frame(None, &b"key"[..]).unwrap(); + let mut group = producer.append_next().unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"key"[..]) + .unwrap(); if delta > 0 { - group.write_frame(None, vec![0u8; delta]).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), vec![0u8; delta]) + .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } /// Poll `demand` for a change without waiting. @@ -21959,7 +22001,7 @@ mod test { refresh_group(&producer, delta); let mut subscriber = producer.subscribe(None); assert_eq!(producer.demand().wanted(), Some(wanted), "{delta} byte delta"); - assert_eq!(subscriber.assert_group().sequence, 0, "{delta} byte delta"); + assert_eq!(subscriber.assert_group().start, 0, "{delta} byte delta"); } } @@ -21969,14 +22011,14 @@ mod test { fn a_group_read_keeps_its_track_wanted_until_its_end() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); - let mut writing = producer.append_group().unwrap(); - writing.write_frame(Timestamp::ZERO, &b"key"[..]).unwrap(); + let mut writing = producer.append_next().unwrap(); + writing.write_frame(writing.key_time(), &b"key"[..]).unwrap(); let mut group = subscriber.assert_group(); let other = group.clone(); drop(subscriber); assert_eq!(producer.demand().wanted(), Some(0), "the group is read"); - writing.finish().unwrap(); + writing.close().unwrap(); assert_eq!(producer.demand().wanted(), Some(0), "not read to its end yet"); let waiter = kio::Waiter::noop(); assert!(matches!(group.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); @@ -21995,8 +22037,8 @@ mod test { fn a_range_cap_is_not_the_groups_end() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(None); - let mut writing = producer.append_group().unwrap(); - writing.write_frame(Timestamp::ZERO, &b"key"[..]).unwrap(); + let mut writing = producer.append_next().unwrap(); + writing.write_frame(writing.key_time(), &b"key"[..]).unwrap(); let group = subscriber.assert_group(); let mut capped = group.clone().with_range(..1); drop(subscriber); @@ -22011,10 +22053,10 @@ mod test { .abort_unwanted(Error::Cancel) .expect_err("the capped reader can raise its cap"); - writing.write_frame(Timestamp::ZERO, &b"delta"[..]).unwrap(); + writing.write_frame(writing.key_time(), &b"delta"[..]).unwrap(); let mut capped = capped.with_range(..); assert!(matches!(capped.poll_read_frame(&waiter), Poll::Ready(Ok(Some(_))))); - writing.finish().unwrap(); + writing.close().unwrap(); assert!(matches!(capped.poll_read_frame(&waiter), Poll::Ready(Ok(None)))); assert_eq!(producer.demand().wanted(), None, "read to the group's end"); } @@ -22024,8 +22066,8 @@ mod test { #[cfg(feature = "async")] fn a_fetched_group_keeps_its_track_wanted() { let producer = track_producer("test", Info::timed()); - let mut writing = producer.append_group().unwrap(); - writing.write_frame(Timestamp::ZERO, &b"key"[..]).unwrap(); + let mut writing = producer.append_next().unwrap(); + writing.write_frame(writing.key_time(), &b"key"[..]).unwrap(); let group = producer .consume() .fetch_group(0, None) @@ -22175,7 +22217,7 @@ mod test { matches!(kio::Task::poll(&mut *fetch, &waiter), Poll::Ready(Ok(_))), "the fetch gets the accepted group" ); - group.finish().unwrap(); + group.close().unwrap(); } /// A feed declared to start at group 2, with groups 2 and 3 written, read from group 0. @@ -22195,7 +22237,7 @@ mod test { group .write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } #[cfg(feature = "async")] @@ -22218,7 +22260,7 @@ mod test { assert!(drained_drops(&mut subscriber).is_empty()); let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { - read.push(group.sequence); + read.push(group.start); } assert_eq!(read, [2, 3]); let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { @@ -22230,7 +22272,7 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the filled group is read"); }; - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert!(drained_drops(&mut subscriber).is_empty(), "group 1 may still arrive"); @@ -22248,10 +22290,12 @@ mod test { #[test] #[cfg(feature = "async")] fn no_fill_below_a_start_resolved_from_the_budget() { - let write = |producer: &Producer, sequence| { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + let write = |producer: &Producer, key| { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); }; let mut producer = track_producer("test", None); producer.fills_from_start(); @@ -22337,13 +22381,13 @@ mod test { let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { panic!("below the start is a miss"); }; - while let Some(sequence) = fill.wanted() { - write_filled(fill.accept(sequence, None), sequence); + while let Some(key) = fill.wanted() { + write_filled(fill.accept(key, None), key); } let mut newcomer = producer.subscribe(replay().with_range(0..)); let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = newcomer.poll_recv_group(&waiter) { - read.push(group.sequence); + read.push(group.start); } read.sort(); assert_eq!(read, [0, 1, 2, 3]); @@ -22369,12 +22413,12 @@ mod test { let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { panic!("below the start is a miss"); }; - while let Some(sequence) = fill.wanted() { - write_filled(fill.accept(sequence, None), sequence); + while let Some(key) = fill.wanted() { + write_filled(fill.accept(key, None), key); } let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { - read.push(group.sequence); + read.push(group.start); } let expected = match order { Order::Descending => [1, 0], @@ -22392,7 +22436,7 @@ mod test { let (producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&waiter) { - Poll::Ready(Ok(Some(group))) => Some(group.sequence), + Poll::Ready(Ok(Some(group))) => Some(group.start), Poll::Pending => None, _ => panic!("the read ended"), }; @@ -22440,16 +22484,16 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { panic!("groups 0 and 1 went stale behind group 2"); }; - assert_eq!(group.sequence, 2, "superseded by group 5 only 11s ago"); + assert_eq!(group.start, 2, "superseded by group 5 only 11s ago"); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 3 is still owed" ); - for sequence in [3, 4] { - assert!(fill.is_wanted(sequence), "group {sequence} is superseded by group 5"); + for key in [3, 4] { + assert!(fill.is_wanted(key), "group {key} is superseded by group 5"); } - for sequence in [0, 1, 6, 7, 9] { - assert!(!fill.is_wanted(sequence), "group {sequence} went stale"); + for key in [0, 1, 6, 7, 9] { + assert!(!fill.is_wanted(key), "group {key} went stale"); } } @@ -22491,8 +22535,8 @@ mod test { fn a_refused_group_is_asked_again_by_a_later_reader() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - for sequence in 0..5 { - write_group(&producer, sequence); + for key in 0..5 { + write_group(&producer, key); } evict(&producer, 3); let waiter = kio::Waiter::noop(); @@ -22510,7 +22554,9 @@ mod test { assert!(matches!(tail.now_or_never(), Some(Err(Error::Unsupported)))); drop(first); - let mut subscriber = producer.subscribe(replay().with_range(3..4)); + // From group 2, which names the start, so group 3 is owed rather than looked up. + let mut subscriber = producer.subscribe(replay().with_range(2..4)); + assert!(matches!(subscriber.poll_next_group(&waiter), Poll::Ready(Ok(Some(_))))); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 3 is being filled" @@ -22528,8 +22574,8 @@ mod test { fn a_source_miss_is_asked_again_by_a_later_reader() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - for sequence in 0..5 { - write_group(&producer, sequence); + for key in 0..5 { + write_group(&producer, key); } evict(&producer, 3); let waiter = kio::Waiter::noop(); @@ -22541,7 +22587,9 @@ mod test { assert!(matches!(fetch.now_or_never(), Some(Err(Error::NotFound)))); drop(first); - let mut subscriber = producer.subscribe(replay().with_range(3..4)); + // From group 2, which names the start, so group 3 is owed rather than looked up. + let mut subscriber = producer.subscribe(replay().with_range(2..4)); + assert!(matches!(subscriber.poll_next_group(&waiter), Poll::Ready(Ok(Some(_))))); assert!( subscriber.poll_next_group(&waiter).is_pending(), "group 3 is being filled" @@ -22602,7 +22650,7 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { panic!("the filled group is read"); }; - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); drop(group); assert!(fill.poll_withdrawn(&waiter).is_ready(), "nobody waits on the fill now"); } @@ -22614,8 +22662,9 @@ mod test { fn fill_wanted_changed_follows_the_group_owed() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); - let mut subscriber = producer.subscribe(ascending(0..)); + // Descending, so the reader narrows with its end, which settles at once. + let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); + let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let noop = kio::Waiter::noop(); @@ -22626,12 +22675,12 @@ mod test { let (flag, waiter) = Flag::waiter(); assert_eq!( fill.poll_wanted_changed(&waiter), - Poll::Ready(Some(0)), + Poll::Ready(Some(9)), "the first poll reports" ); assert!(fill.poll_wanted_changed(&waiter).is_pending()); - subscriber.update(ascending(8..)).unwrap(); + subscriber.update(descending(Range::groups(0..9))).unwrap(); assert!(subscriber.poll_recv_group(&noop).is_pending()); assert!(flag.take(), "narrowing woke the parked poll"); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(8))); @@ -22641,11 +22690,11 @@ mod test { flag.take(); assert!( fill.poll_wanted_changed(&waiter).is_pending(), - "group 9 waits for the reader to pick up 8" + "group 7 waits for the reader to pick up 8" ); assert!(matches!(subscriber.poll_recv_group(&noop), Poll::Ready(Ok(Some(_))))); assert!(flag.take(), "the reader picking it up woke the parked poll"); - assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(9))); + assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(7))); assert!(fill.poll_wanted_changed(&waiter).is_pending()); drop(subscriber); @@ -22662,19 +22711,19 @@ mod test { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); - let mut subscriber = producer.subscribe(ascending(9..)); - producer.start_at(10).unwrap(); - write_group(&producer, 10); + let mut subscriber = producer.subscribe(ascending(0..)); + producer.start_at(1).unwrap(); + write_group(&producer, 1); let noop = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&noop) {} let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_fill(&noop) else { panic!("below the start is a miss"); }; let (flag, waiter) = Flag::waiter(); - assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(9))); + assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(0))); assert!(fill.poll_wanted_changed(&waiter).is_pending()); - write_group(&producer, 9); + write_group(&producer, 0); assert!(flag.take(), "the owed group arriving live woke the parked poll"); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(None)); assert_eq!(fill.wanted(), None); @@ -22689,8 +22738,9 @@ mod test { fn a_narrowed_reader_leaves_the_rest_of_its_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); - let mut subscriber = producer.subscribe(ascending(0..)); + // Descending, so the reader narrows with its end, which settles at once. + let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); + let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); @@ -22698,30 +22748,30 @@ mod test { let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { panic!("below the start is a miss"); }; - assert_eq!(fill.wanted(), Some(0)); - assert!(fill.is_wanted(0)); - assert!(fill.poll_unwanted(0, &waiter).is_pending()); + assert_eq!(fill.wanted(), Some(9)); + assert!(fill.is_wanted(9)); + assert!(fill.poll_unwanted(9, &waiter).is_pending()); - subscriber.update(ascending(8..)).unwrap(); + subscriber.update(descending(Range::groups(0..8))).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); - assert!(fill.poll_unwanted(0, &waiter).is_ready(), "nobody reads group 0 now"); - assert!(!fill.is_wanted(0)); - assert!(fill.is_wanted(8)); - assert!(fill.poll_unwanted(8, &waiter).is_pending()); - assert_eq!(fill.wanted(), Some(8), "nobody reads below 8 any more"); - write_filled(fill.accept(8, None), 8); - assert_eq!(fill.wanted(), Some(9)); + assert!(fill.poll_unwanted(9, &waiter).is_ready(), "nobody reads group 9 now"); + assert!(!fill.is_wanted(9)); + assert!(fill.is_wanted(7)); + assert!(fill.poll_unwanted(7, &waiter).is_pending()); + assert_eq!(fill.wanted(), Some(7), "nobody reads above 7 any more"); + write_filled(fill.accept(7, None), 7); + assert_eq!(fill.wanted(), Some(6)); // Over, so what the fill refuses is a drop rather than a group still to come. producer.finish_shared().unwrap(); - fill.drop_groups(0..9); + fill.drop_groups(7..10); drop(fill); let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the filled group is read"); }; - assert_eq!(group.sequence, 8); - assert_eq!(drained_drops(&mut subscriber), [(9, 10)], "only what it still read"); - assert!(producer.state.read().drops.overlapping(&(0..8)).next().is_none()); + assert_eq!(group.start, 7); + assert_eq!(drained_drops(&mut subscriber), [(0, 7)], "only what it still read"); + assert!(producer.state.read().drops.overlapping(&(8..10)).next().is_none()); } /// A group the fill accepted stays wanted until its reader takes it, even once the rest of @@ -22742,13 +22792,13 @@ mod test { panic!("below the start is a miss"); }; let _writing = [fill.accept(0, None).unwrap(), fill.accept(1, None).unwrap()]; - for sequence in 2..10 { - fill.group(sequence).reject(Error::NotFound); + for key in 2..10 { + fill.group(key).reject(Error::NotFound); } let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the first filled group is read"); }; - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); assert!(!fill.is_wanted(0), "taken: its reader holds the group itself"); assert!(fill.is_wanted(1), "filled, not taken yet"); @@ -22765,8 +22815,9 @@ mod test { fn an_accepted_groups_readers_leave_before_its_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); - let mut subscriber = producer.subscribe(ascending(0..)); + // Descending, so the reader narrows with its end, which settles at once. + let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); + let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); @@ -22774,32 +22825,31 @@ mod test { let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { panic!("below the start is a miss"); }; - let group = fill.group(0); + let group = fill.group(9); let readers = group.readers(); let _writing = group.accept(None).unwrap(); assert!(readers.poll_unwanted(&waiter).is_pending()); - subscriber.update(ascending(1..)).unwrap(); + subscriber.update(descending(Range::groups(0..9))).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); - assert!(readers.poll_unwanted(&waiter).is_ready(), "nobody reads group 0 now"); + assert!(readers.poll_unwanted(&waiter).is_ready(), "nobody reads group 9 now"); assert!( fill.poll_withdrawn(&waiter).is_pending(), - "the fill still owes 1 and up" + "the fill still owes 8 and down" ); } /// A time start before every group held looks up its group through a fill that owes /// none: a handler that can't resolve times drops it, and the start reaches back to the - /// oldest group held, asking no fill below it, since which of those groups it reaches is - /// unknown until they start. Another range still asks for its own misses. + /// group containing its time, which is unknown, so it asks no fill below the time itself: + /// every key from there is in range. Another range still asks for its own misses. #[test] #[cfg(feature = "async")] fn a_time_start_before_the_cache_asks_nothing_below_it() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let at = |millis| Timestamp::from_millis(millis).unwrap(); let since = Range { - start: Anchor::Time(at(5)), + start: Anchor::Time(5), end: None, }; let mut subscriber = producer.subscribe( @@ -22818,42 +22868,38 @@ mod test { let Poll::Ready(Ok(lookup)) = dynamic.poll_requested_fill(&waiter) else { panic!("the time start is looked up"); }; - assert_eq!( - (lookup.time(), lookup.start().group, lookup.end()), - (Some(at(5)), 0, 10) - ); + assert_eq!((lookup.time(), lookup.start().group, lookup.end()), (Some(5), 0, 10)); assert_eq!(lookup.wanted(), None, "a lookup owes no group"); drop(lookup); let mut read = drain_groups(&mut subscriber); - let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { - panic!("the group range misses its groups"); - }; + let mut asked = Vec::new(); + while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { + asked.push((fill.time(), fill.start().group, fill.end())); + while let Some(key) = fill.wanted() { + write_filled(fill.accept(key, None), key); + } + drop(fill); + read.extend(drain_groups(&mut subscriber)); + } assert_eq!( - (fill.time(), fill.start().group, fill.end()), - (None, 0, 2), - "nothing below 10 for the time start" + asked, + [(None, 0, 2), (None, 5, 10)], + "nothing below 5 for the time start" ); - while let Some(sequence) = fill.wanted() { - write_filled(fill.accept(sequence, None), sequence); - } - drop(fill); - read.extend(drain_groups(&mut subscriber)); - assert_eq!(read, [10, 0, 1]); - assert!(dynamic.poll_requested_fill(&waiter).is_pending()); + assert_eq!(read, [10, 0, 1, 5, 6, 7, 8, 9]); } - /// A time start before every group held starts at the group its lookup answers, once that - /// group's first frame says where it starts, asking nothing below it; a time end walks the - /// history below the feed's start through fills, stopping just past the group containing it. + /// A time start before every group held starts at the group its lookup answers, whose key + /// says where it starts, asking nothing below it; the misses past it through the time end + /// are filled. #[test] #[cfg(feature = "async")] fn a_looked_up_time_range_reads_the_history() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let at = |millis| Timestamp::from_millis(millis).unwrap(); let mut subscriber = producer.subscribe(replay().with_ranges([Range { - start: Anchor::Time(at(5)), - end: Some(Anchor::Time(at(7))), + start: Anchor::Time(5), + end: Some(Anchor::Time(7)), }])); producer.start_at(10).unwrap(); write_group(&producer, 10); @@ -22866,27 +22912,24 @@ mod test { let Poll::Ready(Ok(lookup)) = dynamic.poll_requested_fill(&waiter) else { panic!("the time start is looked up"); }; - assert_eq!(lookup.time(), Some(at(5))); + assert_eq!(lookup.time(), Some(5)); let found = lookup.accept(5, None); drop(lookup); - assert_eq!( - drain_groups(&mut subscriber), - Vec::::new(), - "nothing reads until the group found starts" - ); - write_filled(found, 5); let mut read = drain_groups(&mut subscriber); - // One group at a time past the start, until one starts after the end. + assert_eq!(read, [5], "the group found is read at once: its key is its start"); + write_filled(found, 5); + // The misses past the start, through the end. while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { assert_eq!(fill.time(), None); - while let Some(sequence) = fill.wanted() { - assert!(sequence >= 5, "group {sequence} is below the looked-up start"); - write_filled(fill.accept(sequence, None), sequence); + while let Some(key) = fill.wanted() { + assert!(key >= 5, "group {key} is below the looked-up start"); + write_filled(fill.accept(key, None), key); } drop(fill); read.extend(drain_groups(&mut subscriber)); } - assert_eq!(read, [5, 6, 7]); + // The misses come newest first, as the reader orders them. + assert_eq!(read, [5, 7, 6]); } /// A time end walking the history a handler serves takes no group the live feed @@ -22897,29 +22940,28 @@ mod test { fn a_time_end_walks_evicted_arrivals_through_the_history() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let at = |millis| Timestamp::from_millis(millis).unwrap(); - for sequence in 0..10 { - write_group(&producer, sequence); + for key in 0..10 { + write_group(&producer, key); } - for sequence in [0, 1, 2, 3, 5, 6, 7, 8] { - evict(&producer, sequence); + for key in [0, 1, 2, 3, 5, 6, 7, 8] { + evict(&producer, key); } let mut subscriber = producer.subscribe(replay().with_ranges([Range { - start: Anchor::Time(at(5)), - end: Some(Anchor::Time(at(6))), + start: Anchor::Time(5), + end: Some(Anchor::Time(6)), }])); let waiter = kio::Waiter::noop(); assert_eq!(drain_groups(&mut subscriber), Vec::::new()); let Poll::Ready(Ok(lookup)) = dynamic.poll_requested_fill(&waiter) else { panic!("the time start in the evicted gap is looked up"); }; - assert_eq!(lookup.time(), Some(at(5))); + assert_eq!(lookup.time(), Some(5)); write_filled(lookup.accept(5, None), 5); drop(lookup); let mut read = drain_groups(&mut subscriber); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { - while let Some(sequence) = fill.wanted() { - write_filled(fill.accept(sequence, None), sequence); + while let Some(key) = fill.wanted() { + write_filled(fill.accept(key, None), key); } drop(fill); read.extend(drain_groups(&mut subscriber)); @@ -22934,22 +22976,21 @@ mod test { fn a_time_end_parked_on_a_gap_wakes_when_its_fill_refuses_it() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let at = |millis| Timestamp::from_millis(millis).unwrap(); - for sequence in 0..7 { - write_group(&producer, sequence); + for key in 0..7 { + write_group(&producer, key); } - for sequence in [3, 4] { - evict(&producer, sequence); + for key in [3, 4] { + evict(&producer, key); } let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending).with_ranges([Range { - start: Anchor::Time(at(0)), - end: Some(Anchor::Time(at(4))), + start: Anchor::Time(0), + end: Some(Anchor::Time(4)), }])); // Polled as an executor would, again whenever the read wakes itself. let (flag, waiter) = Flag::waiter(); let mut poll = || loop { match subscriber.poll_group(true, &waiter) { - Poll::Ready(res) => return Some(res.expect("no error").map(|group| group.sequence)), + Poll::Ready(res) => return Some(res.expect("no error").map(|group| group.start)), Poll::Pending if flag.take() => continue, Poll::Pending => return None, } @@ -22958,15 +22999,15 @@ mod test { loop { while !ended && let Some(next) = poll() { match next { - Some(sequence) => read.push(sequence), + Some(key) => read.push(key), None => ended = true, } } let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&kio::Waiter::noop()) else { break; }; - while let Some(sequence) = fill.wanted() { - fill.group(sequence).reject(Error::App(sequence as u16)); + while let Some(key) = fill.wanted() { + fill.group(key).reject(Error::App(key as u16)); } } assert_eq!(read, [0, 1, 2]); @@ -22989,11 +23030,11 @@ mod test { panic!("below the start is a miss"); }; assert_eq!((fill.start().group, fill.end()), (0, 100)); - for sequence in 0..50 { - assert_eq!(fill.wanted(), Some(sequence)); - fill.group(sequence).reject(Error::App(sequence as u16)); + for key in 0..50 { + assert_eq!(fill.wanted(), Some(key)); + fill.group(key).reject(Error::App(key as u16)); let _ = subscriber.poll_recv_group(&waiter); - assert!(fill.answers() <= 1, "group {sequence}: {} answers held", fill.answers()); + assert!(fill.answers() <= 1, "group {key}: {} answers held", fill.answers()); } } @@ -23004,8 +23045,9 @@ mod test { fn a_later_reader_asks_afresh_for_what_a_fill_stopped_owing() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); - let mut narrowed = producer.subscribe(ascending(0..)); + // Descending, so the first reader narrows with its end, which settles at once. + let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); + let mut narrowed = producer.subscribe(descending(Range::groups(0..))); producer.start_at(10).unwrap(); write_group(&producer, 10); let waiter = kio::Waiter::noop(); @@ -23013,17 +23055,17 @@ mod test { let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) else { panic!("below the start is a miss"); }; - narrowed.update(ascending(8..)).unwrap(); + narrowed.update(descending(Range::groups(0..8))).unwrap(); assert!(narrowed.poll_recv_group(&waiter).is_pending()); - let mut later = producer.subscribe(ascending(0..)); + let mut later = producer.subscribe(descending(Range::groups(0..))); while let Poll::Ready(Ok(Some(_))) = later.poll_recv_group(&waiter) {} let Poll::Ready(Ok(again)) = dynamic.poll_requested_fill(&waiter) else { panic!("what the first fill stopped owing is asked afresh"); }; - assert_eq!((again.start().group, again.end()), (0, 8)); - assert_eq!(fill.wanted(), Some(8), "the first fill still owes only 8 and up"); - assert!(!fill.is_wanted(0), "a group stops being wanted for good"); + assert_eq!((again.start().group, again.end()), (8, 10)); + assert_eq!(fill.wanted(), Some(7), "the first fill still owes only 7 and down"); + assert!(!fill.is_wanted(9), "a group stops being wanted for good"); } /// A descending reader's disjoint misses queue newest first, so a handler serving one fill @@ -23037,13 +23079,15 @@ mod test { producer.subscribe(replay().with_ranges([Range::groups(0..2), Range::groups(4..6), Range::groups(8..)])); producer.start_at(8).unwrap(); write_group(&producer, 8); + // Held, so the middle range's time start names it and only 5 is a miss there. + write_group(&producer, 4); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = subscriber.poll_recv_group(&waiter) {} let mut asked = Vec::new(); while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { asked.push((fill.start().group, fill.end())); } - assert_eq!(asked, [(4, 6), (0, 2)]); + assert_eq!(asked, [(5, 6), (0, 2)]); } /// A group evicted after an arrival-order reader judged it cached, before it took it, is @@ -23055,14 +23099,14 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay()); - for sequence in 0..3 { - write_group(&producer, sequence); + for key in 0..3 { + write_group(&producer, key); } let waiter = kio::Waiter::noop(); let Poll::Ready(Ok(Some(first))) = subscriber.poll_recv_group(&waiter) else { panic!("group 0 is cached"); }; - assert_eq!(first.sequence, 0); + assert_eq!(first.start, 0); evict(&producer, 0); evict(&producer, 1); assert_eq!(drain_groups(&mut subscriber), [2]); @@ -23091,13 +23135,13 @@ mod test { let mut producer = track_producer("test", None); let dynamic = producer.requests(); producer.received(); - for sequence in 0..5 { - framed(&producer, sequence, 1).finish().unwrap(); + for key in 0..5 { + framed(&producer, key, 1).close().unwrap(); } producer.set_idle(); // The feed resumes at 8, above the cached 0..=4: its live floor is 5. producer.set_live(Some(Position::group(8)), None).unwrap(); - framed(&producer, 8, 1).finish().unwrap(); + framed(&producer, 8, 1).close().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_range(0..) @@ -23106,17 +23150,17 @@ mod test { ); let waiter = kio::Waiter::noop(); let next = |reader: &mut Subscriber| match reader.poll_next_group(&waiter) { - Poll::Ready(Ok(Some(group))) => Some(group.sequence), + Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; assert_eq!(next(&mut reader), Some(0)); // The gap the feed may still deliver arrives late, so nothing is deferred after. - for sequence in 5..8 { - framed(&producer, sequence, 1).finish().unwrap(); + for key in 5..8 { + framed(&producer, key, 1).close().unwrap(); } assert_eq!(next(&mut reader), Some(1)); // Only the newest group moves before this read: judged on its own. - framed(&producer, 9, 1).finish().unwrap(); + framed(&producer, 9, 1).close().unwrap(); assert_eq!(next(&mut reader), Some(2)); evict(&producer, 3); let _ = next(&mut reader); @@ -23151,10 +23195,10 @@ mod test { drain_groups(&mut stalled); drain_groups(&mut active); // Late groups below the stalled reader's newest, cached after it judged. - for sequence in (2..200).step_by(2) { - write_group(&producer, sequence); + for key in (2..200).step_by(2) { + write_group(&producer, key); drain_groups(&mut active); - evict(&producer, sequence); + evict(&producer, key); } // The active reader judged each one cached, so it holds the last until it judges again. assert_eq!(held(), [vec![], vec![(198, 199)]], "the stalled reader judged none"); @@ -23190,15 +23234,15 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); - for sequence in 0..3 { - write_group(&producer, sequence); + for key in 0..3 { + write_group(&producer, key); } let waiter = kio::Waiter::noop(); for expected in 0..2 { let Poll::Ready(Ok(Some(group))) = subscriber.poll_next_group(&waiter) else { panic!("group {expected} is cached"); }; - assert_eq!(group.sequence, expected); + assert_eq!(group.start, expected); } evict(&producer, 0); evict(&producer, 1); @@ -23207,19 +23251,19 @@ mod test { } /// An ordered reader waits on the refill of a group evicted at or above its next - /// sequence, and reads it in order. + /// key, and reads it in order. #[test] #[cfg(feature = "async")] fn an_ordered_reader_refills_an_evicted_group_in_order() { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); - for sequence in 0..4 { - write_group(&producer, sequence); + for key in 0..4 { + write_group(&producer, key); } let waiter = kio::Waiter::noop(); let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&kio::Waiter::noop()) { - Poll::Ready(Ok(Some(group))) => Some(group.sequence), + Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; assert_eq!(next(&mut subscriber), Some(0)); @@ -23230,8 +23274,8 @@ mod test { panic!("groups 1 and 2 are misses now"); }; assert_eq!((fill.start().group, fill.end()), (1, 3)); - while let Some(sequence) = fill.wanted() { - write_filled(fill.accept(sequence, None), sequence); + while let Some(key) = fill.wanted() { + write_filled(fill.accept(key, None), key); } drop(fill); assert_eq!([next(&mut subscriber), next(&mut subscriber)], [Some(1), Some(2)]); @@ -23246,12 +23290,12 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); - for sequence in 0..4 { - write_group(&producer, sequence); + for key in 0..4 { + write_group(&producer, key); } let waiter = kio::Waiter::noop(); let next = |subscriber: &mut Subscriber| match subscriber.poll_next_group(&kio::Waiter::noop()) { - Poll::Ready(Ok(Some(group))) => Some(group.sequence), + Poll::Ready(Ok(Some(group))) => Some(group.start), _ => None, }; assert_eq!(next(&mut subscriber), Some(0)); @@ -23280,8 +23324,8 @@ mod test { let producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay()); - for sequence in 0..3 { - write_group(&producer, sequence); + for key in 0..3 { + write_group(&producer, key); } let waiter = kio::Waiter::noop(); assert!(matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(Some(_))))); @@ -23308,12 +23352,12 @@ mod test { let (producer, _pool) = pooled_producer((cached * size) as u64); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending)); - let write = |group: Result, sequence: u64| { + let write = |group: Result, key: u64| { let mut group = group.unwrap(); group - .write_frame(Timestamp::from_millis(sequence).unwrap(), vec![0u8; size]) + .write_frame(Timestamp::from_millis(key).unwrap(), vec![0u8; size]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; let waiter = kio::Waiter::noop(); let mut refilled = 0; @@ -23325,8 +23369,8 @@ mod test { fills.retain_mut(|fill| { loop { match fill.poll_wanted_changed(&waiter) { - Poll::Ready(Some(sequence)) => { - write(fill.accept(sequence, None), sequence); + Poll::Ready(Some(key)) => { + write(fill.accept(key, None), key); refilled += 1; } Poll::Ready(None) => break false, @@ -23340,7 +23384,7 @@ mod test { let mut take = |subscriber: &mut Subscriber| { let taken = match subscriber.poll_next_group(&kio::Waiter::noop()) { Poll::Ready(Ok(Some(group))) => { - read.push(group.sequence); + read.push(group.start); true } _ => false, @@ -23348,9 +23392,9 @@ mod test { drops.extend(drained_drops(subscriber)); taken }; - for sequence in 0..GROUPS { - write(producer.create_group(group::Info { sequence }), sequence); - if sequence % 4 < pace { + for key in 0..GROUPS { + write(producer.create_group(group::Info { start: key }), key); + if key % 4 < pace { take(&mut subscriber); } serve(); @@ -23395,7 +23439,7 @@ mod test { assert_eq!(seen, read, "none twice"); assert!(read.is_sorted(), "in order"); assert!(refilled <= GROUPS, "{refilled} refills for {GROUPS} groups"); - let lost: Vec = (0..GROUPS).filter(|sequence| !read.contains(sequence)).collect(); + let lost: Vec = (0..GROUPS).filter(|key| !read.contains(key)).collect(); let named: Vec = drops.iter().flat_map(|&(start, end)| start..end).collect(); assert_eq!(lost, named, "every group lost is named dropped"); } @@ -23410,15 +23454,15 @@ mod test { let (mut producer, _pool) = pooled_producer(4 * size as u64); let dynamic = producer.requests(); let mut subscriber = producer.subscribe(replay().with_range(0..).with_order(Order::Ascending)); - let write = |group: Result, sequence: u64| { + let write = |group: Result, key: u64| { let mut group = group.unwrap(); group - .write_frame(Timestamp::from_millis(sequence).unwrap(), vec![0u8; size]) + .write_frame(Timestamp::from_millis(key).unwrap(), vec![0u8; size]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; producer.start_at(GROUPS).unwrap(); - write(producer.create_group(group::Info { sequence: GROUPS }), GROUPS); + write(producer.create_group(group::Info { start: GROUPS }), GROUPS); let waiter = kio::Waiter::noop(); assert!(subscriber.poll_next_group(&waiter).is_pending(), "group 0 is a miss"); let Poll::Ready(Ok(mut fill)) = dynamic.poll_requested_fill(&waiter) else { @@ -23428,13 +23472,13 @@ mod test { let (mut read, mut ahead) = (Vec::new(), 0); loop { let mut served = 0; - while let Poll::Ready(Some(sequence)) = fill.poll_wanted_changed(&waiter) { - write(fill.accept(sequence, None), sequence); + while let Poll::Ready(Some(key)) = fill.poll_wanted_changed(&waiter) { + write(fill.accept(key, None), key); served += 1; } ahead = ahead.max(served); match subscriber.poll_next_group(&waiter) { - Poll::Ready(Ok(Some(group))) => read.push(group.sequence), + Poll::Ready(Ok(Some(group))) => read.push(group.start), _ => break, } } @@ -23443,7 +23487,7 @@ mod test { assert_eq!(ahead, 1, "one group served at a time"); } - /// A reader asking newest first but reading in sequence picks up each group the fill writes + /// A reader asking newest first but reading in key picks up each group the fill writes /// while it waits on the oldest, so the fill never waits on a group the reader cannot take /// yet. #[test] @@ -23462,11 +23506,11 @@ mod test { }; assert_eq!(fill.order(), Order::Descending); let (mut served, mut read) = (Vec::new(), Vec::new()); - while let Poll::Ready(Some(sequence)) = fill.poll_wanted_changed(&waiter) { - write_filled(fill.accept(sequence, None), sequence); - served.push(sequence); + while let Poll::Ready(Some(key)) = fill.poll_wanted_changed(&waiter) { + write_filled(fill.accept(key, None), key); + served.push(key); while let Poll::Ready(Ok(Some(group))) = ordered.poll_next_group(&waiter) { - read.push(group.sequence); + read.push(group.start); } } assert_eq!(served, [3, 2, 1, 0]); @@ -23498,7 +23542,7 @@ mod test { "group 1 waits for a read" ); } - assert!(matches!(subscriber.poll_next_group(&waiter), Poll::Ready(Ok(Some(group))) if group.sequence == 0)); + assert!(matches!(subscriber.poll_next_group(&waiter), Poll::Ready(Ok(Some(group))) if group.start == 0)); assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(1))); } @@ -23510,8 +23554,9 @@ mod test { fn a_fetch_picks_up_its_group_in_a_batch_fill() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let ascending = |groups| replay().with_range(groups).with_order(Order::Ascending); - let mut subscriber = producer.subscribe(ascending(0..)); + // Descending, so the reader narrows past group 2 with its end, which settles at once. + let descending = |groups: Range| replay().with_ranges([groups]).with_order(Order::Descending); + let mut subscriber = producer.subscribe(descending(Range::groups(0..))); producer.start_at(4).unwrap(); write_group(&producer, 4); let waiter = kio::Waiter::noop(); @@ -23531,7 +23576,7 @@ mod test { "the fetch joined the batch" ); - subscriber.update(ascending(3..)).unwrap(); + subscriber.update(descending(Range::groups(0..2))).unwrap(); assert!(subscriber.poll_recv_group(&waiter).is_pending()); assert_eq!( fill.poll_wanted_changed(&waiter), @@ -23541,10 +23586,10 @@ mod test { write_filled(fill.accept(2, None), 2); assert!( fill.poll_wanted_changed(&waiter).is_pending(), - "3 waits for 2 to be picked up" + "1 waits for 2 to be picked up" ); assert!(matches!(kio::Task::poll(&mut *fetch, &waiter), Poll::Ready(Ok(_)))); - assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(3))); + assert_eq!(fill.poll_wanted_changed(&waiter), Poll::Ready(Some(1))); } /// A reader forgets the misses it gave up once its window moves past them, as it forgets @@ -23560,8 +23605,8 @@ mod test { let ascending = |groups| Subscription::default().with_range(groups).with_order(Order::Ascending); let mut subscriber = producer.subscribe(ascending(0..)); producer.start_at(0).unwrap(); - for sequence in (0..20).step_by(2) { - write_group(&producer, sequence); + for key in (0..20).step_by(2) { + write_group(&producer, key); drain_groups(&mut subscriber); elapse(&producer, Duration::from_secs(2)); drain_groups(&mut subscriber); @@ -23588,9 +23633,11 @@ mod test { fn a_descending_reader_asks_its_newest_fragment_first() { let mut producer = track_producer("test", Info::timed()); let dynamic = producer.requests(); - let mut middle = producer.subscribe(replay().with_range(5..6)); + let mut middle = producer.subscribe(replay().with_range(4..6)); producer.start_at(8).unwrap(); write_group(&producer, 8); + // Held, so the middle reader's time start names it and only 5 is a miss. + write_group(&producer, 4); let waiter = kio::Waiter::noop(); while let Poll::Ready(Ok(Some(_))) = middle.poll_recv_group(&waiter) {} let Poll::Ready(Ok(taken)) = dynamic.poll_requested_fill(&waiter) else { @@ -23604,7 +23651,7 @@ mod test { while let Poll::Ready(Ok(fill)) = dynamic.poll_requested_fill(&waiter) { asked.push((fill.start().group, fill.end())); } - assert_eq!(asked, [(6, 8), (0, 5)]); + assert_eq!(asked, [(6, 8), (0, 4)]); } /// A group the handler refused that still arrives live is delivered. @@ -23623,7 +23670,7 @@ mod test { let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) else { panic!("the late group is read"); }; - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); elapse(&producer, Duration::from_secs(31)); assert_eq!(drained_drops(&mut subscriber), [(0, 1)]); } @@ -23659,8 +23706,8 @@ mod test { // A received copy, whose missing groups may still arrive. producer.received(); let mut subscriber = producer.subscribe(replay().with_range(0..)); - for sequence in [0, 1, 2, 4, 5] { - write_group(&producer, sequence); + for key in [0, 1, 2, 4, 5] { + write_group(&producer, key); } producer.finish_at(Position::group(10), None).unwrap(); assert_eq!(drain_groups(&mut subscriber), [0, 1, 2, 4, 5]); @@ -23680,7 +23727,7 @@ mod test { assert!(drained_drops(&mut subscriber).is_empty()); let mut read = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&waiter) { - read.push(group.sequence); + read.push(group.start); } assert_eq!(read, [2, 3]); assert!(drained_drops(&mut subscriber).is_empty()); @@ -23730,8 +23777,8 @@ mod test { panic!("the miss below the start is asked"); }; for refused in 1..=REFUSED { - let sequence = fill.wanted().unwrap(); - fill.drop_groups(sequence..sequence + 1); + let key = fill.wanted().unwrap(); + fill.drop_groups(key..key + 1); assert!(subscriber.poll_recv_group(&kio::Waiter::noop()).is_pending()); assert_eq!(fill.wanted(), Some(WIDE - refused - 1)); assert_eq!(fill.answers(), 0, "the reader consumed the refusal"); @@ -23851,7 +23898,7 @@ mod test { let (_producer, dynamic, mut subscriber) = started_late(); let waiter = kio::Waiter::noop(); let mut next = || match subscriber.poll_next_group(&waiter) { - Poll::Ready(Ok(Some(group))) => Some(group.sequence), + Poll::Ready(Ok(Some(group))) => Some(group.start), Poll::Pending => None, _ => panic!("the read ended"), }; @@ -23954,31 +24001,31 @@ mod test { drop(dynamic); } - /// Write a finished one-frame group, presented at its sequence in milliseconds. - fn write_group(producer: &Producer, sequence: u64) { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + /// Write a finished one-frame group, presented at its key in milliseconds. + fn write_group(producer: &Producer, key: u64) { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } /// Every group the subscriber hands out right now, in arrival order. fn drain_groups(subscriber: &mut Subscriber) -> Vec { - let mut sequences = Vec::new(); + let mut keys = Vec::new(); while let Poll::Ready(Ok(Some(group))) = subscriber.poll_recv_group(&kio::Waiter::noop()) { - sequences.push(group.sequence); + keys.push(group.start); } - sequences + keys } - /// A sequence the publisher aborts and re-creates is handed out once. + /// A key the publisher aborts and re-creates is handed out once. #[test] fn a_recreated_group_is_handed_out_once() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); write_group(&producer, 0); - let open = producer.create_group(group::Info { sequence: 1 }).unwrap(); + let open = producer.create_group(group::Info { start: 1 }).unwrap(); assert_eq!(drain_groups(&mut subscriber), [0, 1]); open.abort(Error::Cancel).unwrap(); @@ -24006,16 +24053,16 @@ mod test { let producer = track_producer("test", info); let mut subscriber = producer.subscribe(subscription); // Group 3 never arrives. - for sequence in [0, 1, 2, 4, 5, 6] { - write_group(&producer, sequence); + for key in [0, 1, 2, 4, 5, 6] { + write_group(&producer, key); } assert_eq!(drain_groups(&mut subscriber), [0, 1, 2, 4, 5, 6]); // Group 4 reaches until group 5 presents at 5ms, so it is past a 10ms budget once // the edge presents at 15ms. - for sequence in 7..=14 { - write_group(&producer, sequence); - assert_eq!(drain_groups(&mut subscriber), [sequence]); - assert_eq!(subscriber.delivered_runs(), 2, "the gap at 3 is kept at {sequence}"); + for key in 7..=14 { + write_group(&producer, key); + assert_eq!(drain_groups(&mut subscriber), [key]); + assert_eq!(subscriber.delivered_runs(), 2, "the gap at 3 is kept at {key}"); } write_group(&producer, 15); assert_eq!(drain_groups(&mut subscriber), [15]); @@ -24034,13 +24081,13 @@ mod test { let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(10)))); let mut most = 0; - for sequence in 0..3000u64 { + for key in 0..3000u64 { // Every third group never arrives. - if sequence % 3 == 1 { + if key % 3 == 1 { continue; } - write_group(&producer, sequence); - assert_eq!(drain_groups(&mut subscriber), [sequence]); + write_group(&producer, key); + assert_eq!(drain_groups(&mut subscriber), [key]); most = most.max(subscriber.delivered_runs()); } assert_eq!(most, 5, "the floor and the gaps within the budget"); @@ -24051,11 +24098,13 @@ mod test { assert_eq!(drain_groups(&mut subscriber), [2998]); } - /// A group of one untimed frame at `sequence`. - fn write_untimed_group(producer: &Producer, sequence: u64) { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + /// A group of one untimed frame at `key`. + fn write_untimed_group(producer: &Producer, key: u64) { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); } /// The wall clock forgets the gaps on an untimed track, which has no media time to @@ -24067,14 +24116,14 @@ mod test { let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(10)))); let mut most = 0; - for sequence in 0..3000u64 { + for key in 0..3000u64 { elapse(&producer, Duration::from_millis(1)); // Every third group never arrives. - if sequence % 3 == 1 { + if key % 3 == 1 { continue; } - write_untimed_group(&producer, sequence); - assert_eq!(drain_groups(&mut subscriber), [sequence]); + write_untimed_group(&producer, key); + assert_eq!(drain_groups(&mut subscriber), [key]); most = most.max(subscriber.delivered_runs()); } assert_eq!(most, 5, "the floor and the gaps within the budget"); @@ -24085,6 +24134,27 @@ mod test { assert_eq!(drain_groups(&mut subscriber), [2998]); } + /// A scale-less track's max age counts bare key units as milliseconds over the whole key + /// range, so keys past a scaled timestamp's bound age by their distance as low ones do. + #[test] + fn a_scale_less_track_ages_wide_keys_by_their_distance() { + for base in [0, 1u64 << 61, u64::MAX - 30_000] { + let producer = track_producer("test", None); + for offset in [0, 10_000, 20_000] { + write_untimed_group(&producer, base + offset); + } + // All three arrived at once, so only the keys' distance makes the oldest stale: its + // successor starts 10 s behind the edge. The middle one reaches the edge. + let mut subscriber = producer + .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(5)))); + assert_eq!( + drain_groups(&mut subscriber), + [base + 10_000, base + 20_000], + "from {base}" + ); + } + } + /// An empty successor gives the lowest group above a gap no reach, so media time never /// forgets that gap or any after it. The wall clock still does. #[test] @@ -24093,23 +24163,23 @@ mod test { let mut subscriber = producer .subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_millis(10)))); let mut most = 0; - for sequence in 0..3000u64 { + for key in 0..3000u64 { elapse(&producer, Duration::from_millis(1)); - if sequence % 3 == 1 { + if key % 3 == 1 { continue; } // Group 2 sits above the first gap, and its successor never presents a frame. - match sequence { + match key { 3 => { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } - _ => write_group(&producer, sequence), + _ => write_group(&producer, key), } - assert_eq!(drain_groups(&mut subscriber), [sequence]); + assert_eq!(drain_groups(&mut subscriber), [key]); most = most.max(subscriber.delivered_runs()); } assert_eq!(most, 5, "the floor and the gaps within the budget"); @@ -24126,7 +24196,7 @@ mod test { write_group(&producer, 0); // Group 1 never arrives. let open: Vec<_> = (2..2 + RUN) - .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) + .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); assert_eq!( drain_groups(&mut subscriber), @@ -24150,10 +24220,10 @@ mod test { let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(Duration::from_secs(1)))); let mut handed = Vec::new(); - for sequence in (0..20).step_by(2) { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + for key in (0..20).step_by(2) { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key).unwrap(), &b"x"[..]) .unwrap(); handed.push(group); } @@ -24180,9 +24250,11 @@ mod test { let producer = track_producer("test", None); let mut subscriber = producer.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); let mut open = Vec::new(); - for sequence in (0..=10).step_by(2) { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); + for key in (0..=10).step_by(2) { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); open.push(group); } assert_eq!(drain_groups(&mut subscriber), [0, 2, 4, 6, 8, 10]); @@ -24215,11 +24287,13 @@ mod test { fn an_untimed_sparse_feed_keeps_its_gaps() { let producer = untimed_producer(); let mut subscriber = producer.subscribe(replay()); - for sequence in (0..2000u64).step_by(2) { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); - assert_eq!(drain_groups(&mut subscriber), [sequence]); + for key in (0..2000u64).step_by(2) { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); + assert_eq!(drain_groups(&mut subscriber), [key]); } assert_eq!(subscriber.delivered_runs(), 1000); } @@ -24234,8 +24308,9 @@ mod test { producer } - /// A live range merged with one above the live edge still starts at the newest group, - /// and the aggregate keeps both rather than flooring the live one at the higher start. + /// A live range merged with one starting past the live edge still starts at the newest + /// group, and the aggregate keeps both rather than flooring the live one at the higher + /// start. The later start is a time the newest group contains, so both read it. #[test] #[cfg(feature = "async")] fn a_live_range_merged_with_one_above_the_live_edge_gets_the_newest_group() { @@ -24244,9 +24319,9 @@ mod test { let mut live = producer.subscribe(replay().with_ranges([Range::live()])); assert_eq!( producer.subscription().unwrap().ranges, - vec![Range::groups(4..), Range::live()] + vec![Range::live(), Range::groups(4..)] ); - assert_eq!(drain(&mut ahead), Vec::::new()); + assert_eq!(drain(&mut ahead), vec![3]); assert_eq!(drain(&mut live), vec![3]); append_at(&mut producer, 4); @@ -24277,11 +24352,11 @@ mod test { let mut early = producer.subscribe(replay().with_ranges([Range::live()])); let _fetching = consumer.fetch_group(5, None); let req = dynamic.requested_fill_group().now_or_never().unwrap().unwrap(); - req.accept(None).unwrap().finish().unwrap(); + req.accept(None).unwrap().close().unwrap(); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); let mut late = producer.subscribe(replay().with_ranges([Range::live()])); assert_eq!(drain(&mut early), vec![0]); @@ -24299,10 +24374,12 @@ mod test { end: None, }; let mut behind = producer.subscribe(replay().with_ranges([buffering])); - let write = |sequence| { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + let write = |key| { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); }; write(10); elapse(&producer, cache::DEFAULT_EXPIRY + Duration::from_secs(1)); @@ -24315,7 +24392,8 @@ mod test { } /// A relative start buffers that many groups behind the live edge, whatever a range - /// above the edge merged beside it asks for. + /// starting past the edge merged beside it asks for: that one reads from the newest + /// group, which contains its start. #[test] #[cfg(feature = "async")] fn a_buffering_range_merged_with_one_above_the_live_edge_keeps_its_backlog() { @@ -24328,7 +24406,7 @@ mod test { let mut behind = producer.subscribe(replay().with_ranges([buffering])); assert_eq!( producer.subscription().unwrap().ranges, - vec![Range::groups(8..), buffering] + vec![buffering, Range::groups(8..)] ); assert_eq!(drain(&mut behind), vec![3, 4, 5]); @@ -24336,7 +24414,7 @@ mod test { append_at(&mut producer, millis); } assert_eq!(drain(&mut behind), vec![6, 7, 8]); - assert_eq!(drain(&mut ahead), vec![8]); + assert_eq!(drain(&mut ahead), vec![5, 6, 7, 8]); } /// An untimed track replays from group 0 only what the wall clock hasn't made stale. @@ -24357,10 +24435,10 @@ mod test { assert_eq!(drain(&mut subscriber), vec![3, 4]); } - /// A group `sequence` whose first frame lands at `millis`, or the write's refusal. A + /// A group `key` whose first frame lands at `millis`, or the write's refusal. A /// refused group is aborted, as a caller would. - fn start_at_millis(producer: &Producer, sequence: u64, millis: u64) -> Result { - let mut group = producer.create_group(group::Info { sequence })?; + fn start_at_millis(producer: &Producer, key: u64, millis: u64) -> Result { + let mut group = producer.create_group(group::Info { start: key })?; match group.write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) { Ok(()) => Ok(group), Err(err) => { @@ -24370,9 +24448,9 @@ mod test { } } - /// Take group `sequence` out of the cache as eviction does. - fn evict(producer: &Producer, sequence: u64) { - producer.modify().unwrap().evict_group(sequence, Error::Evicted); + /// Take group `key` out of the cache as eviction does. + fn evict(producer: &Producer, key: u64) { + producer.modify().unwrap().evict_group(key, Error::Evicted); } /// A rewound first frame is refused and leaves the group as it was, so the publisher can @@ -24382,15 +24460,17 @@ mod test { let mut producer = track_producer("test", Info::timed()); append_at(&mut producer, 1000); - let mut group = producer.append_group().unwrap(); + let mut group = producer.append_next().unwrap(); let refused = group.write_frame(Timestamp::from_millis(500).unwrap(), &b"x"[..]); assert!(matches!(refused, Err(Error::TimestampMismatch)), "{refused:?}"); assert_eq!(group.frame_count(), 0); assert_eq!(group.handle().timestamp(), None); - // An equal start is a keyframe at the previous group's start. + // Starts strictly increase, so an equal start is refused like a rewind. + let tie = group.write_frame(Timestamp::from_millis(1000).unwrap(), &b"x"[..]); + assert!(matches!(tie, Err(Error::TimestampMismatch)), "{tie:?}"); group - .write_frame(Timestamp::from_millis(1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(1001).unwrap(), &b"x"[..]) .unwrap(); } @@ -24402,24 +24482,26 @@ mod test { let refused = producer.write_frame(Timestamp::from_millis(500).unwrap(), &b"x"[..]); assert!(matches!(refused, Err(Error::TimestampMismatch)), "{refused:?}"); let state = producer.state.read(); - assert!(state.lookup.get(&1).unwrap().group.is_aborted()); + assert!(state.lookup.get(&500).unwrap().group.is_aborted()); } - /// Groups land out of sequence order, so a start is bounded from above as well as below. + /// Groups land out of key order, so a start is bounded from above as well as below. #[test] fn a_rewind_is_refused_whichever_group_lands_first() { let producer = track_producer("test", Info::timed()); - start_at_millis(&producer, 5, 1000).unwrap().finish().unwrap(); + start_at_millis(&producer, 5, 1000).unwrap().close().unwrap(); assert!(matches!( start_at_millis(&producer, 3, 2000), Err(Error::TimestampMismatch) )); - start_at_millis(&producer, 3, 500).unwrap().finish().unwrap(); - assert!(matches!( - start_at_millis(&producer, 4, 400), - Err(Error::TimestampMismatch) - )); - start_at_millis(&producer, 4, 1000).unwrap().finish().unwrap(); + start_at_millis(&producer, 3, 500).unwrap().close().unwrap(); + for millis in [400, 500, 1000] { + assert!(matches!( + start_at_millis(&producer, 4, millis), + Err(Error::TimestampMismatch) + )); + } + start_at_millis(&producer, 4, 800).unwrap().close().unwrap(); } /// The batched and streamed writes check frame 0 the same way, before writing anything. @@ -24429,7 +24511,7 @@ mod test { append_at(&mut producer, 1000); let rewound = Timestamp::from_millis(500).unwrap(); - let mut batched = producer.append_group().unwrap(); + let mut batched = producer.append_next().unwrap(); let mut frames = frame::Buffer::<4>::new(); frames .push(frame::Frame::new(rewound, bytes::Bytes::from_static(b"x"))) @@ -24441,12 +24523,12 @@ mod test { assert_eq!(frames.len(), 1, "the batch is left to retry"); assert_eq!(batched.frame_count(), 0); - let streamed = producer.append_group().unwrap(); + let streamed = producer.append_next().unwrap(); let handle = streamed.handle(); let refused = streamed .create_frame(frame::Info { size: 1, - timestamp: Some(rewound), + timestamp: rewound, }) .err(); assert!(matches!(refused, Some(Error::TimestampMismatch)), "{refused:?}"); @@ -24454,20 +24536,20 @@ mod test { assert!(handle.is_aborted(), "a refused frame aborts its group"); } - /// Eviction is not a sequence prefix: a group stamped after its predecessor was evicted + /// Eviction is not a key prefix: a group stamped after its predecessor was evicted /// is still held above it, though an older group survives below. #[test] fn an_evicted_predecessor_still_bounds_a_delayed_stamp() { let producer = track_producer("test", Info::timed()); - start_at_millis(&producer, 0, 0).unwrap().finish().unwrap(); - start_at_millis(&producer, 1, 1000).unwrap().finish().unwrap(); - let mut delayed = producer.create_group(group::Info { sequence: 2 }).unwrap(); + start_at_millis(&producer, 0, 0).unwrap().close().unwrap(); + start_at_millis(&producer, 1, 1000).unwrap().close().unwrap(); + let mut delayed = producer.create_group(group::Info { start: 2 }).unwrap(); evict(&producer, 1); let refused = delayed.write_frame(Timestamp::from_millis(500).unwrap(), &b"x"[..]); assert!(matches!(refused, Err(Error::TimestampMismatch)), "{refused:?}"); delayed - .write_frame(Timestamp::from_millis(1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(1500).unwrap(), &b"x"[..]) .unwrap(); } @@ -24477,8 +24559,8 @@ mod test { #[test] fn a_late_group_below_the_watermark_is_unverified() { let producer = track_producer("test", Info::timed()); - for (sequence, millis) in [(0, 1000), (1, 2000), (2, 3000), (3, 4000)] { - start_at_millis(&producer, sequence, millis).unwrap().finish().unwrap(); + for (key, millis) in [(0, 1000), (1, 2000), (2, 3000), (3, 4000)] { + start_at_millis(&producer, key, millis).unwrap().close().unwrap(); } evict(&producer, 1); evict(&producer, 2); @@ -24495,10 +24577,10 @@ mod test { assert_eq!(late.handle().timestamp(), None, "nor drives media-time expiry"); // Group 2 really started before late group 1 claimed to. - start_at_millis(&producer, 2, 2500).unwrap().finish().unwrap(); + start_at_millis(&producer, 2, 2500).unwrap().close().unwrap(); } - /// Subscribers may have seen an aborted group's start, so a replacement at its sequence + /// Subscribers may have seen an aborted group's start, so a replacement at its key /// leaves that start as the watermark, and no later group can start before it. #[test] fn a_replaced_aborted_group_still_bounds_later_ones() { @@ -24513,7 +24595,7 @@ mod test { start_at_millis(&producer, 1, 600), Err(Error::TimestampMismatch) )); - start_at_millis(&producer, 1, 1000).unwrap().finish().unwrap(); + start_at_millis(&producer, 1, 1500).unwrap().close().unwrap(); } /// A group received from past its first frame stamps a later frame, which may overlap @@ -24521,27 +24603,29 @@ mod test { #[test] fn a_resumed_partial_group_is_neither_checked_nor_a_bound() { let producer = track_producer("test", Info::timed()); - start_at_millis(&producer, 0, 1000).unwrap().finish().unwrap(); - let mut partial = producer.create_group(group::Info { sequence: 1 }).unwrap(); + start_at_millis(&producer, 0, 1000).unwrap().close().unwrap(); + let mut partial = producer.create_group(group::Info { start: 1 }).unwrap(); partial.start_at(3).unwrap(); partial .write_frame(Timestamp::from_millis(2100).unwrap(), &b"x"[..]) .unwrap(); - start_at_millis(&producer, 2, 2000).unwrap().finish().unwrap(); + start_at_millis(&producer, 2, 2000).unwrap().close().unwrap(); } /// An untimed track has no timeline, so its groups land in any order. #[test] fn an_untimed_track_takes_groups_in_any_order() { let producer = untimed_producer(); - for sequence in [5, 3, 4] { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); + for key in [5, 3, 4] { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(crate::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); } } /// A subscriber reads only its own ranges, whatever another subscriber's widen the - /// aggregate to, in arrival order and in sequence order alike. + /// aggregate to, in arrival order and in key order alike. #[test] #[cfg(feature = "async")] fn a_subscriber_reads_only_its_own_ranges() { @@ -24590,11 +24674,11 @@ mod test { assert_eq!(drain(&mut late), vec![4]); } - /// A time end whose cap moves beside fixed ranges owes, of every cached group, exactly - /// what was owed and is still admitted, and what is newly admitted: what the walks' runs - /// gain is taken in by their runs alone, and must still owe each cached group it reaches. + /// A cap moving beside fixed ranges owes, of every cached group, exactly what was owed and + /// is still admitted, and what is newly admitted: what the cap gains is owed where the + /// cache holds a group now, and what it loses is cut from what is owed. #[test] - fn a_moving_time_cap_owes_what_it_newly_admits() { + fn a_moving_cap_owes_what_it_newly_admits() { let seed = std::cell::Cell::new(0x2545_f491_4f6c_dd1d_u64); let next = |bound: u64| { let mut x = seed.get(); @@ -24605,41 +24689,46 @@ mod test { x % bound }; let at = |millis| Timestamp::from_millis(millis).unwrap(); - const SEQUENCES: u64 = 40; + const KEYS: u64 = 40; let mut owed = 0; for round in 0..150 { let producer = track_producer("test", Info::timed()); - // One time end, and fixed ranges around and above where the feed starts. + // One open range, and fixed ranges around and above where the feed starts. let ranges = std::iter::once(Range { - start: Anchor::Time(at(next(100))), - end: Some(Anchor::Time(at(200 + next(400)))), + start: Anchor::Time(next(KEYS)), + end: None, }) .chain((0..1 + next(6)).map(|_| { - let start = next(SEQUENCES); + let start = next(KEYS); Range::groups(start..start + 1 + next(3)) })); let mut subscriber = producer.subscribe(Subscription::default().with_ranges(ranges)); let mut groups = Vec::new(); for step in 0..30 { - let sequence = next(SEQUENCES); - match next(4) { + let key = next(KEYS); + match next(5) { 0 | 1 => { - if let Ok(mut group) = producer.create_group(group::Info { sequence }) { + if let Ok(mut group) = producer.create_group(group::Info { start: key }) { if next(2) == 0 { group - .write_frame(at(sequence * 20), bytes::Bytes::from_static(b"x")) + .write_frame(at(key * 20), bytes::Bytes::from_static(b"x")) .unwrap(); } groups.push(group); } } 2 => { - if let Some(group) = groups.iter_mut().find(|group| group.sequence == sequence) + if let Some(group) = groups.iter_mut().find(|group| group.start == key) && group.handle().timestamp().is_none() { - let _ = group.write_frame(at(sequence * 20), bytes::Bytes::from_static(b"x")); + let _ = group.write_frame(at(key * 20), bytes::Bytes::from_static(b"x")); } } + 3 => match next(4) { + // The cap moves, and now and then lifts. + 0 => subscriber.set_range(..), + _ => subscriber.set_range(..next(KEYS + 1)), + }, _ => { // Read now and then, so the walk passes groups and owes them again. let _ = subscriber.poll_recv_group(&kio::Waiter::noop()); @@ -24656,16 +24745,14 @@ mod test { ); cursor.admit(); let now = cursor.filter.admitted.runs(); - let held = - |runs: &[ops::RangeInclusive], sequence: u64| runs.iter().any(|run| run.contains(&sequence)); + let held = |runs: &[ops::RangeInclusive], key: u64| runs.iter().any(|run| run.contains(&key)); let cached: Vec = cursor.state.read().lookup.keys().copied().collect(); - for sequence in cached { - let expected = - primed && held(&now, sequence) && (was.contains(&sequence) || !held(&before, sequence)); + for key in cached { + let expected = primed && held(&now, key) && (was.contains(&key) || !held(&before, key)); assert_eq!( - cursor.filter.owed.contains(&sequence), + cursor.filter.owed.contains(&key), expected, - "round {round} step {step}: group {sequence}, admitted {before:?} then {now:?}, owed {was:?}" + "round {round} step {step}: group {key}, admitted {before:?} then {now:?}, owed {was:?}" ); owed += usize::from(expected); } @@ -24689,11 +24776,11 @@ mod test { let producer = track_producer("test", Info::timed()); let _wide = producer.subscribe(replay()); let mut narrow = producer.subscribe(replay().with_range(0..2)); - for sequence in 0..1000 { + for key in 0..1000 { producer - .create_group(group::Info { sequence }) + .create_group(group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } assert_eq!(drain(&mut narrow), vec![0, 1]); @@ -24767,13 +24854,18 @@ mod test { assert_eq!(drain(&mut subscriber), vec![6]); } - /// Only the sequences a subscriber asked for are named dropped to it. + /// Only the keys a subscriber asked for are named dropped to it. #[test] #[cfg(feature = "async")] fn drops_are_named_only_within_the_ranges() { let producer = track_producer("test", Info::timed()); + // Held, so each range's time start names its own group. + write_group(&producer, 2); + write_group(&producer, 6); let mut subscriber = producer.subscribe(replay().with_ranges([Range::groups(2..4), Range::groups(6..8)])); - producer.drop_groups(0..10, StreamError::NotFound); + for keys in [0..2, 3..6, 7..10] { + producer.drop_groups(keys, StreamError::NotFound); + } let waiter = kio::Waiter::noop(); let mut dropped = Vec::new(); while let Poll::Ready(range) = subscriber.poll_drop(&waiter) { @@ -24782,8 +24874,8 @@ mod test { assert_eq!( dropped, vec![ - (2..4, Hole::drop(StreamError::NotFound)), - (6..8, Hole::drop(StreamError::NotFound)) + (3..4, Hole::drop(StreamError::NotFound)), + (7..8, Hole::drop(StreamError::NotFound)) ] ); } @@ -24809,10 +24901,12 @@ mod test { let mut covering = reader(budget, Range::groups(0..)); let mut narrower = reader(Duration::from_millis(100), Range::groups(0..)); let mut wider = reader(Duration::from_secs(5), Range::groups(0..)); - let mut capped = reader(budget, Range::groups(0..4)); + let mut capped = reader(budget, Range::groups(0..5)); + // Held, so a time start there names it rather than reaching below. + write_group(&producer, 3); let mut above = reader(budget, Range::groups(3..)); producer.give_up(1..3, Hole::stale(budget)); - producer.skip_groups(3..4); + producer.skip_groups(4..5); let waiter = kio::Waiter::noop(); let dropped = |subscriber: &mut Subscriber| { let mut runs = Vec::new(); @@ -24821,18 +24915,18 @@ mod test { } runs }; - let settled = vec![(1..3, Hole::stale(budget)), (3..4, Hole::Skip)]; + let settled = vec![(1..3, Hole::stale(budget)), (4..5, Hole::Skip)]; assert_eq!(dropped(&mut covering), settled); assert_eq!(dropped(&mut narrower), settled, "a narrower budget is settled too"); - assert_eq!(dropped(&mut wider), [(3..4, Hole::Skip)], "a wider budget waits"); + assert_eq!(dropped(&mut wider), [(4..5, Hole::Skip)], "a wider budget waits"); assert_eq!( dropped(&mut capped), - [(3..4, Hole::Skip)], + [(4..5, Hole::Skip)], "a capped range may judge otherwise" ); assert_eq!( dropped(&mut above), - [(3..4, Hole::Skip)], + [(4..5, Hole::Skip)], "the stale run is outside its range" ); } @@ -24995,7 +25089,8 @@ mod test { } /// A stale verdict settles a reader that reads every range the upstream judged it - /// against, however many there are, with nothing received and no fill asked. Mirrors JS. + /// against, however many there are, with no fill asked. Group 3 is received, so the + /// later range's time start names it. Mirrors JS. #[test] #[cfg(feature = "async")] fn a_stale_verdict_settles_a_reader_of_every_range_it_judged() { @@ -25007,6 +25102,7 @@ mod test { producer.supersede_stale(judged([(0, 2), (4, 6)])); producer.give_up(0..2, Hole::stale(budget)); producer.give_up(4..6, Hole::stale(budget)); + producer.create_group(3u64.into()).unwrap().close().unwrap(); let mut subscriber = producer.subscribe( Subscription::default() .with_delay(Delay::default().with_max(budget)) @@ -25015,17 +25111,21 @@ mod test { let waiter = kio::Waiter::noop(); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((0..2, Hole::stale(budget)))); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((4..6, Hole::stale(budget)))); + assert!(matches!( + subscriber.poll_recv_group(&waiter), + Poll::Ready(Ok(Some(group))) if group.start == 3 + )); assert!( matches!(subscriber.poll_recv_group(&waiter), Poll::Ready(Ok(None))), - "nothing is owed" + "nothing else is owed" ); assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "nothing is asked"); } /// A reader whose ranges only partly overlap the ones a stale verdict was judged against /// is settled where it reads every judged group above the run, and asks a fill for the - /// rest, even with nothing received: no update brings it, since its ranges add nothing to - /// the upstream's terms. Mirrors JS. + /// rest, even with only group 3 received: no update brings it, since its ranges add + /// nothing to the upstream's terms. Mirrors JS. #[test] #[cfg(feature = "async")] fn a_stale_verdict_settles_a_partly_overlapping_reader_only_where_it_reads_above() { @@ -25036,6 +25136,8 @@ mod test { producer.supersede_stale(judged([(0, 2), (4, 6)])); producer.give_up(0..2, Hole::stale(budget)); producer.give_up(4..6, Hole::stale(budget)); + // Held, so a time start at 4 names it. + producer.create_group(3u64.into()).unwrap().close().unwrap(); let reader = |ranges: [Range; 2]| { producer.subscribe( Subscription::default() @@ -25044,15 +25146,20 @@ mod test { ) }; let waiter = kio::Waiter::noop(); - // Reads every judged group above each run, and more past them: settled. + // Reads every judged group above each run, and more past them: settled. Its start at + // time 1 reaches back to the oldest group, since none held contains it. let mut wider = reader([Range::groups(1..2), Range::groups(4..8)]); - assert_eq!(wider.poll_drop(&waiter), Poll::Ready((1..2, Hole::stale(budget)))); + assert_eq!(wider.poll_drop(&waiter), Poll::Ready((0..2, Hole::stale(budget)))); assert_eq!(wider.poll_drop(&waiter), Poll::Ready((4..6, Hole::stale(budget)))); assert!(wider.poll_drop(&waiter).is_pending()); assert!(dynamic.poll_requested_fill(&waiter).is_pending(), "nothing is asked"); // Stops at 4, which the upstream judged against 5: its own newest, which no verdict // convicts. Both runs are unknown to it, and asked for. let mut narrower = reader([Range::groups(0..2), Range::groups(4..5)]); + assert!(matches!( + narrower.poll_recv_group(&waiter), + Poll::Ready(Ok(Some(group))) if group.start == 3 + )); assert!(narrower.poll_recv_group(&waiter).is_pending()); assert!(narrower.poll_drop(&waiter).is_pending()); let mut asked = Vec::new(); @@ -25094,12 +25201,12 @@ mod test { drop(dynamic); let mut joined = reader(); assert!(joined.poll_recv_group(&waiter).is_pending(), "it joins the taken fill"); - for sequence in 0..2 { - let mut group = fill.accept(sequence, None).unwrap(); + for key in 0..2 { + let mut group = fill.accept(key, None).unwrap(); group - .write_frame(Timestamp::from_millis(sequence).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } drop(fill); let mut late = reader(); @@ -25111,7 +25218,7 @@ mod test { } } - /// Creating a group past the next sequence says nothing about the gap: groups may be + /// Creating a group past the next key says nothing about the gap: groups may be /// created out of order, so a late one in it still reaches readers, and nothing is skipped. #[test] #[cfg(feature = "async")] @@ -25130,7 +25237,7 @@ mod test { assert!(producer.state.read().drops.is_empty()); } - /// Appending says the producer will not go back: every sequence below the new group it + /// Appending says the producer will not go back: every key below the new group it /// never produced since its first group is skipped, and readers are told at once. #[test] #[cfg(feature = "async")] @@ -25141,14 +25248,14 @@ mod test { write_group(&producer, 2); write_group(&producer, 5); write_group(&producer, 7); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((3..5, Hole::Skip))); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((6..7, Hole::Skip))); assert_eq!(drain_groups(&mut subscriber), [2, 5, 7, 8]); // The next append only names what was left since this one. write_group(&producer, 10); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((9..10, Hole::Skip))); let skipped: Vec<_> = producer .state @@ -25160,7 +25267,7 @@ mod test { assert_eq!(skipped, [(3..5, Hole::Skip), (6..7, Hole::Skip), (9..10, Hole::Skip)]); } - /// An append's skip replaces a drop below it, whatever its error: the sequence was never + /// An append's skip replaces a drop below it, whatever its error: the key was never /// produced, which is the most authoritative verdict. #[test] #[cfg(feature = "async")] @@ -25169,7 +25276,7 @@ mod test { write_group(&producer, 0); write_group(&producer, 4); producer.give_up(1..2, Hole::drop(StreamError::Cancel)); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); let drops: Vec<_> = producer .state .read() @@ -25180,34 +25287,34 @@ mod test { assert_eq!(drops, [(1..4, Hole::Skip)]); } - /// A skip is final: creating or filling a group at a sequence an append or `skip_groups` + /// A skip is final: creating or filling a group at a key an append or `skip_groups` /// skipped is refused, while a dropped one may be created again. #[test] #[cfg(feature = "async")] - fn creating_a_skipped_sequence_is_refused() { + fn creating_a_skipped_key_is_refused() { let producer = track_producer("test", Info::timed()); write_group(&producer, 0); write_group(&producer, 3); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert!(matches!( - producer.create_group(group::Info { sequence: 1 }), + producer.create_group(group::Info { start: 1 }), Err(Error::Skipped) )); producer.skip_groups(5..7); assert!(matches!( - producer.create_group(group::Info { sequence: 6 }), + producer.create_group(group::Info { start: 6 }), Err(Error::Skipped) )); - // The next sequence, 5, was skipped too. - assert!(matches!(producer.append_group(), Err(Error::Skipped))); - // A later drop leaves a skipped sequence skipped. + // The next key, 5, was skipped too. + assert!(matches!(producer.append_next(), Err(Error::Skipped))); + // A later drop leaves a skipped key skipped. producer.give_up(1..2, Hole::drop(StreamError::Cancel)); assert!(matches!( - producer.create_group(group::Info { sequence: 1 }), + producer.create_group(group::Info { start: 1 }), Err(Error::Skipped) )); - // No fill holds a skipped sequence either. + // No fill holds a skipped key either. assert!(matches!( producer .state @@ -25217,7 +25324,7 @@ mod test { .insert_group_request(6, group::Bounds::default(), None), Err(Error::Skipped) )); - // A dropped sequence may be created again. + // A dropped key may be created again. producer.give_up(9..10, Hole::drop(StreamError::Cancel)); write_group(&producer, 9); } @@ -25359,7 +25466,7 @@ mod test { let Poll::Ready(Ok(Some(first))) = subscriber.poll_recv_group(&waiter) else { panic!("group 0 is handed out"); }; - assert_eq!(first.sequence, 0); + assert_eq!(first.start, 0); assert!( dynamic.poll_requested_fill(&waiter).is_pending(), "a dropped run asks no fill" @@ -25421,7 +25528,7 @@ mod test { state.bill(producer.bill()); } let now = crate::time::sim().now(); - let finish = |sequence: u64| crate::tail::Reading::open(&tail, Some(sequence), now).finish(); + let finish = |key: u64| crate::tail::Reading::open(&tail, Some(key), now).finish(); for run in 0..RUNS { finish(2 * run); } @@ -25508,14 +25615,14 @@ mod test { let capacity = 64 * (RUN_OVERHEAD + BORN_OVERHEAD); pool.resize(capacity); let mut read = None; - for sequence in 2 * RUNS..2 * RUNS + 64 { - let mut group = producer.create_group(group::Info { sequence }).unwrap(); + for key in 2 * RUNS..2 * RUNS + 64 { + let mut group = producer.create_group(group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence).unwrap(), vec![0u8; 16 * 1024]) + .write_frame(Timestamp::from_millis(key).unwrap(), vec![0u8; 16 * 1024]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); match subscriber.poll_recv_group(&waiter) { - Poll::Ready(Ok(Some(group))) => assert_eq!(group.sequence, sequence, "stale={stale}"), + Poll::Ready(Ok(Some(group))) => assert_eq!(group.start, key, "stale={stale}"), res => { read = Some(res.map(|res| res.map(|_| ()))); break; @@ -25637,7 +25744,7 @@ mod test { assert!(producer.state.read().skipped(0), "a skip above the edge stays"); // The write's own expiry scan finds the skip, with nothing else to rearm it. - let _group = producer.create_group(group::Info { sequence: 10 }).unwrap(); + let _group = producer.create_group(group::Info { start: 10 }).unwrap(); let state = producer.state.read(); assert!(!state.skipped(0), "the edge passed the skip"); assert_eq!(state.drops_bill.size(), 0, "its bill went with it"); @@ -25650,13 +25757,13 @@ mod test { fn a_huge_skip_costs_one_run() { const FAR: u64 = 1 << 61; let waiter = kio::Waiter::noop(); - // Presented just after group 0, since its sequence in milliseconds overflows. + // Presented just after group 0, since its key in milliseconds overflows. let write_far = |producer: &Producer| { - let mut group = producer.create_group(group::Info { sequence: FAR }).unwrap(); + let mut group = producer.create_group(group::Info { start: FAR }).unwrap(); group .write_frame(Timestamp::from_millis(1).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); @@ -25672,23 +25779,23 @@ mod test { let mut subscriber = producer.subscribe(replay().with_range(0..)); write_group(&producer, 0); write_far(&producer); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert_eq!(producer.state.read().drops.iter().count(), 1); assert_eq!(subscriber.poll_drop(&waiter), Poll::Ready((1..FAR, Hole::Skip))); assert_eq!(drain_groups(&mut subscriber), [0, FAR, FAR + 1]); } - /// A dropped sequence may be created again, as a group stream wins over its drop, such as + /// A dropped key may be created again, as a group stream wins over its drop, such as /// one whose group was aborted: only a skip is final. #[test] #[cfg(feature = "async")] - fn an_aborted_sequence_is_created_again() { + fn an_aborted_key_is_created_again() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); - let group = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let group = producer.create_group(group::Info { start: 0 }).unwrap(); group.abort(Error::Stream(StreamError::Cancel)).unwrap(); write_group(&producer, 0); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert!(producer.state.read().drops.is_empty()); let mut read = drain_groups(&mut subscriber); read.dedup(); @@ -25759,7 +25866,7 @@ mod test { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay().with_range(0..)); write_group(&producer, 3); - producer.append_group().unwrap().finish().unwrap(); + producer.append_next().unwrap().close().unwrap(); assert!(producer.state.read().drops.is_empty()); assert!(drained_drops(&mut subscriber).is_empty()); } @@ -25925,11 +26032,11 @@ mod test { fn a_drop_ends_a_held_group_with_its_error() { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); - let mut open = producer.create_group(group::Info { sequence: 0 }).unwrap(); - open.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); - let mut finished = producer.create_group(group::Info { sequence: 1 }).unwrap(); - finished.write_frame(Timestamp::ZERO, &b"b"[..]).unwrap(); - finished.finish().unwrap(); + let mut open = producer.create_group(group::Info { start: 0 }).unwrap(); + open.write_frame(open.key_time(), &b"a"[..]).unwrap(); + let mut finished = producer.create_group(group::Info { start: 1 }).unwrap(); + finished.write_frame(finished.key_time(), &b"b"[..]).unwrap(); + finished.close().unwrap(); let mut held = subscriber.assert_group(); assert_eq!( @@ -25947,7 +26054,7 @@ mod test { Some(Err(Error::Stream(StreamError::App(5)))) )); let mut whole = subscriber.assert_group(); - assert_eq!(whole.sequence, 1); + assert_eq!(whole.start, 1); assert_eq!( &whole.read_frame().now_or_never().unwrap().unwrap().unwrap().payload[..], b"b" @@ -25964,17 +26071,17 @@ mod test { let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(1).unwrap(); - live.write_frame(Timestamp::ZERO, &b"delta"[..]).unwrap(); + live.write_frame(live.key_time(), &b"delta"[..]).unwrap(); let _fetch = consumer.fetch_group(0, None); let fill = dynamic.requested_fill().await.unwrap(); let mut head = fill.accept(0, Info::timed()).unwrap(); - head.write_frame(Timestamp::ZERO, &b"snapshot"[..]).unwrap(); + head.write_frame(head.key_time(), &b"snapshot"[..]).unwrap(); producer.drop_groups(0..1, StreamError::App(5)); - head.write_frame(Timestamp::ZERO, &b"delta"[..]).unwrap(); - head.finish().unwrap(); + head.write_frame(head.key_time(), &b"delta"[..]).unwrap(); + head.close().unwrap(); let mut group = consumer.peek_group(0).expect("the head still serves group 0"); for payload in [&b"snapshot"[..], b"delta"] { @@ -25984,7 +26091,7 @@ mod test { } /// A group the publisher cuts before a reader takes it is still handed out, so the - /// reader sees why it ended rather than a group never sent: in arrival and in sequence + /// reader sees why it ended rather than a group never sent: in arrival and in key /// order, and on a track that then finishes, before the clean end. #[test] #[cfg(feature = "async")] @@ -25992,24 +26099,24 @@ mod test { let producer = track_producer("test", Info::timed()); let mut arrival = producer.subscribe(replay()); let mut ordered = producer.subscribe(replay()).ordered(); - let mut cut = producer.create_group(group::Info { sequence: 0 }).unwrap(); - cut.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); + let mut cut = producer.create_group(group::Info { start: 0 }).unwrap(); + cut.write_frame(cut.key_time(), &b"a"[..]).unwrap(); cut.abort(Error::App(5)).unwrap(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); let _producer = producer.finish().unwrap(); let mut first = arrival.assert_group(); - assert_eq!(first.sequence, 0); + assert_eq!(first.start, 0); assert!(matches!(first.read_frame().now_or_never(), Some(Err(Error::App(5))))); - assert_eq!(arrival.assert_group().sequence, 1); + assert_eq!(arrival.assert_group().start, 1); assert!(matches!(arrival.recv_group().now_or_never(), Some(Ok(None)))); let mut first = ordered.next_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(first.sequence, 0); + assert_eq!(first.start, 0); assert!(matches!(first.read_frame().now_or_never(), Some(Err(Error::App(5))))); assert_eq!(drain_ordered(&mut ordered), [1]); assert!(matches!(ordered.next_group().now_or_never(), Some(Ok(None)))); @@ -26031,15 +26138,15 @@ mod test { ] { let producer = track_producer("test", Info::timed()); let mut subscriber = producer.subscribe(replay()); - let mut group = producer.create_group(group::Info { sequence: 0 }).unwrap(); - group.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); + let mut group = producer.create_group(group::Info { start: 0 }).unwrap(); + group.write_frame(group.key_time(), &b"a"[..]).unwrap(); group.abort(err.clone()).unwrap(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); - assert_eq!(subscriber.assert_group().sequence, 1, "{err:?}"); + assert_eq!(subscriber.assert_group().start, 1, "{err:?}"); } } @@ -26051,21 +26158,21 @@ mod test { fn a_cut_outlives_the_sweep_until_it_expires() { let pool = cache::Pool::new(cache::Config::default().with_expiry(Duration::from_secs(10))); let producer = track_producer_pooled("test", pool.clone()); - let mut cut = producer.create_group(group::Info { sequence: 0 }).unwrap(); - cut.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); + let mut cut = producer.create_group(group::Info { start: 0 }).unwrap(); + cut.write_frame(cut.key_time(), &b"a"[..]).unwrap(); cut.abort(Error::App(5)).unwrap(); producer - .create_group(group::Info { sequence: 1 }) + .create_group(group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); - let _live = producer.create_group(group::Info { sequence: 2 }).unwrap(); + let _live = producer.create_group(group::Info { start: 2 }).unwrap(); elapse(&producer, Duration::from_secs(1)); pool.sweep(); let mut late = producer.subscribe(replay()); let mut first = late.assert_group(); - assert_eq!(first.sequence, 0, "the sweep kept the cut"); + assert_eq!(first.start, 0, "the sweep kept the cut"); assert!(matches!(first.read_frame().now_or_never(), Some(Err(Error::App(5))))); elapse(&producer, Duration::from_secs(20)); @@ -26092,19 +26199,19 @@ mod test { let mut arrival = producer.subscribe(replay()).with_range(..1); let mut ordered = producer.subscribe(replay()).ordered().with_range(..1); producer - .create_group(group::Info { sequence: 0 }) + .create_group(group::Info { start: 0 }) .unwrap() - .finish() + .close() .unwrap(); - let mut cut = producer.create_group(group::Info { sequence: 1 }).unwrap(); - cut.write_frame(Timestamp::ZERO, &b"a"[..]).unwrap(); + let mut cut = producer.create_group(group::Info { start: 1 }).unwrap(); + cut.write_frame(cut.key_time(), &b"a"[..]).unwrap(); cut.abort(Error::App(5)).unwrap(); - assert_eq!(arrival.assert_group().sequence, 0); + assert_eq!(arrival.assert_group().start, 0); assert!(arrival.recv_group().now_or_never().is_none(), "capped"); arrival = arrival.with_range(..); let mut reoffered = arrival.assert_group(); - assert_eq!(reoffered.sequence, 1); + assert_eq!(reoffered.start, 1); assert!(matches!( reoffered.read_frame().now_or_never(), Some(Err(Error::App(5))) @@ -26113,317 +26220,130 @@ mod test { assert_eq!(drain_ordered(&mut ordered), [0]); ordered = ordered.with_range(..); let mut reoffered = ordered.next_group().now_or_never().unwrap().unwrap().unwrap(); - assert_eq!(reoffered.sequence, 1); + assert_eq!(reoffered.start, 1); assert!(matches!( reoffered.read_frame().now_or_never(), Some(Err(Error::App(5))) )); } - /// A time start is the cached group containing it and a time end stops just past that - /// group; one past every start held stays open until a group starts after it. + /// A time start is the group containing it, the newest starting at or before it, and a + /// time end includes the group containing it. #[test] #[cfg(feature = "async")] - fn time_anchors_resolve_against_the_cache() { - let mut producer = track_producer("test", Info::timed()); - for second in 0..4 { - append_at(&mut producer, second * 1000); - } - let time = |millis| Anchor::Time(Timestamp::from_millis(millis).unwrap()); + fn time_anchors_resolve_against_the_keys() { + let producer = track_producer("test", Info::timed()); + for key in [0, 1000, 2000, 3000] { + write_group(&producer, key); + } + let time = |start, end| { + replay().with_ranges([Range { + start: Anchor::Time(start), + end: Some(Anchor::Time(end)), + }]) + }; - let mut seek = producer.subscribe(replay().with_ranges([Range { - start: time(1500), - end: Some(time(2000)), - }])); - assert_eq!(drain(&mut seek), vec![1, 2]); + let mut seek = producer.subscribe(time(1500, 2000)); + assert_eq!(drain(&mut seek), vec![1000, 2000]); - // A start before every group reaches back to the oldest one. - let mut early = producer.subscribe(replay().with_ranges([Range { - start: Anchor::Time(Timestamp::from_micros(0).unwrap()), - end: Some(time(500)), - }])); + let mut early = producer.subscribe(time(0, 500)); assert_eq!(drain(&mut early), vec![0]); - // Past the newest start: open until a group is known to start after the end. - let mut live = producer.subscribe(replay().with_ranges([Range { - start: time(3000), - end: Some(time(4500)), - }])); - assert_eq!(drain(&mut live), vec![3]); - append_at(&mut producer, 4000); - append_at(&mut producer, 5000); - append_at(&mut producer, 6000); - assert_eq!(drain(&mut live), vec![4]); + // Past the newest start: open for the groups that start inside the end. + let mut live = producer.subscribe(time(3000, 4500)); + assert_eq!(drain(&mut live), vec![3000]); + for key in [4000, 5000, 6000] { + write_group(&producer, key); + } + assert_eq!(drain(&mut live), vec![4000]); } - /// A group joined mid-group stamps a later frame than its start, so it never proves a - /// time end falls before it. + /// A group joined mid-group still starts at its key, whatever its first frame held says: + /// a time start inside it names it. The range is open, since a bounded read takes whole + /// groups only. #[test] #[cfg(feature = "async")] - fn a_partial_group_proves_no_start() { - let mut producer = track_producer("test", Info::timed()); - append_at(&mut producer, 0); - append_at(&mut producer, 1000); - let mut partial = producer.append_group().unwrap(); + fn a_partial_group_starts_at_its_key() { + let producer = track_producer("test", Info::timed()); + write_group(&producer, 0); + write_group(&producer, 1000); + let mut partial = producer.create_group(group::Info { start: 2000 }).unwrap(); partial.start_at(1).unwrap(); partial .write_frame(Timestamp::from_millis(2500).unwrap(), bytes::Bytes::from_static(b"x")) .unwrap(); - partial.finish().unwrap(); + partial.close().unwrap(); - let time = |millis| Anchor::Time(Timestamp::from_millis(millis).unwrap()); let mut seek = producer.subscribe(replay().with_ranges([Range { - start: time(1500), - end: Some(time(2200)), + start: Anchor::Time(2200), + end: None, }])); - assert_eq!(drain(&mut seek), vec![1, 2]); + assert_eq!(drain(&mut seek), vec![2000]); } - /// A time start resolves once a group starts, not at the latest group: a copy can show - /// the route's answer (here an empty group) before the groups its publisher resolved the - /// time to start, and a producer can open groups before their first frames. + /// A time start resolves on keys alone: groups opened before their first frames are read + /// from the one containing it. #[test] #[cfg(feature = "async")] - fn a_time_start_waits_for_a_group_to_start() { - let at = |millis| Timestamp::from_millis(millis).unwrap(); - let since = [Range { - start: Anchor::Time(at(2500)), - end: None, - }]; - for (copy, order) in [ - (false, Order::Ascending), - (true, Order::Ascending), - (true, Order::Descending), - ] { - let mut producer = track_producer("test", Info::timed()); - if copy { - producer.set_idle(); - producer.set_live(Some(Position::group(9)), Some(2)).unwrap(); - } - let mut groups: Vec<_> = (2..9) - .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) + fn a_time_start_needs_no_frame() { + for order in [Order::Ascending, Order::Descending] { + let producer = track_producer("test", Info::timed()); + let _groups: Vec<_> = (2..10) + .map(|second| producer.create_group(group::Info { start: second * 1000 }).unwrap()) .collect(); - producer - .create_group(group::Info { sequence: 9 }) - .unwrap() - .finish() - .unwrap(); - + let since = [Range { + start: Anchor::Time(2500), + end: None, + }]; let mut subscriber = producer.subscribe(replay().with_ranges(since).with_order(order)); - assert_eq!(drain(&mut subscriber), Vec::::new(), "copy: {copy}, {order:?}"); - for (group, sequence) in groups.iter_mut().zip(2..) { - group - .write_frame(at(sequence * 1000), bytes::Bytes::from_static(b"x")) - .unwrap(); - } let mut read = drain(&mut subscriber); read.sort_unstable(); - assert_eq!(read, (2..10).collect::>(), "copy: {copy}, {order:?}"); - } - } - - /// A time end stays open across a group that has yet to start: it may start before the - /// end, so a later group starting after it settles nothing. - #[test] - #[cfg(feature = "async")] - fn a_time_end_waits_on_an_unstarted_group_below_it() { - let producer = track_producer("test", Info::timed()); - let at = |millis| Timestamp::from_millis(millis).unwrap(); - let mut zero = producer.create_group(group::Info { sequence: 0 }).unwrap(); - zero.write_frame(at(0), bytes::Bytes::from_static(b"x")).unwrap(); - let mut hole = producer.create_group(group::Info { sequence: 1 }).unwrap(); - let mut two = producer.create_group(group::Info { sequence: 2 }).unwrap(); - two.write_frame(at(2000), bytes::Bytes::from_static(b"x")).unwrap(); - - let mut subscriber = producer.subscribe(replay().with_ranges([Range { - start: Anchor::Absolute(0), - end: Some(Anchor::Time(at(500))), - }])); - hole.write_frame(at(1000), bytes::Bytes::from_static(b"x")).unwrap(); - assert_eq!(drain(&mut subscriber), vec![0]); - } - - /// A time end stays open across a sequence yet to arrive: it may start before the end, - /// so a later group starting after it settles nothing. Once that sequence is reported - /// dropped, nothing below the later group can start before the end, so it settles. - #[test] - #[cfg(feature = "async")] - fn a_time_end_waits_on_an_absent_group_below_it() { - let at = |millis| Timestamp::from_millis(millis).unwrap(); - let ranges = [Range { - start: Anchor::Absolute(0), - end: Some(Anchor::Time(at(500))), - }]; - for dropped in [false, true] { - let producer = track_producer("test", Info::timed()); - // A received copy whose wire carries media time, so group 1 may still arrive. - producer.received(); - producer.anchored(); - let mut zero = producer.create_group(group::Info { sequence: 0 }).unwrap(); - zero.write_frame(at(0), bytes::Bytes::from_static(b"x")).unwrap(); - let mut two = producer.create_group(group::Info { sequence: 2 }).unwrap(); - two.write_frame(at(2000), bytes::Bytes::from_static(b"x")).unwrap(); - - let mut subscriber = producer.subscribe(replay().with_ranges(ranges)); assert_eq!( - subscriber.resolved_ranges(), - vec![Some(Resolved { - start: 0, - end: None, - floored: true - })], - "group 1 may still start before the end" + read, + (2..10).map(|second| second * 1000).collect::>(), + "{order:?}" ); - if dropped { - producer.drop_groups(1..2, StreamError::NotFound); - assert_eq!(drain(&mut subscriber), vec![0]); - assert_eq!( - subscriber.resolved_ranges(), - vec![Some(Resolved { - start: 0, - end: Some(2), - floored: true - })], - "the drop settles the end" - ); - continue; - } - let mut one = producer.create_group(group::Info { sequence: 1 }).unwrap(); - one.write_frame(at(1000), bytes::Bytes::from_static(b"x")).unwrap(); - assert_eq!(drain(&mut subscriber), vec![0]); } } - /// A time end at the live edge reads at most one group past it: groups opened before - /// their first frame wait on the one below them starting, rather than every one being - /// handed out while the end is unsettled. + /// A time end settles on keys alone: a group opened before its first frame is inside the + /// end or past it by its key. #[test] #[cfg(feature = "async")] - fn a_time_end_reads_one_group_past_it_at_the_live_edge() { - let at = |millis| Timestamp::from_millis(millis).unwrap(); - let ranges = [Range { - start: Anchor::Time(at(3000)), - end: Some(Anchor::Time(at(3500))), - }]; - for ordered in [false, true] { - let mut producer = track_producer("test", Info::timed()); - append_at(&mut producer, 3000); - let mut groups: Vec<_> = (1..4) - .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) - .collect(); - let subscriber = producer.subscribe(replay().with_ranges(ranges)); - let mut reader = Reader::new(subscriber, ordered); - assert_eq!(reader.drain(), vec![0, 1], "1 is read on speculation, 2 and 3 wait"); - groups[0] - .write_frame(at(3600), bytes::Bytes::from_static(b"x")) - .unwrap(); - groups[1] - .write_frame(at(4000), bytes::Bytes::from_static(b"x")) - .unwrap(); - assert_eq!(reader.drain(), Vec::::new(), "ordered: {ordered}"); - assert_eq!( - reader.resolved(), - vec![Some(Resolved { - start: 0, - end: Some(1), - floored: true - })], - "1 starting past the end settles it" - ); - } - } - - /// A read parked on a time end's cap wakes when the speculative group starts, and when - /// a later group starts inside the end while the speculative one stalls. - #[kio::test] - #[cfg(feature = "async")] - async fn a_read_parked_on_a_time_end_wakes_as_groups_start() { - let at = |millis| Timestamp::from_millis(millis).unwrap(); + fn a_time_end_needs_no_frame() { let producer = track_producer("test", Info::timed()); - let mut groups: Vec<_> = (0..4) - .map(|sequence| producer.create_group(group::Info { sequence }).unwrap()) + let _groups: Vec<_> = [0, 400, 600] + .into_iter() + .map(|key| producer.create_group(group::Info { start: key }).unwrap()) .collect(); - let mut subscriber = producer.subscribe(replay().with_order(Order::Ascending).with_ranges([Range { - start: Anchor::Absolute(0), - end: Some(Anchor::Time(at(3500))), + let mut subscriber = producer.subscribe(replay().with_ranges([Range { + start: Anchor::Time(0), + end: Some(Anchor::Time(500)), }])); - assert_eq!(drain(&mut subscriber), vec![0]); - - let parked = kio::clock::sim::spawn(async move { - let one = subscriber.recv_group().await.unwrap().unwrap().sequence; - let two = subscriber.recv_group().await.unwrap().unwrap().sequence; - (one, two, subscriber) - }); - kio::clock::sim::yield_now().await; - assert!(!parked.is_finished(), "1 waits on 0 starting"); - groups[0] - .write_frame(at(3000), bytes::Bytes::from_static(b"x")) - .unwrap(); - kio::clock::sim::yield_now().await; - assert!(!parked.is_finished(), "1 is read, and 2 waits on 1 starting"); - - // 1 stalls, so only 2 starting inside the end lifts the cap past it. - groups[2] - .write_frame(at(3200), bytes::Bytes::from_static(b"x")) - .unwrap(); - let (one, two, mut subscriber) = parked.await.unwrap(); - assert_eq!((one, two), (1, 2)); - assert_eq!(drain(&mut subscriber), vec![3], "3 is the speculative one now"); - groups[3] - .write_frame(at(3600), bytes::Bytes::from_static(b"x")) - .unwrap(); - groups[1] - .write_frame(at(3100), bytes::Bytes::from_static(b"x")) - .unwrap(); + assert_eq!(drain(&mut subscriber), vec![0, 400]); assert_eq!( subscriber.resolved_ranges(), vec![Some(Resolved { start: 0, - end: Some(3), + end: Some(501), floored: true })] ); } - /// An untimed track refuses a time anchor wherever the ranges reach it. + /// A scale-less track's keys are bare units, so a time anchor reads them as they are. #[test] #[cfg(feature = "async")] - fn an_untimed_track_refuses_time_anchors() { + fn an_untimed_track_reads_time_anchors_as_keys() { let producer = untimed_producer(); - let timed = replay().with_ranges([Range { - start: Anchor::Time(Timestamp::from_millis(0).unwrap()), - end: None, - }]); - - let pending = producer.consume().subscribe(timed.clone()); - assert!(matches!( - pending.poll_ok(&kio::Waiter::noop()), - Poll::Ready(Err(Error::TimestampMismatch)) - )); - - let mut subscriber = producer.subscribe(replay()); - assert!(matches!( - subscriber.update(timed.clone()), - Err(Error::TimestampMismatch) - )); - assert!(matches!( - subscriber.control().update(timed.clone()), - Err(Error::TimestampMismatch) - )); - assert_eq!(subscriber.subscription(), replay(), "a refused update changes nothing"); - - let mut refused = producer.subscribe(timed); - assert!(matches!( - refused.recv_group().now_or_never(), - Some(Err(Error::TimestampMismatch)) - )); - - // Ranges it can serve lift the refusal. - refused.update(replay()).unwrap(); - let group = producer.append_group().unwrap(); - group.finish().unwrap(); - let read = refused.recv_group().now_or_never().expect("the group is cached"); - assert_eq!(read.unwrap().map(|group| group.sequence), Some(0)); + for key in [0, 5, 10, 15] { + producer.append_group(key).unwrap().close().unwrap(); + } + let mut subscriber = producer.subscribe(replay().with_ranges([Range { + start: Anchor::Time(7), + end: Some(Anchor::Time(10)), + }])); + assert_eq!(drain(&mut subscriber), vec![5, 10]); } /// A widened budget re-offers the cached groups the cursor skipped as stale, newest @@ -26628,9 +26548,9 @@ mod test { let pool = producer.broadcast.pool.clone(); let dynamic = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(32).unwrap(); - live.write_frame(Timestamp::ZERO, vec![32; size]).unwrap(); + live.write_frame(live.key_time(), vec![32; size]).unwrap(); assert!( live.handle().content().is_none(), "a single copy keeps its fast representation" @@ -26652,9 +26572,9 @@ mod test { let fill = dynamic.requested_fill_group().await.unwrap(); let mut tail = fill.accept(Info::timed()).unwrap(); for frame in start..end { - tail.write_frame(Timestamp::ZERO, vec![frame as u8; size]).unwrap(); + tail.write_frame(tail.key_time(), vec![frame as u8; size]).unwrap(); } - let tail = tail.finish().unwrap(); + let tail = tail.close().unwrap(); deliveries.push(tail.handle()); held.push(pending.await.unwrap()); } @@ -26710,9 +26630,9 @@ mod test { #[cfg(feature = "async")] async fn tail_owner_survives_slot_removal_but_not_live_recreation() { let producer = track_producer("test", Info::timed()); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(8).unwrap(); - live.write_frame(Timestamp::ZERO, b"eight".as_slice()).unwrap(); + live.write_frame(live.key_time(), b"eight".as_slice()).unwrap(); let mut tails = Vec::new(); let mut held = Vec::new(); for start in [4, 2] { @@ -26731,8 +26651,8 @@ mod test { None, ) .unwrap(); - tail.write_frame(Timestamp::ZERO, vec![start as u8; 4]).unwrap(); - let tail = tail.finish().unwrap(); + tail.write_frame(tail.key_time(), vec![start as u8; 4]).unwrap(); + let tail = tail.close().unwrap(); held.push(tail.consume().with_range(start..)); tails.push(tail); } @@ -26765,12 +26685,14 @@ mod test { ) .unwrap(); assert!(Arc::ptr_eq(&owner, &refill.handle().content().unwrap())); - refill.write_frame(Timestamp::ZERO, b"zero".as_slice()).unwrap(); - let refill = refill.finish().unwrap(); + refill.write_frame(refill.key_time(), b"zero".as_slice()).unwrap(); + let refill = refill.close().unwrap(); refill.abort(Error::Cancel).unwrap(); - let mut recreated = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut recreated = producer.create_group(group::Info { start: 0 }).unwrap(); recreated.start_at(8).unwrap(); - recreated.write_frame(Timestamp::ZERO, b"new eight".as_slice()).unwrap(); + recreated + .write_frame(recreated.key_time(), b"new eight".as_slice()) + .unwrap(); assert!( recreated.handle().content().is_none(), "an explicit new live incarnation starts independently" @@ -26790,7 +26712,7 @@ mod test { None, ) .unwrap(); - new.write_frame(Timestamp::ZERO, b"new zero".as_slice()).unwrap(); + new.write_frame(new.key_time(), b"new zero".as_slice()).unwrap(); assert!(!Arc::ptr_eq(&owner, &new.handle().content().unwrap())); assert_eq!(held[0].read_frame().await.unwrap().unwrap().payload.as_ref(), &[4; 4]); } @@ -26807,13 +26729,13 @@ mod test { ..Default::default() }); let producer = Producer::new(broadcast, "test", Info::timed()); - let mut first = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut first = producer.create_group(group::Info { start: 0 }).unwrap(); first.start_at(4).unwrap(); let mut first_read = first.consume().with_range(4..); let mut first = first .create_frame(frame::Info { size: 1024, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); first.write(vec![4; 512]).unwrap(); @@ -26835,12 +26757,12 @@ mod test { None, ) .unwrap(); - second.write_frame(Timestamp::ZERO, b"three".as_slice()).unwrap(); + second.write_frame(second.key_time(), b"three".as_slice()).unwrap(); let mut second_read = second.consume().with_range(4..); let mut second = second .create_frame(frame::Info { size: 1024, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); assert!( @@ -26857,7 +26779,7 @@ mod test { assert!(budget.reserve(1).is_none(), "the survivor still holds admission"); second.write(vec![4; 768]).unwrap(); let second = second.finish().unwrap(); - second.finish().unwrap(); + second.close().unwrap(); assert!(matches!(streaming.read_chunk().await, Err(Error::Cancel))); let frame = second_read.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload.as_ref(), &[4; 1024]); @@ -26869,7 +26791,7 @@ mod test { } else { second.write(vec![4; 768]).unwrap(); let second = second.finish().unwrap(); - second.finish().unwrap(); + second.close().unwrap(); let later_copy = second_read.read_frame().await.unwrap().unwrap(); assert!( budget.reserve(1).is_none(), @@ -26880,7 +26802,7 @@ mod test { kio::clock::sim::advance(Duration::from_millis(10)).await; first.write(vec![4; 512]).unwrap(); let first = first.finish().unwrap(); - let first = first.finish().unwrap(); + let first = first.close().unwrap(); let original_copy = first.consume().with_range(4..).read_frame().await.unwrap().unwrap(); assert_eq!( original_copy.arrived().duration_since(later_copy.arrived()), @@ -26917,7 +26839,7 @@ mod test { .create_object(frame::Header { info: frame::Info { size: payload.len() as u64, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }, fields: Some(Arc::new(super::super::object::Fields { id, @@ -26933,7 +26855,7 @@ mod test { frame.finish().unwrap() } let producer = track_producer("test", Info::timed()); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(2).unwrap(); live.set_wire(wire(100)).unwrap(); let live = write(live, 100, b"a"); @@ -26960,7 +26882,7 @@ mod test { properties: Default::default(), }) .unwrap(); - let head = head.finish().unwrap(); + let head = head.close().unwrap(); let earlier = producer .state .write() @@ -26979,7 +26901,7 @@ mod test { earlier.set_wire(wire(90)).unwrap(); let earlier = write(earlier, 90, b"b"); let earlier = write(earlier, 95, b"c"); - let earlier = earlier.finish().unwrap(); + let earlier = earlier.close().unwrap(); assert!(head.consume().end_object().is_some_and(|fields| fields.id == 101)); assert!( earlier.consume().end_object().is_none(), @@ -27057,14 +26979,14 @@ mod test { } let info = Info::timed().with_compression(Compression::Deflate); let reference = track_producer("reference", info.clone()); - let mut source = reference.create_group(group::Info { sequence: 0 }).unwrap(); + let mut source = reference.create_group(group::Info { start: 0 }).unwrap(); let expected: Vec<_> = (0..9) .map(|index| bytes::Bytes::from(format!("shared dictionary frame {index}; ").repeat(64))) .collect(); for payload in &expected { - source.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + source.write_frame(source.key_time(), payload.clone()).unwrap(); } - let source = source.finish().unwrap(); + let source = source.close().unwrap(); let mut stored = arrayvec::ArrayVec::::new(); assert!(matches!( source @@ -27076,7 +26998,7 @@ mod test { let producer = track_producer("test", info.clone()); let requests = producer.requests(); let consumer = producer.consume(); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(8).unwrap(); let live = ingest(live, &stored[8]); let mut raw_live = live.consume().with_range(8..).compressed(); @@ -27107,7 +27029,7 @@ mod test { assert_eq!(first.read_frame().await.unwrap().unwrap().payload, expected[0]); kio::clock::sim::advance(Duration::from_millis(20)).await; head = ingest(head, &stored[8]); - let head = head.finish().unwrap(); + let head = head.close().unwrap(); pending.await.unwrap(); let mut raw_head = head.consume().with_range(8..).compressed(); let raw_head = raw_head.read_frame().await.unwrap().unwrap(); @@ -27170,14 +27092,14 @@ mod test { #[cfg(feature = "async")] async fn canonical_owner_follows_an_evicted_streaming_frame_reader() { let producer = track_producer("test", Info::timed()); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); live.start_at(4).unwrap(); let observing = live.handle(); let mut reader = live.consume().with_range(4..); let mut live = live .create_frame(frame::Info { size: 1024, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); live.write(vec![4; 512]).unwrap(); @@ -27200,8 +27122,8 @@ mod test { None, ) .unwrap(); - head.write_frame(Timestamp::ZERO, b"three".as_slice()).unwrap(); - head.finish().unwrap(); + head.write_frame(head.key_time(), b"three".as_slice()).unwrap(); + head.close().unwrap(); let identity = Arc::downgrade(&observing.content().unwrap()); producer .state @@ -27238,12 +27160,12 @@ mod test { let mut refill = refill .create_frame(frame::Info { size: 1024, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); refill.write(vec![4; 1024]).unwrap(); let refill = refill.finish().unwrap(); - let refill = refill.finish().unwrap(); + let refill = refill.close().unwrap(); let complete = refill.consume().with_range(4..).read_frame().await.unwrap().unwrap(); assert_eq!( complete.payload.as_ptr() as usize, @@ -27258,11 +27180,11 @@ mod test { #[test] fn canonical_content_preserves_individually_valid_disjoint_ranges() { let producer = track_producer("test", Info::timed()); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); const FRAMES: usize = 8192; live.start_at(FRAMES as u64).unwrap(); for _ in 0..FRAMES { - live.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(); + live.write_frame(live.key_time(), bytes::Bytes::new()).unwrap(); } let mut head = producer .state @@ -27279,8 +27201,8 @@ mod test { None, ) .unwrap(); - head.write_frame(Timestamp::ZERO, bytes::Bytes::new()).unwrap(); - let head = head.finish().unwrap(); + head.write_frame(head.key_time(), bytes::Bytes::new()).unwrap(); + let head = head.close().unwrap(); assert_eq!(live.handle().content().unwrap().cost().0, FRAMES + 1); assert_eq!(live.frame_count(), 2 * FRAMES); assert_eq!(head.frame_count(), 1); @@ -27300,12 +27222,12 @@ mod test { const POOL: u64 = 4 * 1024 * 1024; let pool = cache::Pool::new(cache::Config::default().with_capacity(POOL)); let producer = track_producer_pooled("test", pool.clone()); - let mut live = producer.create_group(group::Info { sequence: 0 }).unwrap(); + let mut live = producer.create_group(group::Info { start: 0 }).unwrap(); let mut reader = live.consume(); let mut late = live.consume(); // 48 MiB, past the 32 MiB tail: pool debt sheds first. for index in 0..3_000 { - live.write_frame(Timestamp::ZERO, numbered(index)).unwrap(); + live.write_frame(live.key_time(), numbered(index)).unwrap(); let frame = reader.read_frame().now_or_never().unwrap(); let frame = frame.unwrap_or_else(|err| panic!("frame {index}: {err}")).unwrap(); assert_eq!(frame.payload, numbered(index)); @@ -27328,9 +27250,10 @@ mod test { "test", Info::default().with_refresh(1_000_000), ); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); for index in 0..1_000 { - live.write_frame(None, numbered(index)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) + .unwrap(); } let _subscriber = producer.subscribe(None); assert_eq!(producer.demand().wanted(), Some(1)); @@ -27352,10 +27275,11 @@ mod test { ); // Joined before the group grew, so the join asks for nothing. let _subscriber = producer.subscribe(None); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); for index in 0..1_000 { - live.write_frame(None, numbered(index)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) + .unwrap(); } assert_eq!(producer.demand().wanted(), Some(0), "nothing asked yet"); assert!(matches!( @@ -27385,13 +27309,14 @@ mod test { fn a_late_joiner_rides_a_flight_and_hands_over() { let producer = track_producer("test", Info::default()); let requests = producer.requests(); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); let mut late = live.consume(); // 48 MiB, half again the live tail. const FRAMES: u32 = 3_000; for index in 0..FRAMES { - live.write_frame(None, numbered(index)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut reader), Some(index)); } @@ -27410,7 +27335,9 @@ mod test { // The handler writes as the rider reads, until the rider leaves for the live copy. let mut index = 0; while !unwanted(&flight) { - flight.write_frame(None, numbered(index)).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut late), Some(index)); index += 1; assert!(index < FRAMES, "handed over before the flight caught up"); @@ -27422,7 +27349,8 @@ mod test { .poll_ready(&kio::Waiter::noop(), &frame_info(&numbered(0))) .is_ready() { - live.write_frame(None, numbered(written)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(written)) + .unwrap(); written += 1; } assert!(written > FRAMES + 900, "a whole window: {written}"); @@ -27445,12 +27373,13 @@ mod test { fn a_compressed_flight_starts_at_frame_0_and_its_rider_skips_ahead() { let producer = track_producer("test", Info::default().with_compression(Compression::Deflate)); let requests = producer.requests(); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); let mut behind = live.consume(); const FRAMES: u32 = 3_000; for index in 0..FRAMES { - live.write_frame(None, numbered(index)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut reader), Some(index)); if index < 10 { assert_eq!(read_numbered(&mut behind), Some(index)); @@ -27464,12 +27393,16 @@ mod test { let mut flight = fill.accept(0, None).unwrap(); let unwanted = |flight: &group::Producer| flight.handle().poll_unwanted(&kio::Waiter::noop()).is_ready(); for index in 0..10 { - flight.write_frame(None, numbered(index)).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut behind), None, "frame {index} is skipped"); } let mut index = 10; while !unwanted(&flight) { - flight.write_frame(None, numbered(index)).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut behind), Some(index)); index += 1; assert!(index < FRAMES, "handed over before the flight caught up"); @@ -27485,12 +27418,13 @@ mod test { assert_eq!(read_numbered(&mut behind), None, "at the live edge"); } - /// The size of `payload` as a frame, for [`group::Producer::poll_ready`]. + /// The size of `payload` as a frame, for [`group::Producer::poll_ready`], which weighs + /// nothing else. #[cfg(feature = "async")] fn frame_info(payload: &bytes::Bytes) -> frame::Info { frame::Info { size: payload.len() as u64, - timestamp: None, + timestamp: Timestamp::unscaled(0), } } @@ -27500,10 +27434,11 @@ mod test { fn shed_live_group(frames: u32) -> (Producer, Requests, group::Producer, group::Consumer) { let producer = track_producer("test", Info::default()); let requests = producer.requests(); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); let mut reader = live.consume(); for index in 0..frames { - live.write_frame(None, numbered(index)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut reader), Some(index)); } (producer, requests, live, reader) @@ -27533,7 +27468,9 @@ mod test { let mut flight = fill.accept(0, None).unwrap(); // 8 MiB in, half the reach. for index in 0..500 { - flight.write_frame(None, numbered(index)).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut first), Some(index)); } let mut near = live.consume(); @@ -27542,7 +27479,9 @@ mod test { assert!(asked(&requests).is_none(), "one upstream fill"); // 17 MiB in, past the reach. for index in 500..1_100 { - flight.write_frame(None, numbered(index)).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut first), Some(index)); } let mut far = live.consume(); @@ -27570,16 +27509,19 @@ mod test { Info::default(), ); let requests = producer.requests(); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); let mut late = live.consume(); for index in 0..1_000 { - live.write_frame(None, numbered(index)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) + .unwrap(); } assert_eq!(read_numbered(&mut late), None); let mut flight = asked(&requests).expect("a flight").accept(0, None).unwrap(); assert_eq!(read_numbered(&mut late), None); for index in 0..1_000 { - flight.write_frame(None, numbered(index)).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) + .unwrap(); assert_eq!(read_numbered(&mut late), Some(index)); assert!(pool.used() < 2 * POOL, "frame {index}: {} bytes held", pool.used()); } @@ -27593,7 +27535,7 @@ mod test { let producer = track_producer("test", Info::timed()); let requests = producer.requests(); let mut sub = producer.subscribe(None); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); for index in 0..3_000u32 { let at = Timestamp::from_millis(index.into()).unwrap(); live.write_frame(at, numbered(index)).unwrap(); @@ -27601,7 +27543,7 @@ mod test { let mut late = sub.recv_group().now_or_never().unwrap().unwrap().expect("group 0"); assert!(late.read_frame().now_or_never().is_none(), "boarding"); let _flight = asked(&requests).expect("a flight, never answered"); - let mut newer = producer.append_group().unwrap(); + let mut newer = producer.append_next().unwrap(); newer .write_frame(Timestamp::from_millis(100_000).unwrap(), numbered(0)) .unwrap(); @@ -27626,22 +27568,27 @@ mod test { Info::default(), ); let requests = producer.requests(); - let mut old = producer.append_group().unwrap(); + let mut old = producer.append_next().unwrap(); let mut late = old.consume(); for index in 0..1_000 { - old.write_frame(None, numbered(index)).unwrap(); + old.write_frame(crate::Timestamp::unscaled(old.start), numbered(index)) + .unwrap(); } assert_eq!(read_numbered(&mut late), None); let mut flight = asked(&requests).expect("a flight").accept(0, None).unwrap(); assert_eq!(read_numbered(&mut late), None); // A newer group leaves group 0 cold, and its cached copy goes whole. - let mut newer = producer.append_group().unwrap(); + let mut newer = producer.append_next().unwrap(); for index in 0..1_000 { - newer.write_frame(None, numbered(index)).unwrap(); + newer + .write_frame(crate::Timestamp::unscaled(newer.start), numbered(index)) + .unwrap(); } assert!(!producer.state.read().lookup.contains_key(&0), "group 0 evicted"); for index in 0..1_000 { - flight.write_frame(None, numbered(index)).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), numbered(index)) + .unwrap(); assert!(pool.used() < 2 * POOL, "frame {index}: {} bytes held", pool.used()); } // Ending every copy frees what they held, though the read still holds their slots. @@ -27659,10 +27606,11 @@ mod test { let requests = producer.requests(); // Joined before the group grew, so the join asks for nothing. let _subscriber = producer.subscribe(None); - let mut live = producer.append_group().unwrap(); + let mut live = producer.append_next().unwrap(); let mut late = live.consume(); for index in 0..3_000 { - live.write_frame(None, numbered(index)).unwrap(); + live.write_frame(crate::Timestamp::unscaled(live.start), numbered(index)) + .unwrap(); } assert_eq!(read_numbered(&mut late), None, "asks a flight"); assert_eq!(producer.demand().wanted(), Some(0), "nothing refreshed yet"); @@ -27692,7 +27640,9 @@ mod test { .is_ready(), "frame {index}: paced by the fast rider only" ); - flight.write_frame(None, payload).unwrap(); + flight + .write_frame(crate::Timestamp::unscaled(flight.start), payload) + .unwrap(); assert_eq!(read_numbered(&mut fast), Some(index)); } assert_eq!(read_numbered(&mut slow), None, "fell off"); @@ -27700,11 +27650,11 @@ mod test { assert_eq!(own.start(), Position { group: 0, frame: 0 }); } - /// What a read returns right now: `Some(Some(sequence))` for a group, `Some(None)` for + /// What a read returns right now: `Some(Some(key))` for a group, `Some(None)` for /// its end, `None` while it parks. fn read_now(subscriber: &mut Subscriber, ordered: bool) -> Option> { match subscriber.poll_group(ordered, &kio::Waiter::noop()) { - Poll::Ready(res) => Some(res.expect("would have errored").map(|group| group.sequence)), + Poll::Ready(res) => Some(res.expect("would have errored").map(|group| group.start)), Poll::Pending => None, } } @@ -27742,8 +27692,8 @@ mod test { reader.update(subscription.clone().with_range(0..6)).unwrap(); let newly_covered = if ordered { &[4, 5][..] } else { &[0, 2, 4, 5][..] }; - for sequence in newly_covered { - assert_eq!(read_now(&mut reader, ordered), Some(Some(*sequence))); + for key in newly_covered { + assert_eq!(read_now(&mut reader, ordered), Some(Some(*key))); } for _ in 0..3 { assert_eq!(read_now(&mut reader, ordered), end); @@ -27761,8 +27711,8 @@ mod test { #[test] fn a_bounded_read_stays_ended_until_widened() { let mut producer = track_producer("test", Info::timed()); - for sequence in 0..6 { - write_group(&producer, sequence); + for key in 0..6 { + write_group(&producer, key); } bounded_read_reopens(&producer.consume(), &mut producer); } @@ -27774,8 +27724,8 @@ mod test { .routes(routes.consume()) .accept(Info::timed()); let mut producer = track_producer("test", Info::timed()); - for sequence in 0..6 { - write_group(&producer, sequence); + for key in 0..6 { + write_group(&producer, key); } routes.serve(producer.consume()); bounded_read_reopens(&logical.consume(), &mut producer); @@ -27786,7 +27736,7 @@ mod test { #[test] fn a_read_with_more_to_come_parks() { let producer = track_producer("test", Info::timed()); - producer.create_group(0u64.into()).unwrap().finish().unwrap(); + producer.create_group(0u64.into()).unwrap().close().unwrap(); let mut open = producer.subscribe(replay().with_range(0..)); assert_eq!(read_now(&mut open, false), Some(Some(0))); @@ -27799,7 +27749,7 @@ mod test { let mut owed = producer.subscribe(replay().with_range(0..2)); assert_eq!(read_now(&mut owed, false), Some(Some(0))); assert_eq!(read_now(&mut owed, false), None, "group 1 may still arrive"); - producer.create_group(1u64.into()).unwrap().finish().unwrap(); + producer.create_group(1u64.into()).unwrap().close().unwrap(); assert_eq!(read_now(&mut owed, false), Some(Some(1))); assert_eq!(read_now(&mut owed, false), Some(None)); } @@ -27810,7 +27760,7 @@ mod test { fn a_bounded_read_ends_past_dropped_and_stale_groups() { let producer = track_producer("test", Info::timed()); let mut dropped = producer.subscribe(replay().with_range(0..2)); - producer.create_group(0u64.into()).unwrap().finish().unwrap(); + producer.create_group(0u64.into()).unwrap().close().unwrap(); assert_eq!(read_now(&mut dropped, false), Some(Some(0))); assert_eq!(read_now(&mut dropped, false), None); producer.drop_groups(1..2, StreamError::NotFound); @@ -27820,14 +27770,14 @@ mod test { let producer = track_producer("test", Info::timed()); // Group 1 runs on past where group 0 could still present. - for (sequence, frames) in [(0u64, &[0u64][..]), (1, &[1_000, 1_100])] { - let mut group = producer.create_group(sequence.into()).unwrap(); + for (key, frames) in [(0u64, &[0u64][..]), (1, &[1_000, 1_100])] { + let mut group = producer.create_group(key.into()).unwrap(); for at in frames { group .write_frame(Timestamp::from_millis(*at).unwrap(), bytes::Bytes::from_static(b"f")) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let mut stale = producer.subscribe( Subscription::default() @@ -27846,7 +27796,7 @@ mod test { #[test] fn a_parked_bounded_read_waits_past_its_end() { let producer = track_producer("test", Info::timed()); - producer.create_group(0u64.into()).unwrap().finish().unwrap(); + producer.create_group(0u64.into()).unwrap().close().unwrap(); let mut reader = producer.subscribe(replay().with_range(0..1)).parked(); assert_eq!(read_now(&mut reader, false), Some(Some(0))); assert_eq!(read_now(&mut reader, false), None); @@ -27860,22 +27810,24 @@ mod test { // A received copy, whose missing groups may still arrive. producer.received(); let dynamic = producer.requests(); - producer.create_group(0u64.into()).unwrap().finish().unwrap(); - producer.create_group(2u64.into()).unwrap().finish().unwrap(); + producer.create_group(0u64.into()).unwrap().close().unwrap(); + producer.create_group(2u64.into()).unwrap().close().unwrap(); let producer = producer.finish().unwrap(); - let mut reader = producer.subscribe(replay().with_order(Order::Ascending).with_range(1..2)); + // From group 0, which a time start names without a lookup, through the hole at 1. + let mut reader = producer.subscribe(replay().with_order(Order::Ascending).with_range(0..2)); + assert_eq!(read_now(&mut reader, false), Some(Some(0))); assert_eq!(read_now(&mut reader, false), None, "the fill is out"); let fill = dynamic .requested_fill_group() .now_or_never() .expect("the miss asks a fill") .unwrap(); - assert_eq!(fill.sequence(), 1); + assert_eq!(fill.key(), 1); let mut group = fill.accept(Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"filled")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"filled")) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(read_now(&mut reader, false), Some(Some(1))); assert_eq!(read_now(&mut reader, false), Some(None)); } @@ -27889,13 +27841,13 @@ mod test { async fn a_hole_below_an_undeclared_start_is_asked_after_a_bounded_grace() { let (producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); - producer.create_group(10u64.into()).unwrap().finish().unwrap(); + producer.create_group(10u64.into()).unwrap().close().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) - .with_range(5..6), + .with_range(0..1), ); - assert_eq!(read_now(&mut reader, false), None, "group 5 may still arrive"); + assert_eq!(read_now(&mut reader, false), None, "group 0 may still arrive"); assert!(dynamic.requested_fill_group().now_or_never().is_none()); elapse(&producer, ledger::GRACE); assert_eq!(read_now(&mut reader, false), None, "the fill is out"); @@ -27904,17 +27856,17 @@ mod test { .now_or_never() .expect("the hole is asked once the grace passed") .unwrap(); - assert_eq!(fill.sequence(), 5); + assert_eq!(fill.key(), 0); fill.reject(Error::NotFound); assert_eq!(read_now(&mut reader, false), Some(None), "the refusal is final"); // With no fill handler, the hole is given up once the grace passed. let (producer, _pool) = pooled_producer(1 << 40); - producer.create_group(10u64.into()).unwrap().finish().unwrap(); + producer.create_group(10u64.into()).unwrap().close().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) - .with_range(5..6), + .with_range(0..1), ); assert_eq!(read_now(&mut reader, false), None); elapse(&producer, ledger::GRACE); @@ -27934,11 +27886,11 @@ mod test { } let dynamic = producer.requests(); producer.start_at(10).unwrap(); - producer.create_group(10u64.into()).unwrap().finish().unwrap(); + producer.create_group(10u64.into()).unwrap().close().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) - .with_range(5..6), + .with_range(0..1), ); assert_eq!(read_now(&mut reader, false), None, "received: {received}"); let fill = dynamic @@ -27946,7 +27898,7 @@ mod test { .now_or_never() .expect("a hole below the start is asked at once") .unwrap(); - assert_eq!(fill.sequence(), 5, "received: {received}"); + assert_eq!(fill.key(), 0, "received: {received}"); fill.reject(Error::NotFound); assert_eq!( read_now(&mut reader, false), @@ -27967,7 +27919,7 @@ mod test { let (mut producer, _pool) = pooled_producer(1 << 40); let dynamic = producer.requests(); producer.set_idle(); - let mut reader = producer.subscribe(replay().with_order(Order::Ascending).with_range(1..3)); + let mut reader = producer.subscribe(replay().with_order(Order::Ascending).with_range(0..2)); assert_eq!(read_now(&mut reader, false), None); assert!( dynamic.requested_fill().now_or_never().is_none(), @@ -27981,14 +27933,14 @@ mod test { .now_or_never() .expect("the holes are asked at once") .unwrap(); - assert_eq!((fill.start(), fill.end()), (Position::group(1), 3)); - let mut group = fill.accept(1, Info::timed()).unwrap(); + assert_eq!((fill.start(), fill.end()), (Position::group(0), 2)); + let mut group = fill.accept(0, Info::timed()).unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from_static(b"filled")) + .write_frame(group.key_time(), bytes::Bytes::from_static(b"filled")) .unwrap(); - group.finish().unwrap(); - fill.refuse_groups(2..3, Error::Unsupported); - assert_eq!(read_now(&mut reader, false), Some(Some(1))); + group.close().unwrap(); + fill.refuse_groups(1..2, Error::Unsupported); + assert_eq!(read_now(&mut reader, false), Some(Some(0))); assert_eq!(read_now(&mut reader, false), Some(None), "the refusal is final"); } @@ -28023,14 +27975,14 @@ mod test { for round in 0..8u64 { // Group 2k+1 runs on past where group 2k could still present, so 2k is stale. let at = 2_000 * round; - for (sequence, frames) in [(2 * round, vec![at]), (2 * round + 1, vec![at + 1_000, at + 1_100])] { - let mut group = producer.create_group(sequence.into()).unwrap(); + for (key, frames) in [(2 * round, vec![at]), (2 * round + 1, vec![at + 1_000, at + 1_100])] { + let mut group = producer.create_group(key.into()).unwrap(); for at in frames { group .write_frame(Timestamp::from_millis(at).unwrap(), bytes::Bytes::from_static(b"f")) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } assert_eq!(read_now(&mut reader, false), Some(Some(2 * round + 1)), "round {round}"); assert!( @@ -28054,14 +28006,14 @@ mod test { .with_order(Order::Ascending) .with_range(0..), ); - for (sequence, frames) in [(0u64, &[0u64][..]), (1, &[1_000, 1_100])] { - let mut group = producer.create_group(sequence.into()).unwrap(); + for (key, frames) in [(0u64, &[0u64][..]), (1, &[1_000, 1_100])] { + let mut group = producer.create_group(key.into()).unwrap(); for at in frames { group .write_frame(Timestamp::from_millis(*at).unwrap(), bytes::Bytes::from_static(b"f")) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } assert_eq!(read_now(&mut reader, false), Some(Some(1)), "evict: {evict}"); reader @@ -28083,11 +28035,11 @@ mod test { async fn an_unfillable_hole_below_the_start_wakes_its_reader() { let (mut producer, _pool) = pooled_producer(1 << 40); producer.start_at(10).unwrap(); - producer.create_group(10u64.into()).unwrap().finish().unwrap(); + producer.create_group(10u64.into()).unwrap().close().unwrap(); let mut reader = producer.subscribe( Subscription::default() .with_delay(crate::track::Delay::default().with_max(None)) - .with_range(5..6), + .with_range(0..1), ); let (flag, waiter) = Flag::waiter(); assert!(reader.poll_group(false, &waiter).is_pending()); diff --git a/rs/moq-net/src/model/track/demand.rs b/rs/moq-net/src/model/track/demand.rs index df0d27cc32c..8a7adf99c3e 100644 --- a/rs/moq-net/src/model/track/demand.rs +++ b/rs/moq-net/src/model/track/demand.rs @@ -29,11 +29,11 @@ pub(super) struct Subscriptions { /// The lowest group consumers want while no refresh is pending, never lowered. See /// [`super::Demand::wanted`]. wanted: u64, - /// A join over the track's refresh ratio asked for a group past every sequence written, + /// A join over the track's refresh ratio asked for a group past every key written, /// and no such group has landed yet. Read against `next` rather than stored as a number, - /// so a sequence written without answering it can never leave the ask behind. + /// so a key written without answering it can never leave the ask behind. refresh: bool, - /// One past the highest sequence written, which a pending refresh asks for. + /// One past the highest key written, which a pending refresh asks for. next: u64, waiters: kio::WaiterList, } @@ -95,7 +95,7 @@ impl Subscriptions { } /// The lowest group consumers want, whether or not anyone consumes now: the next free - /// sequence while a refresh is pending. + /// key while a refresh is pending. pub(super) fn wanted(&self) -> u64 { match self.refresh { true => self.next.max(self.wanted), @@ -109,7 +109,7 @@ impl Subscriptions { self.wanted() } - /// A join over the refresh ratio: ask for a group past every sequence written. Joins + /// A join over the refresh ratio: ask for a group past every key written. Joins /// while one is pending coalesce into it. pub(super) fn refresh(&mut self) -> kio::WaiterList { if std::mem::replace(&mut self.refresh, true) { @@ -118,18 +118,18 @@ impl Subscriptions { self.waiters.take() } - /// Sequence `sequence` was written. One at or past the next free sequence moves it: a + /// Key `key` was written. One at or past the next free key moves it: a /// group subscribers receive answers a pending refresh, and anything else moves the - /// sequence it asks for. - pub(super) fn wrote(&mut self, sequence: u64, answers: bool) -> kio::WaiterList { + /// key it asks for. + pub(super) fn wrote(&mut self, key: u64, answers: bool) -> kio::WaiterList { let next = self.next; - self.next = next.max(sequence.saturating_add(1)); - if !self.refresh || sequence < next { + self.next = next.max(key.saturating_add(1)); + if !self.refresh || key < next { return Default::default(); } if answers { self.refresh = false; - self.wanted = self.wanted.max(sequence); + self.wanted = self.wanted.max(key); } self.waiters.take() } @@ -315,7 +315,7 @@ mod tests { let anchor = |rng: &mut rand::rngs::StdRng| match rng.random_range(0..4) { 0 => Anchor::Relative(rng.random_range(0..3)), 1 => Anchor::Oldest, - _ => Anchor::Absolute(rng.random_range(0..8)), + _ => Anchor::Time(rng.random_range(0..8)), }; let ranges: Vec = (0..rng.random_range(0..3)) .map(|_| Range { @@ -373,13 +373,13 @@ mod tests { } /// An oldest end is refused wherever it is read, so it asks nothing upstream, even - /// past an absolute start. + /// past a time start. #[test] fn an_oldest_end_adds_no_demand() { let broadcast = crate::broadcast::Info::default().produce(); let producer = broadcast.create_track("demand", None).unwrap(); let _sub = producer.subscribe(Subscription::default().with_ranges([Range { - start: Anchor::Absolute(3), + start: Anchor::Time(3), end: Some(Anchor::Oldest), }])); assert_eq!(producer.subscription().map(|sub| sub.ranges), Some(Vec::new())); diff --git a/rs/moq-net/src/model/track/demand/ranges.rs b/rs/moq-net/src/model/track/demand/ranges.rs index 4cdcbb5a1c8..ab03f25996d 100644 --- a/rs/moq-net/src/model/track/demand/ranges.rs +++ b/rs/moq-net/src/model/track/demand/ranges.rs @@ -1,11 +1,12 @@ //! The union of every subscriber's ranges, kept current per change rather than refolded. //! -//! Absolute ranges become signed boundaries over groups, +1 where one starts and -1 -//! where it ends, so a group is covered while the running sum stays above zero. A treap -//! keeps each subtree's sum and lowest running sum, so a change touches two boundaries and -//! asks one range-minimum question, whatever else the audience holds. Ranges anchored to -//! the latest group or to a time can't be placed on that line and are counted by value -//! instead. +//! Time ranges become signed boundaries over group starts, +1 where one starts and -1 +//! past where it ends, so a start is covered while the running sum stays above zero. A +//! start between groups stands for the group containing it, which a range starting at or +//! before it covers too, so spans merge as they do over starts. A treap keeps each +//! subtree's sum and lowest running sum, so a change touches two boundaries and asks one +//! range-minimum question, whatever else the audience holds. Ranges anchored to the +//! latest group can't be placed on that line and are counted by value instead. use std::cell::OnceCell; use std::collections::BTreeMap; @@ -19,7 +20,8 @@ use super::{add, remove}; pub(super) struct Ranges { /// Holders of [`Range::oldest`], which covers every other range. oldest: usize, - absolute: Coverage, + /// Time ranges, as spans of starts. + time: Coverage, /// Open relative ranges by how far back they start: only the farthest matters. relative: BTreeMap, /// Ranges with one relative end, or a bounded relative span, kept as they are. @@ -32,7 +34,8 @@ pub(super) struct Ranges { enum Shape { Empty, Oldest, - Absolute(u64, Option), + /// A time range, as the exclusive span of starts it covers. + Time(u64, Option), Relative(u64), Mixed, } @@ -42,8 +45,9 @@ impl Shape { match (range.start, range.end) { _ if range.is_empty() => Self::Empty, (Anchor::Oldest, None) => Self::Oldest, - (Anchor::Absolute(start), None) => Self::Absolute(start, None), - (Anchor::Absolute(start), Some(Anchor::Absolute(end))) => Self::Absolute(start, Some(end)), + (Anchor::Time(start), None) => Self::Time(start, None), + // An end at the last start covers every group, as an open one does. + (Anchor::Time(start), Some(Anchor::Time(end))) => Self::Time(start, end.checked_add(1)), (Anchor::Relative(back), None) => Self::Relative(back), _ => Self::Mixed, } @@ -59,9 +63,9 @@ impl Ranges { self.oldest += 1; self.oldest == 1 } - Shape::Absolute(start, end) => { - let uncovered = self.absolute.min(start, end) == 0; - self.absolute.add(start, end, 1); + Shape::Time(start, end) => { + let uncovered = self.time.min(start, end) == 0; + self.time.add(start, end, 1); uncovered && self.oldest == 0 } Shape::Relative(back) => { @@ -82,9 +86,9 @@ impl Ranges { self.oldest -= 1; self.oldest == 0 } - Shape::Absolute(start, end) => { - self.absolute.add(start, end, -1); - self.oldest == 0 && self.absolute.min(start, end) == 0 + Shape::Time(start, end) => { + self.time.add(start, end, -1); + self.oldest == 0 && self.time.min(start, end) == 0 } Shape::Relative(back) => { let farthest = self.farthest(); @@ -114,12 +118,13 @@ impl Ranges { return vec![Range::oldest()]; } let mut union: Vec = self - .absolute + .time .runs() .into_iter() .map(|(start, end)| Range { - start: Anchor::Absolute(start), - end: end.map(Anchor::Absolute), + start: Anchor::Time(start), + // A run past its start ends at or past one more. + end: end.map(|end| Anchor::Time(end - 1)), }) .collect(); union.extend(self.mixed.keys().filter(|range| !self.covered(range))); @@ -138,17 +143,16 @@ impl Ranges { /// Whether an open range already delivers everything a mixed `range` could. fn covered(&self, range: &Range) -> bool { match range.start { - Anchor::Absolute(start) => self.absolute.tail().is_some_and(|tail| tail <= start), + // The group containing a start at or before this one's comes no later. + Anchor::Time(start) => self.time.tail().is_some_and(|tail| tail <= start), Anchor::Relative(back) => self.farthest().is_some_and(|farthest| back <= farthest), // Only the open oldest range, which hides every other, covers the oldest group. Anchor::Oldest => false, - // Where a time falls among the groups is the serving hop's to say. - Anchor::Time(_) => false, } } } -/// Signed boundary counts over groups: the coverage of the absolute ranges. +/// Signed boundary counts over group starts: the coverage of the time ranges. #[derive(Default)] struct Coverage { root: Link, @@ -357,7 +361,7 @@ mod tests { let anchor = |rng: &mut rand::rngs::StdRng| match rng.random_range(0..6) { 0 => Anchor::Relative(rng.random_range(0..4)), 1 => Anchor::Oldest, - _ => Anchor::Absolute(rng.random_range(0..12)), + _ => Anchor::Time(rng.random_range(0..12)), }; let mut ranges = Ranges::default(); let mut held: Vec = Vec::new(); diff --git a/rs/moq-net/src/model/track/fill.rs b/rs/moq-net/src/model/track/fill.rs index 8ab3a7c32d2..0272ec0ca1d 100644 --- a/rs/moq-net/src/model/track/fill.rs +++ b/rs/moq-net/src/model/track/fill.rs @@ -1,6 +1,6 @@ //! Fills: the groups readers miss, asked of a [`Requests`](super::Requests) handler by range. //! -//! A miss is a sequence a reader wants that the cache doesn't hold and the live feed will +//! A miss is a key a reader wants that the cache doesn't hold and the live feed will //! never deliver. A reader joins the fill already covering it, or queues a new one, and a //! handler serves each fill it takes as one bounded run: [`Fill::accept`] for each group it //! has, [`Fill::drop_groups`] for the rest. @@ -21,7 +21,7 @@ use std::{ use rangemap::{RangeMap, RangeSet}; use super::{Info, Order, Position, TrackState, clamp_max_delay}; -use crate::{Error, Result, StreamError, Timestamp, group, util::SendCell}; +use crate::{Error, Result, StreamError, group, util::SendCell}; /// The fills a track's readers asked for, queued or taken by a handler, under one lock. #[derive(Default)] @@ -33,7 +33,7 @@ pub(in crate::model) struct FillState { pending: HashMap, // The queued subset, oldest first. queue: VecDeque, - // The fill each unresolved sequence waits on, so a second reader joins it rather than + // The fill each unresolved key waits on, so a second reader joins it rather than // asking again. covers: RangeMap, } @@ -47,20 +47,20 @@ struct Pending { // Exclusive frame cap in the last group; None asks through the original end. frame_end: Option, // The time a lookup asks for, which owes no group. - time: Option, + time: Option, // What its joins ask with now, shared with them and its handler. asks: kio::Shared, queued: bool, // A flight: a separate copy of one group from its start frame, which no other reader - // joins by sequence; see [`FillState::fly`]. + // joins by key; see [`FillState::fly`]. flight: bool, - // How many joins still read each sequence: one a reader narrowed away is no longer owed. + // How many joins still read each key: one a reader narrowed away is no longer owed. wants: RangeMap, // What its handler accepted that no join has picked up yet, which holds back the next // group: see [`Fill::poll_wanted_changed`]. unseen: RangeSet, // Readers hold consumers, so it counts the fill's demand; the handler's copy says what - // became of each sequence. + // became of each key. result: kio::Producer, } @@ -184,11 +184,11 @@ impl Asks { } } -/// What became of a fill's sequences, shared with every reader that joined it. +/// What became of a fill's keys, shared with every reader that joined it. #[derive(Default)] pub(crate) struct Outcome { accepted: RangeSet, - // Why each refused run was refused; refusing a sequence again replaces why. + // Why each refused run was refused; refusing a key again replaces why. refused: RangeMap, refusals: u64, // Accepted or refused, kept as answers arrive so a fill answered group by group costs @@ -200,13 +200,13 @@ pub(crate) struct Outcome { } impl Outcome { - /// Why `sequence` was refused, if it was, and the budget its source was asked under. - pub(crate) fn refused(&self, sequence: u64) -> Option<(&Error, Option)> { - self.refused.get(&sequence).map(|refusal| (&refusal.err, refusal.asked)) + /// Why `key` was refused, if it was, and the budget its source was asked under. + pub(crate) fn refused(&self, key: u64) -> Option<(&Error, Option)> { + self.refused.get(&key).map(|refusal| (&refusal.err, refusal.asked)) } - pub(super) fn is_accepted(&self, sequence: u64) -> bool { - self.accepted.contains(&sequence) + pub(super) fn is_accepted(&self, key: u64) -> bool { + self.accepted.contains(&key) } /// The lowest group accepted: a lookup's answer. @@ -214,9 +214,9 @@ impl Outcome { self.accepted.iter().next().map(|run| run.start) } - /// The partial copy accepted for `sequence`, if it still holds `frames`. - pub(super) fn partial(&self, sequence: u64, frames: group::Bounds) -> Option { - self.partial.get(&sequence).filter(|copy| copy.covers(frames)).cloned() + /// The partial copy accepted for `key`, if it still holds `frames`. + pub(super) fn partial(&self, key: u64, frames: group::Bounds) -> Option { + self.partial.get(&key).filter(|copy| copy.covers(frames)).cloned() } /// What was answered within `wanted`: accepted, or refused with why and the budget @@ -286,14 +286,14 @@ pub(super) struct Ask { pub frame_end: Option, pub asking: Asking, /// A lookup of the group holding this time within the run, which owes none of it. - pub time: Option, + pub time: Option, } /// A reader's place in one fill: the part of it the reader wants. pub(super) struct Join { fills: kio::Shared, id: u64, - /// The sequences of the fill this reader asked for. + /// The keys of the fill this reader asked for. pub groups: ops::Range, // The part of `groups` it still reads, counted in the fill's wants. held: RangeSet, @@ -456,10 +456,7 @@ impl FillState { .collect(); let mut gaps: Vec> = self.covers.gaps(&span).collect(); for (mut groups, id) in covered { - let pending = self - .pending - .get_mut(&id) - .expect("a covered sequence names a pending fill"); + let pending = self.pending.get_mut(&id).expect("a covered key names a pending fill"); if groups.start == pending.start.group { // A reader starting in an earlier group wants this one whole. let (frame, byte) = match ask.start.group == groups.start { @@ -559,25 +556,25 @@ impl FillState { (joins, unfillable) } - /// Join or queue a flight of group `sequence` from `frame`: a fill answered with a copy - /// of its own, which no reader joins by sequence, since a copy shed past its readers + /// Join or queue a flight of group `key` from `frame`: a fill answered with a copy + /// of its own, which no reader joins by key, since a copy shed past its readers /// holds nothing for one asking earlier. Readers asking from the same frame before it is /// answered share it. `None` while no handler can take it. pub(super) fn fly( &mut self, shared: &kio::Shared, - sequence: u64, + key: u64, frame: u64, asking: Asking, ) -> Option { - let groups = sequence..sequence.checked_add(1)?; + let groups = key..key.checked_add(1)?; if self.handlers == 0 { return None; } let boarding = self.pending.iter_mut().find(|(_, pending)| { pending.flight - && pending.start == Position { group: sequence, frame } - && !pending.result.read().resolved.contains(&sequence) + && pending.start == Position { group: key, frame } + && !pending.result.read().resolved.contains(&key) }); let id = match boarding { Some((id, pending)) => { @@ -593,7 +590,7 @@ impl FillState { self.pending.insert( id, Pending { - start: Position { group: sequence, frame }, + start: Position { group: key, frame }, byte: 0, end: groups.end, frame_end: None, @@ -624,7 +621,7 @@ impl FillState { } /// Queue a fill looking up `time` within `ask`'s run, with the join waiting on its answer. - fn queue_lookup(&mut self, shared: &kio::Shared, ask: &Ask, time: Timestamp) -> Join { + fn queue_lookup(&mut self, shared: &kio::Shared, ask: &Ask, time: u64) -> Join { let id = self.next; self.next += 1; let result = kio::Producer::::default(); @@ -663,7 +660,7 @@ impl FillState { /// Count a join out of reading `groups` of fill `id`. What nobody reads any more is no /// longer routed to it: a reader wanting it later asks afresh rather than joining a fetch - /// being cancelled, so a sequence a fill stops owing stays unowed. Its answers are + /// being cancelled, so a key a fill stops owing stays unowed. Its answers are /// forgotten too, so a wide fill holds the answers its readers have yet to take, not one /// per group its source answered. fn unwant(&mut self, id: u64, groups: ops::Range) { @@ -690,11 +687,11 @@ impl FillState { } } - /// Whether a join of fill `id` still reads `sequence`. - fn wants(&self, id: u64, sequence: u64) -> bool { + /// Whether a join of fill `id` still reads `key`. + fn wants(&self, id: u64, key: u64) -> bool { self.pending .get(&id) - .is_some_and(|pending| pending.wants.contains_key(&sequence)) + .is_some_and(|pending| pending.wants.contains_key(&key)) } /// Whether a join of fill `id` still reads any of `groups`. @@ -864,7 +861,7 @@ struct Inner { end: u64, // Exclusive frame cap in the last group; None asks through the original end. frame_end: Option, - time: Option, + time: Option, // Answered with a copy of its own rather than the cache's; see [`FillState::fly`]. flight: bool, asks: kio::Shared, @@ -877,10 +874,10 @@ impl Fill { self.inner.start } - /// The frame group `sequence` starts from: [`start`](Self::start)'s frame for the first + /// The frame group `key` starts from: [`start`](Self::start)'s frame for the first /// group, 0 for the rest. - pub fn frame_start(&self, sequence: u64) -> u64 { - self.inner.frame_start(sequence) + pub fn frame_start(&self, key: u64) -> u64 { + self.inner.frame_start(key) } /// The exclusive end group. @@ -888,17 +885,19 @@ impl Fill { self.inner.end } - /// The presentation time a reader's range starts at that no group the cache holds - /// resolves, so this fill looks it up: [`accept`](Self::accept) the group containing it, - /// the newest one starting at or before it, or the oldest group when every group starts - /// after it. The reader's range then starts there, once that group's first frame is - /// written. + /// The time a reader's range starts at that no group the cache holds resolves, in the + /// track's timescale (a bare start on a scale-less track), so this fill looks it up: + /// [`accept`](Self::accept) the group containing it, the newest one starting at or before + /// it, or the oldest group when every group starts after it. The reader's range then + /// starts there. /// /// A lookup owes no group, so [`wanted`](Self::wanted) is `None` and /// [`is_wanted`](Self::is_wanted) false throughout; it waits only on its first accept. /// Dropped unanswered, as a handler that can't resolve times does, the reader's range - /// reaches back to the oldest group the cache holds. `None` for a fill of groups. - pub fn time(&self) -> Option { + /// reaches back to the newest group the cache holds at or before the time, or, with none, + /// reads every group the cache holds and asks none starting before the time. `None` for + /// a fill of groups. + pub fn time(&self) -> Option { self.inner.time } @@ -918,7 +917,7 @@ impl Fill { self.inner.asks.read().order() } - /// Insert group `sequence` into the track's cache, handing it to the readers waiting on + /// Insert group `key` into the track's cache, handing it to the readers waiting on /// it, and return a [`group::Producer`] to fill. /// /// The group starts at [`start`](Self::start)'s frame when it is the first group, so the @@ -927,8 +926,8 @@ impl Fill { /// ignored otherwise. Errors with [`Error::InvalidRange`] outside the fill, /// [`Error::Duplicate`] if the group was already answered or cached, [`Error::Skipped`] if /// the track skipped it, or the track's abort error. - pub fn accept(&self, sequence: u64, info: impl Into>) -> Result { - self.inner.accept(sequence, info.into()) + pub fn accept(&self, key: u64, info: impl Into>) -> Result { + self.inner.accept(key, info.into()) } /// Name `groups` as never to be delivered: the source no longer has them. Readers stop @@ -959,18 +958,18 @@ impl Fill { self.inner.result.poll_wanted_changed(waiter, true).map(drop) } - /// Whether a reader still waits for group `sequence` from this fill: false once every + /// Whether a reader still waits for group `key` from this fill: false once every /// reader took it or narrowed past it, or nobody waits on the fill. A group stops being /// wanted for good; once accepted and taken, [`group::Producer::demand`](crate::group::Producer::demand) /// says who reads it. - pub fn is_wanted(&self, sequence: u64) -> bool { - self.inner.is_wanted(sequence) + pub fn is_wanted(&self, key: u64) -> bool { + self.inner.is_wanted(key) } - /// Resolves once no reader waits for group `sequence` from this fill, so stop serving it. + /// Resolves once no reader waits for group `key` from this fill, so stop serving it. #[cfg_attr(feature = "async", kio::wrapper)] - pub fn poll_unwanted(&self, sequence: u64, waiter: &kio::Waiter) -> Poll<()> { - self.inner.poll_unwanted(sequence, waiter) + pub fn poll_unwanted(&self, key: u64, waiter: &kio::Waiter) -> Poll<()> { + self.inner.poll_unwanted(key, waiter) } /// How many answer runs the fill still holds for its readers. @@ -1001,7 +1000,7 @@ impl Fill { } loop { let wanted = self.inner.wanted(); - let sequence = { + let key = { let outcome = self.inner.result.read(); let mut gaps = wanted .iter() @@ -1019,17 +1018,17 @@ impl Fill { .inner .state .read() - .covering_range(sequence, self.inner.bounds(sequence)) + .covering_range(key, self.inner.bounds(key)) .is_some(); if !cached { - return Some(sequence); + return Some(key); } let Ok(mut outcome) = self.inner.result.write() else { - return Some(sequence); + return Some(key); }; - outcome.accept(sequence..sequence + 1); + outcome.accept(key..key + 1); drop(outcome); - uncover(&self.inner.fills, self.inner.id, sequence..sequence + 1); + uncover(&self.inner.fills, self.inner.id, key..key + 1); } } @@ -1038,7 +1037,7 @@ impl Fill { /// when it next reads. `None`, once the fill is withdrawn or answered, never waits. The /// first call returns at once, unless a group already accepted has yet to reach a reader. /// - /// So `while let Some(sequence) = fill.wanted_changed().await` writes one group at a time + /// So `while let Some(key) = fill.wanted_changed().await` writes one group at a time /// as the fastest reader reads on, rather than a batch the cache may evict before it is /// read. A reader reading in another order than the fill's picks up groups it cannot take /// yet, so those wait in the cache. @@ -1090,46 +1089,46 @@ impl Fill { /// session asks its source for the whole run at once. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn next_run(&self) -> Option> { - let sequence = self.wanted()?; - Some(self.run_from(sequence, self.inner.owed_around(sequence))) + let key = self.wanted()?; + Some(self.run_from(key, self.inner.owed_around(key))) } /// [`Self::next_run`], passing over `skip`: a group another request is already bringing, /// so the groups around it are served while it waits. `None` once nothing else is owed. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) fn next_run_except(&self, skip: u64) -> Option> { - let sequence = self.wanted_except(Some(skip))?; - let owed = self.inner.owed_around(sequence); - let owed = match skip < sequence { + let key = self.wanted_except(Some(skip))?; + let owed = self.inner.owed_around(key); + let owed = match skip < key { true => owed.start.max(skip + 1)..owed.end, false => owed.start..owed.end.min(skip), }; - Some(self.run_from(sequence, owed)) + Some(self.run_from(key, owed)) } - /// The run from `sequence` over the `owed` groups beside it, in the fill's order, up to + /// The run from `key` over the `owed` groups beside it, in the fill's order, up to /// the nearest one the cache holds. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn run_from(&self, sequence: u64, owed: ops::Range) -> ops::Range { + fn run_from(&self, key: u64, owed: ops::Range) -> ops::Range { let state = self.inner.state.read(); let held = |(_, slot): &(&u64, &super::Slot)| !slot.is_aborted(); match self.order() { Order::Ascending => { let end = state .lookup - .range(sequence + 1..owed.end) + .range(key + 1..owed.end) .find(held) .map_or(owed.end, |(held, _)| *held); - sequence..end + key..end } Order::Descending => { let start = state .lookup - .range(owed.start..sequence) + .range(owed.start..key) .rev() .find(held) .map_or(owed.start, |(held, _)| held + 1); - start..sequence + 1 + start..key + 1 } } } @@ -1187,23 +1186,23 @@ impl Fill { self.inner.reject(groups, err, self.max_delay()); } - /// Declare the track's exclusive final sequence, as the source answering a request + /// Declare the track's exclusive final key, as the source answering a request /// reported it. A no-op once the track declared an end, or holds a later group. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn finish_track_at(&self, final_sequence: u64) { - self.inner.finish_track_at(final_sequence); + pub(crate) fn finish_track_at(&self, final_key: u64) { + self.inner.finish_track_at(final_key); } - /// A handle on group `sequence` alone, for a session serving the fill one group at a time. + /// A handle on group `key` alone, for a session serving the fill one group at a time. /// It keeps what the readers ask now, which is what its source gets asked. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - pub(crate) fn group(&self, sequence: u64) -> GroupFill { + pub(crate) fn group(&self, key: u64) -> GroupFill { GroupFill { inner: self.inner.clone(), - sequence, - frame_start: self.inner.frame_start(sequence), - byte_start: self.inner.byte_start(sequence), - frame_end: self.inner.bounds(sequence).end, + key, + frame_start: self.inner.frame_start(key), + byte_start: self.inner.byte_start(key), + frame_end: self.inner.bounds(key).end, priority: self.priority(), max_delay: self.max_delay(), } @@ -1211,12 +1210,12 @@ impl Fill { } impl Inner { - fn contains(&self, sequence: u64) -> bool { - (self.start.group..self.end).contains(&sequence) + fn contains(&self, key: u64) -> bool { + (self.start.group..self.end).contains(&key) } - fn is_wanted(&self, sequence: u64) -> bool { - self.result.wanted() && self.fills.read().wants(self.id, sequence) + fn is_wanted(&self, key: u64) -> bool { + self.result.wanted() && self.fills.read().wants(self.id, key) } /// Whether a group this fill accepted has yet to reach a reader. @@ -1232,12 +1231,12 @@ impl Inner { self.asks.read().max_delay() } - fn poll_unwanted(&self, sequence: u64, waiter: &kio::Waiter) -> Poll<()> { + fn poll_unwanted(&self, key: u64, waiter: &kio::Waiter) -> Poll<()> { if self.result.poll_wanted_changed(waiter, true).is_ready() { return Poll::Ready(()); } self.fills - .poll(waiter, |fills| match fills.wants(self.id, sequence) { + .poll(waiter, |fills| match fills.wants(self.id, key) { true => Poll::Pending, false => Poll::Ready(()), }) @@ -1254,26 +1253,26 @@ impl Inner { .map_or_else(RangeSet::new, |pending| pending.wanted(&span)) } - fn frame_start(&self, sequence: u64) -> u64 { - match sequence == self.start.group { + fn frame_start(&self, key: u64) -> u64 { + match key == self.start.group { true => self.start.frame, false => 0, } } - /// The first byte group `sequence` starts from within its first frame. - fn byte_start(&self, sequence: u64) -> u64 { - match sequence == self.start.group { + /// The first byte group `key` starts from within its first frame. + fn byte_start(&self, key: u64) -> u64 { + match key == self.start.group { true => self.byte, false => 0, } } - fn bounds(&self, sequence: u64) -> group::Bounds { + fn bounds(&self, key: u64) -> group::Bounds { group::Bounds { - start: self.frame_start(sequence), - byte: self.byte_start(sequence), - end: if sequence.checked_add(1) == Some(self.end) { + start: self.frame_start(key), + byte: self.byte_start(key), + end: if key.checked_add(1) == Some(self.end) { self.frame_end } else { None @@ -1284,22 +1283,22 @@ impl Inner { /// A writer that never says it answers from a byte writes whole frames, so its copy /// starts at a frame boundary even when the fill asked from a byte: only /// [`GroupFill::accept_from`] starts one partway into a frame. - fn accept(&self, sequence: u64, info: Option) -> Result { + fn accept(&self, key: u64, info: Option) -> Result { let frames = group::Bounds { byte: 0, - ..self.bounds(sequence) + ..self.bounds(key) }; - self.accept_from(sequence, info, frames) + self.accept_from(key, info, frames) } - fn accept_from(&self, sequence: u64, info: Option, frames: group::Bounds) -> Result { - if !self.contains(sequence) { + fn accept_from(&self, key: u64, info: Option, frames: group::Bounds) -> Result { + if !self.contains(key) { return Err(Error::InvalidRange); } - let Some(end) = sequence.checked_add(1) else { + let Some(end) = key.checked_add(1) else { return Err(Error::InvalidRange); }; - if self.result.read().resolved.contains(&sequence) { + if self.result.read().resolved.contains(&key) { return Err(Error::Duplicate); } // Written for a reader that has yet to pick it up, which the next group waits on. @@ -1308,8 +1307,8 @@ impl Inner { let unseen = { let mut fills = self.fills.lock(); match fills.pending.get_mut(&self.id) { - Some(pending) if pending.wants.contains_key(&sequence) => { - pending.unseen.insert(sequence..end); + Some(pending) if pending.wants.contains_key(&key) => { + pending.unseen.insert(key..end); true } _ => false, @@ -1318,8 +1317,8 @@ impl Inner { // Cache the group before answering: readers resolve an accept through the cache. let res = TrackState::modify(&self.state).and_then(|mut state| { let res = match self.flight { - true => state.insert_flight(sequence, frames, info), - false => state.insert_group_request(sequence, frames, info), + true => state.insert_flight(key, frames, info), + false => state.insert_group_request(key, frames, info), }; let deferred = state.deferred(); drop(state); @@ -1330,17 +1329,17 @@ impl Inner { && res.is_err() && let Some(pending) = self.fills.lock().pending.get_mut(&self.id) { - pending.unseen.remove(sequence..end); + pending.unseen.remove(key..end); } if let Ok(mut outcome) = self.result.write() { - outcome.accept(sequence..end); + outcome.accept(key..end); if let Ok(group) = &res && (self.flight || frames.start > 0 || frames.byte > 0 || frames.end.is_some()) { - outcome.partial.insert(sequence, group.handle()); + outcome.partial.insert(key, group.handle()); } } - uncover(&self.fills, self.id, sequence..end); + uncover(&self.fills, self.id, key..end); res } @@ -1439,24 +1438,24 @@ impl Inner { } } - /// The run of owed sequences around `sequence`: read by a reader and unanswered. + /// The run of owed keys around `key`: read by a reader and unanswered. #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn owed_around(&self, sequence: u64) -> ops::Range { + fn owed_around(&self, key: u64) -> ops::Range { let wanted = self.wanted(); let outcome = self.result.read(); wanted - .overlapping(&(sequence..sequence + 1)) + .overlapping(&(key..key + 1)) .flat_map(|run| outcome.resolved.gaps(run)) - .find(|gap| gap.contains(&sequence)) - .unwrap_or(sequence..sequence + 1) + .find(|gap| gap.contains(&key)) + .unwrap_or(key..key + 1) } #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] - fn finish_track_at(&self, final_sequence: u64) { + fn finish_track_at(&self, final_key: u64) { if let Ok(mut state) = TrackState::modify(&self.state) && state.declared().is_none() { - let _ = state.set_end(Position::group(final_sequence), None); + let _ = state.set_end(Position::group(final_key), None); } } } @@ -1519,7 +1518,7 @@ impl TrackState { pub(crate) struct Readers { fills: kio::Shared, id: u64, - sequence: u64, + key: u64, outcome: kio::Producer, } @@ -1533,7 +1532,7 @@ impl Readers { } self.fills .poll(waiter, |fills| match fills.pending.get(&self.id) { - Some(pending) if !pending.wants.contains_key(&self.sequence) => Poll::Ready(()), + Some(pending) if !pending.wants.contains_key(&self.key) => Poll::Ready(()), _ => Poll::Pending, }) .map(drop) @@ -1545,7 +1544,7 @@ impl Readers { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] pub(crate) struct GroupFill { inner: Arc>, - sequence: u64, + key: u64, frame_start: u64, byte_start: u64, frame_end: Option, @@ -1555,9 +1554,9 @@ pub(crate) struct GroupFill { #[cfg_attr(not(feature = "async"), allow(dead_code, reason = "async-only"))] impl GroupFill { - /// The group sequence to fetch. - pub(crate) fn sequence(&self) -> u64 { - self.sequence + /// The group key to fetch. + pub(crate) fn key(&self) -> u64 { + self.key } /// The delivery priority the readers asked for when the group was taken. @@ -1588,12 +1587,12 @@ impl GroupFill { /// Accept the actual wire range, which an older or compressed upstream may have widened. pub(crate) fn accept_from(&self, info: Info, frames: group::Bounds) -> Result { - self.inner.accept_from(self.sequence, Some(info), frames) + self.inner.accept_from(self.key, Some(info), frames) } /// Poll until no reader waits on this group, as [`Fill::poll_unwanted`]. pub(crate) fn poll_unwanted(&self, waiter: &kio::Waiter) -> Poll<()> { - self.inner.poll_unwanted(self.sequence, waiter) + self.inner.poll_unwanted(self.key, waiter) } /// Whether a reader still waits on the fill. @@ -1608,7 +1607,7 @@ impl GroupFill { Readers { fills: self.inner.fills.clone(), id: self.inner.id, - sequence: self.sequence, + key: self.key, outcome: self.inner.result.clone(), } } @@ -1616,13 +1615,13 @@ impl GroupFill { /// Insert the group into the cache, as [`Fill::accept`]. #[cfg(test)] pub(crate) fn accept(self, info: impl Into>) -> Result { - self.inner.accept(self.sequence, info.into()) + self.inner.accept(self.key, info.into()) } /// Refuse the group with the source's answer, for the readers waiting now. Only /// [`Error::Old`] under the publisher's max age (with every older group of the fill) /// is recorded, as a stale verdict under that age. pub(crate) fn reject(self, err: Error) { - self.inner.reject(self.sequence..self.sequence + 1, err, self.max_delay); + self.inner.reject(self.key..self.key + 1, err, self.max_delay); } } diff --git a/rs/moq-net/src/model/track/hole.rs b/rs/moq-net/src/model/track/hole.rs index 52312784100..aa8a52a6c05 100644 --- a/rs/moq-net/src/model/track/hole.rs +++ b/rs/moq-net/src/model/track/hole.rs @@ -2,7 +2,7 @@ use std::time::Duration; use crate::StreamError; -/// Why a group sequence will not be delivered. +/// Why a group key will not be delivered. /// /// lite-07 carries the two kinds in their own messages, SUBSCRIBE_SKIP and SUBSCRIBE_DROP; /// earlier versions name both in SUBSCRIBE_DROP. @@ -43,7 +43,7 @@ impl Hole { } } - /// The error a reader waiting on the sequence ends with: a sequence never produced is + /// The error a reader waiting on the key ends with: a key never produced is /// simply not found. pub fn error(&self) -> StreamError { match self { diff --git a/rs/moq-net/src/model/track/miss.rs b/rs/moq-net/src/model/track/miss.rs index b5ea111fecc..510c649b731 100644 --- a/rs/moq-net/src/model/track/miss.rs +++ b/rs/moq-net/src/model/track/miss.rs @@ -1,4 +1,4 @@ -//! A reader's misses: the sequences it reads that the cache doesn't hold and the live feed +//! A reader's misses: the keys it reads that the cache doesn't hold and the live feed //! will never deliver, asked of the track's fills until each arrives or is given up. //! //! Nothing above the newest group the feed produced is a miss, since it arrives live, so a @@ -8,7 +8,7 @@ //! judged again as the feed moves, or once its wait ends: the arrival grace, bounded by //! [`ledger::GRACE`] when nothing else bounds it, so every hole is asked in finite time. //! -//! Each sequence is judged once, except a group evicted after the reader judged it cached +//! Each key is judged once, except a group evicted after the reader judged it cached //! and before it took it: judged again as a miss, once. A refill evicted again before it is //! taken is given up, so a slow reader's refills cannot evict each other without bound. //! @@ -48,29 +48,29 @@ pub(super) struct Misses { budget: Option, grace: Option, ended: bool, - /// The sequences judged already, within what the reader reads up to the newest group. + /// The keys judged already, within what the reader reads up to the newest group. judged: RangeSet, /// Holes the live feed may still deliver, revisited on a ledger change or deadline. deferred: Deferred, /// Asked under the feed's start and refused, never asked again: given up once the feed /// gives up on them, unless they arrive first. refused: RangeSet, - /// The fills this reader waits on, under the keys `holders` names them by. + /// The fills this reader waits on, under the ids `holders` names them by. joins: BTreeMap, - /// The join holding each sequence: a reader's joins hold disjoint groups, so a change + /// The join holding each key: a reader's joins hold disjoint groups, so a change /// touches only the joins holding what it changes, never every fill the reader asked. holders: RangeMap, - /// What each join parks on, under its key: a waiter of its own, which records that its + /// What each join parks on, under its id: a waiter of its own, which records that its /// fill answered before waking the reader, so [`Self::resolve`] looks only at the fills /// that answered, never at every fill still out. parked: BTreeMap, /// The joins whose fills answered since the reader last looked. answered: Arc, - /// The key the next join takes. + /// The id the next join takes. next_join: u64, /// What the reader asked its fills with last. asking: Option, - /// The sequences of `joins` still unanswered. + /// The keys of `joins` still unanswered. wanted: RangeSet, /// Answered by a fill and cached, not handed out yet. filled: BTreeSet, @@ -79,9 +79,6 @@ pub(super) struct Misses { picked: BTreeMap>, /// Never asked again: named dropped to the reader. given_up: RangeSet, - /// What became [`Self::gone`] since the cursor last took it, for its window's parked time - /// ends to wake on (see [`Self::take_gone`]). - newly_gone: RangeSet, /// The window revision and cap what this holds was last clipped to, with the window's /// missable groups then, `None` while every group was. clipped: Option<(u64, Option, Option)>, @@ -94,7 +91,7 @@ pub(super) struct Misses { evictions: u64, /// Judged again after an eviction, within the judged span: never refilled twice. refilled: RangeSet, - /// The last sequence of the highest received stale run above the newest group that this + /// The last key of the highest received stale run above the newest group that this /// reader cannot trust, as of [`Self::unknown_at`]; see [`Self::unknown`]. unknown: Option, /// The drops revision, judged revision, budget, and coverage [`Self::unknown`] was found @@ -117,13 +114,13 @@ pub(super) type GivenUp = Vec<(ops::Range, Option)>; /// parked on a want already settled. #[derive(Default)] struct Answered { - keys: Mutex>, + ids: Mutex>, tasks: Mutex, } -/// Wakes the reader once join `key`'s fill answers, recording which join it was. +/// Wakes the reader once join `id`'s fill answers, recording which join it was. struct JoinWake { - key: u64, + id: u64, answered: Arc, } @@ -136,10 +133,10 @@ impl Wake for JoinWake { // Recorded before the task wakes, so the resolve it triggers sees the change. let answered = &self.answered; answered - .keys + .ids .lock() .unwrap_or_else(PoisonError::into_inner) - .insert(self.key); + .insert(self.id); // Woken with no lock held: an executor may poll inline. let mut tasks = answered.tasks.lock().unwrap_or_else(PoisonError::into_inner).take(); tasks.wake(); @@ -184,7 +181,7 @@ impl Forgotten { } } -/// The current deferred runs, indexed by sequence and by their next judgement deadline. +/// The current deferred runs, indexed by key and by their next judgement deadline. /// Splits remove the old age entry before publishing either piece; no stale timers remain. #[derive(Default)] struct Deferred { @@ -284,7 +281,7 @@ impl Deferred { } impl Misses { - /// The lowest sequence still to come from a fill: asked, or answered and not handed out. + /// The lowest key still to come from a fill: asked, or answered and not handed out. pub fn lowest(&self) -> Option { let wanted = self.wanted.iter().next().map(|run| run.start); let filled = self.filled.first().copied(); @@ -294,7 +291,7 @@ impl Misses { } } - /// The highest sequence still to come from a fill. + /// The highest key still to come from a fill. fn highest(&self) -> Option { let wanted = self.wanted.iter().next_back().map(|run| run.end - 1); wanted.max(self.filled.last().copied()) @@ -305,8 +302,8 @@ impl Misses { fn lose(&mut self, lost: &RangeSet) { for run in lost.iter() { let filled: Vec = self.filled.range(run.clone()).copied().collect(); - for sequence in filled { - self.filled.remove(&sequence); + for key in filled { + self.filled.remove(&key); } self.wanted.remove(run.clone()); } @@ -329,7 +326,7 @@ impl Misses { !self.filled.is_empty() } - /// The lowest sequence asked and not answered yet. + /// The lowest key asked and not answered yet. pub fn lowest_wanted(&self) -> Option { self.wanted.iter().next().map(|run| run.start) } @@ -339,21 +336,15 @@ impl Misses { &self.given_up } - /// The exclusive end of the run from `sequence` this reader's fills refused or it gave + /// The exclusive end of the run from `key` this reader's fills refused or it gave /// up, `None` when it may still be read. - pub fn gone(&self, sequence: u64) -> Option { + pub fn gone(&self, key: u64) -> Option { [&self.refused, &self.given_up] .into_iter() - .find_map(|runs| runs.get(&sequence)) + .find_map(|runs| runs.get(&key)) .map(|run| run.end) } - /// What became [`Self::gone`] since the last call: a time end parked on a gap this - /// reader's own fills gave up on walks on once told. - pub fn take_gone(&mut self) -> RangeSet { - std::mem::take(&mut self.newly_gone) - } - /// What was refilled after an eviction and not handed out yet. #[cfg(all(test, feature = "async"))] pub fn refilled(&self) -> &RangeSet { @@ -369,32 +360,32 @@ impl Misses { given_up: &mut GivenUp, ) -> Option { loop { - let sequence = match order { + let key = match order { Order::Ascending => self.filled.first(), Order::Descending => self.filled.last(), } .copied()?; - if let Some(slot) = state.lookup.get(&sequence).filter(|slot| slot.serves_end()) { + if let Some(slot) = state.lookup.get(&key).filter(|slot| slot.serves_end()) { return Some(slot.serving().clone()); } - self.filled.remove(&sequence); + self.filled.remove(&key); let mut lost = RangeSet::new(); - lost.insert(sequence..sequence + 1); + lost.insert(key..key + 1); self.release(&lost); - self.give_up(sequence..sequence + 1, given_up); + self.give_up(key..key + 1, given_up); } } - /// Take `sequence` as handed out, if a fill answered it. A refill handed out is never + /// Take `key` as handed out, if a fill answered it. A refill handed out is never /// judged again, so it is forgotten too. - pub fn take_filled(&mut self, sequence: u64) { - // The last sequence is never judged, so never refilled. - if let Some(end) = sequence.checked_add(1) { - self.refilled.remove(sequence..end); + pub fn take_filled(&mut self, key: u64) { + // The last key is never judged, so never refilled. + if let Some(end) = key.checked_add(1) { + self.refilled.remove(key..end); } - if self.filled.remove(&sequence) { + if self.filled.remove(&key) { let mut lost = RangeSet::new(); - lost.insert(sequence..sequence + 1); + lost.insert(key..key + 1); self.release(&lost); } } @@ -406,21 +397,21 @@ impl Misses { self.holders.overlapping(&join.groups).next().is_none(), "a reader asks only for what no join holds" ); - let key = self.next_join; + let id = self.next_join; self.next_join += 1; - self.holders.insert(join.groups.clone(), key); - self.joins.insert(key, join); + self.holders.insert(join.groups.clone(), id); + self.joins.insert(id, join); let wake = Arc::new(JoinWake { - key, + id, answered: self.answered.clone(), }); - self.parked.insert(key, kio::Waiter::new(Waker::from(wake))); + self.parked.insert(id, kio::Waiter::new(Waker::from(wake))); // Looked at once, which parks it. self.answered - .keys + .ids .lock() .unwrap_or_else(PoisonError::into_inner) - .insert(key); + .insert(id); } } @@ -428,17 +419,17 @@ impl Misses { /// this reader, and drop each join left holding nothing. fn release(&mut self, lost: &RangeSet) { for run in lost.iter() { - while let Some((held, &key)) = self.holders.overlapping(run).next() { + while let Some((held, &id)) = self.holders.overlapping(run).next() { // Only this piece: a join split by an earlier release has other pieces in // `lost`, and each must still find the join when its turn comes. let piece = held.start.max(run.start)..held.end.min(run.end); self.holders.remove(piece.clone()); - let join = self.joins.get_mut(&key).expect("a holder names a join"); + let join = self.joins.get_mut(&id).expect("a holder names a join"); join.release(&RangeSet::from_iter([piece])); if join.is_empty() { - self.joins.remove(&key); - self.parked.remove(&key); - self.picked.remove(&key); + self.joins.remove(&id); + self.parked.remove(&id); + self.picked.remove(&id); } } } @@ -500,8 +491,8 @@ impl Misses { return; } let admitted = &admitted(); - let outside = |sequence: u64| sequence < from || !admitted.contains(&sequence); - self.filled.retain(|sequence| !outside(*sequence)); + let outside = |key: u64| key < from || !admitted.contains(&key); + self.filled.retain(|key| !outside(*key)); let mut kept = RangeSet::new(); for run in self.wanted.iter() { for inside in clip(run.clone(), from, admitted) { @@ -538,11 +529,11 @@ impl Misses { .unwrap_or_else(PoisonError::into_inner) .register(waiter); // Taken after registering: an answer landing since wakes the waiter registered above. - let keys = std::mem::take(&mut *self.answered.keys.lock().unwrap_or_else(PoisonError::into_inner)); - let mut keys = keys.into_iter().peekable(); - while let Some(&key) = keys.peek() { - let (Some(join), Some(parked)) = (self.joins.get(&key), self.parked.get(&key)) else { - keys.next(); + let ids = std::mem::take(&mut *self.answered.ids.lock().unwrap_or_else(PoisonError::into_inner)); + let mut ids = ids.into_iter().peekable(); + while let Some(&id) = ids.peek() { + let (Some(join), Some(parked)) = (self.joins.get(&id), self.parked.get(&id)) else { + ids.next(); continue; }; let wanted: RangeSet = join @@ -555,7 +546,7 @@ impl Misses { }) .collect(); if wanted.is_empty() { - keys.next(); + ids.next(); continue; } // Only what is unanswered: a join holding only filled groups until they are taken @@ -580,7 +571,7 @@ impl Misses { answers } Poll::Pending => { - keys.next(); + ids.next(); continue; } }; @@ -599,10 +590,10 @@ impl Misses { Ok(()) => { seen.insert(groups.clone()); self.settle(&state, groups.clone(), given_up); - for (&sequence, slot) in state.lookup.range(groups) { + for (&key, slot) in state.lookup.range(groups) { if slot.serves_end() { - self.filled.insert(sequence); - taken.remove(sequence..sequence + 1); + self.filled.insert(key); + taken.remove(key..key + 1); } } // Seen cached, though judging may not run again before it is evicted. @@ -620,7 +611,7 @@ impl Misses { } drop(state); if !seen.is_empty() { - self.picked.entry(key).or_default().extend(seen); + self.picked.entry(id).or_default().extend(seen); } self.release(&taken); // Judged again: the same join may have more to answer, or park the waiter now. @@ -639,12 +630,12 @@ impl Misses { /// handler may serve the next group. Only a read: a drop poll takes in answers too, from /// a publisher that may be unable to send the groups, which must not pace the fill. pub fn pick_up(&mut self) { - for (key, groups) in std::mem::take(&mut self.picked) { - self.joins[&key].saw(&groups); + for (id, groups) in std::mem::take(&mut self.picked) { + self.joins[&id].saw(&groups); } } - /// The last sequence of the highest received stale run above the newest group that `drift` + /// The last key of the highest received stale run above the newest group that `drift` /// does not settle. While the terms hold and the run it found still stands, only the runs /// changed since are looked at, so a drop costs each reader the runs it touches. fn unknown(&mut self, state: &TrackState, drift: &Drift) -> Option { @@ -704,7 +695,7 @@ impl Misses { pub fn judge( &mut self, state: &TrackState, - key: (u64, u64, Option), + id: (u64, u64, Option), view: Option<&Layered>, named: Option, bound: Option, @@ -732,12 +723,12 @@ impl Misses { return asks; }; let at = ( - key.0, - key.1, - key.2, + id.0, + id.1, + id.2, Some(newest), state.live_floor, - // Whole, not just its sequence: a start resolving from a budget turns the history + // Whole, not just its key: a start resolving from a budget turns the history // below it from unasked into fillable. state.start, state.drops_revision, @@ -759,7 +750,7 @@ impl Misses { return asks; } let grace = state.miss_grace(); - if self.judge_arrivals(state, key, at, view, &mut target, taken, drift, grace, ended) { + if self.judge_arrivals(state, id, at, view, &mut target, taken, drift, grace, ended) { self.ledger_revision = ledger.revision(); return asks; } @@ -788,7 +779,7 @@ impl Misses { }) || self.budget != Some(drift.budget) || self.grace != Some(grace) || self.ended != ended; - let low = key.0; + let low = id.0; let top = newest; let mut candidates = RangeSet::new(); if terms { @@ -848,9 +839,7 @@ impl Misses { // What the time ends' runs no longer cover is forgotten, and what they newly cover // judged, within what the probe judges: from the floor up to the newest group, // below the cap. - let last = key - .2 - .map_or(Some(top), |cap| cap.checked_sub(1).map(|cap| cap.min(top))); + let last = id.2.map_or(Some(top), |cap| cap.checked_sub(1).map(|cap| cap.min(top))); if let Some((old, view)) = &layered && let Some(last) = last && self.at.is_some_and(|previous| previous.1 != at.1) @@ -923,11 +912,11 @@ impl Misses { .range(run.start.max(changed.start)..run.end.min(changed.end)) // An aborted or back-cut slot is still a hole, so its refusal stands. .filter(|(_, slot)| slot.serves_end()) - .map(|(&sequence, _)| sequence) + .map(|(&key, _)| key) }) .collect(); - for sequence in arrived { - self.refused.remove(sequence..sequence + 1); + for key in arrived { + self.refused.remove(key..key + 1); } } // What it judged cached and the track evicted since, before this reader took it, is a @@ -987,9 +976,9 @@ impl Misses { { // A fill's answer evicted before it was taken is no longer held for this reader. let filled: Vec = self.filled.range(run.clone()).copied().collect(); - for sequence in filled { - self.filled.remove(&sequence); - self.release(&RangeSet::from_iter(std::iter::once(sequence..sequence + 1))); + for key in filled { + self.filled.remove(&key); + self.release(&RangeSet::from_iter(std::iter::once(key..key + 1))); } match refill { true => { @@ -1009,7 +998,7 @@ impl Misses { for run in candidates.iter() { self.deferred.remove(run.clone()); } - let start = state.start.sequence().unwrap_or(0); + let start = state.start.key().unwrap_or(0); // Below a start its publisher resolved from a budget, which an update may have changed // in flight, every older group is history a fill may still find. A feed that began at // the live edge or the floor the copy asked for leaves older groups as history only a @@ -1112,7 +1101,7 @@ impl Misses { fn judge_arrivals( &mut self, state: &TrackState, - key: (u64, u64, Option), + id: (u64, u64, Option), at: JudgedAt, view: Option<&Layered>, target: &mut impl FnMut(RangeInclusive) -> RangeInclusiveSet, @@ -1147,7 +1136,7 @@ impl Misses { }; // The reader's floor, as the whole judgement takes it: a resumed copy's live floor // can sit above history the reader still reads. - let low = key.0; + let low = id.0; let top = newest; let first = judged_to.saturating_add(1).max(low); let mut fresh = RangeSet::new(); @@ -1160,11 +1149,11 @@ impl Misses { } let walked = fresh.iter().all(|run| { let mut next = run.start; - for (&sequence, slot) in state.lookup.range(run.clone()) { - if sequence != next || !slot.is_walked() { + for (&key, slot) in state.lookup.range(run.clone()) { + if key != next || !slot.is_walked() { return false; } - next = sequence + 1; + next = key + 1; } next == run.end }); @@ -1238,7 +1227,7 @@ impl Misses { .next() .map(|run| run.start.max(floor)); while let Some(from) = next { - let bound = state.arrival(from, servable).map(|slot| slot.group.sequence); + let bound = state.arrival(from, servable).map(|slot| slot.group.start); let drifted = state.poll_drifted(from, servable, drift.budget, None, waiter); next = self.stretches(state, from, bound, drifted, &mut stale, &mut arrived); } @@ -1268,7 +1257,7 @@ impl Misses { .visible .range(lo..hi) .copied() - .find(|sequence| state.lookup.get(sequence).is_some_and(|slot| slot.serves_end())) + .find(|key| state.lookup.get(key).is_some_and(|slot| slot.serves_end())) }; // Where the visit is, and whether a stretch started below it is still running there. let (mut at, mut within) = (from, false); @@ -1361,8 +1350,8 @@ impl Misses { left.remove(run.clone()); } } - for sequence in self.filled.range(hole) { - left.remove(*sequence..*sequence + 1); + for key in self.filled.range(hole) { + left.remove(*key..*key + 1); } left.iter().cloned().collect() } @@ -1381,7 +1370,6 @@ impl Misses { } for run in left.iter() { self.refused.insert(run.clone()); - self.newly_gone.insert(run.clone()); self.deferred .insert(run.clone(), state.arrivals.folds_at(run.start, state.miss_grace())); } @@ -1411,7 +1399,6 @@ impl Misses { return; } self.given_up.insert(groups.clone()); - self.newly_gone.insert(groups.clone()); given_up.push((groups, stale)); } } @@ -1482,10 +1469,10 @@ impl Inboxes { .collect() } - /// Leave `sequence`, cached as count `cached`, with each reader that judged it cached. + /// Leave `key`, cached as count `cached`, with each reader that judged it cached. /// Returns whether any did. - pub(super) fn evicted(&mut self, sequence: u64, cached: u64) -> bool { - let Some(end) = sequence.checked_add(1) else { + pub(super) fn evicted(&mut self, key: u64, cached: u64) -> bool { + let Some(end) = key.checked_add(1) else { return false; }; let mut landed = false; @@ -1495,8 +1482,8 @@ impl Inboxes { return false; } let mut judging = inbox.0.lock().unwrap_or_else(PoisonError::into_inner); - if cached <= judging.cached && judging.span.as_ref().is_some_and(|span| span.contains(&sequence)) { - judging.evicted.insert(sequence..end); + if cached <= judging.cached && judging.span.as_ref().is_some_and(|span| span.contains(&key)) { + judging.evicted.insert(key..end); landed = true; } true @@ -1590,7 +1577,7 @@ fn backfill(state: &TrackState, run: ops::Range) -> RangeSet { .lookup .range(run) .filter(|(_, slot)| !slot.is_walked() && slot.serves_end()) - .map(|(&sequence, _)| sequence..sequence + 1) + .map(|(&key, _)| key..key + 1) .collect() } @@ -1598,14 +1585,14 @@ fn backfill(state: &TrackState, run: ops::Range) -> RangeSet { pub(super) fn holes(state: &TrackState, run: ops::Range) -> Vec> { let mut holes = Vec::new(); let mut from = run.start; - for (&sequence, slot) in state.lookup.range(run.clone()) { + for (&key, slot) in state.lookup.range(run.clone()) { if !slot.serves_end() { continue; } - if from < sequence { - holes.push(from..sequence); + if from < key { + holes.push(from..key); } - from = sequence + 1; + from = key + 1; } if from < run.end { holes.push(from..run.end); @@ -1678,11 +1665,11 @@ mod tests { pool.poll(opened, &noop); let budget = Duration::from_secs(1); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); - for sequence in [0, 2] { + for key in [0, 2] { track - .create_group(crate::group::Info { sequence }) + .create_group(crate::group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let requests = handled.then(|| track.requests()); @@ -1739,11 +1726,11 @@ mod tests { let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); // Newest first, so the publisher skips nothing below it: group 2 is a live gap a fill // may answer. - for sequence in [4, 0] { + for key in [4, 0] { track - .create_group(crate::group::Info { sequence }) + .create_group(crate::group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } track.start_at(3).unwrap(); @@ -1790,7 +1777,7 @@ mod tests { // An arrival and cut happen while the cursor is idle. The drop suppresses asking // again, and reclaiming the cut removes the cached incarnation before the probe. track - .create_group(crate::group::Info { sequence: 2 }) + .create_group(crate::group::Info { start: 2 }) .unwrap() .abort(Error::Cancel) .unwrap(); @@ -1815,9 +1802,9 @@ mod tests { let revision = state.arrivals.changes().revision(); drop(state); track - .create_group(crate::group::Info { sequence: 2 }) + .create_group(crate::group::Info { start: 2 }) .unwrap() - .finish() + .close() .unwrap(); let state = track.state.read(); assert!(state.drops.is_empty(), "recreation removed the last drop run"); @@ -1884,11 +1871,11 @@ mod tests { let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); // Newest first, so the publisher skips nothing below it: group 2 is a live gap a fill // may answer. - for sequence in [4, 0] { + for key in [4, 0] { track - .create_group(crate::group::Info { sequence }) + .create_group(crate::group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } track.start_at(3).unwrap(); @@ -1932,7 +1919,7 @@ mod tests { // The live feed accounts for group 2 but aborts it without a cut, so its slot is // still a hole: the refusal stands and the hole is given up, not asked again. track - .create_group(crate::group::Info { sequence: 2 }) + .create_group(crate::group::Info { start: 2 }) .unwrap() .abort(Error::Cancel) .unwrap(); @@ -1961,11 +1948,11 @@ mod tests { track.received(); let budget = Duration::from_secs(5); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); - for sequence in [0, 2, 4] { + for key in [0, 2, 4] { track - .create_group(crate::group::Info { sequence }) + .create_group(crate::group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let mut misses = Misses::default(); @@ -2014,9 +2001,9 @@ mod tests { ); drop(state); track - .create_group(crate::group::Info { sequence: 1 }) + .create_group(crate::group::Info { start: 1 }) .unwrap() - .finish() + .close() .unwrap(); let state = track.state.read(); assert!( @@ -2098,11 +2085,11 @@ mod tests { track.received(); let budget = Duration::from_secs(5); let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); - for sequence in 0..3 { + for key in 0..3 { track - .create_group(crate::group::Info { sequence }) + .create_group(crate::group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } let mut misses = Misses::default(); @@ -2158,9 +2145,9 @@ mod tests { drop(state); track - .create_group(crate::group::Info { sequence: 4 }) + .create_group(crate::group::Info { start: 4 }) .unwrap() - .finish() + .close() .unwrap(); let state = track.state.read(); let opened = state.cache.pool().now_wall(); @@ -2207,7 +2194,7 @@ mod tests { .lookup .range(run.clone()) .filter(|(_, slot)| slot.is_walked()) - .map(|(&sequence, _)| sequence); + .map(|(&key, _)| key); for at in cached.chain(std::iter::once(run.end)) { if from < at && state.poll_drifted(from, &drift.edge.servable, drift.budget, None, &waiter) { stale.insert(from..at); @@ -2230,7 +2217,7 @@ mod tests { seed.set(x); x % bound }; - const SEQUENCES: u64 = 32; + const KEYS: u64 = 32; let at = |millis| Timestamp::from_millis(millis).unwrap(); let mut judged = 0; for round in 0..400 { @@ -2245,7 +2232,7 @@ mod tests { let budget = Duration::from_millis(50 + next(200)); let runs = |count: u64| { Runs::new((0..1 + next(count)).map(|_| { - let start = next(SEQUENCES); + let start = next(KEYS); start..=start + next(6) })) }; @@ -2253,39 +2240,39 @@ mod tests { 0 => Servable::ALL, 1 => Servable::Run { start: next(8), - end: Some(SEQUENCES - next(8)), + end: Some(KEYS - next(8)), }, _ => Servable::layered( Layered::new(Arc::new(runs(6)), runs(2)), - (next(3) == 0).then(|| next(SEQUENCES + 2)), + (next(3) == 0).then(|| next(KEYS + 2)), &Servable::NONE, ), }; let mut misses = Misses::default(); for _ in 0..1 + next(6) { - let start = next(SEQUENCES); + let start = next(KEYS); misses.wanted.insert(start..start + 1 + next(6)); } for step in 0..12 { - let sequence = next(SEQUENCES); + let key = next(KEYS); match next(6) { 0..=2 => { - if let Ok(mut group) = track.create_group(crate::group::Info { sequence }) { + if let Ok(mut group) = track.create_group(crate::group::Info { start: key }) { if next(4) > 0 { group - .write_frame(at(sequence * 20), bytes::Bytes::from_static(b"x")) + .write_frame(at(key * 20), bytes::Bytes::from_static(b"x")) .unwrap(); } groups.push(group); } } 3 => { - if let Some(index) = groups.iter().position(|group| group.sequence == sequence) { + if let Some(index) = groups.iter().position(|group| group.start == key) { let _ = groups.swap_remove(index).abort(Error::Cancel); } } 4 => { - let _ = track.modify().unwrap().uncache(sequence); + let _ = track.modify().unwrap().uncache(key); } _ => broadcast.info().pool.step(Duration::from_millis(next(120))), } @@ -2333,10 +2320,10 @@ mod tests { seed.set(x); x % bound }; - const SEQUENCES: u64 = 40; + const KEYS: u64 = 40; let runs = |count: u64| { Runs::new((0..next(count + 1)).map(|_| { - let start = next(SEQUENCES); + let start = next(KEYS); start..=start + next(5) })) }; @@ -2361,21 +2348,21 @@ mod tests { let _reader = track.subscribe(Subscription::default().with_delay(Delay::default().with_max(budget))); // A feed with holes, some of them under where it starts, created newest first so the // publisher skips none of them. - let sequences: Vec<_> = (0..SEQUENCES).filter(|_| next(3) > 0).collect(); - for &sequence in sequences.iter().rev() { + let keys: Vec<_> = (0..KEYS).filter(|_| next(3) > 0).collect(); + for &key in keys.iter().rev() { track - .create_group(crate::group::Info { sequence }) + .create_group(crate::group::Info { start: key }) .unwrap() - .finish() + .close() .unwrap(); } if next(2) == 0 { - track.start_at(next(SEQUENCES / 2)).unwrap(); + track.start_at(next(KEYS / 2)).unwrap(); } let fills = track.state.read().fills.clone(); fills.lock().add_handler(); let fixed = Arc::new(runs(6)); - let cap = (next(3) == 0).then(|| next(SEQUENCES + 4)); + let cap = (next(3) == 0).then(|| next(KEYS + 4)); let asking = Asking { priority: 0, max_delay: Some(budget), @@ -2387,7 +2374,7 @@ mod tests { let layer = runs(3); let shared = Layered::new(fixed.clone(), layer.clone()); let unshared = Layered::new(Arc::new((*fixed).clone()), layer); - let key = (0, revision, cap); + let id = (0, revision, cap); for (misses, view, given_up) in [ (&mut moved, &shared, &mut given_moved), (&mut whole, &unshared, &mut given_whole), @@ -2402,7 +2389,7 @@ mod tests { .map(|run| *run.start()..=(*run.end()).min(end)) .collect() }; - misses.retain(key, Some(view), || target(0..=u64::MAX - 1)); + misses.retain(id, Some(view), || target(0..=u64::MAX - 1)); let state = track.state.read(); let drift = Drift { budget, @@ -2412,7 +2399,7 @@ mod tests { }; let asks = misses.judge( &state, - key, + id, Some(view), None, None, @@ -2464,13 +2451,16 @@ mod tests { .unwrap(); let requests = track.requests(); track.start_at(8).unwrap(); - track - .create_group(crate::group::Info { sequence: 8 }) - .unwrap() - .finish() - .unwrap(); - // Four groups below where the feed starts, each a fill of its own. - let ranges = [0..1, 2..3, 4..5, 6..7].map(crate::track::Range::groups); + // Every range starts at a group held, so each knows where it starts without a lookup. + for key in [0, 2, 4, 6, 8] { + track + .create_group(crate::group::Info { start: key }) + .unwrap() + .close() + .unwrap(); + } + // Four groups below where the feed starts, 1, 3, 5, and 7, each a fill of its own. + let ranges = [0..2, 2..4, 4..6, 6..8].map(crate::track::Range::groups); let mut reader = track.subscribe( Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(60))) @@ -2488,7 +2478,7 @@ mod tests { let misses = cursor.misses.as_ref().expect("the reader asked fills"); assert_eq!(misses.joins.len(), 4, "one join per fill"); assert!( - misses.answered.keys.lock().unwrap().is_empty(), + misses.answered.ids.lock().unwrap().is_empty(), "every join was looked at once and parked" ); @@ -2505,7 +2495,7 @@ mod tests { let super::super::Inner::Plain(cursor) = &reader.inner else { unreachable!("a local subscriber reads its own cursor"); }; - let answered = cursor.misses.as_ref().unwrap().answered.keys.lock().unwrap().len(); + let answered = cursor.misses.as_ref().unwrap().answered.ids.lock().unwrap().len(); assert_eq!(answered, 1, "only the join that answered is looked at again"); let _ = reader.poll_recv_group(&second_waiter); let super::super::Inner::Plain(cursor) = &reader.inner else { @@ -2540,9 +2530,9 @@ mod tests { let requests = track.requests(); track.start_at(8).unwrap(); track - .create_group(crate::group::Info { sequence: 8 }) + .create_group(crate::group::Info { start: 8 }) .unwrap() - .finish() + .close() .unwrap(); let track = track.finish().unwrap(); // Group 0 lies below where the feed starts: only a fill may deliver it. @@ -2557,7 +2547,7 @@ mod tests { let drops_waiter = kio::Waiter::new(Waker::from(drops.clone())); let mut handed = Vec::new(); while let Poll::Ready(Ok(Some(group))) = reader.poll_recv_group(&read_waiter) { - handed.push(group.sequence); + handed.push(group.start); } assert_eq!(handed, [8]); assert!( diff --git a/rs/moq-net/src/path/mod.rs b/rs/moq-net/src/path/mod.rs index 12ababc0e8d..bc780af569b 100644 --- a/rs/moq-net/src/path/mod.rs +++ b/rs/moq-net/src/path/mod.rs @@ -105,7 +105,7 @@ fn is_escaped(s: &str, end: usize) -> bool { /// This type wraps a string but provides path-aware operations that respect /// delimiter boundaries, preventing issues like "foo" matching "foobar". /// -/// A path is a sequence of non-empty segments joined by `/`, so it has one spelling: +/// A path is a key of non-empty segments joined by `/`, so it has one spelling: /// [`Path::new`] refuses a leading, trailing, or doubled `/` rather than collapsing it. /// A `\` escapes the `\` or `/` after it, which is how a moq-transport namespace field holding /// either one is spelled, and any other `\` is refused. Segments still split at every `/`. diff --git a/rs/moq-net/src/priority.rs b/rs/moq-net/src/priority.rs index efc68f1c843..3aa784f3350 100644 --- a/rs/moq-net/src/priority.rs +++ b/rs/moq-net/src/priority.rs @@ -18,10 +18,10 @@ use crate::track::Order; // - On remove from overflow: remove its priority-and-id key in O(log n) // // Priority ordering: higher track value = higher priority, then the publisher's priority, -// then subscription, then group sequence in that subscription's own direction. +// then subscription, then group key in that subscription's own direction. /// A group stream's rank: its subscription's track priority, the publisher's priority for -/// the track, the subscription it belongs to, its group sequence in that subscription's +/// the track, the subscription it belongs to, its group key in that subscription's /// [`Order`], and the first object it carries. /// /// Higher `track` always wins, and a higher `publisher` breaks a tie between two @@ -30,7 +30,7 @@ use crate::track::Order; /// newest first by default, so a congested track sheds its backlog rather than its live /// edge, or oldest first for an ascending subscription draining one. /// -/// Scoping the tie-break to a subscription is what keeps group sequence from deciding +/// Scoping the tie-break to a subscription is what keeps group key from deciding /// between tracks. The draft gives that job to `Priority` alone, so a session carrying /// two tracks at equal priority interleaves them rather than draining one first. /// @@ -112,11 +112,11 @@ impl PartialOrd for Priority { /// A stream's send order within its subscription's send group, where higher goes first. /// -/// The [`Priority`] ranking minus the parts the group itself carries: the group sequence in +/// The [`Priority`] ranking minus the parts the group itself carries: the group key in /// the subscription's `order`, then a group's first stream ahead of a later `start` in the /// same group. The low bit only splits a stream from the start of its group (`start == 0`) /// from the rest, which is the joining prefix beside the live stream; any other two streams -/// of one group share. Sequences are wire varints below 2^62, so doubling one fits. +/// of one group share. Keys are wire varints below 2^62, so doubling one fits. pub fn in_group(group: u64, start: u64, order: Order) -> i64 { let group = i64::try_from(group).unwrap_or(i64::MAX).saturating_mul(2); let group = match order { @@ -537,17 +537,20 @@ pub const RECV_PARKED: i64 = i64::MIN; /// unidentified one, ranked by its subscription's `priority` and then its `order`, like the /// sender ranks it. A [`Order::Descending`] subscription gives up its oldest group first, /// since a live track that falls behind skips to its newest anyway; an [`Order::Ascending`] -/// one drains a backlog in sequence, so its newest group goes first. -pub fn recv_group(priority: i8, sequence: u64, order: Order) -> i64 { - // Saturating the sequence at 2^48 keeps the band negative; no track gets that far. - const SEQUENCE: u32 = 48; - const MAX: i64 = (1 << SEQUENCE) - 1; - let sequence = (sequence.min(MAX as u64)) as i64; +/// one drains a backlog in key, so its newest group goes first. +pub fn recv_group(priority: i8, key: u64, order: Order) -> i64 { + // Keys are starts, which pass 2^48 on a fine timescale (nanoseconds since an epoch), so + // only their low 48 bits rank, keeping the band negative: groups in flight together + // start close together, so they keep their order except across one wrap every 2^48 + // units, where saturating would tie every later group instead. + const KEY: u32 = 48; + const MAX: i64 = (1 << KEY) - 1; + let key = (key & MAX as u64) as i64; let kept = match order { - Order::Descending => sequence, - Order::Ascending => MAX - sequence, + Order::Descending => key, + Order::Ascending => MAX - key, }; - i64::MIN / 2 + (i64::from(priority) << SEQUENCE) + kept + i64::MIN / 2 + (i64::from(priority) << KEY) + kept } #[cfg(test)] diff --git a/rs/moq-net/src/reconnect/tests.rs b/rs/moq-net/src/reconnect/tests.rs index 1126634bd79..c1473c08ebe 100644 --- a/rs/moq-net/src/reconnect/tests.rs +++ b/rs/moq-net/src/reconnect/tests.rs @@ -1104,9 +1104,9 @@ async fn busy_handshake() -> (Session, Session, Vec>) { .await .expect("the broadcast resolves") .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(crate::Timestamp::ZERO, b"live".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); sub.recv_group().await.unwrap().expect("the track ended"); ( diff --git a/rs/moq-net/src/recv.rs b/rs/moq-net/src/recv.rs index a85ebc2d01e..dd7f502b0af 100644 --- a/rs/moq-net/src/recv.rs +++ b/rs/moq-net/src/recv.rs @@ -10,7 +10,7 @@ use crate::{Error, frame, group}; /// A received group that aborts if its handler is cancelled before finishing or handing it off. pub(crate) struct Group { writer: Option, - pub sequence: u64, + pub key: u64, handle: group::Handle, armed: bool, } @@ -76,7 +76,7 @@ impl Group { /// Own a group for the lifetime of its receive handler. pub fn new(producer: group::Producer) -> Self { Self { - sequence: producer.sequence, + key: producer.start, handle: producer.handle(), writer: Some(producer), armed: true, @@ -91,7 +91,7 @@ impl Group { /// Finish receiving without running cancellation cleanup. pub fn finish(mut self) -> Result<(), Error> { - self.writer.take().expect("group owned until consumed").finish()?; + self.writer.take().expect("group owned until consumed").close()?; self.armed = false; Ok(()) } @@ -131,14 +131,14 @@ mod tests { #[test] fn receive_frame_cancellation_keeps_a_terminal() { - let producer = group::Info { sequence: 0 }.produce(); + let producer = group::Info { start: 0 }.produce(); let mut consumer = producer.consume(); let mut receiving = Group::new(producer); let frame = receiving .create_object( crate::frame::Info { size: 2, - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::ZERO, } .into(), ) @@ -152,14 +152,14 @@ mod tests { } #[test] fn receive_frame_cancellation_preserves_an_explicit_abort() { - let producer = group::Info { sequence: 0 }.produce(); + let producer = group::Info { start: 0 }.produce(); let mut consumer = producer.consume(); let mut receiving = Group::new(producer); let frame = receiving .create_object( crate::frame::Info { size: 2, - timestamp: Some(crate::Timestamp::ZERO), + timestamp: crate::Timestamp::ZERO, } .into(), ) diff --git a/rs/moq-net/src/session/tests.rs b/rs/moq-net/src/session/tests.rs index ad3c551c322..7683e18f3ff 100644 --- a/rs/moq-net/src/session/tests.rs +++ b/rs/moq-net/src/session/tests.rs @@ -119,9 +119,9 @@ async fn a_subscription_keeps_both_ends_busy() { "{version}: an announcement kept the subscriber busy" ); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(Timestamp::ZERO, b"live".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut sub = kio::clock::timeout(TIMEOUT, remote.track("video").unwrap().subscribe(None)) .await .expect("subscribe timeout") @@ -131,7 +131,7 @@ async fn a_subscription_keeps_both_ends_busy() { .expect("recv timeout") .expect("recv") .expect("the track ended"); - assert_eq!(received.sequence, 0, "{version}"); + assert_eq!(received.start, 0, "{version}"); assert!(!idle(&pair.client), "{version}: idle while serving a subscription"); assert!(!idle(&pair.server), "{version}: idle while holding a subscription"); @@ -161,7 +161,7 @@ async fn a_fetch_keeps_both_ends_busy() { broadcast: _broadcast, } = setup(version).await; - let mut writing = track.append_group().unwrap(); + let mut writing = track.append_next().unwrap(); writing.write_frame(Timestamp::ZERO, b"first".as_ref()).unwrap(); let fetching = remote.track("video").unwrap(); let fetch = kio::clock::sim::spawn(async move { @@ -178,7 +178,7 @@ async fn a_fetch_keeps_both_ends_busy() { assert!(!idle(&pair.server), "{version}: idle while a fetch is in flight"); writing.write_frame(Timestamp::ZERO, b"last".as_ref()).unwrap(); - writing.finish().unwrap(); + writing.close().unwrap(); let frames = kio::clock::timeout(TIMEOUT, fetch) .await .expect("fetch timeout") diff --git a/rs/moq-net/src/tail.rs b/rs/moq-net/src/tail.rs index 66ee5f1058a..51ff2736a26 100644 --- a/rs/moq-net/src/tail.rs +++ b/rs/moq-net/src/tail.rs @@ -95,7 +95,7 @@ impl Tail { /// Fold all but the newest finished run into `folded`, returning whether any was: a stale /// drop under this update then stands as the publisher judged it, but the update owes none /// of what it names that [`Self::overtaken`] reports. The newest run stays, so streams - /// finishing in sequence grow nothing to charge again. + /// finishing in key grow nothing to charge again. pub fn fold(&mut self) -> bool { let (Some(first), Some(older), Some(newest)) = ( self.finished.iter().next().cloned(), diff --git a/rs/moq-net/src/test_interop.rs b/rs/moq-net/src/test_interop.rs index 27f7c894d31..d1eac3fbb34 100644 --- a/rs/moq-net/src/test_interop.rs +++ b/rs/moq-net/src/test_interop.rs @@ -270,7 +270,7 @@ fn lite_group(version: lite::Version, frames: impl IntoIterator = groups .iter() - .map(|(name, sequence, group)| { + .map(|(name, key, group)| { let options = SendOptions { group: Some(group), - priority: sequence + round, + priority: key + round, }; (*name, options, Open::default()) }) diff --git a/rs/moq-net/tests/group_charge.rs b/rs/moq-net/tests/group_charge.rs index bb40e8567c8..5b7275ab6c6 100644 --- a/rs/moq-net/tests/group_charge.rs +++ b/rs/moq-net/tests/group_charge.rs @@ -73,19 +73,20 @@ fn measure(frames: usize, payload: usize) -> (usize, u64) { // Warm up outside the measurement so the track's own one-time allocations and // any lazy statics aren't billed to the groups. - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(0).unwrap(); group.write_frame(Timestamp::ZERO, vec![0u8; payload]).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let before_heap = LIVE.with(Cell::get); let before_charge = pool.used(); - for _ in 0..GROUPS { - let mut group = track.append_group().unwrap(); + for key in 1..=GROUPS as u64 { + let mut group = track.append_group(key).unwrap(); + let timestamp = Timestamp::from_millis(key).unwrap(); for _ in 0..frames { - group.write_frame(Timestamp::ZERO, vec![0u8; payload]).unwrap(); + group.write_frame(timestamp, vec![0u8; payload]).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let heap = (LIVE.with(Cell::get) - before_heap) / GROUPS; diff --git a/rs/moq-net/tests/integration/announce_to_serve.rs b/rs/moq-net/tests/integration/announce_to_serve.rs index a340f8c77e2..22e93156f15 100644 --- a/rs/moq-net/tests/integration/announce_to_serve.rs +++ b/rs/moq-net/tests/integration/announce_to_serve.rs @@ -11,11 +11,13 @@ //! is a virtual sleep. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; use futures::{StreamExt, channel::mpsc}; -use moq_net::{Error, Timestamp, Version, origin}; +use moq_net::{Error, Version, origin}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -70,7 +72,7 @@ fn outcome(result: &Result) -> String { /// group's frames and then how the subscription ended. Read continuously, as a /// real subscriber would, so the subscription is driven the whole time. async fn read(broadcast: &moq_net::broadcast::Consumer, name: &str) -> mpsc::UnboundedReceiver { - let subscription = moq_net::track::Subscription::default().with_range(0..); + let subscription = moq_net::track::Subscription::default().with_ranges([support::keys(0..)]); let mut sub = broadcast .track(name) .unwrap() @@ -112,9 +114,9 @@ async fn next(rx: &mut mpsc::UnboundedReceiver) -> String { } fn write_group(track: &moq_net::track::Producer, payload: &'static str) { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.as_bytes()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), payload.as_bytes()).unwrap(); + group.close().unwrap(); } /// Walk one broadcast through its whole announce lifecycle and record what the diff --git a/rs/moq-net/tests/integration/answer_behind_cache.rs b/rs/moq-net/tests/integration/answer_behind_cache.rs index a7050f9c860..b9d1dcbea3f 100644 --- a/rs/moq-net/tests/integration/answer_behind_cache.rs +++ b/rs/moq-net/tests/integration/answer_behind_cache.rs @@ -20,9 +20,11 @@ fn produce_origin(hop: u64) -> origin::Producer { } fn write_group(track: &track::Producer, payload: &'static [u8]) { - let mut group = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); - group.write_frame(None, payload).unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload) + .unwrap(); + group.close().unwrap(); } /// The next group `subscriber` receives, with its first frame. @@ -35,7 +37,7 @@ async fn next_group(subscriber: &mut track::Subscriber) -> moq_net::Result<(u64, .await .unwrap()? .unwrap(); - Ok((group.sequence, frame.payload.to_vec())) + Ok((group.start, frame.payload.to_vec())) } /// The publisher replaces its track with an empty one of the same name, so the lingering @@ -131,8 +133,9 @@ async fn a_reset_newest_group_answers_the_group_before(version: &str) { .create_track("audio", track::Info::default().with_timescale(None)) .unwrap(); route.announce().unwrap(); - let mut g0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); - g0.write_frame(None, &b"0a"[..]).unwrap(); + let mut g0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); + g0.write_frame(moq_net::Timestamp::unscaled(g0.start), &b"0a"[..]) + .unwrap(); // A reader on the upper relay keeps its copy live after the lower relay leaves. let local = kio::clock::timeout(TIMEOUT, upper.consume().follow("room", None).unwrap().available()) @@ -157,16 +160,17 @@ async fn a_reset_newest_group_answers_the_group_before(version: &str) { .unwrap() .unwrap(); assert_eq!(next_group(&mut subscriber).await.unwrap(), (0, b"0a".to_vec())); - g0.finish().unwrap(); - let mut g1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); - g1.write_frame(None, &b"1a"[..]).unwrap(); + g0.close().unwrap(); + let mut g1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); + g1.write_frame(moq_net::Timestamp::unscaled(g1.start), &b"1a"[..]) + .unwrap(); assert_eq!(next_group(&mut subscriber).await.unwrap(), (1, b"1a".to_vec())); let mut kept = kio::clock::timeout(TIMEOUT, keeper.recv_group()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(kept.sequence, 1); + assert_eq!(kept.start, 1); g1.abort(Error::Cancel).unwrap(); // The upper relay's copy stays live with group 1 reset and group 0 held. loop { @@ -187,11 +191,12 @@ async fn a_reset_newest_group_answers_the_group_before(version: &str) { .unwrap(); // The upper relay answers the resubscription before the feed moves on. kio::clock::sleep(Duration::from_secs(1)).await; - let mut g2 = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); - g2.write_frame(None, &b"2a"[..]).unwrap(); + let mut g2 = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); + g2.write_frame(moq_net::Timestamp::unscaled(g2.start), &b"2a"[..]) + .unwrap(); loop { - let (sequence, payload) = next_group(&mut subscriber).await.unwrap(); - if sequence == 2 { + let (key, payload) = next_group(&mut subscriber).await.unwrap(); + if key == 2 { assert_eq!(payload, b"2a".to_vec(), "{version}"); break; } diff --git a/rs/moq-net/tests/integration/auth.rs b/rs/moq-net/tests/integration/auth.rs index fb6df3c3d07..811dc051ca2 100644 --- a/rs/moq-net/tests/integration/auth.rs +++ b/rs/moq-net/tests/integration/auth.rs @@ -5,6 +5,7 @@ //! moq-transport carries tokens only on the requests that name them. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -1212,7 +1213,7 @@ async fn exact_and_subtree_grants_admit_their_broadcast(version: &'static str) { }) .await; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group .write_frame(moq_net::Timestamp::from_millis(0).unwrap(), b"frame".as_ref()) .unwrap(); @@ -1291,7 +1292,7 @@ async fn a_live_subscription_survives_union_renewal(version: &'static str) { .await .unwrap() .unwrap(); - let mut group = published.append_group().unwrap(); + let mut group = published.append_next().unwrap(); group .write_frame(moq_net::Timestamp::from_millis(0).unwrap(), b"first".as_ref()) .unwrap(); @@ -1308,7 +1309,7 @@ async fn a_live_subscription_survives_union_renewal(version: &'static str) { let _renewed = renewed.unwrap(); drop(first); first_grant.closed().await; - let mut group = published.append_group().unwrap(); + let mut group = published.append_next().unwrap(); group .write_frame(moq_net::Timestamp::from_millis(1).unwrap(), b"renewed".as_ref()) .unwrap(); @@ -1363,9 +1364,9 @@ async fn a_revoked_grant_cancels_its_subscriptions(version: &'static str) { }); let (_, setup) = issued.next().await.unwrap(); - let mut group = down_track.append_group().unwrap(); + let mut group = down_track.append_next().unwrap(); group.write_frame(ts(0), b"down".as_ref()).unwrap(); - let mut group = up_track.append_group().unwrap(); + let mut group = up_track.append_next().unwrap(); group.write_frame(ts(0), b"up".as_ref()).unwrap(); let remote = received @@ -1789,9 +1790,9 @@ async fn a_narrowing_deafens_one_path(version: &'static str) { }) .await; - let mut group = audio_track.append_group().unwrap(); + let mut group = audio_track.append_next().unwrap(); group.write_frame(ts(0), b"a".as_ref()).unwrap(); - let mut group = video_track.append_group().unwrap(); + let mut group = video_track.append_next().unwrap(); group.write_frame(ts(0), b"v".as_ref()).unwrap(); let remote = received @@ -1822,10 +1823,10 @@ async fn a_narrowing_deafens_one_path(version: &'static str) { wait_announced(&received.consume(), "room/alice/audio", false).await; // The sibling keeps flowing. - let mut group = video_track.append_group().unwrap(); + let mut group = video_track.append_next().unwrap(); group.write_frame(ts(1), b"v".as_ref()).unwrap(); let group = video_sub.recv_group().await.unwrap().expect("video still flows"); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); // The relay enforced it, not the client: moq-lite resets the subscription with // UNAUTHORIZED, even on a version without the Auth Stream, and neither side @@ -1883,9 +1884,9 @@ async fn a_narrowing_aborts_what_the_peer_published(version: &'static str) { }) .await; - let mut group = mic_track.append_group().unwrap(); + let mut group = mic_track.append_next().unwrap(); group.write_frame(ts(0), b"m".as_ref()).unwrap(); - let mut group = cam_track.append_group().unwrap(); + let mut group = cam_track.append_next().unwrap(); group.write_frame(ts(0), b"c".as_ref()).unwrap(); let remote = relay @@ -1915,10 +1916,10 @@ async fn a_narrowing_aborts_what_the_peer_published(version: &'static str) { assert!(unauthorized(&err), "{err:?}"); wait_announced(&relay.consume(), "room/bob/mic", false).await; - let mut group = cam_track.append_group().unwrap(); + let mut group = cam_track.append_next().unwrap(); group.write_frame(ts(1), b"c".as_ref()).unwrap(); let group = cam_sub.recv_group().await.unwrap().expect("cam still flows"); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); // A narrowing is not a publication outside the grant: the client stays up. assert_eq!(pair.client_transport.close_reason(), None); @@ -1938,7 +1939,7 @@ async fn a_narrowing_resets_a_fetch_in_flight(version: &'static str) { let track = broadcast.broadcast().create_track("video", timed()).unwrap(); broadcast.announce().unwrap(); // The group stays open, so the fetch is still in flight when the grant narrows. - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(0), b"first".as_ref()).unwrap(); let received = produce_origin(3); @@ -1986,11 +1987,11 @@ async fn a_narrowing_resets_a_fetch_in_flight(version: &'static str) { .expect("timed out"); } -/// Read groups until one at `sequence` or later arrives. -async fn recv_through(sub: &mut moq_net::track::Subscriber, sequence: u64) { +/// Read groups until one at `key` or later arrives. +async fn recv_through(sub: &mut moq_net::track::Subscriber, key: u64) { loop { let group = sub.recv_group().await.unwrap().expect("track ended"); - if group.sequence >= sequence { + if group.start >= key { return; } } @@ -2022,7 +2023,7 @@ async fn a_widening_brings_back_a_deafened_path(version: &'static str) { }) .await; - let mut group = audio_track.append_group().unwrap(); + let mut group = audio_track.append_next().unwrap(); group.write_frame(ts(0), b"a".as_ref()).unwrap(); let remote = received .consume() @@ -2052,7 +2053,7 @@ async fn a_widening_brings_back_a_deafened_path(version: &'static str) { .unwrap() .unwrap(); let mut sub = remote.track("opus").unwrap().subscribe(prefs()).await.unwrap(); - let mut group = audio_track.append_group().unwrap(); + let mut group = audio_track.append_next().unwrap(); group.write_frame(ts(1), b"a".as_ref()).unwrap(); recv_through(&mut sub, 1).await; @@ -2113,7 +2114,7 @@ async fn a_widening_brings_back_what_the_peer_published(version: &'static str) { .unwrap() .unwrap(); let mut sub = remote.track("opus").unwrap().subscribe(prefs()).await.unwrap(); - let mut group = mic_track.append_group().unwrap(); + let mut group = mic_track.append_next().unwrap(); group.write_frame(ts(0), b"m".as_ref()).unwrap(); recv_through(&mut sub, 0).await; @@ -2372,7 +2373,7 @@ async fn a_broadcast_relayed_from_another_client_is_not_ours_to_check(version: & .await; assert_eq!(granted(&pair.client).await, grant(&["local"], &[""])); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group .write_frame(moq_net::Timestamp::from_millis(0).unwrap(), b"frame".as_ref()) .unwrap(); diff --git a/rs/moq-net/tests/integration/byte_holes.rs b/rs/moq-net/tests/integration/byte_holes.rs index a200dc706fe..413aa9767bc 100644 --- a/rs/moq-net/tests/integration/byte_holes.rs +++ b/rs/moq-net/tests/integration/byte_holes.rs @@ -63,17 +63,17 @@ async fn a_half_read_frame_continues_from_its_byte() { let mut sub = remote.track("file").unwrap().subscribe(prefs).await.unwrap(); let payload = pattern(SIZE); - let group0 = track.append_group().unwrap(); + let group0 = track.append_group(0).unwrap(); let mut writing = group0 .create_frame(frame::Info { size: SIZE as u64, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); writing.write(payload[..HALF].to_vec()).unwrap(); let mut read0 = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(read0.sequence, 0); + assert_eq!(read0.start, 0); let mut reading = read0.next_frame().await.unwrap().unwrap(); let mut got = Vec::new(); while got.len() < HALF { @@ -81,11 +81,13 @@ async fn a_half_read_frame_continues_from_its_byte() { } assert_eq!(got, payload[..HALF]); - let mut group1 = track.append_group().unwrap(); - group1.write_frame(Timestamp::ZERO, b"next".to_vec()).unwrap(); - group1.finish().unwrap(); + let mut group1 = track.append_group(1).unwrap(); + group1 + .write_frame(Timestamp::from_millis(1).unwrap(), b"next".to_vec()) + .unwrap(); + group1.close().unwrap(); let mut read1 = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(read1.sequence, 1); + assert_eq!(read1.start, 1); assert_eq!(&read1.read_frame().await.unwrap().unwrap().payload[..], b"next"); // The cheaper route through `B` wins, metered so its egress shows what it sent `E`. @@ -103,7 +105,7 @@ async fn a_half_read_frame_continues_from_its_byte() { settle().await; writing.write(payload[HALF..].to_vec()).unwrap(); - writing.finish().unwrap().finish().unwrap(); + writing.finish().unwrap().close().unwrap(); while let Some(chunk) = reading.read_chunk().await.unwrap() { got.extend_from_slice(&chunk); diff --git a/rs/moq-net/tests/integration/catalog_acquisition.rs b/rs/moq-net/tests/integration/catalog_acquisition.rs index 3ebcd556333..39f3bb9f335 100644 --- a/rs/moq-net/tests/integration/catalog_acquisition.rs +++ b/rs/moq-net/tests/integration/catalog_acquisition.rs @@ -3,6 +3,7 @@ //! Initial catalog acquisition while the same publisher gains a second session. use crate::support; +use support::AppendNext; use moq_net::{Error, Version, origin}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; @@ -71,9 +72,11 @@ async fn acquisition(version: &str, carries_instance: bool) { }; assert!(new_cost < old_cost, "the cheaper session wins the route"); let track = request.accept(None); - let mut group = track.append_group().unwrap(); - group.write_frame(None, b"catalog".to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), b"catalog".to_vec()) + .unwrap(); + group.close().unwrap(); let result = pending.await; if carries_instance { let mut subscription = result.expect("the same instance must acquire"); diff --git a/rs/moq-net/tests/integration/catalog_churn.rs b/rs/moq-net/tests/integration/catalog_churn.rs index 3846b21d137..c53d287a32e 100644 --- a/rs/moq-net/tests/integration/catalog_churn.rs +++ b/rs/moq-net/tests/integration/catalog_churn.rs @@ -10,6 +10,7 @@ //! shrunk to the ops it needs, which the report names. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -165,7 +166,7 @@ impl Reader { return; } }; - let frame = format!("{}:{}", group.sequence, String::from_utf8_lossy(&frame.payload)); + let frame = format!("{}:{}", group.start, String::from_utf8_lossy(&frame.payload)); if tx.unbounded_send(Ok(frame)).is_err() { return; } @@ -202,14 +203,14 @@ impl Reader { let joined = self.expected.first().map(group); let mut seen = self.seen.as_slice(); while let (Some(first), Some(joined)) = (seen.first(), joined) { - let sequence = group(first); - if sequence >= joined { + let key = group(first); + if key >= joined { break; } if !first.ends_with(":snapshot") { return Verdict::Wrong; } - let older = seen.iter().take_while(|frame| group(frame) == sequence).count(); + let older = seen.iter().take_while(|frame| group(frame) == key).count(); seen = &seen[older..]; } let mut expected = self.expected.as_slice(); @@ -302,7 +303,7 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { _upstream: upstream, }; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); // The open group as written so far, as `group:payload`. let mut written: Vec = Vec::new(); let mut readers: Vec = Vec::new(); @@ -314,7 +315,7 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { group .write_frame(Timestamp::from_millis(clock).unwrap(), payload.as_bytes()) .unwrap(); - let frame = format!("{}:{payload}", group.sequence); + let frame = format!("{}:{payload}", group.start); for reader in readers { reader.expected.push(frame.clone()); } @@ -326,8 +327,8 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { match *op { Op::Write => write(&mut group, &mut written, &mut readers, &format!("d{step}")), Op::NewGroup => { - group.finish().unwrap(); - group = track.append_group().unwrap(); + group.close().unwrap(); + group = track.append_next().unwrap(); written.clear(); write(&mut group, &mut written, &mut readers, "snapshot"); // Let it reach `R`, so the newest group at the next join is unambiguous. @@ -349,7 +350,7 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { let Some(remote) = topology.request(step).await? else { continue; }; - let fetch = remote.track("catalog.json").unwrap().fetch_group(group.sequence, None); + let fetch = remote.track("catalog.json").unwrap().fetch_group(group.start, None); let mut fetched = match kio::clock::timeout(READ_TIMEOUT, fetch).await { Ok(Ok(fetched)) => fetched, Ok(Err(_)) => continue, @@ -358,11 +359,9 @@ async fn execute(version: Version, ops: &[Op]) -> Result<(), Failure> { let mut got = Vec::new(); while got.len() < written.len() { match kio::clock::timeout(READ_TIMEOUT, fetched.read_frame()).await { - Ok(Ok(Some(frame))) => got.push(format!( - "{}:{}", - fetched.sequence, - String::from_utf8_lossy(&frame.payload) - )), + Ok(Ok(Some(frame))) => { + got.push(format!("{}:{}", fetched.start, String::from_utf8_lossy(&frame.payload))) + } Ok(Ok(None)) => return Err(wrong(format!("{step}: fetch ended at {got:?}"))), Ok(Err(_)) => break, Err(_) => return Err(stuck(format!("{step}: fetched {got:?}, then hung"))), diff --git a/rs/moq-net/tests/integration/catalog_handover.rs b/rs/moq-net/tests/integration/catalog_handover.rs index 78d4c966c5b..2147dfd0166 100644 --- a/rs/moq-net/tests/integration/catalog_handover.rs +++ b/rs/moq-net/tests/integration/catalog_handover.rs @@ -12,6 +12,7 @@ //! no catalog. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -109,7 +110,7 @@ impl Topology { .unwrap(); let track = broadcast.broadcast().create_track("catalog.json", timed()).unwrap(); broadcast.announce().unwrap(); - let group = track.append_group().unwrap(); + let group = track.append_next().unwrap(); let p_to_a = link(version, &publisher, &relay_a, None).await; let p_to_b = link(version, &publisher, &relay_b, None).await; @@ -210,12 +211,12 @@ impl Reader { let sent = match group.read_frame().await { Ok(Some(frame)) => tx.unbounded_send(Ok(format!( "{}:{}", - group.sequence, + group.start, String::from_utf8_lossy(&frame.payload) ))), Ok(None) => break, Err(err) => { - let _ = tx.unbounded_send(Err(format!("group {} frame: {err}", group.sequence))); + let _ = tx.unbounded_send(Err(format!("group {} frame: {err}", group.start))); break; } }; @@ -348,7 +349,7 @@ enum Before { Resume, } -/// The minimal sequences the release-line splice left a late reader stuck on: it +/// The minimal keys the release-line splice left a late reader stuck on: it /// subscribes, and nothing ever arrives. async fn late_reader(version: &str, before: Before) { let version: Version = version.parse().unwrap(); diff --git a/rs/moq-net/tests/integration/claim_continues.rs b/rs/moq-net/tests/integration/claim_continues.rs index b2b94acedd9..80097ae9856 100644 --- a/rs/moq-net/tests/integration/claim_continues.rs +++ b/rs/moq-net/tests/integration/claim_continues.rs @@ -8,10 +8,12 @@ //! subscription through the claim sees another broadcast rather than a splice. use crate::support; +use support::AppendNext; +use support::AtStart; use std::{cell::RefCell, rc::Rc, time::Duration}; -use moq_net::{Error, Instance, Timestamp, Version, origin, track}; +use moq_net::{Error, Instance, Version, origin, track}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; const TEST_TIMEOUT: Duration = Duration::from_secs(600); @@ -69,9 +71,9 @@ impl Worker { ) .unwrap(); let tag = format!("{name}{}", served.borrow().len()); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, tag.into_bytes()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), tag.into_bytes()).unwrap(); + group.close().unwrap(); prepared.announce().unwrap(); served.borrow_mut().push((prepared, track)); } diff --git a/rs/moq-net/tests/integration/compress.rs b/rs/moq-net/tests/integration/compress.rs index 7b3dfe81fe6..34c9388fd30 100644 --- a/rs/moq-net/tests/integration/compress.rs +++ b/rs/moq-net/tests/integration/compress.rs @@ -6,6 +6,7 @@ //! signal compression receive the plain frames the relay inflates once for all of them. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -153,23 +154,25 @@ async fn a_start_group_arrives_whole() { false => moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), }; let published = publish(info); - let mut group = published - .track - .create_group(moq_net::group::Info { sequence: 5 }) - .unwrap(); + let mut group = published.track.create_group(moq_net::group::Info { start: 5 }).unwrap(); write(&mut group, 0..head); - let mut fixture = relay(published, hops, Subscription::default().with_range(5..)).await; + let mut fixture = relay( + published, + hops, + Subscription::default().with_ranges([support::keys(5..)]), + ) + .await; if lite { assert_eq!( fixture.track.subscription_changed().await.unwrap().unwrap().ranges[0].start, - Anchor::Absolute(5), + Anchor::Time(5), "{case}: upstream asks for the start group" ); } write(&mut group, head..5); - group.finish().unwrap(); + group.close().unwrap(); let mut reading = fixture.subscriber.recv_group().await.unwrap().unwrap(); - assert_eq!(reading.sequence, 5, "{case}"); + assert_eq!(reading.start, 5, "{case}"); for index in 0..5 { assert_eq!( reading.read_frame().await.unwrap().unwrap().payload, @@ -200,7 +203,7 @@ async fn relay_serves_every_version() { "{version}: compression declared" ); - let mut group = fixture.track.append_group().unwrap(); + let mut group = fixture.track.append_next().unwrap(); group .write_frame(Timestamp::from_millis(0).unwrap(), payload(0)) .unwrap(); @@ -212,7 +215,7 @@ async fn relay_serves_every_version() { let mut frame = group .create_frame(moq_net::frame::Info { size: chunked.len() as u64, - timestamp: Some(Timestamp::from_millis(2).unwrap()), + timestamp: Timestamp::from_millis(2).unwrap(), }) .unwrap(); for chunk in chunked.chunks(100) { @@ -222,7 +225,7 @@ async fn relay_serves_every_version() { group .write_frame(Timestamp::from_millis(3).unwrap(), bytes::Bytes::new()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut group = fixture.subscriber.recv_group().await.unwrap().expect("group"); for index in 0..3 { @@ -248,13 +251,13 @@ async fn ietf_ingress_stays_compressed_across_protocols() { fixture.subscriber.info().compression == moq_net::track::Compression::Deflate, "{upstream} -> {downstream}" ); - let mut group = fixture.track.append_group().unwrap(); + let mut group = fixture.track.append_next().unwrap(); for index in 0..3 { group .write_frame(Timestamp::from_millis(index).unwrap(), payload(index)) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let mut received = fixture.subscriber.recv_group().await.unwrap().unwrap(); for index in 0..3 { assert_eq!( @@ -280,13 +283,13 @@ async fn ietf_fetch_keeps_the_stored_form() { .route("bcast", moq_net::origin::RouteOptions::hidden()) .unwrap(); let track = broadcast.broadcast().create_track("data", compressed()).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); for index in 0..3 { group .write_frame(Timestamp::from_millis(index).unwrap(), payload(index)) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); broadcast.announce().unwrap(); let origin = produce_origin(2); diff --git a/rs/moq-net/tests/integration/control_stats.rs b/rs/moq-net/tests/integration/control_stats.rs index 23588e9019f..22fd3f30f37 100644 --- a/rs/moq-net/tests/integration/control_stats.rs +++ b/rs/moq-net/tests/integration/control_stats.rs @@ -2,12 +2,14 @@ //! Control counters cross real protocol drivers, including legacy multiplexing. +use crate::support::AppendNext; +use crate::support::AtStart; use crate::support::{ harness::{MockConnectOptions, connect_mock, now, spawn, subscribed}, mock::create_mock_session_pair, }; use moq_net::{ - Timescale, Timestamp, origin, + Timescale, origin, stats::{Message, Registry, Tier}, }; @@ -52,11 +54,11 @@ async fn control_classes_cross_lite_and_ietf_sessions_without_counting_payload() .unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); subscribed(&track).await; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(vec![7; 64 * 1024])) + .write_frame(group.at_start(), bytes::Bytes::from(vec![7; 64 * 1024])) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(group.read_frame().await.unwrap().unwrap().payload.len(), 64 * 1024); let rows = registry.snapshot().control(); diff --git a/rs/moq-net/tests/integration/datagram.rs b/rs/moq-net/tests/integration/datagram.rs index 6294a3f05b2..6fa5ec730f3 100644 --- a/rs/moq-net/tests/integration/datagram.rs +++ b/rs/moq-net/tests/integration/datagram.rs @@ -111,16 +111,17 @@ async fn a_datagram_track_crosses_a_session() { let chain = Chain::new(&["moq-lite-07-wip"]).await; let info = datagram::Info::default().with_timescale(timescale).with_priority(3); let mut producer = chain.publication.broadcast().create_datagrams(TRACK, info).unwrap(); - let timestamp = |ms| timescale.map(|_| millis(ms)); + // A scale-less datagram's timestamp names its sequence. + let timestamp = |ms, sequence| timescale.map_or(moq_net::Timestamp::unscaled(sequence), |_| millis(ms)); // Nobody reads it yet, so nobody ever will. - assert_eq!(producer.append(timestamp(1), &b"early"[..]).unwrap(), 0); + assert_eq!(producer.append(timestamp(1, 0), &b"early"[..]).unwrap(), 0); let mut consumer = chain.consume(&producer, 0).await; assert_eq!(consumer.info().await.unwrap(), info, "{timescale:?}"); - assert_eq!(producer.append(timestamp(2), &b"live"[..]).unwrap(), 1); + assert_eq!(producer.append(timestamp(2, 1), &b"live"[..]).unwrap(), 1); let datagram = recv(&mut consumer).await.unwrap().expect("a datagram"); assert_eq!(datagram.sequence, 1, "{timescale:?}"); - assert_eq!(datagram.timestamp, timestamp(2), "{timescale:?}"); + assert_eq!(datagram.timestamp, timestamp(2, 1), "{timescale:?}"); assert_eq!(&datagram.payload[..], b"live", "{timescale:?}"); producer.finish(); @@ -179,7 +180,7 @@ async fn a_relay_carries_a_datagram_track() { assert_eq!(producer.append(millis(5), &b"relayed"[..]).unwrap(), 0); let datagram = recv(&mut consumer).await.unwrap().expect("a datagram"); assert_eq!(datagram.sequence, 0); - assert_eq!(datagram.timestamp, Some(millis(5))); + assert_eq!(datagram.timestamp, millis(5)); assert_eq!(&datagram.payload[..], b"relayed"); consumer.set_priority(4); @@ -224,7 +225,8 @@ async fn a_stream_track_and_a_datagram_track_share_a_name_independently() { .expect("group timeout") .unwrap() .expect("a group"); - assert_eq!(group.sequence, 0); + // A one-frame group starts at its frame, in the track's milliseconds. + assert_eq!(group.start, 2); assert_eq!( &group.read_frame().await.unwrap().expect("a frame").payload[..], b"frame" @@ -235,7 +237,7 @@ async fn a_stream_track_and_a_datagram_track_share_a_name_independently() { .expect("group timeout") .unwrap() .expect("a group"); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 3); } /// On lite-07 a name that has only a datagram track has no stream track to subscribe to. @@ -283,7 +285,7 @@ async fn legacy_versions_map_a_datagram_track_onto_a_subscription() { assert_eq!(producer.append(millis(9), &b"mapped"[..]).unwrap(), 1); let datagram = recv(&mut consumer).await.unwrap().expect("a datagram"); assert_eq!(datagram.sequence, 1, "{version}"); - assert_eq!(datagram.timestamp, Some(millis(9)), "{version}"); + assert_eq!(datagram.timestamp, millis(9), "{version}"); assert_eq!(&datagram.payload[..], b"mapped", "{version}"); consumer.set_priority(6); @@ -338,7 +340,7 @@ async fn legacy_versions_carry_both_kinds_of_a_name() { producer.append(millis(1), &b"datagram"[..]).unwrap(); let datagram = recv(&mut consumer).await.unwrap().expect("a datagram"); assert_eq!(&datagram.payload[..], b"datagram", "{hops:?}"); - let timestamp = datagram.timestamp.expect("a timestamp"); + let timestamp = datagram.timestamp; assert_eq!(timestamp.convert(Timescale::MILLI).unwrap(), millis(1), "{hops:?}"); track.write_frame(millis(2), &b"frame"[..]).unwrap(); @@ -418,7 +420,7 @@ async fn lite05_and_lite06_finish_a_datagram_track_as_a_stream_subscriber_expect .expect("end timeout"); match end { Ok(None) => {} - Ok(Some(group)) => panic!("{version}: a group {}", group.sequence), + Ok(Some(group)) => panic!("{version}: a group {}", group.start), Err(err) => panic!("{version}: {err:?}"), } } @@ -451,7 +453,7 @@ async fn lite05_and_lite06_answer_a_datagram_request_beside_a_silent_stream_hand producer.append(millis(1), &b"requested"[..]).unwrap(); let datagram = recv(&mut consumer).await.unwrap().expect("a datagram"); assert_eq!(&datagram.payload[..], b"requested", "{version}"); - assert_eq!(datagram.timestamp, Some(millis(1)), "{version}"); + assert_eq!(datagram.timestamp, millis(1), "{version}"); } } @@ -495,7 +497,7 @@ async fn an_old_subscriber_reaches_a_datagram_track_through_a_lite07_relay() { producer.append(millis(4), &b"relayed"[..]).unwrap(); let datagram = recv(&mut consumer).await.unwrap().expect("a datagram"); assert_eq!(&datagram.payload[..], b"relayed"); - assert_eq!(datagram.timestamp, Some(millis(4))); + assert_eq!(datagram.timestamp, millis(4)); } /// A relay's upstream datagram subscription outlives its last demanding reader while an old diff --git a/rs/moq-net/tests/integration/drained_claim.rs b/rs/moq-net/tests/integration/drained_claim.rs index 2d16b97210e..369a389060b 100644 --- a/rs/moq-net/tests/integration/drained_claim.rs +++ b/rs/moq-net/tests/integration/drained_claim.rs @@ -10,10 +10,12 @@ //! the path now, rather than the drained worker. use crate::support; +use support::AppendNext; +use support::AtStart; use std::{cell::RefCell, rc::Rc, time::Duration}; -use moq_net::{Timestamp, Version, goaway::Goaway, origin, track}; +use moq_net::{Version, goaway::Goaway, origin, track}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -62,9 +64,9 @@ impl Worker { while let Some(request) = handler.requested().await { let prepared = request.route(moq_net::origin::RouteOptions::hidden()).unwrap(); let track = prepared.broadcast().create_track("video", timed()).unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, name.as_bytes().to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), name.as_bytes().to_vec()).unwrap(); + group.close().unwrap(); prepared.announce().unwrap(); served.borrow_mut().push((prepared, track)); } diff --git a/rs/moq-net/tests/integration/drop_record.rs b/rs/moq-net/tests/integration/drop_record.rs index 38572511a36..a1b86b05e50 100644 --- a/rs/moq-net/tests/integration/drop_record.rs +++ b/rs/moq-net/tests/integration/drop_record.rs @@ -9,6 +9,7 @@ use std::time::Duration; use moq_net::track::{Delay, Info, Range, Subscription}; use moq_net::{Timescale, Timestamp, Version, cache, origin, track}; +use crate::support; use crate::support::harness::{MockConnectOptions, MockPair, connect_mock, spawn}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -34,13 +35,13 @@ fn timed() -> Info { Info::default().with_timescale(Timescale::MILLI).with_max_age(None) } -/// Write group `sequence` presented at `millis`, and finish it. -fn write(source: &track::Producer, sequence: u64, millis: u64) { - let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); +/// Write group `key` presented at `millis`, and finish it. +fn write(source: &track::Producer, key: u64, millis: u64) { + let mut group = source.create_group(moq_net::group::Info { start: key }).unwrap(); group .write_frame(Timestamp::from_millis(millis).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } /// A publisher, a relay subscribing to it, and a viewer subscribing to the relay. @@ -93,8 +94,8 @@ impl Chain { /// Groups 0 through 9, one per second. fn populate(&self) { - for sequence in 0..GROUPS { - write(&self.source, sequence, sequence * 1000); + for key in 0..GROUPS { + write(&self.source, key, key * 1000); } } @@ -168,7 +169,7 @@ async fn received(subscriber: &mut track::Subscriber) -> BTreeSet { let Some(group) = group.expect("the reader runs on") else { break; }; - assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + assert!(got.insert(group.start), "group {} delivered twice", group.start); } got } @@ -201,7 +202,7 @@ async fn a_relay_eviction_is_filled_by_range_for_every_reader() { let mut widened = chain.subscribe(reading(NARROW, [Range::oldest()])).await; assert_eq!(received(&mut widened).await, (6..GROUPS).collect()); // Owed every group from 0, on the relay's own cache, and reading none of them yet. - let mut owed = chain.subscribe_relay(reading(None, [Range::groups(0..)])).await; + let mut owed = chain.subscribe_relay(reading(None, [support::keys(0..)])).await; // Nobody touches them, so the relay and the viewer reclaim all but the live edge. Every // group is stale by the wall clock now, so only a reader with no budget wants them. @@ -211,7 +212,7 @@ async fn a_relay_eviction_is_filled_by_range_for_every_reader() { widened.update(reading(None, [Range::oldest()])).unwrap(); assert_eq!(received(&mut widened).await, (0..6).collect(), "the widened reader"); - let mut new = chain.subscribe(reading(None, [Range::groups(1..4)])).await; + let mut new = chain.subscribe(reading(None, [support::keys(1..4)])).await; assert_eq!(received(&mut new).await, (1..4).collect(), "the new reader"); assert_eq!(received(&mut owed).await, (0..GROUPS).collect(), "the owed reader"); @@ -237,7 +238,7 @@ enum Huge { Forgotten, } -/// A single run spanning 2^40 sequences is settled by each cursor in one step, whether it is +/// A single run spanning 2^40 keys is settled by each cursor in one step, whether it is /// dropped, skipped, or forgotten and judged again: every reader, narrow or wide, gets the /// groups on either side and ends at once, on mock time, rather than walking the run. #[kio::test] @@ -247,14 +248,14 @@ async fn a_huge_run_settles_in_one_step() { for huge in [Huge::Dropped, Huge::Skipped, Huge::Forgotten] { kio::clock::timeout(TIMEOUT * 10, async { let chain = Chain::new([LITE07, LITE07], [forever(), expiring(EXPIRY), expiring(EXPIRY)]).await; - let everything = || reading(None, [Range::groups(0..)]); - let mut narrow = chain.subscribe(reading(NARROW, [Range::groups(0..)])).await; + let everything = || reading(None, [support::keys(0..)]); + let mut narrow = chain.subscribe(reading(NARROW, [support::keys(0..)])).await; let mut wide = chain.subscribe(everything()).await; write(&chain.source, 0, 0); if let Huge::Skipped = huge { chain.source.skip_groups(1..FAR); } - // Presented just after group 0, since its sequence in milliseconds overflows. + // Presented just after group 0, since its key in milliseconds overflows. write(&chain.source, FAR, 1); let _finished = chain.source.clone().finish().unwrap(); @@ -296,7 +297,7 @@ async fn drain(subscriber: &mut track::Subscriber) -> BTreeSet { .expect("the reader stalled") .expect("the reader runs on") { - assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + assert!(got.insert(group.start), "group {} delivered twice", group.start); } got } @@ -332,8 +333,8 @@ async fn reported_drop_through(versions: [&str; 2]) { let name = format!("{} then {}", versions[0], versions[1]); kio::clock::timeout(TIMEOUT * 10, async { let chain = Chain::new(versions, [forever(), forever(), forever()]).await; - for sequence in (0..GROUPS).filter(|sequence| *sequence != 7) { - write(&chain.source, sequence, sequence * 1000); + for key in (0..GROUPS).filter(|key| *key != 7) { + write(&chain.source, key, key * 1000); } let mut first = chain.subscribe(reading(B1, [Range::oldest()])).await; assert_eq!(received(&mut first).await, [5, 6, 8, 9].into(), "{name}"); @@ -347,7 +348,7 @@ async fn reported_drop_through(versions: [&str; 2]) { // A lite-07 viewer hop leaves the stale report open to a widening update, which fills // the group from upstream; over a lite-06 viewer hop the drop is final. let want: BTreeSet = match versions[1] == LITE06 { - true => (0..GROUPS).filter(|sequence| *sequence != 7).collect(), + true => (0..GROUPS).filter(|key| *key != 7).collect(), false => (0..GROUPS).collect(), }; first.update(reading(B2, [Range::oldest()])).unwrap(); @@ -371,13 +372,13 @@ async fn reported_drop_through(versions: [&str; 2]) { // The shrink reaches the publisher before any group it narrows away exists. kio::clock::timeout(TIMEOUT * 10, async { let chain = Chain::new(versions, [forever(), forever(), forever()]).await; - let mut shrunk = chain.subscribe(reading(None, [Range::groups(0..)])).await; - shrunk.update(reading(None, [Range::groups(8..)])).unwrap(); + let mut shrunk = chain.subscribe(reading(None, [support::keys(0..)])).await; + shrunk.update(reading(None, [support::keys(8..)])).unwrap(); kio::clock::sleep(Duration::from_secs(1)).await; chain.populate(); assert_eq!(received(&mut shrunk).await, [8, 9].into(), "{name}: the shrunk reader"); - let mut later = chain.subscribe(reading(None, [Range::groups(0..)])).await; + let mut later = chain.subscribe(reading(None, [support::keys(0..)])).await; assert_eq!( received(&mut later).await, (0..GROUPS).collect(), @@ -405,17 +406,17 @@ async fn an_origin_eviction_drop_is_forwarded_final() { }; // One stream of credit: group 0's, held from the relay, so the rest queue. let chain = Chain::with_credit([LITE07, LITE07], [pool, forever(), forever()], Some(1)).await; - let mut reader = chain.subscribe(reading(None, [Range::groups(0..)])).await; + let mut reader = chain.subscribe(reading(None, [support::keys(0..)])).await; chain.upstream.server_transport.hold_unis(); - let write = |sequence: u64| { - let mut group = chain.source.create_group(moq_net::group::Info { sequence }).unwrap(); + let write = |key: u64| { + let mut group = chain.source.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), vec![0u8; PAYLOAD]) + .write_frame(Timestamp::from_millis(key * 1000).unwrap(), vec![0u8; PAYLOAD]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); }; - for sequence in 0..4 { - write(sequence); + for key in 0..4 { + write(key); } let last = match expiry { // Nobody reads the queued groups, so the origin reclaims all but the newest. @@ -425,8 +426,8 @@ async fn an_origin_eviction_drop_is_forwarded_final() { } // The newer groups push the queued ones out. false => { - for sequence in 4..8 { - write(sequence); + for key in 4..8 { + write(key); } 7 } @@ -446,11 +447,11 @@ async fn an_origin_eviction_drop_is_forwarded_final() { // A later reader may subscribe afresh once the finished track's copies are gone, // so only the update is held to asking nothing again. - let mut later = chain.subscribe(reading(None, [Range::groups(0..)])).await; + let mut later = chain.subscribe(reading(None, [support::keys(0..)])).await; assert_eq!(drain(&mut later).await, got, "expiry={expiry}: the later reader"); let asked = chain.asked(); reader - .update(reading(Duration::from_secs(3600), [Range::groups(0..)])) + .update(reading(Duration::from_secs(3600), [support::keys(0..)])) .unwrap(); assert_eq!( drain(&mut reader).await, @@ -484,8 +485,8 @@ async fn a_drop_under_a_superseded_budget_settles_no_reader() { const LATENCY: Duration = Duration::from_secs(10); kio::clock::timeout(TIMEOUT * 10, async { let chain = Chain::new([LITE07, LITE07], [expiring(EXPIRY), forever(), forever()]).await; - for sequence in (0..GROUPS).filter(|sequence| *sequence != 7) { - write(&chain.source, sequence, sequence * 1000); + for key in (0..GROUPS).filter(|key| *key != 7) { + write(&chain.source, key, key * 1000); } let mut reader = chain.subscribe(reading(B1, [Range::oldest()])).await; assert_eq!(received(&mut reader).await, [5, 6, 8, 9].into()); @@ -508,7 +509,7 @@ async fn a_drop_under_a_superseded_budget_settles_no_reader() { .await .expect("the reader never settled group 7") .unwrap() - .map(|group| group.sequence); + .map(|group| group.start); assert_eq!(ended, None, "group 7 was reclaimed"); // The B1 drop reached the relay at one latency, the B2 update the publisher at one and // a half, and its drop the relay at two and a half. @@ -528,17 +529,17 @@ async fn a_drop_under_a_superseded_budget_settles_no_reader() { #[kio::test] async fn a_run_dropped_before_the_first_group_settles_a_later_range() { let chain = Chain::new([LITE07, LITE07], [forever(), forever(), forever()]).await; - let mut reader = chain.subscribe(reading(None, [Range::groups(0..)])).await; - for sequence in 0..2 { - let group = chain.source.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut reader = chain.subscribe(reading(None, [support::keys(0..)])).await; + for key in 0..2 { + let group = chain.source.create_group(moq_net::group::Info { start: key }).unwrap(); group.abort(moq_net::Error::App(1)).unwrap(); } write(&chain.source, 2, 0); assert_eq!(received(&mut reader).await, [2].into()); - let mut viewer = chain.subscribe(reading(None, [Range::groups(0..3)])).await; + let mut viewer = chain.subscribe(reading(None, [support::keys(0..3)])).await; assert_eq!(drain(&mut viewer).await, [2].into(), "on the viewer"); - let mut relayed = chain.subscribe_relay(reading(None, [Range::groups(0..3)])).await; + let mut relayed = chain.subscribe_relay(reading(None, [support::keys(0..3)])).await; assert_eq!(drain(&mut relayed).await, [2].into(), "on the relay"); } @@ -557,8 +558,8 @@ async fn relay_pool_pressure_folds_final_runs_and_ends_stale_reports() { kio::clock::timeout(TIMEOUT * 10, async { let pool = cache::Pool::new(cache::Config::default().with_expiry(None)); let chain = Chain::new([LITE07, LITE07], [forever(), pool.clone(), forever()]).await; - let mut local = chain.subscribe_relay(reading(budget, [Range::groups(0..)])).await; - let mut remote = chain.subscribe(reading(budget, [Range::groups(0..)])).await; + let mut local = chain.subscribe_relay(reading(budget, [support::keys(0..)])).await; + let mut remote = chain.subscribe(reading(budget, [support::keys(0..)])).await; let odd = write_odd(&chain, RUNS); for (reader, name) in [(&mut local, "local"), (&mut remote, "remote")] { assert_eq!(received(reader).await, odd, "stale={stale}: the {name} reader"); @@ -575,8 +576,8 @@ async fn relay_pool_pressure_folds_final_runs_and_ends_stale_reports() { // The pool shrinks, and a burst of groups accrues the relay's debt: its own groups // and drop set go first, then its readers' records. pool.resize(4 * 1024); - for sequence in 2 * RUNS..2 * RUNS + BURST { - write(&chain.source, sequence, sequence); + for key in 2 * RUNS..2 * RUNS + BURST { + write(&chain.source, key, key); } let mut read = [0, 0]; let mut ended = [None, None]; @@ -616,7 +617,7 @@ async fn relay_pool_pressure_folds_final_runs_and_ends_stale_reports() { } } -/// Write the odd sequences below `2 * runs`, presented a millisecond apart so only the wall +/// Write the odd keys below `2 * runs`, presented a millisecond apart so only the wall /// clock makes one stale, and return them. fn write_odd(chain: &Chain, runs: u64) -> BTreeSet { for run in 0..runs { @@ -625,7 +626,7 @@ fn write_odd(chain: &Chain, runs: u64) -> BTreeSet { (0..runs).map(|run| 2 * run + 1).collect() } -/// Settle the even sequences below `2 * runs` as disjoint runs. With `stale`, each group lands +/// Settle the even keys below `2 * runs` as disjoint runs. With `stale`, each group lands /// that long after its successor, so a reader with a shorter budget drops it as stale; without, /// each is aborted, a drop no budget changes. async fn settle_even(chain: &Chain, runs: u64, stale: Option) { @@ -640,7 +641,7 @@ async fn settle_even(chain: &Chain, runs: u64, stale: Option) { for run in 0..runs { let group = chain .source - .create_group(moq_net::group::Info { sequence: 2 * run }) + .create_group(moq_net::group::Info { start: 2 * run }) .unwrap(); group.abort(moq_net::Error::App(1)).unwrap(); } diff --git a/rs/moq-net/tests/integration/egress_stats.rs b/rs/moq-net/tests/integration/egress_stats.rs index 6b5b335a246..e987bfd997a 100644 --- a/rs/moq-net/tests/integration/egress_stats.rs +++ b/rs/moq-net/tests/integration/egress_stats.rs @@ -77,7 +77,7 @@ async fn an_open_group_s_last_frame_counts_once_delivered() { subscribed(&track).await; // One frame, in a group left open. - let mut open = track.append_group().unwrap(); + let mut open = track.append_group(0).unwrap(); open.write_frame(Timestamp::ZERO, vec![b'x'; FRAME]).unwrap(); let mut group = kio::clock::timeout(TIMEOUT, live.recv_group()) .await diff --git a/rs/moq-net/tests/integration/empty_group.rs b/rs/moq-net/tests/integration/empty_group.rs index bf97a3e15b0..d3828e62a45 100644 --- a/rs/moq-net/tests/integration/empty_group.rs +++ b/rs/moq-net/tests/integration/empty_group.rs @@ -3,10 +3,12 @@ //! A group finished with no frames reaches the subscriber as an empty group. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Timestamp, Version, origin}; +use moq_net::{Version, origin}; use support::harness::{peer, subscribed}; /// A millisecond track: the frames written here carry timestamps. @@ -49,28 +51,28 @@ async fn an_empty_group_arrives_empty() { .expect("subscribe"); subscribed(&track).await; - track.append_group().unwrap().finish().unwrap(); + track.append_next().unwrap().close().unwrap(); let mut empty = kio::clock::timeout(TIMEOUT, sub.recv_group()) .await .unwrap_or_else(|_| panic!("{version}: the empty group was dropped")) .unwrap() .expect("a group"); - assert_eq!(empty.sequence, 0, "{version}"); + assert_eq!(empty.start, 0, "{version}"); let frame = kio::clock::timeout(TIMEOUT, empty.read_frame()) .await .expect("frame timeout") .unwrap(); assert!(frame.is_none(), "{version}: the empty group carried a frame"); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, &b"ok"[..]).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), &b"ok"[..]).unwrap(); + group.close().unwrap(); let mut next = kio::clock::timeout(TIMEOUT, sub.recv_group()) .await .expect("group timeout") .unwrap() .expect("a group"); - assert_eq!(next.sequence, 1, "{version}"); + assert_eq!(next.start, 1, "{version}"); let frame = next.read_frame().await.unwrap().expect("a frame"); assert_eq!(frame.payload.as_ref(), b"ok", "{version}"); } diff --git a/rs/moq-net/tests/integration/errored_end.rs b/rs/moq-net/tests/integration/errored_end.rs index c1315a61b17..5b55c7cba8c 100644 --- a/rs/moq-net/tests/integration/errored_end.rs +++ b/rs/moq-net/tests/integration/errored_end.rs @@ -6,11 +6,13 @@ use crate::support; use moq_net::track::Delay; +use support::AppendNext; +use support::AtStart; use std::time::Duration; use moq_net::track::{Position, Subscription}; -use moq_net::{Error, StreamError, Timestamp, Version}; +use moq_net::{Error, StreamError, Version}; use support::harness::{MockConnectOptions, connect_mock}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -24,11 +26,11 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer.with_proxy(true) } -/// Read every group from the first, frame by frame, returning each group's sequence and +/// Read every group from the first, frame by frame, returning each group's key and /// payloads, and how the read ended. async fn read(broadcast: &moq_net::broadcast::Consumer) -> (Vec<(u64, Vec>)>, Option) { let subscription = Subscription::default() - .with_range(0..) + .with_ranges([support::keys(0..)]) .with_delay(Delay::default().with_max(Duration::from_secs(30))); let mut sub = kio::clock::timeout(TIMEOUT, broadcast.track("video").unwrap().subscribe(subscription)) .await @@ -45,11 +47,11 @@ async fn read(broadcast: &moq_net::broadcast::Consumer) -> (Vec<(u64, Vec return (got, None), Err(err) => return (got, Some(err)), @@ -60,7 +62,7 @@ async fn read(broadcast: &moq_net::broadcast::Consumer) -> (Vec<(u64, Vec Vec<(u64, Vec>)> { (0..GROUPS) - .map(|sequence| (sequence, FRAMES.iter().map(|frame| frame.to_vec()).collect())) + .map(|key| (key, FRAMES.iter().map(|frame| frame.to_vec()).collect())) .collect() } @@ -88,11 +90,11 @@ async fn an_errored_end_keeps_its_history_across_a_hop() { let mut track = broadcast.broadcast().create_track("video", info).unwrap(); broadcast.announce().unwrap(); for _ in 0..GROUPS { - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); for frame in FRAMES { - group.write_frame(Timestamp::ZERO, frame).unwrap(); + group.write_frame(group.at_start(), frame).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } track.finish_at(Position::group(GROUPS), Some(Error::App(7))).unwrap(); diff --git a/rs/moq-net/tests/integration/failover_chain.rs b/rs/moq-net/tests/integration/failover_chain.rs index bbb92d706fe..3187a5be929 100644 --- a/rs/moq-net/tests/integration/failover_chain.rs +++ b/rs/moq-net/tests/integration/failover_chain.rs @@ -14,9 +14,11 @@ use crate::support::harness::MockPair; use std::time::Duration; +use crate::support::AppendNext; +use crate::support::AtStart; use futures::StreamExt; use futures::channel::mpsc; -use moq_net::{Error, Timestamp, Version, group, origin, track}; +use moq_net::{Error, Version, group, origin, track}; /// Within the subscriber's one-minute budget, so a group that never continues is a hang /// rather than a group given up as too old. @@ -69,20 +71,20 @@ impl Chain { track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(60))); let mut sub = remote.track("video").unwrap().subscribe(prefs).await.unwrap(); - let mut group0 = track.append_group().unwrap(); - group0.write_frame(Timestamp::ZERO, payload(0, 0)).unwrap(); + let mut group0 = track.append_next().unwrap(); + group0.write_frame(group0.at_start(), payload(0, 0)).unwrap(); let mut read0 = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(read0.sequence, 0); + assert_eq!(read0.start, 0); assert_eq!( read0.read_frame().await.unwrap().unwrap().payload.to_vec(), payload(0, 0) ); - let mut group1 = track.append_group().unwrap(); - group1.write_frame(Timestamp::ZERO, payload(1, 0)).unwrap(); - group1.finish().unwrap(); + let mut group1 = track.append_next().unwrap(); + group1.write_frame(group1.at_start(), payload(1, 0)).unwrap(); + group1.close().unwrap(); let mut read1 = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(read1.sequence, 1); + assert_eq!(read1.start, 1); assert_eq!( read1.read_frame().await.unwrap().unwrap().payload.to_vec(), payload(1, 0) @@ -128,8 +130,8 @@ impl Chain { /// The publisher writes the rest of group 0, which every relay can still reach. fn finish_group0(&mut self) { let mut group0 = self.group0.take().expect("finished once"); - group0.write_frame(Timestamp::ZERO, payload(0, 1)).unwrap(); - group0.finish().unwrap(); + group0.write_frame(group0.at_start(), payload(0, 1)).unwrap(); + group0.close().unwrap(); } } diff --git a/rs/moq-net/tests/integration/fastest_reader.rs b/rs/moq-net/tests/integration/fastest_reader.rs index d6f10ec536f..11bd64e880e 100644 --- a/rs/moq-net/tests/integration/fastest_reader.rs +++ b/rs/moq-net/tests/integration/fastest_reader.rs @@ -10,6 +10,8 @@ use std::time::Duration; use bytes::Bytes; use moq_net::{Timestamp, Version, origin, track, transport::ReceiveWindow}; +use crate::support::AppendNext; +use crate::support::AtStart; use crate::support::harness::{MockPair, peer, spawn}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -100,8 +102,8 @@ async fn idle_reader(version: &str, window: ReceiveWindow, expected: std::ops::R let mut relay = relay(version).await; relay.pair.server_transport.set_receive_window(Some(window)); - let mut group = relay.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut group = relay.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); let mut reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") @@ -109,9 +111,9 @@ async fn idle_reader(version: &str, window: ReceiveWindow, expected: std::ops::R .expect("group 0"); // Within the publisher's own 32 MB tail, so only the relay's pacing holds it back. for _ in 1..30 { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); // Simulated time moves only once every task is parked: the relay's ingest stopped. kio::clock::sleep(Duration::from_secs(1)).await; @@ -143,15 +145,15 @@ async fn a_reset_reaches_a_paused_relay() { for version in VERSIONS { let mut relay = relay(version).await; - let mut group = relay.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut group = relay.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); let mut reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() .expect("group 0"); for _ in 1..30 { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); } kio::clock::sleep(Duration::from_secs(1)).await; assert!(reader.frame_count() < 30, "{version}: the relay never paused"); @@ -176,23 +178,23 @@ async fn a_large_frame_sheds_nothing_an_idle_reader_wants() { let mut relay = relay(version).await; relay.pair.server_transport.set_receive_window(Some(uncapped())); - let mut group = relay.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut group = relay.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); let mut reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() .expect("group 0"); for _ in 1..15 { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); } // Let the relay take what fits, so the publisher's egress has sent every frame the // large one sheds from the publisher's own tail. kio::clock::sleep(Duration::from_secs(1)).await; group - .write_frame(Timestamp::ZERO, Bytes::from(vec![7u8; 20 * MB])) + .write_frame(group.at_start(), Bytes::from(vec![7u8; 20 * MB])) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); kio::clock::sleep(Duration::from_secs(1)).await; assert_eq!( @@ -223,17 +225,17 @@ async fn a_session_past_its_credit_cap_reads_on() { let window = moq_net::transport::ReceiveWindow::new(std::num::NonZeroU64::new(MB as u64), MB as u64); relay.pair.server_transport.set_receive_window(Some(window)); - let mut group = relay.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut group = relay.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); let reader = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") .unwrap() .expect("group 0"); for _ in 1..30 { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); kio::clock::sleep(Duration::from_secs(1)).await; let held = reader.frame_count(); @@ -247,8 +249,8 @@ async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { for version in VERSIONS { let mut relay = relay(version).await; - let mut group = relay.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + let mut group = relay.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); let mut fast = kio::clock::timeout(TIMEOUT, relay.subscriber.recv_group()) .await .expect("group timeout") @@ -259,7 +261,7 @@ async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { // Well past the 32 MB tail: the fast reader keeps up the whole way. for index in 0..48 { if index > 0 { - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); } let frame = kio::clock::timeout(TIMEOUT, fast.read_frame()) .await @@ -268,7 +270,7 @@ async fn a_slow_reader_falls_off_without_slowing_the_fast_one() { .expect("a frame"); assert_eq!(frame.payload.len(), MB, "{version}: frame {index}"); } - group.finish().unwrap(); + group.close().unwrap(); let fell = kio::clock::timeout(TIMEOUT, slow.read_frame()) .await @@ -328,7 +330,7 @@ async fn two_relays_bound_their_cache_and_resume_without_a_hole() { .await .unwrap(); crate::support::harness::subscribed(&track).await; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); let written = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); let progress = written.clone(); let writer = kio::clock::sim::spawn(async move { @@ -336,13 +338,13 @@ async fn two_relays_bound_their_cache_and_resume_without_a_hole() { let payload = Bytes::from(vec![(index % 251) as u8; FRAME]); let info = frame::Info { size: FRAME as u64, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }; group.ready(&info).await; - group.write_frame(Timestamp::ZERO, payload).unwrap(); + group.write_frame(group.at_start(), payload).unwrap(); progress.store(index + 1, std::sync::atomic::Ordering::Relaxed); } - group.finish().unwrap(); + group.close().unwrap(); }); let mut reader = subscriber.recv_group().await.unwrap().unwrap(); // Simulated time advances only once all drivers and the paced writer park. diff --git a/rs/moq-net/tests/integration/fetch_cancel.rs b/rs/moq-net/tests/integration/fetch_cancel.rs index 3e0534283fe..22e414550aa 100644 --- a/rs/moq-net/tests/integration/fetch_cancel.rs +++ b/rs/moq-net/tests/integration/fetch_cancel.rs @@ -8,11 +8,12 @@ //! partway through the group. A group cut short is aborted, never cached as whole. use crate::support; +use support::AtStart; use std::time::Duration; use bytes::Bytes; -use moq_net::{Timestamp, Version, origin}; +use moq_net::{Version, origin}; use support::harness::peer; /// A millisecond track: the frames written here carry timestamps. @@ -79,7 +80,9 @@ async fn abandoned_fetch_reaches_the_publisher(version: &str, relays: u64, stage } Stage::MidResponse => { let mut group = request.accept(0, timed()).unwrap(); - group.write_frame(Timestamp::ZERO, Bytes::from_static(b"head")).unwrap(); + group + .write_frame(group.at_start(), Bytes::from_static(b"head")) + .unwrap(); let mut fetched = waiting.await.unwrap(); let frame = fetched.read_frame().await.unwrap().unwrap(); @@ -98,8 +101,10 @@ async fn abandoned_fetch_reaches_the_publisher(version: &str, relays: u64, stage .unwrap(); // No hop cached the head as the whole group: a fresh fetch reads it all. - group.write_frame(Timestamp::ZERO, Bytes::from_static(b"tail")).unwrap(); - group.finish().unwrap(); + group + .write_frame(group.at_start(), Bytes::from_static(b"tail")) + .unwrap(); + group.close().unwrap(); let mut fetched = kio::clock::timeout(Duration::from_secs(5), track.fetch_group(0, None)) .await .unwrap_or_else(|_| panic!("{ctx}: the refetch stalled")) @@ -112,8 +117,12 @@ async fn abandoned_fetch_reaches_the_publisher(version: &str, relays: u64, stage } Stage::Drained => { let mut group = request.accept(0, timed()).unwrap(); - group.write_frame(Timestamp::ZERO, Bytes::from_static(b"head")).unwrap(); - group.write_frame(Timestamp::ZERO, Bytes::from_static(b"tail")).unwrap(); + group + .write_frame(group.at_start(), Bytes::from_static(b"head")) + .unwrap(); + group + .write_frame(group.at_start(), Bytes::from_static(b"tail")) + .unwrap(); let mut fetched = waiting.await.unwrap(); for expected in [&b"head"[..], b"tail"] { @@ -125,7 +134,7 @@ async fn abandoned_fetch_reaches_the_publisher(version: &str, relays: u64, stage // group is whole, so it must be cached as such rather than cancelled. const LATENCY: Duration = Duration::from_millis(100); _pairs.last().unwrap().client_transport.set_latency(LATENCY); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); kio::clock::sleep(LATENCY).await; drop(fetched); kio::clock::sleep(Duration::from_secs(1)).await; @@ -193,9 +202,9 @@ async fn a_refetch_within_the_linger_stays_on_the_relay(version: &str) { }; let mut group = request.accept(0, timed()).unwrap(); group - .write_frame(Timestamp::ZERO, Bytes::from_static(b"segment")) + .write_frame(group.at_start(), Bytes::from_static(b"segment")) .unwrap(); - let group = group.finish().unwrap(); + let group = group.close().unwrap(); let mut fetched = waiting.await.unwrap(); assert_eq!(&fetched.read_frame().await.unwrap().unwrap().payload[..], b"segment"); drop(fetched); diff --git a/rs/moq-net/tests/integration/fetch_fanout.rs b/rs/moq-net/tests/integration/fetch_fanout.rs index cfbf1833764..6b4f7672fd4 100644 --- a/rs/moq-net/tests/integration/fetch_fanout.rs +++ b/rs/moq-net/tests/integration/fetch_fanout.rs @@ -6,6 +6,7 @@ use crate::support; use moq_net::{Version, origin, track}; use std::time::Duration; +use support::AppendNext; use support::harness::{MockConnectOptions, connect_mock}; fn origin(hop: u64) -> origin::Producer { @@ -50,22 +51,17 @@ async fn sequential_viewers_share_the_relays_fetched_groups() { viewers.push((viewer, pair, remote)); } for round in 0..3 { - let mut group = track.append_group().unwrap(); - let sequence = group.sequence; + let mut group = track.append_next().unwrap(); + let key = group.start; group .write_frame( moq_net::Timestamp::from_millis(round * 1000).unwrap(), b"payload".as_ref(), ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); for (i, (_, _, remote)) in viewers.iter().enumerate() { - let mut group = remote - .track("video") - .unwrap() - .fetch_group(sequence, None) - .await - .unwrap(); + let mut group = remote.track("video").unwrap().fetch_group(key, None).await.unwrap(); assert_eq!(group.read_frame().await.unwrap().unwrap().payload.as_ref(), b"payload"); assert!(group.read_frame().await.unwrap().is_none()); // FETCH's lite stream type is 3, and its varint occupies one byte. @@ -75,7 +71,7 @@ async fn sequential_viewers_share_the_relays_fetched_groups() { .into_iter() .filter(|kind| *kind == 3) .count(); - assert_eq!(fetches, round as usize + 1, "{version}: viewer {i}, group {sequence}"); + assert_eq!(fetches, round as usize + 1, "{version}: viewer {i}, group {key}"); } kio::clock::sleep(Duration::from_secs(1)).await; } diff --git a/rs/moq-net/tests/integration/fetch_from_feed.rs b/rs/moq-net/tests/integration/fetch_from_feed.rs index a28fee7a2e8..9e96bebec28 100644 --- a/rs/moq-net/tests/integration/fetch_from_feed.rs +++ b/rs/moq-net/tests/integration/fetch_from_feed.rs @@ -5,6 +5,7 @@ //! never a version error that a caller would retry forever. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -30,18 +31,27 @@ struct Relay { _origins: (moq_net::origin::Producer, moq_net::origin::Producer), } +/// Append the next group of an untimed track, holding one frame. +fn write(track: &moq_net::track::Producer, payload: &'static [u8]) { + let mut group = track.append_next().unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload) + .unwrap(); + group.close().unwrap(); +} + /// A publisher on `version` with group 0 of an untimed track written, and the relay's view of it. async fn relay(version: &str) -> Relay { let publisher = produce_origin(1); let broadcast = publisher .route("bcast", moq_net::origin::RouteOptions::hidden()) .unwrap(); - let mut track = broadcast + let track = broadcast .broadcast() .create_track("video", moq_net::track::Info::default().with_timescale(None)) .unwrap(); broadcast.announce().unwrap(); - track.write_frame(None, &b"0"[..]).unwrap(); + write(&track, b"0"); let client = produce_origin(2); let mut options = MockConnectOptions::new(version.parse::().unwrap()); @@ -64,11 +74,11 @@ async fn relay(version: &str) -> Relay { } } -/// The error a fetch of `sequence` fails with. -async fn fetch_err(relay: &Relay, sequence: u64) -> moq_net::Error { - let fetching = relay.remote.track("video").unwrap().fetch_group(sequence, None); +/// The error a fetch of `key` fails with. +async fn fetch_err(relay: &Relay, key: u64) -> moq_net::Error { + let fetching = relay.remote.track("video").unwrap().fetch_group(key, None); match kio::clock::timeout(TIMEOUT, fetching).await.expect("fetch timed out") { - Ok(_) => panic!("group {sequence} was served"), + Ok(_) => panic!("group {key} was served"), Err(err) => err, } } @@ -77,7 +87,7 @@ async fn fetch_err(relay: &Relay, sequence: u64) -> moq_net::Error { #[kio::test] async fn a_fetch_is_served_from_the_live_feed() { for version in VERSIONS { - let mut relay = relay(version).await; + let relay = relay(version).await; let mut subscriber = kio::clock::timeout(TIMEOUT, relay.remote.track("video").unwrap().subscribe(None)) .await .expect("subscribe timeout") @@ -87,7 +97,7 @@ async fn a_fetch_is_served_from_the_live_feed() { .expect("group timeout") .unwrap() .expect("a group") - .sequence; + .start; // Reaches the relay before the group exists, so it can't be a plain cache hit. let next = first + 1; @@ -98,7 +108,7 @@ async fn a_fetch_is_served_from_the_live_feed() { .is_err(), "{version}: answered before the group exists" ); - relay.track.write_frame(None, &b"1"[..]).unwrap(); + write(&relay.track, b"1"); let mut group = kio::clock::timeout(TIMEOUT, fetching) .await .expect("fetch timed out") @@ -116,7 +126,7 @@ async fn a_fetch_is_served_from_the_live_feed() { #[kio::test] async fn a_fetch_waits_for_a_feed_opened_with_it() { for version in VERSIONS { - let mut relay = relay(version).await; + let relay = relay(version).await; let err = fetch_err(&relay, 0).await; assert!(matches!(err, moq_net::Error::NotFound), "{version}: no feed: {err}"); @@ -128,7 +138,7 @@ async fn a_fetch_waits_for_a_feed_opened_with_it() { .expect("subscribe timeout") .expect("subscribe"); - relay.track.write_frame(None, &b"1"[..]).unwrap(); + write(&relay.track, b"1"); let mut group = kio::clock::timeout(TIMEOUT, fetching) .await .expect("fetch timed out") @@ -145,15 +155,17 @@ async fn a_fetch_waits_for_a_feed_opened_with_it() { #[kio::test] async fn an_unfed_fetch_is_a_miss() { for version in VERSIONS { - let mut relay = relay(version).await; + let relay = relay(version).await; let err = fetch_err(&relay, 0).await; assert!(matches!(err, moq_net::Error::NotFound), "{version}: no feed: {err}"); - relay.track.write_frame(None, &b"1"[..]).unwrap(); + write(&relay.track, b"1"); // The publisher skips group 2, so the feed goes past it. - let mut skipped = relay.track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); - skipped.write_frame(None, &b"3"[..]).unwrap(); - skipped.finish().unwrap(); + let mut skipped = relay.track.create_group(moq_net::group::Info { start: 3 }).unwrap(); + skipped + .write_frame(moq_net::Timestamp::unscaled(skipped.start), &b"3"[..]) + .unwrap(); + skipped.close().unwrap(); let mut subscriber = kio::clock::timeout(TIMEOUT, relay.remote.track("video").unwrap().subscribe(None)) .await .expect("subscribe timeout") @@ -163,7 +175,7 @@ async fn an_unfed_fetch_is_a_miss() { .expect("group timeout") .unwrap() .expect("a group") - .sequence; + .start; assert_eq!(first, 3, "{version}"); let err = fetch_err(&relay, 2).await; assert!(matches!(err, moq_net::Error::NotFound), "{version}: passed: {err}"); diff --git a/rs/moq-net/tests/integration/fetch_only.rs b/rs/moq-net/tests/integration/fetch_only.rs index a78a0c46d7f..c347bb8b33f 100644 --- a/rs/moq-net/tests/integration/fetch_only.rs +++ b/rs/moq-net/tests/integration/fetch_only.rs @@ -7,6 +7,8 @@ //! the fetches that report where the track ends. use crate::support; +use support::AppendNext; +use support::AtStart; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -46,14 +48,14 @@ fn publish_finished(origin: &moq_net::origin::Producer) -> (moq_net::origin::Rou let broadcast = origin.route("bcast", moq_net::origin::RouteOptions::hidden()).unwrap(); let mut track = broadcast.broadcast().create_track("video", timed()).unwrap(); broadcast.announce().unwrap(); - for sequence in 0..3u64 { - let mut group = track.append_group().unwrap(); + for key in 0..3u64 { + let mut group = track.append_next().unwrap(); for frame in 0..2 { group - .write_frame(Timestamp::ZERO, format!("{sequence}-{frame}").into_bytes()) + .write_frame(group.at_start(), format!("{key}-{frame}").into_bytes()) .unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let _finished = track.clone().finish().unwrap(); @@ -69,9 +71,9 @@ fn publish_finished(origin: &moq_net::origin::Producer) -> (moq_net::origin::Rou (broadcast, subscribed) } -async fn fetch(broadcast: &moq_net::broadcast::Consumer, sequence: u64) -> Result>, moq_net::Error> { +async fn fetch(broadcast: &moq_net::broadcast::Consumer, key: u64) -> Result>, moq_net::Error> { let mut group = kio::clock::timeout(TIMEOUT, async { - broadcast.track("video").unwrap().fetch_group(sequence, None).await + broadcast.track("video").unwrap().fetch_group(key, None).await }) .await .expect("fetch timed out")?; @@ -85,8 +87,8 @@ async fn fetch(broadcast: &moq_net::broadcast::Consumer, sequence: u64) -> Resul Ok(frames) } -fn frames(sequence: u64) -> Vec> { - (0..2).map(|frame| format!("{sequence}-{frame}").into_bytes()).collect() +fn frames(key: u64) -> Vec> { + (0..2).map(|frame| format!("{key}-{frame}").into_bytes()).collect() } const VERSIONS: &[&str] = &["moq-transport-18", "moq-transport-22"]; @@ -139,12 +141,8 @@ async fn relayed(version: &str) { let downstream = connect_mock(options).await; let remote = resolve(&client).await; - for sequence in [2, 0] { - assert_eq!( - fetch(&remote, sequence).await.expect("fetch"), - frames(sequence), - "{version}" - ); + for key in [2, 0] { + assert_eq!(fetch(&remote, key).await.expect("fetch"), frames(key), "{version}"); } assert!( !subscribed.load(Ordering::SeqCst), @@ -183,15 +181,15 @@ async fn a_subscribe_after_track_status_keeps_timestamps() { crate::support::wanted(track.demand()).await.unwrap(); // Far from any arrival time, so a frame stamped on arrival cannot pass. let stamp = Timestamp::from_millis(123_456).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(stamp, b"frame".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); let frame = group.read_frame().await.unwrap().unwrap(); assert_eq!( - frame.timestamp.map(Timestamp::as_millis), - Some(stamp.as_millis()), + frame.timestamp.as_millis().unwrap(), + stamp.as_millis().unwrap(), "{version}" ); }) @@ -217,7 +215,7 @@ async fn a_relay_fetches_without_subscribing_upstream() { async fn subscribe_history(track: &moq_net::track::Consumer) -> moq_net::track::Subscriber { let subscription = moq_net::track::Subscription::default() .with_order(moq_net::track::Order::Ascending) - .with_range(1..2); + .with_ranges([support::keys(1..2)]); kio::clock::timeout(TIMEOUT, track.subscribe(subscription)) .await .expect("subscribe timed out") @@ -327,13 +325,13 @@ async fn a_relay_serves_a_bounded_read_of_history_without_subscribing_upstream() assert!(failures.is_empty(), "failed: {failures:?}"); } -/// Write group `sequence` of [`frames`] to a live track. -fn write_group(track: &moq_net::track::Producer, sequence: u64) { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); - for frame in frames(sequence) { - group.write_frame(Timestamp::ZERO, frame).unwrap(); +/// Write group `key` of [`frames`] to a live track. +fn write_group(track: &moq_net::track::Producer, key: u64) { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); + for frame in frames(key) { + group.write_frame(group.at_start(), frame).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } /// Publish a live track of groups `0..groups`, which the publisher keeps writing. @@ -345,8 +343,8 @@ fn publish_live( let broadcast = origin.route("bcast", moq_net::origin::RouteOptions::hidden()).unwrap(); let track = broadcast.broadcast().create_track("video", info).unwrap(); broadcast.announce().unwrap(); - for sequence in 0..groups { - write_group(&track, sequence); + for key in 0..groups { + write_group(&track, key); } (broadcast, track) } @@ -494,8 +492,8 @@ async fn a_resumed_bounded_subscription_reads_below_the_answered_edge() { // The fetch learns Largest Object (1, 1), and leaves the copy unfed. let remote = resolve(&client).await; assert_eq!(fetch(&remote, 0).await.expect("fetch"), frames(0)); - for sequence in 2..4 { - write_group(&track, sequence); + for key in 2..4 { + write_group(&track, key); } // Past the reported edge, so it subscribes; the answer names group 3, which the @@ -504,7 +502,7 @@ async fn a_resumed_bounded_subscription_reads_below_the_answered_edge() { let subscription = moq_net::track::Subscription::default() .with_order(moq_net::track::Order::Ascending) .with_delay(moq_net::track::Delay::default().with_max(None)) - .with_range(1..3); + .with_ranges([support::keys(1..3)]); let mut sub = kio::clock::timeout(TIMEOUT, copy.subscribe(subscription)) .await .expect("subscribe timed out") @@ -561,14 +559,14 @@ async fn a_resumed_bounded_subscription_keeps_its_cached_start() { .expect("unsubscribe timed out") .unwrap(); } - for sequence in 1..4 { - write_group(&track, sequence); + for key in 1..4 { + write_group(&track, key); } // Past the edge the live read learned, so it subscribes; the answer names group 3. let subscription = moq_net::track::Subscription::default() .with_delay(moq_net::track::Delay::default().with_max(None)) - .with_range(0..1); + .with_ranges([support::keys(0..1)]); let mut sub = kio::clock::timeout(TIMEOUT, copy.subscribe(subscription)) .await .expect("subscribe timed out") @@ -597,7 +595,7 @@ async fn a_relay_accounts_for_every_group_in_multi_group_history() { let subscription = moq_net::track::Subscription::default() .with_order(moq_net::track::Order::Ascending) .with_delay(moq_net::track::Delay::default().with_max(None)) - .with_range(1..3); + .with_ranges([support::keys(1..3)]); let mut sub = copy.fresh.subscribe(subscription).await.unwrap(); let mut groups = Vec::new(); while let Some(group) = next_frames(&mut sub).await { diff --git a/rs/moq-net/tests/integration/fetch_pipeline.rs b/rs/moq-net/tests/integration/fetch_pipeline.rs index f3adc7b77bd..8f894629d3f 100644 --- a/rs/moq-net/tests/integration/fetch_pipeline.rs +++ b/rs/moq-net/tests/integration/fetch_pipeline.rs @@ -6,10 +6,12 @@ //! `model::resume`'s staged copies. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Error, Timestamp, Version}; +use moq_net::{Error, Version}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; const TIMEOUT: Duration = Duration::from_secs(5); @@ -100,8 +102,8 @@ async fn the_first_fetch_goes_out_with_the_info_request() { let (request, fill, fetching, _held) = fetch_ahead(version, relays).await; let _track = request.accept(timed()); let mut group = fill.accept(0, timed()).unwrap(); - group.write_frame(Timestamp::ZERO, b"early".as_ref()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.at_start(), b"early".as_ref()).unwrap(); + group.close().unwrap(); let mut fetched = kio::clock::timeout(TIMEOUT, fetching) .await .expect("the fetch resolves") @@ -144,9 +146,9 @@ async fn first_fetch(version: &str, relays: u64, rtt: Duration) -> Duration { .unwrap(); let track = broadcast.broadcast().create_track("video", timed()).unwrap(); broadcast.announce().unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"frame".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"frame".as_ref()).unwrap(); + group.close().unwrap(); let mut links = Vec::new(); let mut upstream = publisher.clone(); diff --git a/rs/moq-net/tests/integration/finished_broadcast_mock.rs b/rs/moq-net/tests/integration/finished_broadcast_mock.rs index f8b24c903d0..c77063c24b7 100644 --- a/rs/moq-net/tests/integration/finished_broadcast_mock.rs +++ b/rs/moq-net/tests/integration/finished_broadcast_mock.rs @@ -15,10 +15,12 @@ //! nothing when the broadcast ends. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Timestamp, Version}; +use moq_net::Version; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -61,7 +63,7 @@ async fn round(finish_broadcast: bool) -> (Vec>, Option) // Resolved before the ending starts: a SUBSCRIBE sent alongside its TRACK reaches the // publisher before the subscriber has the TRACK_INFO it accepts the track with. - let subscription = moq_net::track::Subscription::default().with_range(0..); + let subscription = moq_net::track::Subscription::default().with_ranges([support::keys(0..)]); let mut sub = kio::clock::timeout(TIMEOUT, remote.track("video").unwrap().subscribe(subscription)) .await .expect("subscribe timeout") @@ -90,9 +92,9 @@ async fn round(finish_broadcast: bool) -> (Vec>, Option) // One frame and a textbook clean end, innermost first, with no await in between: // nothing is served until the ending is done. - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), PAYLOAD).unwrap(); + group.close().unwrap(); let track = track.finish().unwrap(); drop(track); if finish_broadcast { diff --git a/rs/moq-net/tests/integration/finite_relay.rs b/rs/moq-net/tests/integration/finite_relay.rs index 724b060e8b0..832aa69a0e5 100644 --- a/rs/moq-net/tests/integration/finite_relay.rs +++ b/rs/moq-net/tests/integration/finite_relay.rs @@ -10,14 +10,16 @@ use crate::support; use moq_net::track::Delay; +use support::AppendNext; +use support::AtStart; use std::time::Duration; use futures::channel::mpsc::TryRecvError; use futures::{SinkExt, StreamExt}; +use moq_net::Version; use moq_net::track::Subscription; -use moq_net::{Timestamp, Version}; use support::harness::{MockConnectOptions, connect_mock}; /// A millisecond track with no publisher max age: the frames written here carry timestamps, @@ -93,7 +95,7 @@ async fn round(name: &str, arrival: Arrival) -> (Vec, Result<(), moq_net::E let reader = kio::clock::sim::spawn(async move { let subscription = Subscription::default() .with_delay(Delay::default().with_max(MAX_DELAY)) - .with_range(0..); + .with_ranges([support::keys(0..)]); let mut sub = remote .track("tail") .unwrap() @@ -113,18 +115,18 @@ async fn round(name: &str, arrival: Arrival) -> (Vec, Result<(), moq_net::E Err(err) => break Err(err), }; assert_eq!(&head.payload[..], b"head"); - heads.send(group.sequence).await.unwrap(); + heads.send(group.start).await.unwrap(); tails.push(kio::clock::sim::spawn(async move { let tail = group.read_frame().await?.expect("tail frame"); assert_eq!(&tail.payload[..], b"tail"); assert!(group.read_frame().await?.is_none(), "extra frame"); - Ok::<_, moq_net::Error>(group.sequence) + Ok::<_, moq_net::Error>(group.start) })); }; let mut seen = Vec::new(); for tail in tails { match tail.await.expect("tail reader panicked") { - Ok(sequence) => seen.push(sequence), + Ok(key) => seen.push(key), Err(err) => return (seen, Err(err)), } } @@ -144,13 +146,13 @@ async fn round(name: &str, arrival: Arrival) -> (Vec, Result<(), moq_net::E match arrival { Arrival::HeldTails => { let mut groups = Vec::new(); - for sequence in 0..GROUPS { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, &b"head"[..]).unwrap(); + for key in 0..GROUPS { + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), &b"head"[..]).unwrap(); groups.push(group); // IETF requests the live edge, so observe each header before advancing it. match kio::clock::timeout(TIMEOUT, opened.next()).await.expect("head timeout") { - Some(head) => assert_eq!(head, sequence, "{name}"), + Some(head) => assert_eq!(head, key, "{name}"), None => panic!("{name}: the reader stopped: {:?}", reader.await), } } @@ -159,17 +161,17 @@ async fn round(name: &str, arrival: Arrival) -> (Vec, Result<(), moq_net::E // the subscriber, so the relay's downstream subscription has seen the end. kio::clock::sleep(Duration::from_millis(100)).await; for mut group in groups { - group.write_frame(Timestamp::ZERO, &b"tail"[..]).unwrap(); - group.finish().unwrap(); + group.write_frame(group.at_start(), &b"tail"[..]).unwrap(); + group.close().unwrap(); } } Arrival::NewestFirst => { upstream.server_transport.hold_unis(); for _ in 0..GROUPS { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, &b"head"[..]).unwrap(); - group.write_frame(Timestamp::ZERO, &b"tail"[..]).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), &b"head"[..]).unwrap(); + group.write_frame(group.at_start(), &b"tail"[..]).unwrap(); + group.close().unwrap(); // IETF serves from the live edge, so let each group open its stream before // the next one supersedes it. kio::clock::sleep(Duration::from_millis(10)).await; diff --git a/rs/moq-net/tests/integration/flights.rs b/rs/moq-net/tests/integration/flights.rs index 594f9545f19..4c8c404f750 100644 --- a/rs/moq-net/tests/integration/flights.rs +++ b/rs/moq-net/tests/integration/flights.rs @@ -15,6 +15,7 @@ use std::time::Duration; use bytes::Bytes; use moq_net::{Version, cache, frame, group, origin, track}; +use crate::support::AppendNext; use crate::support::harness::{MockConnectOptions, connect_mock, spawn}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -33,10 +34,11 @@ fn number(frame: &frame::Frame) -> u32 { u32::from_le_bytes(frame.payload[..4].try_into().unwrap()) } +/// The header admission weighs: only its size. fn info(payload: &Bytes) -> frame::Info { frame::Info { size: payload.len() as u64, - timestamp: None, + timestamp: moq_net::Timestamp::unscaled(0), } } @@ -57,10 +59,10 @@ async fn write_paced( demand: &mut group::Demand, payload: Option, ) -> bool { - let sequence = group.sequence; + let key = group.start; let room = kio::wait(|waiter| { while let Poll::Ready(Ok(_)) = demand.poll_wanted_changed(waiter) {} - if fill.poll_unwanted(sequence, waiter).is_ready() && !demand.wanted() { + if fill.poll_unwanted(key, waiter).is_ready() && !demand.wanted() { return Poll::Ready(false); } match &payload { @@ -69,7 +71,11 @@ async fn write_paced( } }) .await; - room && payload.is_some_and(|payload| group.write_frame(None, payload).is_ok()) + room && payload.is_some_and(|payload| { + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload) + .is_ok() + }) } /// The origin's handler: serves each fill from the frame it starts at, paced by its riders, @@ -86,7 +92,7 @@ fn serve(requests: track::Requests, written: Arc, ended: Arc kio::clock: .unwrap(); let subscription = Subscription::default() .with_delay(Delay::default().with_max(FOREVER)) - .with_range(0..) + .with_ranges([support::keys(0..)]) .with_order(order); let mut sub = remote .track("ordered") @@ -74,7 +74,7 @@ fn read_arrivals(origin: moq_net::origin::Producer, order: Order) -> kio::clock: .expect("group timeout") .expect("recv_group") .expect("track open"); - arrivals.push(group.sequence); + arrivals.push(group.start); } arrivals }) @@ -88,10 +88,10 @@ fn read_arrivals(origin: moq_net::origin::Producer, order: Order) -> kio::clock: fn write_backlog(track: &moq_net::track::Producer) -> Vec { BACKLOG .iter() - .map(|&sequence| { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + .map(|&key| { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 10).unwrap(), &b"frame"[..]) + .write_frame(Timestamp::from_millis(key * 10).unwrap(), &b"frame"[..]) .unwrap(); group }) diff --git a/rs/moq-net/tests/integration/history_groups.rs b/rs/moq-net/tests/integration/history_groups.rs index 12e2f9a5e34..29208b93e9a 100644 --- a/rs/moq-net/tests/integration/history_groups.rs +++ b/rs/moq-net/tests/integration/history_groups.rs @@ -39,7 +39,7 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { /// Publish a finished group 5 (three frames) then an open group 6 (two frames), optionally /// through a relay and with their streams delivered newest first, and return each group's -/// sequence and the frames one subscriber read from it. +/// key and the frames one subscriber read from it. async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usize)> { let version: Version = version.parse().unwrap(); let publisher = produce_origin(1); @@ -85,7 +85,7 @@ async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usiz let reader = kio::clock::sim::spawn(async move { let subscription = Subscription::default() .with_delay(Delay::default().with_max(FOREVER)) - .with_range(0..); + .with_ranges([support::keys(0..)]); let mut sub = remote .track("history") .unwrap() @@ -109,7 +109,7 @@ async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usiz None => break, } } - got.push((group.sequence, frames)); + got.push((group.start, frames)); } got.sort(); got @@ -131,14 +131,16 @@ async fn round(version: &str, relay: bool, newest_first: bool) -> Vec<(u64, usiz first_hop.hold_unis(); } - let mut old = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); + let mut old = track.create_group(moq_net::group::Info { start: 5 }).unwrap(); for _ in 0..3 { - old.write_frame(Timestamp::now(), &b"old"[..]).unwrap(); + old.write_frame(Timestamp::from_millis(5).unwrap(), &b"old"[..]) + .unwrap(); } - old.finish().unwrap(); - let mut live = track.create_group(moq_net::group::Info { sequence: 6 }).unwrap(); + old.close().unwrap(); + let mut live = track.create_group(moq_net::group::Info { start: 6 }).unwrap(); for _ in 0..2 { - live.write_frame(Timestamp::now(), &b"live"[..]).unwrap(); + live.write_frame(Timestamp::from_millis(6).unwrap(), &b"live"[..]) + .unwrap(); } if newest_first { @@ -183,21 +185,21 @@ async fn a_fresh_subscriber_receives_the_finished_older_group() { /// Read known frame counts so the open live group never advances paused time to /// the relay's linger deadline while waiting for a FIN that will not arrive. async fn read_history(sub: &mut moq_net::track::Subscriber) { - let mut sequences = Vec::new(); + let mut keys = Vec::new(); for _ in 0..2 { let mut group = sub.recv_group().await.unwrap().unwrap(); - let frames = match group.sequence { + let frames = match group.start { 5 => 3, 6 => 2, - sequence => panic!("unexpected group {sequence}"), + key => panic!("unexpected group {key}"), }; for _ in 0..frames { assert!(group.read_frame().await.unwrap().is_some()); } - sequences.push(group.sequence); + keys.push(group.start); } - sequences.sort(); - assert_eq!(sequences, [5, 6]); + keys.sort(); + assert_eq!(keys, [5, 6]); } #[kio::test] @@ -236,7 +238,7 @@ async fn a_late_subscriber_receives_the_relays_cached_history() { .unwrap(); let subscription = Subscription::default() .with_delay(Delay::default().with_max(FOREVER)) - .with_range(0..); + .with_ranges([support::keys(0..)]); let first_subscription = subscription.clone(); let first_reader = kio::clock::sim::spawn(async move { let mut sub = remote @@ -250,14 +252,16 @@ async fn a_late_subscriber_receives_the_relays_cached_history() { }); crate::support::wanted(track.demand()).await.unwrap(); upstream.server_transport.hold_unis(); - let mut old = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); + let mut old = track.create_group(moq_net::group::Info { start: 5 }).unwrap(); for _ in 0..3 { - old.write_frame(Timestamp::now(), &b"old"[..]).unwrap(); + old.write_frame(Timestamp::from_millis(5).unwrap(), &b"old"[..]) + .unwrap(); } - old.finish().unwrap(); - let mut live = track.create_group(moq_net::group::Info { sequence: 6 }).unwrap(); + old.close().unwrap(); + let mut live = track.create_group(moq_net::group::Info { start: 6 }).unwrap(); for _ in 0..2 { - live.write_frame(Timestamp::now(), &b"live"[..]).unwrap(); + live.write_frame(Timestamp::from_millis(6).unwrap(), &b"live"[..]) + .unwrap(); } // Both streams open before paused time advances; 10ms stays below linger. kio::clock::sleep(Duration::from_millis(10)).await; diff --git a/rs/moq-net/tests/integration/history_join.rs b/rs/moq-net/tests/integration/history_join.rs index 442dbf5d63e..9d4e7b65390 100644 --- a/rs/moq-net/tests/integration/history_join.rs +++ b/rs/moq-net/tests/integration/history_join.rs @@ -6,6 +6,7 @@ use std::time::Duration; use moq_net::{Timestamp, Version, cache, origin, track}; +use crate::support::AppendNext; use crate::support::harness::{MockConnectOptions, connect_mock, spawn}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -38,11 +39,11 @@ async fn each_joining_viewer_reads_the_held_group() { .with_timescale(moq_net::Timescale::MILLI) .with_max_age(None); let source = broadcast.broadcast().create_track("video", info).unwrap(); - let mut group = source.append_group().unwrap(); + let mut group = source.append_next().unwrap(); group .write_frame(Timestamp::from_millis(0).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); for join in 0..3 { let viewer = origin(3 + join); @@ -55,7 +56,7 @@ async fn each_joining_viewer_reads_the_held_group() { let remote = remote.unwrap().expect("the broadcast resolves"); let mut subscriber = remote.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = subscriber.recv_group().await.unwrap().expect("a group"); - assert_eq!(group.sequence, 0, "{version}: viewer {join}"); + assert_eq!(group.start, 0, "{version}: viewer {join}"); let frame = group.read_frame().await.unwrap().expect("a frame"); assert_eq!(&frame.payload[..], b"x", "{version}: viewer {join}"); }) diff --git a/rs/moq-net/tests/integration/idle_cancel_refetch.rs b/rs/moq-net/tests/integration/idle_cancel_refetch.rs index c1625b63d5e..6466c98bef5 100644 --- a/rs/moq-net/tests/integration/idle_cancel_refetch.rs +++ b/rs/moq-net/tests/integration/idle_cancel_refetch.rs @@ -34,11 +34,13 @@ async fn a_group_cut_by_an_idle_cancel_is_refetched_whole() { .unwrap(); broadcast.announce().unwrap(); - let mut g0 = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); - g0.write_frame(None, &b"0a"[..]).unwrap(); - g0.finish().unwrap(); - let mut g1 = track.create_group(moq_net::group::Info { sequence: 1 }).unwrap(); - g1.write_frame(None, &b"1a"[..]).unwrap(); + let mut g0 = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); + g0.write_frame(moq_net::Timestamp::unscaled(g0.start), &b"0a"[..]) + .unwrap(); + g0.close().unwrap(); + let mut g1 = track.create_group(moq_net::group::Info { start: 1 }).unwrap(); + g1.write_frame(moq_net::Timestamp::unscaled(g1.start), &b"1a"[..]) + .unwrap(); let client = produce_origin(2); let mut options = MockConnectOptions::new("moq-lite-06".parse::().unwrap()); @@ -63,7 +65,7 @@ async fn a_group_cut_by_an_idle_cancel_is_refetched_whole() { .unwrap() .unwrap() .unwrap(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); let frame = kio::clock::timeout(TIMEOUT, group.read_frame()) .await .unwrap() @@ -78,10 +80,12 @@ async fn a_group_cut_by_an_idle_cancel_is_refetched_whole() { .unwrap() .unwrap(); - g1.write_frame(None, &b"1b"[..]).unwrap(); - g1.finish().unwrap(); - let mut g2 = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); - g2.write_frame(None, &b"2a"[..]).unwrap(); + g1.write_frame(moq_net::Timestamp::unscaled(g1.start), &b"1b"[..]) + .unwrap(); + g1.close().unwrap(); + let mut g2 = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); + g2.write_frame(moq_net::Timestamp::unscaled(g2.start), &b"2a"[..]) + .unwrap(); let track = remote.track("audio").unwrap(); let _feed = track.subscribe(None); diff --git a/rs/moq-net/tests/integration/late_lower_group.rs b/rs/moq-net/tests/integration/late_lower_group.rs index fce88922796..2b3e20e9d61 100644 --- a/rs/moq-net/tests/integration/late_lower_group.rs +++ b/rs/moq-net/tests/integration/late_lower_group.rs @@ -31,7 +31,7 @@ const TRANSPORT: &[&str] = &["moq-transport-18", "moq-transport-22"]; /// From `start`, or the oldest group still around when there is none. fn floored(start: Option) -> Range { Range { - start: start.map_or(Range::oldest().start, Anchor::Absolute), + start: start.map_or(Range::oldest().start, Anchor::Time), end: None, } } @@ -43,15 +43,15 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer.with_proxy(true) } -/// Four frames starting at `sequence` seconds, so a group created late still starts +/// Four frames starting at `key` seconds, so a group created late still starts /// below the one after it. -fn write_group(track: &moq_net::track::Producer, sequence: u64) { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); +fn write_group(track: &moq_net::track::Producer, key: u64) { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); for frame in 0..4 { - let timestamp = Timestamp::from_millis(sequence * 1000 + frame).unwrap(); + let timestamp = Timestamp::from_millis(key * 1000 + frame).unwrap(); group.write_frame(timestamp, &b"x"[..]).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } async fn read_all(sub: &mut moq_net::track::Subscriber) -> Vec<(u64, usize)> { @@ -74,7 +74,7 @@ async fn read_all(sub: &mut moq_net::track::Subscriber) -> Vec<(u64, usize)> { None => break, } } - got.push((group.sequence, frames)); + got.push((group.start, frames)); } got } @@ -274,7 +274,7 @@ async fn merged(version: &str) -> (Vec<(u64, usize)>, Vec<(u64, usize)>) { .expect("recv_group timed out") .expect("recv_group") .expect("group 1"); - assert_eq!(first.sequence, 1, "{version}: the live reader starts at group 1"); + assert_eq!(first.start, 1, "{version}: the live reader starts at group 1"); let mut oldest = subscribe(floored(Some(0))).await; // Idle time lets the merged demand reach the publisher before the lower group exists. diff --git a/rs/moq-net/tests/integration/late_tail_reader.rs b/rs/moq-net/tests/integration/late_tail_reader.rs new file mode 100644 index 00000000000..6258f05527b --- /dev/null +++ b/rs/moq-net/tests/integration/late_tail_reader.rs @@ -0,0 +1,277 @@ +#![cfg(feature = "async")] + +//! A finished track whose only group starts well past 0, read through a relay by one reader +//! after another: each reader gets the group, however its subscription names the start and +//! whenever it joins as the reader before it lets go. + +use crate::support; + +use std::time::Duration; + +use moq_net::Version; +use moq_net::track::{Delay, Subscription}; +use support::harness::{MockConnectOptions, connect_mock}; + +const TIMEOUT: Duration = Duration::from_secs(10); + +/// Where the publisher's only group starts, in microseconds. +const START: u64 = 407; +const FRAMES: u64 = 10; + +fn produce_origin(hop: u64) -> moq_net::origin::Producer { + let (producer, driver) = + moq_net::origin::Producer::new(moq_net::origin::Config::new(std::num::NonZeroU64::new(hop).unwrap())); + support::harness::spawn(driver); + producer.with_proxy(true) +} + +/// Read every frame `subscription` delivers from the relay over a fresh `version` session. +async fn read( + relay: &moq_net::origin::Producer, + version: Version, + subscription: Subscription, +) -> (Vec, Result<(), moq_net::Error>) { + let subscriber = produce_origin(3); + let mut options = MockConnectOptions::new(version); + options.server_publish = Some(relay.consume()); + options.client_subscribe = Some(subscriber.clone()); + let session = connect_mock(options).await; + let remote = kio::clock::timeout(TIMEOUT, subscriber.consume().follow("bcast", None).unwrap().available()) + .await + .expect("resolve timeout") + .expect("broadcast resolves") + .unwrap(); + let mut track = remote + .track("video") + .unwrap() + .subscribe(subscription) + .await + .expect("subscribe"); + let mut frames = Vec::new(); + let end = loop { + match kio::clock::timeout(TIMEOUT, track.recv_group()) + .await + .expect("group timeout") + { + // A frame that fails to read is missing from what the caller compares. + Ok(Some(mut group)) => { + while let Ok(Some(_)) = group.read_frame().await { + frames.push(group.start); + } + } + Ok(None) => break Ok(()), + Err(err) => break Err(err), + } + }; + drop((track, session, subscriber)); + (frames, end) +} + +#[kio::test] +async fn each_late_reader_reads_the_finished_group() { + for (first, second) in [ + ("moq-lite-07-wip", "moq-lite-07-wip"), + ("moq-lite-07-wip", "moq-lite-06"), + ("moq-lite-06", "moq-lite-07-wip"), + ("moq-lite-06", "moq-lite-06"), + ] { + for gap in [0, 1, 5, 50] { + let publisher = produce_origin(1); + let broadcast = publisher + .route("bcast", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let track = broadcast + .broadcast() + .create_track( + "video", + moq_net::track::Info::default() + .with_timescale(moq_net::Timescale::MICRO) + .with_max_age(None), + ) + .unwrap(); + let mut group = track.append_group(START).unwrap(); + for frame in 0..FRAMES { + group + .write_frame(moq_net::Timestamp::from_micros(START + frame).unwrap(), &b"frame"[..]) + .unwrap(); + } + group.close().unwrap(); + let finished = track.finish().unwrap(); + broadcast.announce().unwrap(); + + let relay = produce_origin(2); + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse().unwrap()); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay.clone()); + let upstream = connect_mock(options).await; + + let budget = Delay::default().with_max(Duration::from_secs(60)); + let oldest = Subscription::default().with_delay(budget); + let from_zero = Subscription::default() + .with_delay(budget) + .with_ranges([support::keys(0..)]); + + let (frames, end) = read(&relay, first.parse().unwrap(), oldest.clone()).await; + assert_eq!( + (frames.len() as u64, end.is_ok()), + (FRAMES, true), + "{first} first, gap {gap}" + ); + kio::clock::sleep(Duration::from_millis(gap)).await; + for (version, subscription) in [(second, from_zero.clone()), (second, oldest.clone())] { + let (frames, end) = read(&relay, version.parse().unwrap(), subscription).await; + assert_eq!( + (frames, end.is_ok()), + (vec![START; FRAMES as usize], true), + "{first} then {version}, gap {gap}" + ); + } + drop((finished, broadcast, upstream, publisher, relay)); + } + } +} + +/// Readers arrive together and a few at a time, as a fan of late viewers does, each on its +/// own session: every one reads the group. +#[kio::test] +async fn concurrent_late_readers_read_the_finished_group() { + for stagger in [0u64, 1, 2, 3, 5, 8] { + let publisher = produce_origin(1); + let broadcast = publisher + .route("bcast", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let track = broadcast + .broadcast() + .create_track( + "video", + moq_net::track::Info::default() + .with_timescale(moq_net::Timescale::MICRO) + .with_max_age(None), + ) + .unwrap(); + let mut group = track.append_group(START).unwrap(); + for frame in 0..FRAMES { + group + .write_frame(moq_net::Timestamp::from_micros(START + frame).unwrap(), &b"frame"[..]) + .unwrap(); + } + group.close().unwrap(); + let finished = track.finish().unwrap(); + broadcast.announce().unwrap(); + + let relay = produce_origin(2); + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse().unwrap()); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay.clone()); + let upstream = connect_mock(options).await; + + let mut readers = Vec::new(); + for index in 0..9u64 { + let relay = relay.clone(); + let version = ["moq-lite-07-wip", "moq-lite-06"][(index % 2) as usize]; + let budget = Delay::default().with_max(Duration::from_secs([5, 60, 1][(index % 3) as usize])); + let subscription = match index % 3 { + 0 => Subscription::default() + .with_delay(budget) + .with_ranges([support::keys(0..)]), + _ => Subscription::default().with_delay(budget), + }; + readers.push(kio::clock::sim::spawn(async move { + kio::clock::sleep(Duration::from_millis(stagger * index)).await; + ( + index, + version, + read(&relay, version.parse().unwrap(), subscription).await, + ) + })); + } + for reader in readers { + let (index, version, (frames, end)) = reader.await.expect("reader panicked"); + assert_eq!( + (frames, end.is_ok()), + (vec![START; FRAMES as usize], true), + "reader {index} on {version}, stagger {stagger}" + ); + } + drop((finished, broadcast, upstream, publisher, relay)); + } +} + +/// A reader asking only for the history before the group, which holds nothing, leaves the +/// group itself to the readers that join meanwhile. +#[kio::test] +async fn an_empty_history_read_leaves_the_group_to_readers_joining_meanwhile() { + for (history_version, version) in [ + ("moq-lite-07-wip", "moq-lite-07-wip"), + ("moq-lite-07-wip", "moq-lite-06"), + ] { + for stagger in [0u64, 1, 2, 5] { + let publisher = produce_origin(1); + let broadcast = publisher + .route("bcast", moq_net::origin::RouteOptions::hidden()) + .unwrap(); + let track = broadcast + .broadcast() + .create_track( + "video", + moq_net::track::Info::default() + .with_timescale(moq_net::Timescale::MICRO) + .with_max_age(None), + ) + .unwrap(); + let mut group = track.append_group(START).unwrap(); + for frame in 0..FRAMES { + group + .write_frame(moq_net::Timestamp::from_micros(START + frame).unwrap(), &b"frame"[..]) + .unwrap(); + } + group.close().unwrap(); + let finished = track.finish().unwrap(); + broadcast.announce().unwrap(); + + let relay = produce_origin(2); + let mut options = MockConnectOptions::new("moq-lite-07-wip".parse().unwrap()); + options.server_publish = Some(publisher.consume()); + options.client_subscribe = Some(relay.clone()); + let upstream = connect_mock(options).await; + + let budget = Delay::default().with_max(Duration::from_secs(60)); + // A live reader first, as the history reader's own live subscription is. + let (frames, _) = read( + &relay, + version.parse().unwrap(), + Subscription::default().with_delay(budget), + ) + .await; + assert_eq!(frames.len() as u64, FRAMES); + + let history = { + let relay = relay.clone(); + kio::clock::sim::spawn(async move { + let subscription = Subscription::default() + .with_delay(budget) + .with_order(moq_net::track::Order::Ascending) + .with_ranges([support::keys(0..START)]); + read(&relay, history_version.parse().unwrap(), subscription).await + }) + }; + kio::clock::sleep(Duration::from_millis(stagger)).await; + for subscription in [ + Subscription::default().with_delay(budget), + Subscription::default() + .with_delay(budget) + .with_ranges([support::keys(0..)]), + ] { + let (frames, end) = read(&relay, version.parse().unwrap(), subscription).await; + assert_eq!( + (frames, end.is_ok()), + (vec![START; FRAMES as usize], true), + "{version} beside a {history_version} history read, stagger {stagger}" + ); + } + let (frames, end) = history.await.expect("history reader panicked"); + assert_eq!((frames, end.is_ok()), (vec![], true), "the history holds nothing"); + drop((finished, broadcast, upstream, publisher, relay)); + } + } +} diff --git a/rs/moq-net/tests/integration/main.rs b/rs/moq-net/tests/integration/main.rs index 4ddefce3700..d676ca85dc3 100644 --- a/rs/moq-net/tests/integration/main.rs +++ b/rs/moq-net/tests/integration/main.rs @@ -46,6 +46,7 @@ mod idle_session; mod implementation; mod instance; mod late_lower_group; +mod late_tail_reader; mod lazy_discovery; mod mesh_sweep; mod mesh_withdraw; @@ -93,6 +94,6 @@ mod unannounce_release; mod untimed; mod viewer_target; mod warm_backfill; -mod wide_sequence; +mod wide_key; mod widened_max_age; mod zero_copy; diff --git a/rs/moq-net/tests/integration/mesh_sweep.rs b/rs/moq-net/tests/integration/mesh_sweep.rs index e50a2bea0ca..0e07ab17c09 100644 --- a/rs/moq-net/tests/integration/mesh_sweep.rs +++ b/rs/moq-net/tests/integration/mesh_sweep.rs @@ -22,9 +22,11 @@ use std::time::Duration; use futures::channel::mpsc; -use moq_net::{Instance, Timescale, Timestamp, Version, cache, origin, stats, track}; +use moq_net::{Instance, Timescale, Version, cache, origin, stats, track}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; +use crate::support::AppendNext; +use crate::support::AtStart; use crate::support::{self, Rng}; const VERSION: &str = "moq-lite-07-wip"; @@ -361,11 +363,11 @@ impl Mesh { .broadcast() .create_track(TRACK, track::Info::default().with_timescale(Timescale::MILLI)) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group - .write_frame(Timestamp::ZERO, bytes::Bytes::from(path.clone())) + .write_frame(group.at_start(), bytes::Bytes::from(path.clone())) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); route.announce().unwrap(); Held { route, _track: track } }; diff --git a/rs/moq-net/tests/integration/model.rs b/rs/moq-net/tests/integration/model.rs index 34f5a538398..71fa4aeb187 100644 --- a/rs/moq-net/tests/integration/model.rs +++ b/rs/moq-net/tests/integration/model.rs @@ -26,9 +26,9 @@ fn origin_front_serves_media_with_only_explicit_polls() { track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = track.create_group(group::Info { sequence: 7 }).unwrap(); - group.write_frame(Timestamp::ZERO, b"media").unwrap(); - group.finish().unwrap(); + let mut group = track.create_group(group::Info { start: 7 }).unwrap(); + group.write_frame(Timestamp::from_millis(7).unwrap(), b"media").unwrap(); + group.close().unwrap(); broadcast.announce().unwrap(); // Resolving and subscribing pass through the origin's core front tasks. @@ -40,7 +40,7 @@ fn origin_front_serves_media_with_only_explicit_polls() { driver.poll(now, &waiter).unwrap(); let mut subscriber = ready(subscribing.poll_ok(&waiter)).unwrap(); let mut group = ready(subscriber.poll_recv_group(&waiter)).unwrap().unwrap(); - assert_eq!(group.sequence, 7); + assert_eq!(group.start, 7); let media: frame::Frame = ready(group.poll_read_frame(&waiter)).unwrap().unwrap(); assert_eq!(media.payload.as_ref(), b"media"); assert!(ready(group.poll_read_frame(&waiter)).unwrap().is_none()); diff --git a/rs/moq-net/tests/integration/mount.rs b/rs/moq-net/tests/integration/mount.rs index dc3cd74541b..59f59000080 100644 --- a/rs/moq-net/tests/integration/mount.rs +++ b/rs/moq-net/tests/integration/mount.rs @@ -5,10 +5,12 @@ //! itself, and a reader there reaches the target. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Pattern, Patterns, Timestamp, Version, origin, track}; +use moq_net::{Pattern, Patterns, Version, origin, track}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; const TEST_TIMEOUT: Duration = Duration::from_secs(600); @@ -29,9 +31,9 @@ fn publish(origin: &origin::Producer, path: &str) -> (origin::Route, track::Prod track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, path.as_bytes().to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), path.as_bytes().to_vec()).unwrap(); + group.close().unwrap(); (route, track) } diff --git a/rs/moq-net/tests/integration/multi_cdn.rs b/rs/moq-net/tests/integration/multi_cdn.rs index 9f6bb2c0061..5f9face586e 100644 --- a/rs/moq-net/tests/integration/multi_cdn.rs +++ b/rs/moq-net/tests/integration/multi_cdn.rs @@ -11,6 +11,7 @@ //! onto it is a restart that ends the subscription rather than stitching onto it. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -82,7 +83,7 @@ fn timestamp(group: u64, frame: u64) -> Timestamp { } /// A frame as the reader saw it. -type Delivery = (u64, moq_net::Result<(Option, Vec)>); +type Delivery = (u64, moq_net::Result<(Timestamp, Vec)>); /// Read every group in full, reporting each frame as it arrives. fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { @@ -101,7 +102,7 @@ fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { match group.read_frame().await { Ok(Some(frame)) => { if tx - .unbounded_send((group.sequence, Ok((frame.timestamp, frame.payload.to_vec())))) + .unbounded_send((group.start, Ok((frame.timestamp, frame.payload.to_vec())))) .is_err() { return; @@ -109,7 +110,7 @@ fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { } Ok(None) => break, Err(err) => { - let _ = tx.unbounded_send((group.sequence, Err(err))); + let _ = tx.unbounded_send((group.start, Err(err))); break; } } @@ -120,7 +121,7 @@ fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { } /// The next frame the reader reports, failing on an error or a hang. -async fn next(rx: &mut mpsc::UnboundedReceiver) -> (u64, Option, Vec) { +async fn next(rx: &mut mpsc::UnboundedReceiver) -> (u64, Timestamp, Vec) { let (group, frame) = kio::clock::timeout(Duration::from_secs(10), rx.next()) .await .expect("reader hung") @@ -155,7 +156,7 @@ struct Topology { _pushes: Vec, s_to_a: Option, s_to_b: MockPair, - sequence: u64, + key: u64, } impl Topology { @@ -192,7 +193,7 @@ impl Topology { _pushes: pushes, s_to_a: Some(s_to_a), s_to_b, - sequence: 0, + key: 0, } } @@ -205,26 +206,21 @@ impl Topology { /// Write one full group, returning what the reader should see. fn group(&mut self) -> Vec<(u64, Vec)> { - let sequence = self.sequence; - self.sequence += 1; - let mut group = self.track.append_group().unwrap(); + let key = self.key; + self.key += 1; + let mut group = self.track.append_next().unwrap(); let mut written = Vec::new(); for frame in 0..FRAMES { - group - .write_frame(timestamp(sequence, frame), payload(sequence, frame)) - .unwrap(); - written.push((sequence, payload(sequence, frame))); + group.write_frame(timestamp(key, frame), payload(key, frame)).unwrap(); + written.push((key, payload(key, frame))); } - group.finish().unwrap(); + group.close().unwrap(); written } /// Write a group while `B` holds back everything it sends `S`: it arrives only if /// `A` serves it. - async fn group_through_a( - &mut self, - rx: &mut mpsc::UnboundedReceiver, - ) -> Vec<(u64, Vec, Option)> { + async fn group_through_a(&mut self, rx: &mut mpsc::UnboundedReceiver) -> Vec<(u64, Vec, Timestamp)> { self.s_to_b.server_transport.hold_unis(); let written = self.group(); let mut seen = Vec::new(); @@ -237,10 +233,7 @@ impl Topology { } /// Write a group and read it back, whichever CDN serves it. - async fn group_anywhere( - &mut self, - rx: &mut mpsc::UnboundedReceiver, - ) -> Vec<(u64, Vec, Option)> { + async fn group_anywhere(&mut self, rx: &mut mpsc::UnboundedReceiver) -> Vec<(u64, Vec, Timestamp)> { let written = self.group(); let mut seen = Vec::new(); for _ in &written { @@ -263,17 +256,14 @@ impl Topology { } /// Every frame once, in order, with no timestamp rewinding. -fn assert_continuous(seen: &[(u64, Vec, Option)], groups: u64) { +fn assert_continuous(seen: &[(u64, Vec, Timestamp)], groups: u64) { let expected: Vec<_> = (0..groups) .flat_map(|group| (0..FRAMES).map(move |frame| (group, payload(group, frame)))) .collect(); let frames: Vec<_> = seen.iter().map(|(group, frame, _)| (*group, frame.clone())).collect(); assert_eq!(frames, expected); for pair in seen.windows(2) { - assert!( - pair[0].2.expect("timed") < pair[1].2.expect("timed"), - "timestamp rewound at {pair:?}" - ); + assert!(pair[0].2 < pair[1].2, "timestamp rewound at {pair:?}"); } } @@ -376,8 +366,8 @@ async fn restarts_across_protocols(secondary: &str) { "the old route did not end its track: {rest:?}" ); let mut fresh = topology.subscribe().await; - let (sequence, _, bytes) = next(&mut fresh).await; - assert_eq!(sequence, 1, "the fresh subscription joins the current group"); + let (key, _, bytes) = next(&mut fresh).await; + assert_eq!(key, 1, "the fresh subscription joins the current group"); assert_eq!(bytes, payload(1, 0)); drop((topology.publisher, topology.cdn_b)); }) diff --git a/rs/moq-net/tests/integration/partial_group.rs b/rs/moq-net/tests/integration/partial_group.rs index c7259e12496..880531f9c43 100644 --- a/rs/moq-net/tests/integration/partial_group.rs +++ b/rs/moq-net/tests/integration/partial_group.rs @@ -6,6 +6,8 @@ use crate::support; use moq_net::track::Delay; +use support::AppendNext; +use support::AtStart; use std::time::Duration; @@ -62,23 +64,23 @@ async fn round(version: &str) -> Option, moq_net::Error>> { .expect("subscribe"); subscribed(&track).await; - let mut stalled = track.append_group().unwrap(); - stalled.write_frame(Timestamp::ZERO, &b"head"[..]).unwrap(); + let mut stalled = track.append_next().unwrap(); + stalled.write_frame(stalled.at_start(), &b"head"[..]).unwrap(); let mut group = kio::clock::timeout(TIMEOUT, live.recv_group()) .await .expect("group timeout") .unwrap() .expect("group 0"); - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); assert!(group.read_frame().await.unwrap().is_some()); // The live edge moves a full second past the stalled group's reach. kio::clock::sleep(Duration::from_secs(1)).await; for millis in [1000, 2000] { - let mut next = track.append_group().unwrap(); + let mut next = track.append_next().unwrap(); next.write_frame(Timestamp::from_millis(millis).unwrap(), &b"edge"[..]) .unwrap(); - next.finish().unwrap(); + next.close().unwrap(); } let tail = kio::clock::timeout(TIMEOUT, group.read_frame()) .await @@ -92,7 +94,7 @@ async fn round(version: &str) -> Option, moq_net::Error>> { remote.track("data").unwrap().subscribe( Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(3600))) - .with_range(0..), + .with_ranges([support::keys(0..)]), ), ) .await @@ -103,7 +105,7 @@ async fn round(version: &str) -> Option, moq_net::Error>> { .expect("group timeout") .unwrap() .expect("a group"); - if first.sequence != 0 { + if first.start != 0 { stalled.abort(moq_net::Error::Cancel).unwrap(); return None; } diff --git a/rs/moq-net/tests/integration/quiet_catalog.rs b/rs/moq-net/tests/integration/quiet_catalog.rs index d2156d62f7c..329ceb7b593 100644 --- a/rs/moq-net/tests/integration/quiet_catalog.rs +++ b/rs/moq-net/tests/integration/quiet_catalog.rs @@ -6,10 +6,12 @@ //! must still get it. use crate::support; +use support::AppendNext; +use support::AtStart; use std::{future::Future, time::Duration}; -use moq_net::{Error, Timestamp, Version, broadcast, origin, track}; +use moq_net::{Error, Version, broadcast, origin, track}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -51,9 +53,9 @@ fn publish(origin: &origin::Producer) -> (moq_net::origin::Route, track::Produce let broadcast = origin.route("live", moq_net::origin::RouteOptions::hidden()).unwrap(); let track = broadcast.broadcast().create_track("catalog.json", timed()).unwrap(); broadcast.announce().unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"snapshot").unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"snapshot").unwrap(); + group.close().unwrap(); (broadcast, track) } @@ -88,9 +90,9 @@ async fn read_snapshot(remote: &broadcast::Consumer) -> Result Ok(sub), - (sequence, payload) => Err(format!("got group {sequence} {payload:?}")), + (key, payload) => Err(format!("got group {key} {payload:?}")), } } @@ -180,13 +182,13 @@ async fn resume_then_fresh(version: Version, in_process: bool) -> Result<(), Str resume_remote .track("catalog.json") .unwrap() - .subscribe(track::Subscription::default().with_range(1..)) + .subscribe(track::Subscription::default().with_ranges([support::keys(1..)])) .await .map_err(|err| format!("resume subscribe: {err}")) }, async { // The fresh reader widens the existing upstream subscription. - let resumed = vec![track::Range::groups(1..)]; + let resumed = vec![support::keys(1..)]; kio::clock::timeout(TIMEOUT, async { while track.subscription().map(|sub| sub.ranges).as_ref() != Some(&resumed) { track.subscription_changed().await.unwrap(); diff --git a/rs/moq-net/tests/integration/range_anchors.rs b/rs/moq-net/tests/integration/range_anchors.rs index 13892eead29..300492d6fd9 100644 --- a/rs/moq-net/tests/integration/range_anchors.rs +++ b/rs/moq-net/tests/integration/range_anchors.rs @@ -2,9 +2,12 @@ //! A range anchored relative to the latest group or at a presentation time resolves where //! the track is held, and on a copy received over a wire that tells it enough to resolve -//! the anchor the same way. A moq-transport copy resolves a relative anchor from the -//! Largest Object its live join reports. A copy received over any other wire refuses the -//! anchor instead of resolving it against the part of the track it happens to hold. +//! the anchor the same way. Every wire carries a time anchor, as the group ID with that key, +//! but only lite-07 resolves a time start to the group containing it: an older wire's +//! publisher starts at the first group at or after it. A moq-transport copy resolves a +//! relative anchor from the Largest Object its live join reports. A copy received over any +//! other wire refuses a relative anchor other than the live start instead of resolving it +//! against the part of the track it happens to hold. use crate::support; use moq_net::track::Delay; @@ -41,9 +44,9 @@ fn millis(ms: u64) -> Timestamp { Timestamp::from_millis(ms).unwrap() } -/// Groups 0 to 4, group `n` presenting at `n` seconds, then what a subscriber to `range` -/// reads oldest first: through the publisher's own origin with no `version`, otherwise -/// over one session on it. +/// Groups at 0 to 4 seconds, each keyed by its start in milliseconds, then what a subscriber +/// to `range` reads oldest first: through the publisher's own origin with no `version`, +/// otherwise over one session on it. async fn read(version: Option<&str>, range: Range) -> Result, Error> { let publisher = produce_origin(1); let broadcast = publisher @@ -57,10 +60,10 @@ async fn read(version: Option<&str>, range: Range) -> Result, Error> { ) .unwrap(); broadcast.announce().unwrap(); - for sequence in 0..5 { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); - group.write_frame(millis(sequence * 1000), &b"x"[..]).unwrap(); - group.finish().unwrap(); + for key in (0..5).map(|second| second * 1000) { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); + group.write_frame(millis(key), &b"x"[..]).unwrap(); + group.close().unwrap(); } let subscriber = produce_origin(2); @@ -93,7 +96,7 @@ async fn read(version: Option<&str>, range: Range) -> Result, Error> { let mut read = Vec::new(); while let Ok(group) = kio::clock::timeout(TIMEOUT, reader.recv_group()).await { match group? { - Some(group) => read.push(group.sequence), + Some(group) => read.push(group.start), None => break, } } @@ -107,48 +110,55 @@ fn range(start: Anchor, end: Option) -> Range { /// Relative and time anchors at each end, with what a reader of the held track reads. fn anchored() -> [(Range, Vec); 6] { - let zero = Anchor::Absolute(0); + let zero = Anchor::Time(0); [ - (range(Anchor::Relative(2), None), vec![2, 3, 4]), - (range(Anchor::Time(millis(2500)), None), vec![2, 3, 4]), - (range(zero, Some(Anchor::Relative(1))), vec![0, 1, 2, 3]), - (range(zero, Some(Anchor::Time(millis(2500)))), vec![0, 1, 2]), - (range(Anchor::Relative(2), Some(Anchor::Relative(1))), vec![2, 3]), + (range(Anchor::Relative(2), None), vec![2000, 3000, 4000]), + (range(Anchor::Time(2500), None), vec![2000, 3000, 4000]), + (range(zero, Some(Anchor::Relative(1))), vec![0, 1000, 2000, 3000]), + (range(zero, Some(Anchor::Time(2500))), vec![0, 1000, 2000]), + (range(Anchor::Relative(2), Some(Anchor::Relative(1))), vec![2000, 3000]), ( - range(Anchor::Time(millis(1500)), Some(Anchor::Time(millis(3500)))), - vec![1, 2, 3], + range(Anchor::Time(1500), Some(Anchor::Time(3500))), + vec![1000, 2000, 3000], ), ] } /// Each wire resolves the anchors it carries as the held track does and refuses the rest, -/// rather than resolve them against the part of the track its copy happens to hold. +/// rather than resolve them against the part of the track its copy happens to hold. An older +/// wire starts a time anchor at the first group at or after it instead. #[kio::test] async fn each_wire_refuses_only_the_anchors_it_drops() { + let mut failures = Vec::new(); for version in std::iter::once(None).chain(VERSIONS.iter().copied().map(Some)) { // Lite-07 reports the largest and asks for everything the budget allows, so its copy // resolves every anchor where it is held. - let carried = matches!(version, None | Some("moq-lite-07-wip")); - // A wire without timestamps refuses a time anchor for that first. - let timed = !matches!(version, Some("moq-lite-03" | "moq-lite-04")); - for (range, held) in anchored() { + let resolves = matches!(version, None | Some("moq-lite-07-wip")); + // A moq-transport copy joins live and resolves a relative anchor from the Largest + // Object, so the groups below it arrive as fills, after the live one. + let transport = matches!(version, Some(version) if version.starts_with("moq-transport")); + for (range, mut held) in anchored() { + // An older wire's publisher starts a time anchor at the first group at or after it, + // not the group containing it. + if let (false, Anchor::Time(start)) = (resolves, range.start) { + held.retain(|&key| key >= start); + } let mut result = read(version, range).await; - let time = matches!(range.start, Anchor::Time(_)) || matches!(range.end, Some(Anchor::Time(_))); - // A moq-transport copy joins live and resolves a relative anchor from the Largest - // Object, so the groups below it arrive as fills, after the live one. - let transport = matches!(version, Some(version) if version.starts_with("moq-transport")); - let carried = carried || (transport && !time); + // Any other wire carries time anchors and the live start, and drops the rest. + let wire = matches!(range.start, Anchor::Time(_) | Anchor::Oldest | Anchor::Relative(0)) + && matches!(range.end, None | Some(Anchor::Time(_))); + let carried = resolves || transport || wire; if transport && let Ok(read) = &mut result { read.sort_unstable(); } - match (&result, time && !timed, carried) { - (Err(Error::TimestampMismatch), true, _) => {} - (Err(Error::Unsupported), false, false) => {} - (Ok(read), false, true) if *read == held => {} - _ => panic!("{version:?}: {range:?} read {result:?}"), + match (&result, carried) { + (Err(Error::Unsupported), false) => {} + (Ok(read), true) if *read == held => {} + _ => failures.push(format!("{version:?}: {range:?} read {result:?}")), } } } + assert!(failures.is_empty(), "{failures:#?}"); } /// Every wire carries the live and oldest starts, which it spells with no floor. A wire whose @@ -157,12 +167,16 @@ async fn each_wire_refuses_only_the_anchors_it_drops() { async fn every_wire_carries_the_live_and_oldest_starts() { for version in VERSIONS { let live = read(Some(version), Range::live()).await; - assert_eq!(live.unwrap().last(), Some(&4), "{version}: live reads the latest group"); + assert_eq!( + live.unwrap().last(), + Some(&4000), + "{version}: live reads the latest group" + ); let mut oldest = read(Some(version), Range::oldest()).await.unwrap(); oldest.sort_unstable(); let want: &[u64] = match *version { - "moq-lite-03" | "moq-lite-04" | "moq-lite-05" | "moq-transport-18" | "moq-transport-22" => &[4], - _ => &[0, 1, 2, 3, 4], + "moq-lite-03" | "moq-lite-04" | "moq-lite-05" | "moq-transport-18" | "moq-transport-22" => &[4000], + _ => &[0, 1000, 2000, 3000, 4000], }; assert_eq!(oldest, want, "{version}: oldest reads every group its start reaches"); } @@ -170,7 +184,7 @@ async fn every_wire_carries_the_live_and_oldest_starts() { /// A publisher counts a relative start back from its newest group, which may hold no frame yet /// and so counts neither in the largest it reports nor where the copy resolves the start. The -/// copy reports the publisher's start instead, where a reader waiting on each group in sequence +/// copy reports the publisher's start instead, where a reader waiting on each group in key /// starts, rather than one below that never arrives. #[kio::test] async fn a_relative_start_reports_the_publishers_start_over_a_frameless_newest_group() { @@ -183,12 +197,14 @@ async fn a_relative_start_reports_the_publishers_start_over_a_frameless_newest_g .create_track("data", Info::default().with_max_age(FOREVER)) .unwrap(); broadcast.announce().unwrap(); - for sequence in 0..5 { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); - group.write_frame(None, &b"x"[..]).unwrap(); - group.finish().unwrap(); + for key in 0..5 { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), &b"x"[..]) + .unwrap(); + group.close().unwrap(); } - let mut newest = track.create_group(moq_net::group::Info { sequence: 5 }).unwrap(); + let mut newest = track.create_group(moq_net::group::Info { start: 5 }).unwrap(); let subscriber = produce_origin(2); let _pair = peer("moq-lite-07-wip".parse::().unwrap(), &publisher, &subscriber).await; @@ -214,12 +230,14 @@ async fn a_relative_start_reports_the_publishers_start_over_a_frameless_newest_g "the publisher's resolved start, counting its frameless newest group" ); - newest.write_frame(None, &b"x"[..]).unwrap(); - newest.finish().unwrap(); + newest + .write_frame(moq_net::Timestamp::unscaled(newest.start), &b"x"[..]) + .unwrap(); + newest.close().unwrap(); let group = kio::clock::timeout(TIMEOUT, reader.recv_group()) .await .expect("group timeout") .unwrap(); - assert_eq!(group.map(|group| group.sequence), Some(5)); + assert_eq!(group.map(|group| group.start), Some(5)); let _track = track.finish().unwrap(); } diff --git a/rs/moq-net/tests/integration/range_fill.rs b/rs/moq-net/tests/integration/range_fill.rs index bef0116c821..ae8bbbab514 100644 --- a/rs/moq-net/tests/integration/range_fill.rs +++ b/rs/moq-net/tests/integration/range_fill.rs @@ -41,6 +41,12 @@ struct Fixture { impl Fixture { /// A lite-07 publisher holding `groups` for good, and a copy whose cache expires them. async fn new(groups: u64) -> Self { + Self::spaced(groups, 1).await + } + + /// [`Self::new`] with the groups `spacing` keys apart, each appended, so the publisher + /// skips the keys between them. + async fn spaced(groups: u64, spacing: u64) -> Self { let publisher = origin(1, Some(cache::Pool::unbounded())); let broadcast = publisher .route("bcast", moq_net::origin::RouteOptions::hidden()) @@ -53,12 +59,15 @@ impl Fixture { ) .unwrap(); broadcast.announce().unwrap(); - for sequence in 0..groups { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + for index in 0..groups { + let mut group = match spacing { + 1 => track.create_group(moq_net::group::Info { start: index }).unwrap(), + spacing => track.append_group(index * spacing).unwrap(), + }; group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), b"x".as_slice()) + .write_frame(Timestamp::from_millis(index * 1000).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let copy = origin(2, None); @@ -108,7 +117,7 @@ impl Fixture { } } -/// The sequences of the next `count` groups `reader` receives. +/// The keys of the next `count` groups `reader` receives. async fn receive(reader: &mut moq_net::track::Subscriber, count: usize) -> BTreeSet { let mut got = BTreeSet::new(); while got.len() < count { @@ -117,7 +126,7 @@ async fn receive(reader: &mut moq_net::track::Subscriber, count: usize) -> BTree .expect("group timeout") .unwrap() .expect("a group"); - got.insert(group.sequence); + got.insert(group.start); } got } @@ -135,27 +144,24 @@ async fn evicted_history_is_recovered_once_on_its_own_subscription() { // Superseded, the copy expires them, while the reader keeps the subscription live. kio::clock::sleep(cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; let subscribes = fixture.opened(SUBSCRIBE); - let mut second = fixture.subscribe([Range::groups(1..4)]).await; + let mut second = fixture.subscribe([support::keys(1..4)]).await; assert_eq!(receive(&mut second, 3).await, (1..4).collect()); assert_eq!(fixture.opened(SUBSCRIBE), subscribes + 1, "recovered on one SUBSCRIBE"); assert_eq!(fixture.opened(FETCH), 0, "nothing fetched"); // The live subscription goes on. - let mut group = fixture - .track - .create_group(moq_net::group::Info { sequence: 6 }) - .unwrap(); + let mut group = fixture.track.create_group(moq_net::group::Info { start: 6 }).unwrap(); group .write_frame(Timestamp::from_millis(6000).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(receive(&mut first, 1).await, [6].into()); assert_eq!(fixture.opened(SUBSCRIBE), subscribes + 1, "recovered once"); // Expired again, the history is recovered again for a later reader. drop(second); kio::clock::sleep(cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; - let mut third = fixture.subscribe([Range::groups(1..4)]).await; + let mut third = fixture.subscribe([support::keys(1..4)]).await; assert_eq!(receive(&mut third, 3).await, (1..4).collect()); assert_eq!( fixture.opened(SUBSCRIBE), @@ -167,6 +173,37 @@ async fn evicted_history_is_recovered_once_on_its_own_subscription() { .expect("timed out"); } +/// Recovering history over sparse keys gets the publisher's skips of the keys between the +/// groups on the recovery's own subscription, which the copy records rather than refusing. +#[kio::test] +async fn sparse_history_is_recovered_across_its_skips() { + kio::clock::timeout(TIMEOUT + cache::DEFAULT_EXPIRY * 3, async { + let fixture = Fixture::spaced(6, 1000).await; + let mut first = fixture.subscribe([Range::oldest()]).await; + let keys: BTreeSet = (0..6).map(|index| index * 1000).collect(); + assert_eq!(receive(&mut first, 6).await, keys); + + // Superseded, the copy expires them, while the reader keeps the subscription live. + kio::clock::sleep(cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; + let subscribes = fixture.opened(SUBSCRIBE); + let mut second = fixture.subscribe([support::keys(1000..4000)]).await; + assert_eq!(receive(&mut second, 3).await, [1000, 2000, 3000].into()); + assert_eq!(fixture.opened(SUBSCRIBE), subscribes + 1, "recovered on one SUBSCRIBE"); + assert_eq!(fixture.opened(FETCH), 0, "nothing fetched"); + + // The session survives the skips, so the live subscription goes on. + kio::clock::sleep(Duration::from_secs(1)).await; + let mut group = fixture.track.append_group(6000).unwrap(); + group + .write_frame(Timestamp::from_millis(6000).unwrap(), b"x".as_slice()) + .unwrap(); + group.close().unwrap(); + assert_eq!(receive(&mut first, 1).await, [6000].into()); + }) + .await + .expect("timed out"); +} + /// The track finishing while a recovery is still out doesn't cut it: the copy ends only once /// the recovered groups arrived, which the live subscription's tail never waits for. #[kio::test] @@ -178,7 +215,7 @@ async fn a_recovery_outlives_the_track_finishing() { // Superseded, the copy expires them, while the reader keeps the subscription live. kio::clock::sleep(cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; - let mut second = fixture.subscribe([Range::groups(1..4)]).await; + let mut second = fixture.subscribe([support::keys(1..4)]).await; let _track = fixture.track.finish().unwrap(); assert_eq!(receive(&mut second, 3).await, (1..4).collect()); }) @@ -186,19 +223,19 @@ async fn a_recovery_outlives_the_track_finishing() { .expect("timed out"); } -/// A recovery's publisher may skip a sequence inside it, which the copy records as the live +/// A recovery's publisher may skip a key inside it, which the copy records as the live /// subscription's skip is: the reader gets the groups around it, and the session goes on. #[kio::test] async fn a_recovery_takes_a_skip_inside_it() { kio::clock::timeout(TIMEOUT + cache::DEFAULT_EXPIRY * 3, async { let fixture = Fixture::new(2).await; fixture.track.skip_groups(2..3); - for sequence in 3..6 { - let mut group = fixture.track.create_group(moq_net::group::Info { sequence }).unwrap(); + for start in 3..6 { + let mut group = fixture.track.create_group(moq_net::group::Info { start }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), b"x".as_slice()) + .write_frame(Timestamp::from_millis(start * 1000).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let mut first = fixture.subscribe([Range::oldest()]).await; assert_eq!(receive(&mut first, 5).await, [0, 1, 3, 4, 5].into()); @@ -206,18 +243,15 @@ async fn a_recovery_takes_a_skip_inside_it() { // Superseded, the copy expires them, skip and all, while the reader keeps the // subscription live. kio::clock::sleep(cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; - let mut second = fixture.subscribe([Range::groups(1..5)]).await; + let mut second = fixture.subscribe([support::keys(1..5)]).await; assert_eq!(receive(&mut second, 3).await, [1, 3, 4].into()); // The live subscription goes on. - let mut group = fixture - .track - .create_group(moq_net::group::Info { sequence: 6 }) - .unwrap(); + let mut group = fixture.track.create_group(moq_net::group::Info { start: 6 }).unwrap(); group .write_frame(Timestamp::from_millis(6000).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(receive(&mut first, 1).await, [6].into()); }) .await diff --git a/rs/moq-net/tests/integration/range_read.rs b/rs/moq-net/tests/integration/range_read.rs index 345487a135f..1f334b6fbfc 100644 --- a/rs/moq-net/tests/integration/range_read.rs +++ b/rs/moq-net/tests/integration/range_read.rs @@ -5,6 +5,7 @@ //! past the live edge ends once the groups up to its end arrive. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -21,18 +22,18 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer.with_proxy(true) } -fn write_group(track: &mut moq_net::track::Producer, sequence: u64) { - let mut group = track.append_group().unwrap(); - assert_eq!(group.sequence, sequence); +fn write_group(track: &mut moq_net::track::Producer, key: u64) { + let mut group = track.append_next().unwrap(); + assert_eq!(group.start, key); + let at = Timestamp::from_millis(key).unwrap(); for frame in 0..2u64 { - let at = Timestamp::from_millis(sequence * 100 + frame).unwrap(); - group.write_frame(at, payload(sequence, frame)).unwrap(); + group.write_frame(at, payload(key, frame)).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } -fn payload(sequence: u64, frame: u64) -> Vec { - format!("{sequence}-{frame}").into_bytes() +fn payload(key: u64, frame: u64) -> Vec { + format!("{key}-{frame}").into_bytes() } /// Each group a one-range read of `range` gets, read whole, until it ends. @@ -51,8 +52,8 @@ async fn read(sub: &mut moq_net::track::Subscriber) -> Vec { { frames.push(frame.payload.to_vec()); } - assert_eq!(frames, [payload(group.sequence, 0), payload(group.sequence, 1)]); - read.push(group.sequence); + assert_eq!(frames, [payload(group.start, 0), payload(group.start, 1)]); + read.push(group.start); } read } @@ -77,8 +78,8 @@ async fn fixture( ) .unwrap(); broadcast.announce().unwrap(); - for sequence in 0..3 { - write_group(&mut track, sequence); + for key in 0..3 { + write_group(&mut track, key); } let middle = produce_origin(2); @@ -108,7 +109,7 @@ async fn fixture( fn one_range(groups: std::ops::Range) -> Subscription { Subscription::default() - .with_range(groups) + .with_ranges([support::keys(groups)]) .with_delay(Delay::default().with_max(None)) .with_order(Order::Ascending) } @@ -141,8 +142,8 @@ async fn straddling(version: Version, relay: bool) { let mut sub = remote.track("video").unwrap().subscribe(one_range(1..5)).await.unwrap(); let reading = kio::clock::sim::spawn(async move { read(&mut sub).await }); kio::clock::sleep(Duration::from_millis(50)).await; - for sequence in 3..6 { - write_group(&mut track, sequence); + for key in 3..6 { + write_group(&mut track, key); } // Arrival order: an older wire sends a backlog in no particular order. let mut read = reading.await.unwrap(); diff --git a/rs/moq-net/tests/integration/received_rewind.rs b/rs/moq-net/tests/integration/received_rewind.rs index 56ab1bd0d55..899be31b926 100644 --- a/rs/moq-net/tests/integration/received_rewind.rs +++ b/rs/moq-net/tests/integration/received_rewind.rs @@ -1,12 +1,7 @@ #![cfg(feature = "async")] -//! A relay passes on the timestamps it receives over lite-05/06, unchecked. -//! -//! Lite-06 stamps an untimed track's frames with their send time and declares -//! milliseconds, so a group sent after a higher one carries the later stamp. The relay -//! cannot tell that from media time, so its copy skips the rewind check, as a -//! moq-transport one does, and serves the timestamps on over every version. A lite-07 -//! receiver checks them, so it refuses that out-of-order group. +//! A relay passes on the timestamps it receives over lite-06 and moq-transport, with their +//! values, even for a group that arrives after a later one. use crate::support; use moq_net::track::Delay; @@ -19,8 +14,8 @@ use support::harness::{MockConnectOptions, connect_mock}; const TIMEOUT: Duration = Duration::from_secs(10); -/// A group's sequence and its first frame's timestamp, or the error reading it. -type Read = (u64, Result, moq_net::Error>); +/// A group's key and its first frame's timestamp, or the error reading it. +type Read = (u64, Result); fn millis(ms: u64) -> moq_net::Timestamp { moq_net::Timestamp::from_millis(ms).unwrap() @@ -36,29 +31,30 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer.with_proxy(true) } -fn write_group(track: &moq_net::track::Producer, sequence: u64) { - // A timed track's group starts at its sequence in seconds, so none rewinds. - let timestamp = track.info().timescale.map(|_| millis(sequence * 1000)); - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); +/// A group starting at `key` milliseconds. +fn write_group(track: &moq_net::track::Producer, key: u64) { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); + let timestamp = millis(key); group.write_frame(timestamp, &b"x"[..]).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } -/// Groups 1 then 0 of a track at `timescale`, into a relay over `upstream` and out over -/// `downstream`. Returns the scale the subscriber learned and, per group read, its sequence -/// and its first frame's timestamp or the error reading it. -async fn relay( - upstream: &str, - downstream: &str, - timescale: Option, -) -> (Option, Vec) { +/// The groups at one second then at zero of a millisecond track, into a relay over +/// `upstream` and out over `downstream`. Returns the scale the subscriber learned and, per +/// group read, its key and its first frame's timestamp or the error reading it. +async fn relay(upstream: &str, downstream: &str) -> (Option, Vec) { let publisher = produce_origin(1); let broadcast = publisher .route("bcast", moq_net::origin::RouteOptions::hidden()) .unwrap(); let track = broadcast .broadcast() - .create_track("data", Info::default().with_timescale(timescale).with_max_age(None)) + .create_track( + "data", + Info::default() + .with_timescale(moq_net::Timescale::MILLI) + .with_max_age(None), + ) .unwrap(); broadcast.announce().unwrap(); @@ -83,7 +79,7 @@ async fn relay( let subscription = Subscription::default() .with_delay(Delay::default().with_max(BUDGET)) - .with_range(0..); + .with_ranges([support::keys(0..)]); let reader = kio::clock::sim::spawn(async move { let mut sub = remote .track("data") @@ -104,7 +100,7 @@ async fn relay( .await .expect("read_frame timed out") .map(|frame| frame.expect("a frame").timestamp); - got.push((group.sequence, read)); + got.push((group.start, read)); } (timescale, got) }); @@ -114,8 +110,7 @@ async fn relay( .expect("no subscriber appeared") .unwrap(); kio::clock::sleep(Duration::from_millis(10)).await; - write_group(&track, 1); - // Sent later, so group 0's send time lands after group 1's. + write_group(&track, 1000); kio::clock::sleep(Duration::from_millis(10)).await; write_group(&track, 0); let track = track.finish().unwrap(); @@ -128,31 +123,6 @@ async fn relay( got } -/// A relay passes on lite-06 send times unchecked, so a lite-06 or moq-transport reader -/// reads both groups timed and the track stays open. A lite-07 reader checks them and -/// refuses the group that rewinds, the cost of an untimed origin on lite-06. -#[kio::test] -async fn a_relay_forwards_descending_send_times_without_closing_the_track() { - let mut failures = Vec::new(); - for downstream in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { - let (timescale, got) = relay("moq-lite-06", downstream, None).await; - let sequences: Vec<_> = got.iter().map(|(sequence, _)| *sequence).collect(); - let refused = downstream == "moq-lite-07-wip"; - let ok = timescale == Some(moq_net::Timescale::MILLI) - && sequences == [1, 0] - && matches!(got[0].1, Ok(Some(_))) - && match &got[1].1 { - Err(moq_net::Error::TimestampMismatch) => refused, - Ok(Some(_)) => !refused, - _ => false, - }; - if !ok { - failures.push(format!("{downstream}: {timescale:?} {got:?}")); - } - } - assert!(failures.is_empty(), "{failures:#?}"); -} - /// A timed track's timestamps cross a lite-06 or moq-transport hop between two origins with /// their values, and the relay serves them on over every version. #[kio::test] @@ -160,11 +130,11 @@ async fn a_timed_track_keeps_its_timestamps_through_a_relay() { let mut failures = Vec::new(); for upstream in ["moq-lite-06", "moq-transport-22"] { for downstream in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-22"] { - let (timescale, got) = relay(upstream, downstream, Some(moq_net::Timescale::MILLI)).await; + let (timescale, got) = relay(upstream, downstream).await; let ok = timescale == Some(moq_net::Timescale::MILLI) && matches!( got.as_slice(), - [(1, Ok(Some(one))), (0, Ok(Some(zero)))] if *one == millis(1000) && *zero == millis(0) + [(1000, Ok(one)), (0, Ok(zero))] if *one == millis(1000) && *zero == millis(0) ); if !ok { failures.push(format!("{upstream} -> {downstream}: {timescale:?} {got:?}")); @@ -173,63 +143,3 @@ async fn a_timed_track_keeps_its_timestamps_through_a_relay() { } assert!(failures.is_empty(), "{failures:#?}"); } - -/// Fetch groups 1 then 0 of a finished untimed track over `version`, with no live -/// subscription. Each answer carries its send time, so group 0 lands after group 1; the -/// copy the fetches fill is unchecked, so neither is refused. Returns each first frame's -/// timestamp. -async fn fetch_only(version: &str) -> Vec, moq_net::Error>> { - let publisher = produce_origin(1); - let broadcast = publisher - .route("bcast", moq_net::origin::RouteOptions::hidden()) - .unwrap(); - let track = broadcast - .broadcast() - .create_track("data", Info::default().with_max_age(None)) - .unwrap(); - broadcast.announce().unwrap(); - write_group(&track, 0); - write_group(&track, 1); - let _track = track.finish().unwrap(); - - let subscriber = produce_origin(2); - let mut options = MockConnectOptions::new(version.parse::().unwrap()); - options.server_publish = Some(publisher.consume()); - options.client_subscribe = Some(subscriber.clone()); - let pair = connect_mock(options).await; - - let consumer = subscriber.consume(); - let remote = kio::clock::timeout(TIMEOUT, consumer.follow("bcast", None).unwrap().available()) - .await - .expect("resolve timeout") - .expect("broadcast resolves") - .unwrap(); - - let mut got = Vec::new(); - for sequence in [1, 0] { - let read = async { - let mut group = remote.track("data").unwrap().fetch_group(sequence, None).await?; - let timestamp = group.read_frame().await?.expect("a frame").timestamp; - while group.read_frame().await?.is_some() {} - Ok(timestamp) - }; - got.push(kio::clock::timeout(TIMEOUT, read).await.expect("fetch timed out")); - // Answered later, so group 0's send time lands after group 1's. - kio::clock::sleep(Duration::from_millis(10)).await; - } - drop((pair, remote, broadcast, publisher, subscriber)); - got -} - -#[kio::test] -async fn fetches_alone_accept_descending_send_times() { - let mut failures = Vec::new(); - for version in ["moq-lite-05", "moq-lite-06"] { - let got = fetch_only(version).await; - // Both arrive timed with their send times, the later fetch's last. - if !matches!(got.as_slice(), [Ok(Some(one)), Ok(Some(zero))] if zero > one) { - failures.push(format!("{version}: {got:?}")); - } - } - assert!(failures.is_empty(), "{failures:#?}"); -} diff --git a/rs/moq-net/tests/integration/rejoin.rs b/rs/moq-net/tests/integration/rejoin.rs index 8068d12fb05..c3759826411 100644 --- a/rs/moq-net/tests/integration/rejoin.rs +++ b/rs/moq-net/tests/integration/rejoin.rs @@ -3,6 +3,8 @@ //! A reader leaving and rejoining a track relayed over the in-memory mock transport. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; @@ -74,17 +76,17 @@ async fn rejoin_recovers_the_group_reset_on_leave() { track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(10))) }; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(0), b"a0".as_ref()).unwrap(); - group.finish().unwrap(); - let mut open = track.append_group().unwrap(); + group.close().unwrap(); + let mut open = track.append_next().unwrap(); open.write_frame(ts(100), b"b0".as_ref()).unwrap(); let mut sub = remote.track("video").unwrap().subscribe(prefs()).await.unwrap(); loop { let mut group = sub.recv_group().await.unwrap().unwrap(); group.read_frame().await.unwrap().unwrap(); - if group.sequence == 1 { + if group.start == 1 { break; } } @@ -93,12 +95,12 @@ async fn rejoin_recovers_the_group_reset_on_leave() { drop(sub); crate::support::unwanted(track.demand()).await.unwrap(); open.write_frame(ts(133), b"b1".as_ref()).unwrap(); - open.finish().unwrap(); + open.close().unwrap(); if advance { - let mut live = track.append_group().unwrap(); + let mut live = track.append_next().unwrap(); live.write_frame(ts(5000), b"c0".as_ref()).unwrap(); - live.finish().unwrap(); + live.close().unwrap(); } let mut sub = remote.track("video").unwrap().subscribe(prefs()).await.unwrap(); @@ -108,8 +110,8 @@ async fn rejoin_recovers_the_group_reset_on_leave() { let wanted: &[u64] = if advance { &[1, 2] } else { &[1] }; while !wanted.iter().all(|want| rejoined.iter().any(|(seq, _)| seq == want)) { let mut group = sub.recv_group().await.unwrap().unwrap(); - let sequence = group.sequence; - rejoined.push((sequence, read_all(&mut group).await)); + let key = group.start; + rejoined.push((key, read_all(&mut group).await)); } rejoined.sort_by_key(|(seq, _)| *seq); let reset = rejoined.iter().find(|(seq, _)| *seq == 1).expect("reset group"); @@ -175,9 +177,9 @@ async fn rejoin_skips_a_cache_kept_by_another_handle() { // Held for fetches only: it keeps the relay's copy without subscribing. let _held = remote.track("video").unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(0), b"old".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(read_all(&mut group).await.unwrap(), [b"old".to_vec()], "{version}"); @@ -185,16 +187,16 @@ async fn rejoin_skips_a_cache_kept_by_another_handle() { crate::support::unwanted(track.demand()).await.unwrap(); // The publisher moves on while nobody subscribes. - for sequence in 1..=3u64 { - let mut group = track.append_group().unwrap(); - group.write_frame(ts(sequence * 1000), b"new".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 1..=3u64 { + let mut group = track.append_next().unwrap(); + group.write_frame(ts(key * 1000), b"new".as_ref()).unwrap(); + group.close().unwrap(); } let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let group = sub.recv_group().await.unwrap().unwrap(); assert_eq!( - group.sequence, 3, + group.start, 3, "{version}: the rejoining reader got the stale cache first" ); }) @@ -243,9 +245,9 @@ async fn rejoin_keeps_the_cache_when_the_backlog_lands_newest_first() { // Held for fetches only: it keeps the relay's copy without subscribing. let _held = remote.track("video").unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(0), b"0".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut sub = remote.track("video").unwrap().subscribe(prefs()).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(read_all(&mut group).await.unwrap(), [b"0".to_vec()], "{version}"); @@ -253,14 +255,17 @@ async fn rejoin_keeps_the_cache_when_the_backlog_lands_newest_first() { crate::support::unwanted(track.demand()).await.unwrap(); // The publisher moves on while nobody subscribes, all within the max age. - for sequence in 1..=3u64 { - let mut group = track.append_group().unwrap(); - group.write_frame(ts(sequence * 100), sequence.to_string()).unwrap(); - group.finish().unwrap(); + for key in 1..=3u64 { + let mut group = track.append_next().unwrap(); + group.write_frame(ts(key * 100), key.to_string()).unwrap(); + group.close().unwrap(); } pair.server_transport.hold_unis(); - let rejoin = remote.track("video").unwrap().subscribe(prefs().with_range(0..)); + let rejoin = remote + .track("video") + .unwrap() + .subscribe(prefs().with_ranges([support::keys(0..)])); let release = async { kio::clock::sleep(Duration::from_millis(100)).await; pair.server_transport.release_unis_reversed(); @@ -270,7 +275,7 @@ async fn rejoin_keeps_the_cache_when_the_backlog_lands_newest_first() { let mut rejoined = Vec::new(); // Arrival order, newest first: whatever the reader gets once the backlog settled. while let Ok(group) = kio::clock::timeout(Duration::from_secs(1), sub.recv_group()).await { - rejoined.push(group.unwrap().unwrap().sequence); + rejoined.push(group.unwrap().unwrap().start); } rejoined.sort(); assert_eq!(rejoined, [0, 1, 2, 3], "{version}: the rejoin skipped the cache"); @@ -305,8 +310,8 @@ async fn rejoin_mid_group_keeps_the_head_for_later_readers() { let track = broadcast.broadcast().create_track("video", timed()).unwrap(); broadcast.announce().unwrap(); // Left open, as a JSON snapshot group stays open for its deltas. - let mut open = track.append_group().unwrap(); - open.write_frame(Timestamp::ZERO, b"a0".as_ref()).unwrap(); + let mut open = track.append_next().unwrap(); + open.write_frame(open.at_start(), b"a0".as_ref()).unwrap(); let version: Version = version.parse().unwrap(); let mut options = MockConnectOptions::new(version); @@ -340,7 +345,7 @@ async fn rejoin_mid_group_keeps_the_head_for_later_readers() { let later = relay.consume().request("bench", None).await.unwrap(); let mut sub = later.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 0, "{version}"); + assert_eq!(group.start, 0, "{version}"); let frame = group.read_frame().await; assert!( matches!(&frame, Ok(Some(frame)) if frame.payload == b"a0".as_ref()), @@ -384,8 +389,8 @@ async fn rejoin_goes_live_without_the_join_head() { .unwrap(); let track = broadcast.broadcast().create_track("video", timed()).unwrap(); broadcast.announce().unwrap(); - let mut open = track.append_group().unwrap(); - open.write_frame(Timestamp::ZERO, b"a0".as_ref()).unwrap(); + let mut open = track.append_next().unwrap(); + open.write_frame(open.at_start(), b"a0".as_ref()).unwrap(); let version: Version = version.parse().unwrap(); let mut options = MockConnectOptions::new(version); @@ -417,7 +422,7 @@ async fn rejoin_goes_live_without_the_join_head() { let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); loop { let mut group = sub.recv_group().await.unwrap().unwrap(); - if group.sequence == 1 { + if group.start == 1 { return read_all(&mut group).await.unwrap(); } } @@ -426,15 +431,15 @@ async fn rejoin_goes_live_without_the_join_head() { kio::clock::sleep(Duration::from_millis(100)).await; upstream.server_transport.drop_unis(); - let mut next = track.append_group().unwrap(); + let mut next = track.append_next().unwrap(); next.write_frame(Timestamp::from_millis(1000).unwrap(), b"b0".as_ref()) .unwrap(); - next.finish().unwrap(); + next.close().unwrap(); let local = relay.consume().request("bench", None).await.unwrap(); let mut sub = local.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 1, "{version}"); + assert_eq!(group.start, 1, "{version}"); assert_eq!(read_all(&mut group).await.unwrap(), [b"b0".to_vec()], "{version}"); assert_eq!(rejoin.await.unwrap(), [b"b0".to_vec()], "{version}"); @@ -478,9 +483,9 @@ async fn rejoin_during_the_cancel_skips_the_cache() { .unwrap(); let ts = |ms| Timestamp::from_millis(ms).unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(0), b"old".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(read_all(&mut group).await.unwrap(), [b"old".to_vec()], "{version}"); @@ -488,17 +493,17 @@ async fn rejoin_during_the_cancel_skips_the_cache() { crate::support::unwanted(track.demand()).await.unwrap(); // The publisher moves on while the relay's cancel is still in flight. - for sequence in 1..=3u64 { - let mut group = track.append_group().unwrap(); - group.write_frame(ts(sequence * 1000), b"new".as_ref()).unwrap(); - group.finish().unwrap(); + for key in 1..=3u64 { + let mut group = track.append_next().unwrap(); + group.write_frame(ts(key * 1000), b"new".as_ref()).unwrap(); + group.close().unwrap(); } let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let group = sub.recv_group().await.unwrap().unwrap(); // A track that arrives untimed skips the cached group by the wall clock instead. assert_eq!( - group.sequence, 3, + group.start, 3, "{version}: the rejoining reader got the stale cache first" ); }) @@ -551,18 +556,18 @@ async fn rejoin_waits_for_the_answers_first_frame() { // Held for fetches only: it keeps the relay's copy without subscribing. let _held = remote.track("video").unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(0), b"old".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(read_all(&mut group).await.unwrap(), [b"old".to_vec()], "{version}"); drop((group, sub)); crate::support::unwanted(track.demand()).await.unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(500), b"new".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); pair.server_transport.split_unis(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); @@ -570,17 +575,17 @@ async fn rejoin_waits_for_the_answers_first_frame() { assert!( early.is_err(), "{version}: the rejoining reader got the cached group {:?} first", - early.map(|group| group.unwrap().unwrap().sequence) + early.map(|group| group.unwrap().unwrap().start) ); pair.server_transport.release_split(); let mut group = sub.recv_group().await.unwrap().unwrap(); // A track that arrives untimed has no media age to skip the cached group by, so // its reader starts at the newest group cached and reads on from there. - if sub.info().timescale.is_none() && group.sequence == 0 { + if sub.info().timescale.is_none() && group.start == 0 { group = sub.recv_group().await.unwrap().unwrap(); } - assert_eq!(group.sequence, 1, "{version}"); + assert_eq!(group.start, 1, "{version}"); assert_eq!(read_all(&mut group).await.unwrap(), [b"new".to_vec()], "{version}"); }) .await @@ -612,9 +617,9 @@ async fn leaving_after_the_cache_window_keeps_the_latest_group() { .unwrap(); let track = broadcast.broadcast().create_track("catalog", timed()).unwrap(); broadcast.announce().unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"c0".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"c0".as_ref()).unwrap(); + group.close().unwrap(); let mut options = MockConnectOptions::new(version); options.server_publish = Some(publisher.consume()); @@ -645,7 +650,7 @@ async fn leaving_after_the_cache_window_keeps_the_latest_group() { // Holds the relay's upstream subscription, and its copy, open throughout. let (holder, _holder) = session(3).await; let mut held = holder.track("catalog").unwrap().subscribe(None).await.unwrap(); - assert_eq!(held.recv_group().await.unwrap().unwrap().sequence, 0, "{version}"); + assert_eq!(held.recv_group().await.unwrap().unwrap().start, 0, "{version}"); let (leaver, leaver_session) = session(4).await; let mut sub = leaver.track("catalog").unwrap().subscribe(None).await.unwrap(); @@ -668,7 +673,7 @@ async fn leaving_after_the_cache_window_keeps_the_latest_group() { .unwrap_or_else(|_| panic!("{version}: reader {hop} never got the latest group")) .unwrap() .unwrap(); - assert_eq!(group.sequence, 0, "{version}"); + assert_eq!(group.start, 0, "{version}"); assert_eq!(read_all(&mut group).await.unwrap(), [b"c0"], "{version}"); } }) @@ -711,24 +716,24 @@ async fn rejoin_keeps_source_timestamps() { // Far from any arrival time, so a frame stamped on arrival cannot pass. let ts = |ms| Timestamp::from_millis(ms).unwrap(); - for (sequence, ms) in [(0, 123_456), (1, 654_321)] { + for (key, ms) in [(0, 123_456), (1, 654_321)] { let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); crate::support::wanted(track.demand()).await.unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); group.write_frame(ts(ms), b"frame".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut group = loop { let group = sub.recv_group().await.unwrap().unwrap(); - if group.sequence == sequence { + if group.start == key { break group; } }; let frame = group.read_frame().await.unwrap().unwrap(); assert_eq!( - frame.timestamp.map(Timestamp::as_millis), - Some(u128::from(ms)), - "{version} group {sequence}" + frame.timestamp.as_millis().unwrap(), + u128::from(ms), + "{version} group {key}" ); drop((group, sub)); crate::support::unwanted(track.demand()).await.unwrap(); @@ -786,9 +791,9 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { .unwrap(); let track = broadcast.broadcast().create_track("catalog", timed()).unwrap(); broadcast.announce().unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"catalog".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"catalog".as_ref()).unwrap(); + group.close().unwrap(); let upstream = |cost| { let mut options = MockConnectOptions::new(version); @@ -825,7 +830,7 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { // The publisher reconnects while the holder reads, and the old connection goes: // the relay takes the track over from the new one. let (mut holder, remote, pair) = session(3).await; - assert_eq!(holder.recv_group().await.unwrap().unwrap().sequence, 0, "{version}"); + assert_eq!(holder.recv_group().await.unwrap().unwrap().start, 0, "{version}"); // The new connection is cheaper, so the relay reports its route taking over. let mut follow = relay.consume().routes(); assert!( @@ -855,11 +860,11 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { } else { // A reader leaves, so the relay parks the track; the holder re-splices it. let (mut first, remote, pair) = session(3).await; - assert_eq!(first.recv_group().await.unwrap().unwrap().sequence, 0, "{version}"); + assert_eq!(first.recv_group().await.unwrap().unwrap().start, 0, "{version}"); drop((first, remote, pair)); crate::support::unwanted(track.demand()).await.unwrap(); let (mut holder, remote, pair) = session(4).await; - assert_eq!(holder.recv_group().await.unwrap().unwrap().sequence, 0, "{version}"); + assert_eq!(holder.recv_group().await.unwrap().unwrap().start, 0, "{version}"); (holder, remote, pair) }; @@ -874,14 +879,14 @@ async fn quiet_catalog_survives_cache_expiry(version: Version, takeover: bool) { drop(group); // A newer catalog supersedes it, so the old one expires like any other group. - let mut newer = track.append_group().unwrap(); + let mut newer = track.append_next().unwrap(); newer .write_frame(Timestamp::from_millis(1000).unwrap(), b"newer".as_ref()) .unwrap(); - newer.finish().unwrap(); + newer.close().unwrap(); // Unnamed wires end the old holder on takeover. The late reader uses the // current route on every version, and keeps that copy live for the expiry check. - assert_eq!(later.recv_group().await.unwrap().unwrap().sequence, 1, "{version}"); + assert_eq!(later.recv_group().await.unwrap().unwrap().start, 1, "{version}"); kio::clock::sleep(moq_net::cache::DEFAULT_EXPIRY * 2 + Duration::from_secs(2)).await; let old = async { let mut group = later_remote.track("catalog").unwrap().fetch_group(0, None).await?; diff --git a/rs/moq-net/tests/integration/relay_upstream_close.rs b/rs/moq-net/tests/integration/relay_upstream_close.rs index 1bf96c1d8c0..895718ad058 100644 --- a/rs/moq-net/tests/integration/relay_upstream_close.rs +++ b/rs/moq-net/tests/integration/relay_upstream_close.rs @@ -15,6 +15,7 @@ use crate::support; use moq_net::track::Delay; +use support::AppendNext; use std::time::Duration; @@ -80,7 +81,7 @@ async fn read(mut sub: moq_net::track::Subscriber) -> Outcome { match group.read_frame().await { Ok(Some(frame)) => { let index: u64 = std::str::from_utf8(&frame.payload).unwrap().parse().unwrap(); - got.push((group.sequence, index)); + got.push((group.start, index)); } Ok(None) => break, Err(err) => return (got, Err(err)), @@ -131,7 +132,7 @@ async fn round(name: &str, end: End) -> Vec { for (index, track) in tracks.iter().enumerate() { let subscription = Subscription::default() .with_delay(Delay::default().with_max(MAX_DELAY)) - .with_range(0..); + .with_ranges([support::keys(0..)]); let sub = remote.track(&format!("track{index}")).unwrap().subscribe(subscription); let sub = kio::clock::timeout(TIMEOUT, sub) .await @@ -147,13 +148,13 @@ async fn round(name: &str, end: End) -> Vec { // The subscriber now reads nothing until the upstream session is gone. downstream.server_transport.hold_unis(); for track in &tracks { - for sequence in 0..GROUPS { - let mut group = track.append_group().unwrap(); + for key in 0..GROUPS { + let mut group = track.append_next().unwrap(); for index in 0..FRAMES { - let at = Timestamp::from_millis(sequence * FRAMES + index).unwrap(); + let at = Timestamp::from_millis(key * FRAMES + index).unwrap(); group.write_frame(at, index.to_string().into_bytes()).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } let _finished = track.clone().finish().unwrap(); } diff --git a/rs/moq-net/tests/integration/replacement.rs b/rs/moq-net/tests/integration/replacement.rs index 3cf56444a66..9a5c34436cc 100644 --- a/rs/moq-net/tests/integration/replacement.rs +++ b/rs/moq-net/tests/integration/replacement.rs @@ -11,10 +11,12 @@ //! from then on, so its holder follows the `Restart` instead. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Error, Instance, Timestamp, Version, broadcast, origin, track}; +use moq_net::{Error, Instance, Version, broadcast, origin, track}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -48,9 +50,9 @@ async fn next_event(announced: &mut origin::Routes) -> origin::RouteEvent { /// Write one group of one frame tagged `tag`. fn write(track: &track::Producer, tag: &str) { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, tag.as_bytes().to_vec()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), tag.as_bytes().to_vec()).unwrap(); + group.close().unwrap(); } /// The next frame a subscription delivers, as text. diff --git a/rs/moq-net/tests/integration/route_change.rs b/rs/moq-net/tests/integration/route_change.rs index 26b9053b214..23df5a1a333 100644 --- a/rs/moq-net/tests/integration/route_change.rs +++ b/rs/moq-net/tests/integration/route_change.rs @@ -17,6 +17,8 @@ //! same way. use crate::support; +use support::AppendNext; +use support::AtStart; use std::collections::BTreeMap; use std::time::Duration; @@ -192,7 +194,7 @@ impl Topology { } /// A frame as the reader saw it: its group, then its timestamp and payload. -pub(super) type Delivery = (u64, moq_net::Result<(Option, Vec)>); +pub(super) type Delivery = (u64, moq_net::Result<(Timestamp, Vec)>); /// Read every group in full, reporting each frame as it arrives. pub(super) fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { @@ -211,13 +213,13 @@ pub(super) fn read(mut sub: track::Subscriber) -> mpsc::UnboundedReceiver { let frame = (frame.timestamp, frame.payload.to_vec()); - if tx.unbounded_send((group.sequence, Ok(frame))).is_err() { + if tx.unbounded_send((group.start, Ok(frame))).is_err() { return; } } Ok(None) => break, Err(err) => { - let _ = tx.unbounded_send((group.sequence, Err(err))); + let _ = tx.unbounded_send((group.start, Err(err))); break; } } @@ -234,7 +236,7 @@ async fn next(rx: &mut mpsc::UnboundedReceiver) -> (u64, Vec) { } /// [`next`], keeping the frame's timestamp. -async fn next_timed(rx: &mut mpsc::UnboundedReceiver) -> (u64, Option, Vec) { +async fn next_timed(rx: &mut mpsc::UnboundedReceiver) -> (u64, Timestamp, Vec) { let (group, frame) = kio::clock::timeout(Duration::from_secs(10), rx.next()) .await .expect("reader hung") @@ -253,13 +255,13 @@ async fn route_change(version: &str, trigger: Trigger, position: Position) { let mut seen = Vec::new(); // Group 0 in full through `A`. - let mut group = topology.track.append_group().unwrap(); + let mut group = topology.track.append_next().unwrap(); for frame in 0..FRAMES { - group.write_frame(Timestamp::ZERO, payload(0, frame)).unwrap(); + group.write_frame(group.at_start(), payload(0, frame)).unwrap(); expected.push((0, payload(0, frame))); seen.push(next(&mut rx).await); } - group.finish().unwrap(); + group.close().unwrap(); // Group 1: half through `A`, then the route changes. Between groups, the change // lands before the group starts at all. @@ -270,25 +272,25 @@ async fn route_change(version: &str, trigger: Trigger, position: Position) { if split == 0 { topology.trigger(trigger).await; } - let mut group = topology.track.append_group().unwrap(); + let mut group = topology.track.append_next().unwrap(); for frame in 0..FRAMES { if frame == split && split != 0 { topology.trigger(trigger).await; } - group.write_frame(Timestamp::ZERO, payload(1, frame)).unwrap(); + group.write_frame(group.at_start(), payload(1, frame)).unwrap(); expected.push((1, payload(1, frame))); seen.push(next(&mut rx).await); } - group.finish().unwrap(); + group.close().unwrap(); // Group 2 entirely through the replacement. - let mut group = topology.track.append_group().unwrap(); + let mut group = topology.track.append_next().unwrap(); for frame in 0..FRAMES { - group.write_frame(Timestamp::ZERO, payload(2, frame)).unwrap(); + group.write_frame(group.at_start(), payload(2, frame)).unwrap(); expected.push((2, payload(2, frame))); seen.push(next(&mut rx).await); } - group.finish().unwrap(); + group.close().unwrap(); let render = |frames: &[(u64, Vec)]| { frames @@ -319,18 +321,18 @@ async fn route_flaps(version: &str) { let triggers = [Trigger::BetterRoute, Trigger::BetterRouteDies, Trigger::Disconnect]; let mut expected = Vec::new(); let mut seen = Vec::new(); - let mut group = topology.track.append_group().unwrap(); + let mut group = topology.track.append_next().unwrap(); for frame in 0..FRAMES * (triggers.len() as u64 + 1) { if frame % FRAMES == FRAMES / 2 && let Some(trigger) = triggers.get((frame / FRAMES) as usize) { topology.trigger(*trigger).await; } - group.write_frame(Timestamp::ZERO, payload(0, frame)).unwrap(); + group.write_frame(group.at_start(), payload(0, frame)).unwrap(); expected.push((0, payload(0, frame))); seen.push(next(&mut rx).await); } - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(seen, expected, "{version}"); settle().await; @@ -346,16 +348,16 @@ async fn route_dies_without_instances(version: &str, trigger: Trigger) { let mut rx = read(sub); topology.standby().await; - let mut group = topology.track.append_group().unwrap(); + let mut group = topology.track.append_next().unwrap(); for frame in 0..FRAMES { - group.write_frame(Timestamp::ZERO, payload(0, frame)).unwrap(); + group.write_frame(group.at_start(), payload(0, frame)).unwrap(); assert_eq!(next(&mut rx).await, (0, payload(0, frame)), "{version}"); } - group.finish().unwrap(); + group.close().unwrap(); topology.trigger(trigger).await; - let mut group = topology.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload(1, 0)).unwrap(); + let mut group = topology.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload(1, 0)).unwrap(); settle().await; match kio::clock::timeout(Duration::from_secs(10), rx.next()) .await @@ -377,10 +379,10 @@ async fn route_dies_without_instances(version: &str, trigger: Trigger) { .expect("B serves a re-request"); let sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); let mut rx = read(sub); - group.write_frame(Timestamp::ZERO, payload(1, 1)).unwrap(); - group.finish().unwrap(); - let mut group = topology.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload(2, 0)).unwrap(); + group.write_frame(group.at_start(), payload(1, 1)).unwrap(); + group.close().unwrap(); + let mut group = topology.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload(2, 0)).unwrap(); while next(&mut rx).await != (2, payload(2, 0)) {} } @@ -435,9 +437,9 @@ async fn route_dies_across_instances(version: &str, trigger: Trigger) { let _b_to_r = link(version, &relay_b, &standby).await; settle().await; - let mut group = track_a.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, tagged("A", 0, 0)).unwrap(); - group.finish().unwrap(); + let mut group = track_a.append_next().unwrap(); + group.write_frame(group.at_start(), tagged("A", 0, 0)).unwrap(); + group.close().unwrap(); assert_eq!(next(&mut rx).await, (0, tagged("A", 0, 0)), "{version}"); match trigger { @@ -504,9 +506,9 @@ async fn route_dies_across_instances(version: &str, trigger: Trigger) { assert!(!second.is_clone(&first), "{version}: joined the replaced broadcast"); let sub = second.track("video").unwrap().subscribe(None).await.unwrap(); let mut rx = read(sub); - let mut group = track_b.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, tagged("B", 0, 0)).unwrap(); - group.finish().unwrap(); + let mut group = track_b.append_next().unwrap(); + group.write_frame(group.at_start(), tagged("B", 0, 0)).unwrap(); + group.close().unwrap(); assert_eq!(next(&mut rx).await, (0, tagged("B", 0, 0)), "{version} {trigger:?}"); } @@ -527,8 +529,8 @@ async fn better_route_restarts(version: &str) { other => panic!("{version}: expected the route, got {other:?}"), } - let mut group = topology.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload(0, 0)).unwrap(); + let mut group = topology.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload(0, 0)).unwrap(); assert_eq!(next(&mut rx).await, (0, payload(0, 0)), "{version}"); let incumbent = consumer.request("live", None).await.unwrap(); @@ -542,7 +544,7 @@ async fn better_route_restarts(version: &str) { } // The incumbent's subscription carries on through `A`. - group.write_frame(Timestamp::ZERO, payload(0, 1)).unwrap(); + group.write_frame(group.at_start(), payload(0, 1)).unwrap(); assert_eq!(next(&mut rx).await, (0, payload(0, 1)), "{version}"); // A re-request resolves the better route on a fresh broadcast. @@ -550,9 +552,9 @@ async fn better_route_restarts(version: &str) { assert!(!replaced.is_clone(&incumbent), "{version}: joined the replaced front"); let sub = replaced.track("video").unwrap().subscribe(None).await.unwrap(); let mut fresh = read(sub); - group.finish().unwrap(); - let mut group = topology.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload(1, 0)).unwrap(); + group.close().unwrap(); + let mut group = topology.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload(1, 0)).unwrap(); while next(&mut fresh).await != (1, payload(1, 0)) {} assert_eq!( next(&mut rx).await, @@ -710,9 +712,9 @@ async fn lagging_route_dies(version: Version) { track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(60))); remote.track("video").unwrap().subscribe(preferences).await.unwrap() } - fn write(group: &mut moq_net::group::Producer, sequence: u64, frame: u64) { - let timestamp = Timestamp::from_micros(1_000_000 + sequence * 100_000 + frame * 1_000).unwrap(); - group.write_frame(timestamp, payload(sequence, frame)).unwrap(); + fn write(group: &mut moq_net::group::Producer, key: u64, frame: u64) { + let timestamp = Timestamp::from_micros(1_000_000 + key * 100_000 + frame * 1_000).unwrap(); + group.write_frame(timestamp, payload(key, frame)).unwrap(); } let (p, a, b, r, w) = ( @@ -734,16 +736,16 @@ async fn lagging_route_dies(version: Version) { let b_r = lagged(version, &b, &r, Duration::ZERO).await; settle().await; - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); for frame in 0..4 { write(&mut group, 0, frame); } - group.finish().unwrap(); + group.close().unwrap(); for frame in 0..4 { assert_eq!(next(&mut rx).await, (0, payload(0, frame)), "{version}"); } - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); write(&mut group, 1, 0); write(&mut group, 1, 1); assert_eq!(next(&mut rx).await, (1, payload(1, 0)), "{version}"); @@ -751,24 +753,24 @@ async fn lagging_route_dies(version: Version) { // P moves on while A is still behind. write(&mut group, 1, 2); write(&mut group, 1, 3); - group.finish().unwrap(); - let mut group = track.append_group().unwrap(); + group.close().unwrap(); + let mut group = track.append_next().unwrap(); for frame in 0..FRAMES { write(&mut group, 2, frame); } - let group = group.finish().unwrap(); + let group = group.close().unwrap(); a_r.server.abort(Error::Cancel); a_r.client.abort(Error::Cancel); settle().await; - for (sequence, frame) in (2..FRAMES) + for (key, frame) in (2..FRAMES) .map(|frame| (1, frame)) .chain((0..FRAMES).map(|frame| (2, frame))) { - let timestamp = Timestamp::from_millis(1_000 + sequence * 100 + frame).unwrap(); + let timestamp = Timestamp::from_millis(1_000 + key * 100 + frame).unwrap(); assert_eq!( next_timed(&mut rx).await, - (sequence, Some(timestamp), payload(sequence, frame)), + (key, timestamp, payload(key, frame)), "{version}" ); } @@ -831,27 +833,28 @@ async fn undetected_outage(budget: Duration) -> BTreeMap> { settle().await; let mut after = BTreeMap::>::new(); - for sequence in 0..=E_NOTICES + 3 { - let mut group = track.append_group().unwrap(); + for key in 0..=E_NOTICES + 3 { + // Each group starts at its first frame, a second after the last. + let mut group = track.append_group(key * 1_000).unwrap(); for frame in 0..FRAMES { - let timestamp = Timestamp::from_millis(sequence * 1_000 + frame * STEP.as_millis() as u64).unwrap(); - group.write_frame(timestamp, payload(sequence, frame)).unwrap(); - if sequence < KILLED || (sequence, frame) == (KILLED, 0) { - assert_eq!(next(&mut rx).await, (sequence, payload(sequence, frame))); + let timestamp = Timestamp::from_millis(key * 1_000 + frame * STEP.as_millis() as u64).unwrap(); + group.write_frame(timestamp, payload(key, frame)).unwrap(); + if key < KILLED || (key, frame) == (KILLED, 0) { + assert_eq!(next(&mut rx).await, (key * 1_000, payload(key, frame))); } - if (sequence, frame) == (KILLED, 0) { + if (key, frame) == (KILLED, 0) { for pair in [&o_m1, &m1_e] { pair.client_transport.set_latency(Duration::from_secs(3600)); } } kio::clock::sleep(STEP).await; } - group.finish().unwrap(); - if sequence == O_NOTICES { + group.close().unwrap(); + if key == O_NOTICES { o_m1.server.abort(Error::Cancel); o_m1.client.abort(Error::Cancel); } - if sequence == E_NOTICES { + if key == E_NOTICES { // Off the group boundary, so no group's age sits on the budget. kio::clock::sleep(STEP * 5 / 2).await; // The groups `O` asks for again reach it newest first, as a congested `P` sends @@ -865,7 +868,7 @@ async fn undetected_outage(budget: Duration) -> BTreeMap> { while let Ok((group, frame)) = rx.try_recv() { let (_, payload) = frame.unwrap_or_else(|err| panic!("group {group} failed: {err}")); after - .entry(group) + .entry(group / 1_000) .or_default() .push(String::from_utf8(payload).unwrap()); } @@ -971,16 +974,16 @@ async fn redundant_pair_follows_the_next_instance(version: &str, loss: Loss) { let first = follow.available().await.unwrap().unwrap(); let prefs = track::Subscription::default().with_delay(track::Delay::default().with_max(Duration::from_secs(60))); let mut rx = read(first.track("video").unwrap().subscribe(prefs.clone()).await.unwrap()); - let write = |replicas: &mut Vec, sequence: u64| { + let write = |replicas: &mut Vec, key: u64| { for replica in replicas.iter_mut() { - let mut group = replica.track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = replica.track.create_group(moq_net::group::Info { start: key }).unwrap(); group .write_frame( - Timestamp::from_micros(1_000_000 + sequence * 1_000).unwrap(), - tagged(replica.name, sequence, 0), + Timestamp::from_micros(1_000_000 + key * 1_000).unwrap(), + tagged(replica.name, key, 0), ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } }; write(&mut replicas, 0); @@ -1094,12 +1097,12 @@ async fn replaced_behind_a_relay(version: &str) { let mut rx = read(remote.track("video").unwrap().subscribe(prefs).await.unwrap()); let mut dead = None; - for sequence in 0..3 { + for key in 0..3 { for replica in &mut replicas { - replica.group = Some(replica.track.append_group().unwrap()); + replica.group = Some(replica.track.append_next().unwrap()); } for frame in 0..FRAMES { - if sequence == 1 && frame == FRAMES / 2 { + if key == 1 && frame == FRAMES / 2 { let replica = replicas.remove(0); let link = replica.link.as_ref().unwrap(); link.server.abort(Error::Cancel); @@ -1107,20 +1110,18 @@ async fn replaced_behind_a_relay(version: &str) { dead = Some(replica); settle().await; } - let timestamp = Timestamp::from_micros(1_000_000 + sequence * 100_000 + frame * 1_000).unwrap(); + let timestamp = Timestamp::from_micros(1_000_000 + key * 100_000 + frame * 1_000).unwrap(); for replica in &mut replicas { let group = replica.group.as_mut().unwrap(); - group - .write_frame(timestamp, tagged(replica.name, sequence, frame)) - .unwrap(); + group.write_frame(timestamp, tagged(replica.name, key, frame)).unwrap(); } if dead.is_none() { - let expected = (sequence, tagged(replicas[0].name, sequence, frame)); + let expected = (key, tagged(replicas[0].name, key, frame)); assert_eq!(next(&mut rx).await, expected, "{version}"); } } for replica in &mut replicas { - replica.group.take().unwrap().finish().unwrap(); + replica.group.take().unwrap().close().unwrap(); } } settle().await; @@ -1209,8 +1210,8 @@ async fn compression_changes_behind_a_relay(compress: bool) { let mut rx = read(remote.track("video").unwrap().subscribe(prefs).await.unwrap()); let mut dead = None; - for sequence in 0..3 { - if sequence == 1 { + for key in 0..3 { + if key == 1 { let replica = replicas.remove(0); let link = replica.link.as_ref().unwrap(); link.server.abort(Error::Cancel); @@ -1219,18 +1220,16 @@ async fn compression_changes_behind_a_relay(compress: bool) { settle().await; } for replica in &mut replicas { - let mut group = replica.track.append_group().unwrap(); + let mut group = replica.track.append_next().unwrap(); for frame in 0..FRAMES { - let timestamp = Timestamp::from_micros(1_000_000 + sequence * 100_000 + frame * 1_000).unwrap(); - group - .write_frame(timestamp, tagged(replica.name, sequence, frame)) - .unwrap(); + let timestamp = Timestamp::from_micros(1_000_000 + key * 100_000 + frame * 1_000).unwrap(); + group.write_frame(timestamp, tagged(replica.name, key, frame)).unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); } if compress || dead.is_none() { for frame in 0..FRAMES { - let expected = (sequence, tagged(replicas[0].name, sequence, frame)); + let expected = (key, tagged(replicas[0].name, key, frame)); assert_eq!(next(&mut rx).await, expected, "compress {compress}"); } } @@ -1268,7 +1267,7 @@ async fn different_compression_is_refused_behind_a_relay() { /// `P` serves `video` on demand and replaces its producer partway through the broadcast, /// the way a publisher restarts an encoder. The instance lets `P`'s own front resume the /// logical track onto the replacement, so `A` and `R` never see the abort. The -/// replacement continues the track's sequences explicitly, so its first group reaches +/// replacement continues the track's keys explicitly, so its first group reaches /// `R` at once instead of being skipped as one `R` already read. async fn producer_replaced(version: &str) { let version: Version = version.parse().unwrap(); @@ -1306,14 +1305,14 @@ async fn producer_replaced(version: &str) { let mut rx = read(sub); let first = producers.next().await.unwrap(); - for sequence in 0..2 { - let mut group = first.append_group().unwrap(); - assert_eq!(group.sequence, sequence, "{version}"); + for key in 0..2 { + let mut group = first.append_next().unwrap(); + assert_eq!(group.start, key, "{version}"); for frame in 0..FRAMES { - group.write_frame(Timestamp::ZERO, payload(sequence, frame)).unwrap(); - assert_eq!(next(&mut rx).await, (sequence, payload(sequence, frame)), "{version}"); + group.write_frame(group.at_start(), payload(key, frame)).unwrap(); + assert_eq!(next(&mut rx).await, (key, payload(key, frame)), "{version}"); } - group.finish().unwrap(); + group.close().unwrap(); } // The encoder restarts: the old producer goes away mid-broadcast, and the resumed @@ -1325,12 +1324,12 @@ async fn producer_replaced(version: &str) { .unwrap(); // The application owns where the replacement starts, so it continues explicitly. - let mut group = second.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); + let mut group = second.create_group(moq_net::group::Info { start: 2 }).unwrap(); for frame in 0..FRAMES { - group.write_frame(Timestamp::ZERO, payload(2, frame)).unwrap(); + group.write_frame(group.at_start(), payload(2, frame)).unwrap(); assert_eq!(next(&mut rx).await, (2, payload(2, frame)), "{version}"); } - group.finish().unwrap(); + group.close().unwrap(); settle().await; assert!( diff --git a/rs/moq-net/tests/integration/route_health.rs b/rs/moq-net/tests/integration/route_health.rs index e573ab5e8b9..c1039dcc981 100644 --- a/rs/moq-net/tests/integration/route_health.rs +++ b/rs/moq-net/tests/integration/route_health.rs @@ -12,6 +12,7 @@ use super::route_change::{Delivery, link, payload, priced_link, produce_origin, use std::collections::BTreeSet; use std::time::Duration; +use crate::support::AppendNext; use futures::channel::mpsc; use moq_net::{Error, StreamError, Timestamp, Version, origin, track}; @@ -30,14 +31,15 @@ const IDLE_TIMEOUT: Duration = Duration::from_secs(10); const KILLED: u64 = 4; const KILLED_AT: u64 = 3; -/// Collect the frames the reader has delivered so far. A group that fails, such as -/// one cut as too old, just stops contributing frames. +/// Collect the frames the reader has delivered so far, by the group's index and the +/// frame's, as each payload names them. A group that fails, such as one cut as too old, +/// just stops contributing frames. fn drain(rx: &mut mpsc::UnboundedReceiver, into: &mut BTreeSet<(u64, u64)>) { - while let Ok((group, frame)) = rx.try_recv() { + while let Ok((_, frame)) = rx.try_recv() { let Ok((_, payload)) = frame else { continue }; let payload = String::from_utf8(payload).unwrap(); - let (_, frame) = payload.split_once('.').unwrap(); - into.insert((group, frame.parse().unwrap())); + let (group, frame) = payload.split_once('.').unwrap(); + into.insert((group.parse().unwrap(), frame.parse().unwrap())); } } @@ -83,11 +85,12 @@ async fn silent_kill(budget: Duration, detect: Duration) -> Vec { let mut seen = BTreeSet::new(); let mut tick = 0; - for sequence in 0..groups { - let mut group = track.append_group().unwrap(); + for key in 0..groups { + // Each group starts at its first frame's time. + let mut group = track.append_group(tick * STEP.as_millis() as u64).unwrap(); for frame in 0..FRAMES { let timestamp = Timestamp::from_millis(tick * STEP.as_millis() as u64).unwrap(); - group.write_frame(timestamp, payload(sequence, frame)).unwrap(); + group.write_frame(timestamp, payload(key, frame)).unwrap(); kio::clock::sleep(STEP).await; tick += 1; if tick == kill { @@ -106,7 +109,7 @@ async fn silent_kill(budget: Duration, detect: Duration) -> Vec { m1_e.client.abort(Error::Cancel); } } - group.finish().unwrap(); + group.close().unwrap(); } settle().await; drain(&mut rx, &mut seen); @@ -115,7 +118,7 @@ async fn silent_kill(budget: Duration, detect: Duration) -> Vec { // The rest of the killed group is left out: whether `E` keeps reading it depends on // whether `O`'s copy went idle before the resume, not on the detection delay. (KILLED + 1..groups) - .filter(|sequence| (0..FRAMES).any(|frame| !seen.contains(&(*sequence, frame)))) + .filter(|key| (0..FRAMES).any(|frame| !seen.contains(&(*key, frame)))) .collect() } @@ -226,15 +229,15 @@ async fn payload_stall(stall: Stall, span: Duration) -> (Observed, Observed) { let mut tick = 0u64; let mut write = async |groups: u64| { for _ in 0..groups { - let sequence = tick / FRAMES; - let mut group = track.append_group().unwrap(); + let key = tick / FRAMES; + let mut group = track.append_next().unwrap(); for frame in 0..FRAMES { let timestamp = Timestamp::from_millis(tick * STEP.as_millis() as u64).unwrap(); - group.write_frame(timestamp, payload(sequence, frame)).unwrap(); + group.write_frame(timestamp, payload(key, frame)).unwrap(); kio::clock::sleep(STEP).await; tick += 1; } - group.finish().unwrap(); + group.close().unwrap(); } }; write(2).await; diff --git a/rs/moq-net/tests/integration/segmented_frame.rs b/rs/moq-net/tests/integration/segmented_frame.rs index ff855daa112..971dd6741c9 100644 --- a/rs/moq-net/tests/integration/segmented_frame.rs +++ b/rs/moq-net/tests/integration/segmented_frame.rs @@ -8,6 +8,7 @@ use std::time::Duration; use bytes::Bytes; use moq_net::{Timestamp, Version, frame, origin, track}; +use crate::support::AppendNext; use crate::support::harness::{MockConnectOptions, connect_mock, spawn, subscribed}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -51,12 +52,12 @@ async fn a_segmented_frame_crosses_whole() { subscribed(&track).await; // Two frames: one read whole, one streamed. - let mut group = track.append_group().unwrap(); + let mut group = track.append_next().unwrap(); for _ in 0..2 { let mut frame = group .create_frame(frame::Info { size: SIZE as u64, - timestamp: Some(Timestamp::ZERO), + timestamp: Timestamp::ZERO, }) .unwrap(); for chunk in payload.chunks(300_000) { @@ -64,7 +65,7 @@ async fn a_segmented_frame_crosses_whole() { } group = frame.finish().unwrap(); } - group.finish().unwrap(); + group.close().unwrap(); let mut group = kio::clock::timeout(TIMEOUT, sub.recv_group()) .await diff --git a/rs/moq-net/tests/integration/session_close.rs b/rs/moq-net/tests/integration/session_close.rs index 617e527e1d2..a1f02df019d 100644 --- a/rs/moq-net/tests/integration/session_close.rs +++ b/rs/moq-net/tests/integration/session_close.rs @@ -6,10 +6,12 @@ //! Deterministic: simulated time over the mock transport. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Error, Timestamp, Version}; +use moq_net::{Error, Version}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -152,9 +154,9 @@ async fn close_delivers_a_finished_track_for(version: &str) { pair, track, reader, .. } = setup(version).await; - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"last".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"last".as_slice()).unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let started = kio::clock::now(); @@ -296,9 +298,9 @@ async fn close_tail_for(version: &str) { } = setup(version).await; pair.client_transport.ack_fins(); pair.client_transport.hold_unis(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"last".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"last".as_slice()).unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let close = kio::clock::sim::spawn(pair.client.close()); kio::clock::sleep(Duration::from_millis(100)).await; @@ -334,9 +336,9 @@ async fn close_requires_subscriber_fin_for(version: &str) { } = setup(version).await; pair.client_transport.ack_fins(); pair.server_transport.withhold_bidi_fins(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"last".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"last".as_slice()).unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let started = kio::clock::now(); let res = pair.client.close().await; @@ -367,9 +369,9 @@ async fn peer_close_reads_buffered_tail(version: &str) { // Delivery to the transport acknowledges the FIN before the receiving // protocol driver has consumed the buffered stream. pair.client_transport.ack_fins(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"last".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"last".as_slice()).unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); pair.client.close().await.expect("released draft still drains on ACKs"); let (got, _) = reader.await.unwrap(); @@ -393,9 +395,9 @@ async fn peer_close_before_receiver_poll_keeps_buffered_tail() { } = setup_with_gate(version, Some(gate.consume())).await; *gate.write().ok().expect("receiver gate remains open") = false; pair.client_transport.ack_fins(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"last".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"last".as_slice()).unwrap(); + group.close().unwrap(); let _track = track.finish().unwrap(); let res = pair.client.close().await; // From lite-05 the close waits for the subscriber's FIN, which the gate holds back. diff --git a/rs/moq-net/tests/integration/settled_end_survives_abort.rs b/rs/moq-net/tests/integration/settled_end_survives_abort.rs index 4a53b6acb9b..af5063ee4d3 100644 --- a/rs/moq-net/tests/integration/settled_end_survives_abort.rs +++ b/rs/moq-net/tests/integration/settled_end_survives_abort.rs @@ -6,8 +6,10 @@ //! consumer that has not read them yet still gets them, then the clean end. use crate::support; +use support::AppendNext; +use support::AtStart; -use moq_net::{Error, Timestamp}; +use moq_net::Error; /// A millisecond track with no publisher max age: the frames written here carry timestamps, /// and superseded groups stay readable. @@ -27,8 +29,8 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { const GROUPS: u64 = 2; /// Publish `GROUPS` finished groups, declare the end at `GROUPS`, then abort the track. -/// Returns the sequences a consumer that starts reading only now receives, in arrival -/// or in sequence order, and how the read ended. +/// Returns the keys a consumer that starts reading only now receives, in arrival +/// or in key order, and how the read ended. async fn round(abort: bool, ordered: bool) -> (Vec, Option) { let origin = produce_origin(1); let broadcast = origin.route("bcast", moq_net::origin::RouteOptions::hidden()).unwrap(); @@ -36,14 +38,14 @@ async fn round(abort: bool, ordered: bool) -> (Vec, Option) { // From the first group with a replay window: a late reader is owed the whole track, // not the live edge. let subscription = moq_net::track::Subscription::default() - .with_range(0..) + .with_ranges([support::keys(0..)]) .with_delay(moq_net::track::Delay::default().with_max(std::time::Duration::from_secs(30))); let mut consumer = track.subscribe(subscription); for _ in 0..GROUPS { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"frame".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"frame".as_slice()).unwrap(); + group.close().unwrap(); } track.finish_at(moq_net::track::Position::group(GROUPS), None).unwrap(); // Every group below the end is finished: the end has settled and the track is @@ -57,7 +59,7 @@ async fn round(abort: bool, ordered: bool) -> (Vec, Option) { let mut consumer = consumer.ordered(); loop { match consumer.next_group().await { - Ok(Some(group)) => got.push(group.sequence), + Ok(Some(group)) => got.push(group.start), Ok(None) => return (got, None), Err(err) => return (got, Some(err)), } @@ -65,7 +67,7 @@ async fn round(abort: bool, ordered: bool) -> (Vec, Option) { } loop { match consumer.recv_group().await { - Ok(Some(group)) => got.push(group.sequence), + Ok(Some(group)) => got.push(group.start), Ok(None) => return (got, None), Err(err) => return (got, Some(err)), } @@ -86,15 +88,15 @@ async fn an_abort_after_the_end_settled_keeps_the_groups_for_a_late_reader() { assert_whole(round(true, false).await, "arrival order, aborted after the end settled"); } -/// The same in sequence order: the ordered cursor ends clean too, not with the abort. +/// The same in key order: the ordered cursor ends clean too, not with the abort. #[kio::test] async fn an_abort_after_the_end_settled_keeps_the_groups_for_a_late_ordered_reader() { - assert_whole(round(true, true).await, "sequence order, aborted after the end settled"); + assert_whole(round(true, true).await, "key order, aborted after the end settled"); } /// Control: with no abort the same late reader gets every group, then the clean end. #[kio::test] async fn a_settled_track_delivers_every_group_to_a_late_reader() { assert_whole(round(false, false).await, "arrival order, no abort"); - assert_whole(round(false, true).await, "sequence order, no abort"); + assert_whole(round(false, true).await, "key order, no abort"); } diff --git a/rs/moq-net/tests/integration/slow_joiner.rs b/rs/moq-net/tests/integration/slow_joiner.rs index 4c28c27812d..e660527eb4e 100644 --- a/rs/moq-net/tests/integration/slow_joiner.rs +++ b/rs/moq-net/tests/integration/slow_joiner.rs @@ -68,12 +68,12 @@ async fn stalled_read(version: Version, budget: Duration) -> BTreeSet { kio::clock::timeout(TIMEOUT, subscribed(&source)).await.unwrap(); *gate.write().ok().unwrap() = false; - for sequence in 0..GROUPS { - let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in 0..GROUPS { + let mut group = source.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key * 1000).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); kio::clock::sleep(Duration::from_secs(1)).await; } let _source = source.finish().unwrap(); @@ -86,13 +86,13 @@ async fn stalled_read(version: Version, budget: Duration) -> BTreeSet { .unwrap_or_else(|_| panic!("{version}: the track never finished")) .unwrap() { - let sequence = group.sequence; + let key = group.start; let frame = group.read_frame().await; assert!( frame.is_ok_and(|frame| frame.is_some()), - "{version}: group {sequence} was cut short" + "{version}: group {key} was cut short" ); - assert!(got.insert(sequence), "{version}: group {sequence} delivered twice"); + assert!(got.insert(key), "{version}: group {key} delivered twice"); } got } diff --git a/rs/moq-net/tests/integration/socket.rs b/rs/moq-net/tests/integration/socket.rs index 79250dc5367..5b91e4351c9 100644 --- a/rs/moq-net/tests/integration/socket.rs +++ b/rs/moq-net/tests/integration/socket.rs @@ -9,6 +9,7 @@ use moq_net::broadcast::TracksEvent; use moq_net::socket::{Endpoint, TrackStream}; use moq_net::{Error, StreamError, Version, group, origin, track}; +use crate::support::AppendNext; use crate::support::harness::{MockConnectOptions, MockPair, connect_mock, spawn}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -97,9 +98,11 @@ async fn connect_track(dialer: &Endpoint, acceptor: &Endpoint) -> (TrackStream, /// Write `payload` as one message of a session: one group. fn send(track: &track::Producer, payload: &'static [u8]) { - let mut group = track.append_group().unwrap(); - group.write_frame(None, payload).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload) + .unwrap(); + group.close().unwrap(); } /// Read the next message from a session, or `None` once the peer finished. @@ -167,17 +170,19 @@ async fn an_aborted_half_fails_the_read() { /// Messages are read in sequence even when they are written, and so arrive, out of order. #[kio::test] -async fn track_messages_are_read_in_sequence() { +async fn track_messages_are_read_in_key() { let mut fixture = Fixture::new(); let alice = fixture.endpoint(LITE, "alice").await; let bob = fixture.endpoint(LITE, "bob").await; let (dialed, accepted) = connect_track(&alice, &bob).await; let mut reader = accepted.consumer.ordered(); - let write = |sequence: u64, payload: &'static [u8]| { - let mut group = dialed.producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, payload).unwrap(); - group.finish().unwrap(); + let write = |key: u64, payload: &'static [u8]| { + let mut group = dialed.producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload) + .unwrap(); + group.close().unwrap(); }; write(1, b"second"); // Let the later message arrive first. @@ -202,10 +207,12 @@ async fn a_lost_track_message_does_not_block_later_ones() { accepted.consumer.control().update(lossy).unwrap(); let mut reader = accepted.consumer.ordered(); - let write = |sequence: u64, payload: &'static [u8]| { - let mut group = dialed.producer.create_group(group::Info { sequence }).unwrap(); - group.write_frame(None, payload).unwrap(); - group.finish().unwrap(); + let write = |key: u64, payload: &'static [u8]| { + let mut group = dialed.producer.create_group(group::Info { start: key }).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), payload) + .unwrap(); + group.close().unwrap(); }; // Message 0 is never written: message 1 waits out the budget, then is read. write(1, b"one"); @@ -513,7 +520,7 @@ async fn closing_an_endpoint_ends_its_sessions() { matches!(ended, Err(Error::Stream(StreamError::Session(_)))), "bob's half outlived his endpoint: {ended:?}" ); - assert!(accepted.producer.append_group().is_err()); + assert!(accepted.producer.append_group(0).is_err()); assert!(matches!(bob.accept_track().await, Err(Error::Closed))); assert!(matches!(bob.dial_track("alice", None), Err(Error::Closed))); } diff --git a/rs/moq-net/tests/integration/source_errors.rs b/rs/moq-net/tests/integration/source_errors.rs index 2734869f9b5..eba457f7a62 100644 --- a/rs/moq-net/tests/integration/source_errors.rs +++ b/rs/moq-net/tests/integration/source_errors.rs @@ -7,10 +7,12 @@ //! moq-transport's PUBLISH_DONE registry has no code for either error. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Error, StreamError, Timestamp, Version}; +use moq_net::{Error, StreamError, Version}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; const VERSIONS: [&str; 3] = ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"]; @@ -135,9 +137,9 @@ async fn track_in_flight_through_a_dead_upstream_is_unroutable() { let setup = relayed(version).await; let track = setup.broadcast.broadcast().create_track("video", timed()).unwrap(); let mut sub = setup.remote.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"head".as_ref()).unwrap(); + group.close().unwrap(); kio::clock::timeout(TIMEOUT, sub.recv_group()) .await .expect("group timeout") @@ -163,9 +165,9 @@ async fn withdrawn_broadcast_keeps_the_source_error() { let setup = relayed(version).await; let track = setup.broadcast.broadcast().create_track("video", timed()).unwrap(); let mut sub = setup.remote.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"head".as_ref()).unwrap(); + group.close().unwrap(); kio::clock::timeout(TIMEOUT, sub.recv_group()) .await .expect("group timeout") diff --git a/rs/moq-net/tests/integration/stream_credit.rs b/rs/moq-net/tests/integration/stream_credit.rs index ca6899841c9..811d32fb332 100644 --- a/rs/moq-net/tests/integration/stream_credit.rs +++ b/rs/moq-net/tests/integration/stream_credit.rs @@ -5,8 +5,10 @@ use std::time::Duration; -use moq_net::{Timestamp, Version, origin, track}; +use moq_net::{Version, origin, track}; +use crate::support::AppendNext; +use crate::support::AtStart; use crate::support::harness::{MockConnectOptions, connect_mock, spawn}; fn origin(hop: u64) -> origin::Producer { @@ -62,9 +64,9 @@ async fn one_stream_credit_serves_every_backlogged_group() { for _ in 0..4 { for track in &tracks { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"x".as_slice()).unwrap(); + group.close().unwrap(); } } for subscriber in &mut subscribers { @@ -76,7 +78,7 @@ async fn one_stream_credit_serves_every_backlogged_group() { .unwrap_or_else(|_| panic!("{version}: only {received:?} arrived")) .unwrap() .unwrap(); - received.push(group.sequence); + received.push(group.start); } received.sort(); assert_eq!(received, [0, 1, 2, 3], "{version}"); @@ -122,24 +124,24 @@ async fn a_newer_group_resets_an_unfinished_older_one() { let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); // Never finished, so its stream holds the only credit. - let mut older = track.append_group().unwrap(); - older.write_frame(Timestamp::ZERO, b"old".as_slice()).unwrap(); + let mut older = track.append_next().unwrap(); + older.write_frame(older.at_start(), b"old".as_slice()).unwrap(); let first = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .unwrap_or_else(|_| panic!("{version}: the older group arrives")) .unwrap() .unwrap(); - assert_eq!(first.sequence, 0, "{version}"); + assert_eq!(first.start, 0, "{version}"); - let mut newer = track.append_group().unwrap(); - newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); - newer.finish().unwrap(); + let mut newer = track.append_next().unwrap(); + newer.write_frame(newer.at_start(), b"new".as_slice()).unwrap(); + newer.close().unwrap(); let group = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .unwrap_or_else(|_| panic!("{version}: the newer group waits behind the older one")) .unwrap() .unwrap(); - assert_eq!(group.sequence, 1, "{version}"); + assert_eq!(group.start, 1, "{version}"); drop(older); } } @@ -180,34 +182,34 @@ async fn a_reset_group_resumes_without_loss() { let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); // Unfinished, so its stream holds the only credit until the newer group resets it. - let mut older = track.append_group().unwrap(); - older.write_frame(Timestamp::ZERO, b"a".as_slice()).unwrap(); + let mut older = track.append_next().unwrap(); + older.write_frame(older.at_start(), b"a".as_slice()).unwrap(); let mut paused = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .expect("the older group arrives") .unwrap() .unwrap(); - assert_eq!(paused.sequence, 0); + assert_eq!(paused.start, 0); let first = kio::clock::timeout(Duration::from_secs(5), paused.read_frame()) .await .expect("its first frame arrives") .unwrap(); assert_eq!(first.map(|frame| frame.payload), Some(bytes::Bytes::from_static(b"a"))); - let mut newer = track.append_group().unwrap(); - newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); - newer.finish().unwrap(); + let mut newer = track.append_next().unwrap(); + newer.write_frame(newer.at_start(), b"new".as_slice()).unwrap(); + newer.close().unwrap(); let group = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .expect("the newer group takes the older one's credit") .unwrap() .unwrap(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); // Written after the reset: a continuation carries them, behind the frame already held. - older.write_frame(Timestamp::ZERO, b"b".as_slice()).unwrap(); - older.write_frame(Timestamp::ZERO, b"c".as_slice()).unwrap(); - older.finish().unwrap(); + older.write_frame(older.at_start(), b"b".as_slice()).unwrap(); + older.write_frame(older.at_start(), b"c".as_slice()).unwrap(); + older.close().unwrap(); let mut rest = Vec::new(); while let Some(frame) = kio::clock::timeout(Duration::from_secs(5), paused.read_frame()) .await @@ -255,8 +257,8 @@ async fn a_reset_group_finishing_without_more_frames_ends_cleanly() { .with_delay(track::Delay::default().with_max(Duration::from_secs(30))); let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); - let mut older = track.append_group().unwrap(); - older.write_frame(Timestamp::ZERO, b"a".as_slice()).unwrap(); + let mut older = track.append_next().unwrap(); + older.write_frame(older.at_start(), b"a".as_slice()).unwrap(); let mut paused = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .expect("the older group arrives") @@ -268,18 +270,18 @@ async fn a_reset_group_finishing_without_more_frames_ends_cleanly() { .unwrap(); assert_eq!(first.map(|frame| frame.payload), Some(bytes::Bytes::from_static(b"a"))); - let mut newer = track.append_group().unwrap(); - newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); - newer.finish().unwrap(); + let mut newer = track.append_next().unwrap(); + newer.write_frame(newer.at_start(), b"new".as_slice()).unwrap(); + newer.close().unwrap(); let group = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .expect("the newer group takes the older one's credit") .unwrap() .unwrap(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); // No frame follows the reset: the group just ends. - older.finish().unwrap(); + older.close().unwrap(); let end = kio::clock::timeout(Duration::from_secs(5), paused.read_frame()) .await .expect("the paused group continues"); @@ -334,25 +336,25 @@ async fn a_waiting_continuation_lets_a_newer_group_through() { // Groups 0 and 1 stay unfinished: 1 resets 0, whose continuation then waits for credit. let mut groups = Vec::new(); let mut readers = Vec::new(); - for sequence in 0..2 { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"a".as_slice()).unwrap(); + for key in 0..2 { + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"a".as_slice()).unwrap(); let reader = next!(); - assert_eq!(reader.sequence, sequence); + assert_eq!(reader.start, key); groups.push(group); readers.push(reader); } // The newest outranks group 1's stream, so it resets that one too. - let mut newest = track.append_group().unwrap(); - newest.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); - newest.finish().unwrap(); - assert_eq!(next!().sequence, 2, "the newest group waits behind a continuation"); + let mut newest = track.append_next().unwrap(); + newest.write_frame(newest.at_start(), b"new".as_slice()).unwrap(); + newest.close().unwrap(); + assert_eq!(next!().start, 2, "the newest group waits behind a continuation"); // The paused groups continue highest-ranked first: newest first in descending order. for (mut group, mut reader) in groups.into_iter().zip(readers).rev() { - group.write_frame(Timestamp::ZERO, b"b".as_slice()).unwrap(); - group.finish().unwrap(); + group.write_frame(group.at_start(), b"b".as_slice()).unwrap(); + group.close().unwrap(); let mut frames = Vec::new(); while let Some(frame) = kio::clock::timeout(Duration::from_secs(5), reader.read_frame()) .await @@ -361,7 +363,7 @@ async fn a_waiting_continuation_lets_a_newer_group_through() { { frames.push(frame.payload); } - assert_eq!(frames, [b"a".as_slice(), b"b".as_slice()], "group {}", reader.sequence); + assert_eq!(frames, [b"a".as_slice(), b"b".as_slice()], "group {}", reader.start); } } @@ -417,15 +419,15 @@ async fn a_subscription_past_the_credit_is_served_once_credit_returns() { .await .unwrap_or_else(|_| panic!("{version}: the waiting subscription never got credit")) .unwrap_or_else(|err| panic!("{version}: the waiting subscription failed: {err}")); - let mut group = second.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = second.append_next().unwrap(); + group.write_frame(group.at_start(), b"x".as_slice()).unwrap(); + group.close().unwrap(); let group = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .unwrap_or_else(|_| panic!("{version}: no group after credit returned")) .unwrap() .unwrap(); - assert_eq!(group.sequence, 0, "{version}"); + assert_eq!(group.start, 0, "{version}"); } } @@ -480,27 +482,27 @@ async fn an_abandoned_setup_returns_credit_with_the_publisher_still_live() { .await .unwrap_or_else(|_| panic!("{version}: the waiting subscription never got credit")) .unwrap_or_else(|err| panic!("{version}: the waiting subscription failed: {err}")); - let mut group = third.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = third.append_next().unwrap(); + group.write_frame(group.at_start(), b"x".as_slice()).unwrap(); + group.close().unwrap(); let group = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .unwrap_or_else(|_| panic!("{version}: no group after credit returned")) .unwrap() .unwrap(); - assert_eq!(group.sequence, 0, "{version}"); + assert_eq!(group.start, 0, "{version}"); drop(group); drop(subscriber); let mut resumed = remote.track("second").unwrap().subscribe(None).await.unwrap(); - let mut group = second.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"still live".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = second.append_next().unwrap(); + group.write_frame(group.at_start(), b"still live".as_slice()).unwrap(); + group.close().unwrap(); let group = kio::clock::timeout(Duration::from_secs(5), resumed.recv_group()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); // Keep the first publisher alive through the resumed wire request too. drop(first); } @@ -553,23 +555,19 @@ async fn a_waiting_late_group_lets_a_newer_group_through() { } // Unfinished, so its stream holds the only credit. - let mut held = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); - held.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); - assert_eq!(next!("the held group arrives").sequence, 2, "{version}"); + let mut held = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); + held.write_frame(held.at_start(), b"held".as_slice()).unwrap(); + assert_eq!(next!("the held group arrives").start, 2, "{version}"); // Late and below the held stream: it waits for credit it cannot make room for. - let mut late = track.create_group(moq_net::group::Info { sequence: 0 }).unwrap(); - late.write_frame(Timestamp::ZERO, b"late".as_slice()).unwrap(); - late.finish().unwrap(); - - let mut newer = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); - newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); - newer.finish().unwrap(); - assert_eq!( - next!("the newer group waits behind the late one").sequence, - 3, - "{version}" - ); + let mut late = track.create_group(moq_net::group::Info { start: 0 }).unwrap(); + late.write_frame(late.at_start(), b"late".as_slice()).unwrap(); + late.close().unwrap(); + + let mut newer = track.create_group(moq_net::group::Info { start: 3 }).unwrap(); + newer.write_frame(newer.at_start(), b"new".as_slice()).unwrap(); + newer.close().unwrap(); + assert_eq!(next!("the newer group waits behind the late one").start, 3, "{version}"); drop(held); } } @@ -621,22 +619,22 @@ async fn a_shed_late_group_lets_a_newer_group_through() { } // Unfinished, so its stream holds the only credit. - let mut held = track.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); - held.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); - assert_eq!(next!("the held group arrives").sequence, 3, "{version}"); + let mut held = track.create_group(moq_net::group::Info { start: 3 }).unwrap(); + held.write_frame(held.at_start(), b"held".as_slice()).unwrap(); + assert_eq!(next!("the held group arrives").start, 3, "{version}"); // Group 1 waits for credit it cannot make room for, and group 0 ranks below it. - for sequence in [1, 0] { - let mut late = track.create_group(moq_net::group::Info { sequence }).unwrap(); - late.write_frame(Timestamp::ZERO, b"late".as_slice()).unwrap(); - late.finish().unwrap(); + for key in [1, 0] { + let mut late = track.create_group(moq_net::group::Info { start: key }).unwrap(); + late.write_frame(late.at_start(), b"late".as_slice()).unwrap(); + late.close().unwrap(); } - let mut newer = track.create_group(moq_net::group::Info { sequence: 4 }).unwrap(); - newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); - newer.finish().unwrap(); + let mut newer = track.create_group(moq_net::group::Info { start: 4 }).unwrap(); + newer.write_frame(newer.at_start(), b"new".as_slice()).unwrap(); + newer.close().unwrap(); assert_eq!( - next!("the newer group waits behind the late ones").sequence, + next!("the newer group waits behind the late ones").start, 4, "{version}" ); @@ -691,22 +689,22 @@ async fn an_ascending_backlog_waits_behind_a_waiting_group() { } // Unfinished, so its stream holds the only credit, and outranks every newer group. - let mut held = track.append_group().unwrap(); - held.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); - assert_eq!(next!("the held group arrives").sequence, 0, "{version}"); + let mut held = track.append_next().unwrap(); + held.write_frame(held.at_start(), b"held".as_slice()).unwrap(); + assert_eq!(next!("the held group arrives").start, 0, "{version}"); for _ in 0..3 { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"x".as_slice()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"x".as_slice()).unwrap(); + group.close().unwrap(); } // Let the publisher take what it would before credit returns. kio::clock::sleep(Duration::from_millis(100)).await; - held.finish().unwrap(); + held.close().unwrap(); let mut received = Vec::new(); while received.len() < 3 { - received.push(next!("the backlog arrives").sequence); + received.push(next!("the backlog arrives").start); } assert_eq!(received, [1, 2, 3], "{version}"); } @@ -752,15 +750,15 @@ async fn a_paused_group_given_up_while_it_waits_is_dropped() { // 1's continuation waits for it, and group 0 waits among the paused behind that. let mut groups = Vec::new(); let mut readers = Vec::new(); - for sequence in 0..3 { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"a".as_slice()).unwrap(); + for start in 0..3 { + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"a".as_slice()).unwrap(); let mut reader = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) .await .expect("a group arrives") .unwrap() .unwrap(); - assert_eq!(reader.sequence, sequence); + assert_eq!(reader.start, start); let first = kio::clock::timeout(Duration::from_secs(5), reader.read_frame()) .await .expect("its first frame arrives") @@ -820,22 +818,22 @@ async fn a_closing_stream_does_not_rank_a_waiting_group() { let mut subscriber = remote.track("0").unwrap().subscribe(prefs).await.unwrap(); // Group 7 finishes, so its stream waits for the acknowledgement; group 10 holds the other. - let mut closing = track.create_group(moq_net::group::Info { sequence: 7 }).unwrap(); - closing.write_frame(Timestamp::ZERO, b"closing".as_slice()).unwrap(); - closing.finish().unwrap(); - let mut held = track.create_group(moq_net::group::Info { sequence: 10 }).unwrap(); - held.write_frame(Timestamp::ZERO, b"held".as_slice()).unwrap(); + let mut closing = track.create_group(moq_net::group::Info { start: 7 }).unwrap(); + closing.write_frame(closing.at_start(), b"closing".as_slice()).unwrap(); + closing.close().unwrap(); + let mut held = track.create_group(moq_net::group::Info { start: 10 }).unwrap(); + held.write_frame(held.at_start(), b"held".as_slice()).unwrap(); kio::clock::sleep(SETTLE).await; // Ranked above group 7 but below group 10: it can only wait for group 7's credit. - let mut late = track.create_group(moq_net::group::Info { sequence: 8 }).unwrap(); - late.write_frame(Timestamp::ZERO, b"late".as_slice()).unwrap(); - late.finish().unwrap(); + let mut late = track.create_group(moq_net::group::Info { start: 8 }).unwrap(); + late.write_frame(late.at_start(), b"late".as_slice()).unwrap(); + late.close().unwrap(); kio::clock::sleep(SETTLE).await; - let mut newer = track.create_group(moq_net::group::Info { sequence: 11 }).unwrap(); - newer.write_frame(Timestamp::ZERO, b"new".as_slice()).unwrap(); - newer.finish().unwrap(); + let mut newer = track.create_group(moq_net::group::Info { start: 11 }).unwrap(); + newer.write_frame(newer.at_start(), b"new".as_slice()).unwrap(); + newer.close().unwrap(); loop { let group = kio::clock::timeout(Duration::from_secs(5), subscriber.recv_group()) @@ -843,11 +841,8 @@ async fn a_closing_stream_does_not_rank_a_waiting_group() { .unwrap_or_else(|_| panic!("{version}: the newer group never arrives")) .unwrap() .unwrap(); - assert_ne!( - group.sequence, 8, - "{version}: the newer group waits behind the late one" - ); - if group.sequence == 11 { + assert_ne!(group.start, 8, "{version}: the newer group waits behind the late one"); + if group.start == 11 { break; } } diff --git a/rs/moq-net/tests/integration/subscribe_ranges.rs b/rs/moq-net/tests/integration/subscribe_ranges.rs index 6e79f7f5313..a335e9441e5 100644 --- a/rs/moq-net/tests/integration/subscribe_ranges.rs +++ b/rs/moq-net/tests/integration/subscribe_ranges.rs @@ -32,22 +32,24 @@ fn timed() -> Info { .with_max_age(None) } -fn write_group(track: &moq_net::track::Producer, sequence: u64) { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); - group.write_frame(Timestamp::now(), &b"x"[..]).unwrap(); - group.finish().unwrap(); +fn write_group(track: &moq_net::track::Producer, key: u64) { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); + group + .write_frame(Timestamp::from_millis(key).unwrap(), &b"x"[..]) + .unwrap(); + group.close().unwrap(); } -/// The next group's sequence, or `None` once the track ends or nothing arrives in time. +/// The next group's key, or `None` once the track ends or nothing arrives in time. async fn next(sub: &mut moq_net::track::Subscriber) -> Option { let group = kio::clock::timeout(TIMEOUT, sub.recv_group()).await.ok()?; - group.expect("recv_group").map(|group| group.sequence) + group.expect("recv_group").map(|group| group.start) } async fn read_all(sub: &mut moq_net::track::Subscriber) -> Vec { let mut got = Vec::new(); - while let Some(sequence) = next(sub).await { - got.push(sequence); + while let Some(key) = next(sub).await { + got.push(key); } got } @@ -73,8 +75,8 @@ async fn ahead_then_live(version: Option<&str>, relay: bool) -> (Vec, Vec, relay: bool) -> (Vec, Vec, relay: bool) -> (Vec, Vec Rang .unwrap(); let track = broadcast.broadcast().create_track("quiet", timed()).unwrap(); broadcast.announce().unwrap(); - for sequence in 0..history { - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in 0..history { + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), b"x".as_slice()) + .write_frame(Timestamp::from_millis(key).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let relay_origin = produce_origin(2); let subscriber = produce_origin(3); @@ -294,12 +296,16 @@ async fn initial_mixed_ranges_keep_the_absolute_history() { assert_eq!(next(&mut warm).await, None); } let subscription = Subscription::default() - .with_ranges([Range::groups(1..2), Range::live()]) + .with_ranges([support::keys(1..2), Range::live()]) .with_delay(Delay::default().with_max(None)); let mut reader = subscribe(&fixture.remote, subscription).await; let mut got = Vec::new(); for _ in 0..2 { - got.push(next(&mut reader).await.expect("a mixed range group is missing")); + got.push(next(&mut reader).await.unwrap_or_else(|| { + panic!( + "{version} relay={relay} finished={finished}: a mixed range group is missing; got {got:?}" + ) + })); } got.sort_unstable(); assert_eq!(got, [1, 5], "{version} relay={relay} finished={finished}"); @@ -316,7 +322,7 @@ async fn draft22_updates_both_bounds_across_cached_groups() { let mut fixture = range_fixture(relay).await; // Reaching the live group 5, so the copy subscribes: history alone is fetched. let initial = Subscription::default() - .with_ranges([Range::groups(2..4), Range::groups(5..6)]) + .with_ranges([support::keys(2..4), support::keys(5..6)]) .with_delay(Delay::default().with_max(None)) .with_order(moq_net::track::Order::Ascending); let mut reader = subscribe(&fixture.remote, initial.clone()).await; @@ -326,7 +332,9 @@ async fn draft22_updates_both_bounds_across_cached_groups() { } seen.sort_unstable(); assert_eq!(seen, [2, 3, 5]); - reader.update(initial.clone().with_range(0..6)).unwrap(); + reader + .update(initial.clone().with_ranges([support::keys(0..6)])) + .unwrap(); let mut got = Vec::new(); for _ in 0..3 { let seen = next(&mut reader).await; @@ -334,7 +342,9 @@ async fn draft22_updates_both_bounds_across_cached_groups() { } got.sort_unstable(); assert_eq!(got, [0, 1, 4], "relay={relay}"); - reader.update(initial.clone().with_range(4..5)).unwrap(); + reader + .update(initial.clone().with_ranges([support::keys(4..5)])) + .unwrap(); kio::clock::timeout( TIMEOUT, kio::wait(|waiter| { @@ -346,7 +356,7 @@ async fn draft22_updates_both_bounds_across_cached_groups() { if fixture .track .subscription() - .is_some_and(|subscription| subscription.ranges == [Range::groups(4..5)]) + .is_some_and(|subscription| subscription.ranges == [support::keys(4..5)]) { return std::task::Poll::Ready(()); } @@ -358,27 +368,25 @@ async fn draft22_updates_both_bounds_across_cached_groups() { ) .await .expect("the narrowed range never reached the publisher"); - reader.update(initial.with_range(0..)).unwrap(); - let mut group = fixture - .track - .create_group(moq_net::group::Info { sequence: 6 }) - .unwrap(); + reader.update(initial.with_ranges([support::keys(0..)])).unwrap(); + let mut group = fixture.track.create_group(moq_net::group::Info { start: 6 }).unwrap(); group - .write_frame(Timestamp::from_millis(6000).unwrap(), b"later".as_slice()) + .write_frame(Timestamp::from_millis(6).unwrap(), b"later".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(next(&mut reader).await, Some(6), "relay={relay}"); } } -/// A warm copy resolves a time end before updating upstream; an unknown end is refused. +/// Draft 22 names groups by ID, and a time anchor is the group ID with that key, so a time +/// end goes upstream as is, whether or not the copy holds the group it names. #[kio::test] -async fn draft22_resolves_cached_time_ends() { +async fn draft22_carries_time_ends_as_group_ids() { for relay in [false, true] { let mut fixture = range_fixture(relay).await; // Reaching the live group 5, so the copy subscribes: history alone is fetched. let initial = Subscription::default() - .with_ranges([Range::groups(2..4), Range::groups(5..6)]) + .with_ranges([support::keys(2..4), support::keys(5..6)]) .with_delay(Delay::default().with_max(None)) .with_order(moq_net::track::Order::Ascending); let mut reader = subscribe(&fixture.remote, initial.clone()).await; @@ -388,16 +396,10 @@ async fn draft22_resolves_cached_time_ends() { } seen.sort_unstable(); assert_eq!(seen, [2, 3, 5]); - let unknown = initial.clone().with_ranges([Range { - start: moq_net::track::Anchor::Absolute(0), - end: Some(moq_net::track::Anchor::Time(Timestamp::from_millis(10_000).unwrap())), - }]); - assert!(matches!(reader.update(unknown), Err(moq_net::Error::Unsupported))); - assert_eq!(reader.subscription().ranges, initial.ranges); reader .update(initial.with_ranges([Range { - start: moq_net::track::Anchor::Absolute(0), - end: Some(moq_net::track::Anchor::Time(Timestamp::from_millis(2500).unwrap())), + start: moq_net::track::Anchor::Time(0), + end: Some(moq_net::track::Anchor::Time(2)), }])) .unwrap(); let mut got = vec![next(&mut reader).await.unwrap(), next(&mut reader).await.unwrap()]; @@ -408,7 +410,7 @@ async fn draft22_resolves_cached_time_ends() { if fixture .track .subscription() - .is_some_and(|subscription| subscription.ranges == [Range::groups(0..3)]) + .is_some_and(|subscription| subscription.ranges == [support::keys(0..3)]) { break; } @@ -416,17 +418,20 @@ async fn draft22_resolves_cached_time_ends() { } }) .await - .expect("the resolved group end never reached the publisher"); + .expect("the group end never reached the publisher"); } + // A cold copy holds none of the groups the end names. let fixture = range_fixture(false).await; - let requested = Subscription::default().with_ranges([Range { - start: moq_net::track::Anchor::Absolute(0), - end: Some(moq_net::track::Anchor::Time(Timestamp::from_millis(2500).unwrap())), - }]); - assert!(matches!( - fixture.remote.track("quiet").unwrap().subscribe(requested).await, - Err(moq_net::Error::Unsupported) - )); + let requested = Subscription::default() + .with_ranges([Range { + start: moq_net::track::Anchor::Time(0), + end: Some(moq_net::track::Anchor::Time(2)), + }]) + .with_delay(Delay::default().with_max(None)); + let mut reader = subscribe(&fixture.remote, requested).await; + let mut got = read_all(&mut reader).await; + got.sort_unstable(); + assert_eq!(got, [0, 1, 2]); } /// A quiet current group does not hold its filled head while a later range update @@ -435,12 +440,9 @@ async fn draft22_resolves_cached_time_ends() { async fn draft22_can_widen_while_the_current_group_is_quiet() { for relay in [false, true] { let fixture = range_fixture(relay).await; - let mut quiet = fixture - .track - .create_group(moq_net::group::Info { sequence: 6 }) - .unwrap(); + let mut quiet = fixture.track.create_group(moq_net::group::Info { start: 6 }).unwrap(); quiet - .write_frame(Timestamp::from_millis(6000).unwrap(), b"head".as_slice()) + .write_frame(Timestamp::from_millis(6).unwrap(), b"head".as_slice()) .unwrap(); let mut live = subscribe(&fixture.remote, Subscription::default().with_ranges([Range::live()])).await; let mut current = kio::clock::timeout(TIMEOUT, live.recv_group()) @@ -448,7 +450,7 @@ async fn draft22_can_widen_while_the_current_group_is_quiet() { .unwrap() .unwrap() .unwrap(); - assert_eq!(current.sequence, 6); + assert_eq!(current.start, 6); assert_eq!( kio::clock::timeout(TIMEOUT, current.read_frame()) .await @@ -461,14 +463,14 @@ async fn draft22_can_widen_while_the_current_group_is_quiet() { let mut history = subscribe( &fixture.remote, Subscription::default() - .with_range(0..2) + .with_ranges([support::keys(0..2)]) .with_delay(Delay::default().with_max(None)), ) .await; let mut got = [next(&mut history).await.unwrap(), next(&mut history).await.unwrap()]; got.sort_unstable(); assert_eq!(got, [0, 1], "relay={relay}"); - quiet.finish().unwrap(); + quiet.close().unwrap(); assert!( kio::clock::timeout(TIMEOUT, current.read_frame()) .await @@ -487,7 +489,7 @@ async fn draft18_updates_narrow_then_widen() { for relay in [false, true] { let mut fixture = range_fixture_version(version, relay).await; let initial = Subscription::default() - .with_range(5..) + .with_ranges([support::keys(5..)]) .with_delay(Delay::default().with_max(None)); let mut reader = subscribe(&fixture.remote, initial.clone()).await; let mut seen = Vec::new(); @@ -496,7 +498,15 @@ async fn draft18_updates_narrow_then_widen() { } seen.sort_unstable(); assert_eq!(seen, [5], "{version} relay={relay}"); - reader.update(initial.clone().with_range(6..7)).unwrap(); + reader + .update(initial.clone().with_ranges([support::keys(6..7)])) + .unwrap(); + // Group 5 is the newest, so until a later one starts it holds time 6: the narrowed + // range still asks from it. + let narrowed = Range { + start: moq_net::track::Anchor::Time(5), + end: Some(moq_net::track::Anchor::Time(6)), + }; kio::clock::timeout( TIMEOUT, kio::wait(|waiter| { @@ -505,7 +515,7 @@ async fn draft18_updates_narrow_then_widen() { if fixture .track .subscription() - .is_some_and(|subscription| subscription.ranges == [Range::groups(6..7)]) + .is_some_and(|subscription| subscription.ranges == [narrowed]) { return std::task::Poll::Ready(()); } @@ -517,15 +527,12 @@ async fn draft18_updates_narrow_then_widen() { ) .await .unwrap_or_else(|_| panic!("{version} relay={relay}: narrowing did not reach the publisher")); - for sequence in 6..8 { - let mut group = fixture.track.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in 6..8 { + let mut group = fixture.track.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame( - Timestamp::from_millis(sequence * 1000).unwrap(), - sequence.to_be_bytes().as_slice(), - ) + .write_frame(Timestamp::from_millis(key).unwrap(), key.to_be_bytes().as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } reader.update(initial).unwrap(); let mut seen = Vec::new(); @@ -535,13 +542,13 @@ async fn draft18_updates_narrow_then_widen() { .unwrap_or_else(|_| panic!("{version} relay={relay}: widening lost history; got {seen:?}")) .unwrap() .unwrap(); - seen.push(group.sequence); + seen.push(group.start); let frame = kio::clock::timeout(TIMEOUT, group.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload, group.sequence.to_be_bytes().as_slice()); + assert_eq!(frame.payload, group.start.to_be_bytes().as_slice()); assert!( kio::clock::timeout(TIMEOUT, group.read_frame()) .await @@ -573,14 +580,11 @@ async fn empty_ranges_retire_the_wire_and_can_widen_again() { reader.poll_recv_group(&kio::Waiter::noop()), std::task::Poll::Ready(Ok(None)) )); - let mut group = fixture - .track - .create_group(moq_net::group::Info { sequence: 6 }) - .unwrap(); + let mut group = fixture.track.create_group(moq_net::group::Info { start: 6 }).unwrap(); group - .write_frame(Timestamp::from_millis(6000).unwrap(), b"sibling".as_slice()) + .write_frame(Timestamp::from_millis(6).unwrap(), b"sibling".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert_eq!(next(&mut sibling).await, Some(6), "{version} relay={relay}"); reader.update(live.clone()).unwrap(); assert_eq!(next(&mut reader).await, Some(6)); @@ -605,14 +609,11 @@ async fn empty_ranges_retire_the_wire_and_can_widen_again() { ) .await .unwrap_or_else(|_| panic!("{version} relay={relay}: empty demand retained a wire subscription")); - let mut group = fixture - .track - .create_group(moq_net::group::Info { sequence: 7 }) - .unwrap(); + let mut group = fixture.track.create_group(moq_net::group::Info { start: 7 }).unwrap(); group - .write_frame(Timestamp::from_millis(7000).unwrap(), b"resumed".as_slice()) + .write_frame(Timestamp::from_millis(7).unwrap(), b"resumed".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); reader.update(live).unwrap(); assert_eq!(next(&mut reader).await, Some(7), "{version} relay={relay}"); } @@ -625,12 +626,17 @@ async fn draft22_cold_widening_finishes_the_largest_group() { for relay in [false, true] { let mut fixture = range_fixture_history("moq-transport-22", relay, 0).await; let initial = Subscription::default() - .with_range(2..) + .with_ranges([support::keys(2..)]) .with_delay(Delay::default().with_max(None)); let mut reader = subscribe(&fixture.remote, initial.clone()).await; for start in [0, 1, 2] { let future = start + 1; - reader.update(initial.clone().with_range(future..)).unwrap(); + reader + .update(initial.clone().with_ranges([support::keys(future..)])) + .unwrap(); + // A time start names the newest group starting at or before it, so once the copy + // holds a group, the narrowing still asks from the newest one. + let narrowed = support::keys(start.checked_sub(1).unwrap_or(future)..); kio::clock::timeout( TIMEOUT, kio::wait(|waiter| { @@ -639,7 +645,7 @@ async fn draft22_cold_widening_finishes_the_largest_group() { if fixture .track .subscription() - .is_some_and(|subscription| subscription.ranges == [Range::groups(future..)]) + .is_some_and(|subscription| subscription.ranges == [narrowed]) { return std::task::Poll::Ready(()); } @@ -651,110 +657,42 @@ async fn draft22_cold_widening_finishes_the_largest_group() { ) .await .unwrap_or_else(|_| panic!("relay={relay}: narrowing to {future} stalled")); - for sequence in start..future { - let mut group = fixture.track.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in start..future { + let mut group = fixture.track.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::ZERO, sequence.to_be_bytes().as_slice()) + .write_frame(Timestamp::from_millis(key).unwrap(), key.to_be_bytes().as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } - reader.update(initial.clone().with_range(start..)).unwrap(); - let mut sequences = Vec::new(); + reader + .update(initial.clone().with_ranges([support::keys(start..)])) + .unwrap(); + let mut keys = Vec::new(); for _ in start..future { let mut group = kio::clock::timeout(TIMEOUT, reader.recv_group()) .await .unwrap_or_else(|_| { - panic!("relay={relay}: a filled group from {start} did not arrive; got {sequences:?}") + panic!("relay={relay}: a filled group from {start} did not arrive; got {keys:?}") }) .unwrap() .unwrap(); - sequences.push(group.sequence); + keys.push(group.start); let frame = kio::clock::timeout(TIMEOUT, group.read_frame()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(frame.payload, group.sequence.to_be_bytes().as_slice()); + assert_eq!(frame.payload, group.start.to_be_bytes().as_slice()); assert!( kio::clock::timeout(TIMEOUT, group.read_frame()) .await - .unwrap_or_else(|_| panic!("relay={relay}: group {} never finished", group.sequence)) + .unwrap_or_else(|_| panic!("relay={relay}: group {} never finished", group.start)) .unwrap() .is_none() ); } - sequences.sort_unstable(); - assert_eq!(sequences, [start]); - } - } -} - -/// A pending refusal is local to the new reader even before it is awaited. -#[kio::test] -async fn unresolved_time_reader_never_changes_sibling_demand() { - for version in ["moq-transport-18", "moq-transport-22"] { - for relay in [false, true] { - let mut fixture = range_fixture_version(version, relay).await; - let mut healthy = subscribe(&fixture.remote, Subscription::default()).await; - assert_eq!(next(&mut healthy).await, Some(5)); - healthy.update(Subscription::default().with_range(4..8)).unwrap(); - kio::clock::timeout( - TIMEOUT, - kio::wait(|waiter| { - assert!(healthy.poll_recv_group(waiter).is_pending()); - loop { - if fixture - .track - .subscription() - .is_some_and(|s| s.ranges == [Range::groups(4..8)]) - { - return std::task::Poll::Ready(()); - } - if fixture.track.poll_subscription_changed(waiter).is_pending() { - return std::task::Poll::Pending; - } - } - }), - ) - .await - .unwrap(); - - let track = fixture.remote.track("quiet").unwrap(); - // Pin the public track instance through an info request before creating the pending reader. - let query = track.info(); - let track = query.track(); - - let mut refused = Some(track.subscribe(Subscription::default().with_ranges([Range { - start: moq_net::track::Anchor::Absolute(0), - end: Some(moq_net::track::Anchor::Time(Timestamp::from_millis(10000).unwrap())), - }]))); - for sequence in 6..8 { - if sequence == 7 { - assert!(matches!( - refused.take().unwrap().await, - Err(moq_net::Error::Unsupported) - )); - } - let mut group = fixture.track.create_group(moq_net::group::Info { sequence }).unwrap(); - group - .write_frame( - Timestamp::from_millis(sequence * 1000).unwrap(), - sequence.to_be_bytes().as_slice(), - ) - .unwrap(); - group.finish().unwrap(); - let mut got = kio::clock::timeout(TIMEOUT, healthy.recv_group()) - .await - .unwrap() - .unwrap_or_else(|err| panic!("{version} relay={relay}: pending refusal aborted sibling: {err:?}")) - .unwrap(); - assert_eq!(got.sequence, sequence); - assert_eq!( - got.read_frame().await.unwrap().unwrap().payload, - sequence.to_be_bytes().as_slice() - ); - assert!(got.read_frame().await.unwrap().is_none()); - } + keys.sort_unstable(); + assert_eq!(keys, [start]); } } } @@ -773,8 +711,8 @@ async fn a_covered_bounded_subscription_stays_open() { .unwrap(); let track = broadcast.broadcast().create_track("quiet", timed()).unwrap(); broadcast.announce().unwrap(); - for sequence in [0, 1, 2, 4, 5] { - write_group(&track, sequence); + for key in [0, 1, 2, 4, 5] { + write_group(&track, key); } let subscriber = produce_origin(2); @@ -793,7 +731,7 @@ async fn a_covered_bounded_subscription_stays_open() { Subscription::default() .with_delay(Delay::default().with_max(Duration::from_secs(60))) .with_order(moq_net::track::Order::Ascending) - .with_ranges([Range::groups(range)]) + .with_ranges([support::keys(range)]) }; let mut reader = subscribe(&remote, bounded(1..3)).await; assert_eq!(next(&mut reader).await, Some(1), "{version}"); @@ -857,7 +795,7 @@ async fn holes_through_a_relay(version: &str, budget: Option) { // The publisher drops group 4 and never produces group 7. Group 8 lies outside both ranges. const DROPPED: u64 = 4; const ABSENT: u64 = 7; - const WANT: [u64; 8] = [0, 1, 2, 3, 5, 6, 9, 10]; + const WANT: [u64; 8] = [0, 1, 2, 3, 5, 6, 9, LIVE_MILLIS]; // History is stamped within a second, so the whole past range is fresh when read. The // live group lands far past the budget, which then rules both holes out. const LIVE_MILLIS: u64 = 100_000; @@ -865,16 +803,16 @@ async fn holes_through_a_relay(version: &str, budget: Option) { for order in [Order::Ascending, Order::Descending] { let name = format!("{version} {order:?} budget={budget:?}"); let fixture = range_fixture_history(version, true, 0).await; - for sequence in (0..=9).filter(|sequence| *sequence != ABSENT) { - let mut group = fixture.track.create_group(moq_net::group::Info { sequence }).unwrap(); - if sequence == DROPPED { + for key in (0..=9).filter(|key| *key != ABSENT) { + let mut group = fixture.track.create_group(moq_net::group::Info { start: key }).unwrap(); + if key == DROPPED { group.abort(moq_net::Error::App(1)).unwrap(); continue; } group - .write_frame(Timestamp::from_millis(sequence).unwrap(), b"x".as_slice()) + .write_frame(Timestamp::from_millis(key).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } // Warm the relay's cache with groups 1 and 5 alone, one-group runs every wire fills. @@ -887,7 +825,7 @@ async fn holes_through_a_relay(version: &str, budget: Option) { &relay, Subscription::default() .with_delay(Delay::default().with_max(None)) - .with_ranges([Range::groups(1..2), Range::groups(5..6)]), + .with_ranges([keys(1..2), keys(5..6)]), ) .await; let mut warmed = read_all(&mut warm).await; @@ -900,7 +838,7 @@ async fn holes_through_a_relay(version: &str, budget: Option) { Subscription::default() .with_delay(Delay::default().with_max(budget)) .with_order(order) - .with_ranges([Range::groups(0..8), Range::live()]), + .with_ranges([keys(0..8), Range::live()]), ) .await; // Everything in range that exists now, before the live edge moves. @@ -910,17 +848,17 @@ async fn holes_through_a_relay(version: &str, budget: Option) { } let mut group = fixture .track - .create_group(moq_net::group::Info { sequence: 10 }) + .create_group(moq_net::group::Info { start: LIVE_MILLIS }) .unwrap(); group .write_frame(Timestamp::from_millis(LIVE_MILLIS).unwrap(), b"x".as_slice()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let _finished = fixture.track.clone().finish().unwrap(); - // The read ends only once every sequence in range was delivered or skipped. + // The read ends only once every key in range was delivered or skipped. loop { match kio::clock::timeout(TIMEOUT, reader.recv_group()).await { - Ok(Ok(Some(group))) => got.push(group.sequence), + Ok(Ok(Some(group))) => got.push(group.start), Ok(Ok(None)) => break, Ok(Err(err)) => { failures.push(format!("{name}: {err} after {got:?}")); @@ -940,3 +878,11 @@ async fn holes_through_a_relay(version: &str, budget: Option) { } assert!(failures.is_empty(), "{failures:#?}"); } + +/// The groups keyed in `range`, as a time range: a time end is the last key it holds. +fn keys(range: std::ops::Range) -> Range { + Range { + start: moq_net::track::Anchor::Time(range.start), + end: Some(moq_net::track::Anchor::Time(range.end - 1)), + } +} diff --git a/rs/moq-net/tests/integration/subscribe_tracks.rs b/rs/moq-net/tests/integration/subscribe_tracks.rs index 0f1d964679c..b48c4aa4284 100644 --- a/rs/moq-net/tests/integration/subscribe_tracks.rs +++ b/rs/moq-net/tests/integration/subscribe_tracks.rs @@ -5,13 +5,15 @@ //! Deterministic: paused time and a single-threaded runtime over the mock transport. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; use futures::StreamExt; +use moq_net::Version; use moq_net::transport::poll::{RecvStream as _, SendStream as _, Session as _}; -use moq_net::{Timestamp, Version}; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -93,9 +95,9 @@ async fn setup(version: Version) -> Setup { } async fn deliver(setup: &mut Setup, payload: &'static [u8]) { - let mut group = setup.track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload).unwrap(); - group.finish().unwrap(); + let mut group = setup.track.append_next().unwrap(); + group.write_frame(group.at_start(), payload).unwrap(); + group.close().unwrap(); let got = kio::clock::timeout(TIMEOUT, setup.frames.next()) .await .expect("frame timed out") diff --git a/rs/moq-net/tests/integration/subscribe_with_track.rs b/rs/moq-net/tests/integration/subscribe_with_track.rs index 4cb6c031f08..8a1e088b163 100644 --- a/rs/moq-net/tests/integration/subscribe_with_track.rs +++ b/rs/moq-net/tests/integration/subscribe_with_track.rs @@ -10,6 +10,8 @@ //! stream and are unchanged. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; @@ -74,9 +76,9 @@ async fn first_frame(version: &str, relays: usize, arrival: Arrival) -> Duration moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut cached = track.append_group().unwrap(); - cached.write_frame(Timestamp::ZERO, b"cached".as_ref()).unwrap(); - cached.finish().unwrap(); + let mut cached = track.append_next().unwrap(); + cached.write_frame(cached.at_start(), b"cached".as_ref()).unwrap(); + cached.close().unwrap(); broadcast.announce().unwrap(); let consumer = nodes.last().unwrap().consume(); @@ -121,10 +123,10 @@ async fn first_frame(version: &str, relays: usize, arrival: Arrival) -> Duration let elapsed = kio::clock::now() - start; assert_eq!(frame.payload.as_ref(), b"cached", "{label}"); - let mut live = track.append_group().unwrap(); + let mut live = track.append_next().unwrap(); live.write_frame(Timestamp::from_millis(1).unwrap(), b"live".as_ref()) .unwrap(); - live.finish().unwrap(); + live.close().unwrap(); let mut group = kio::clock::timeout(TIMEOUT, subscriber.recv_group()) .await .unwrap_or_else(|_| panic!("{label}: the live group did not arrive")) diff --git a/rs/moq-net/tests/integration/subscriber_max_age.rs b/rs/moq-net/tests/integration/subscriber_max_age.rs index ef30fec46ad..cd829ed8c0d 100644 --- a/rs/moq-net/tests/integration/subscriber_max_age.rs +++ b/rs/moq-net/tests/integration/subscriber_max_age.rs @@ -4,8 +4,10 @@ use std::time::Duration; -use moq_net::{Timestamp, Version, origin, track}; +use moq_net::{Version, origin, track}; +use crate::support::AppendNext; +use crate::support::AtStart; use crate::support::harness::{peer, spawn}; fn origin(hop: u64) -> origin::Producer { @@ -49,9 +51,9 @@ async fn relay_forwards_the_most_tolerant_subscriber_max_age() { track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = source.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"catalog").unwrap(); - group.finish().unwrap(); + let mut group = source.append_next().unwrap(); + group.write_frame(group.at_start(), b"catalog").unwrap(); + group.close().unwrap(); broadcast.announce().unwrap(); let consumer = viewer.consume(); let remote = consumer @@ -77,7 +79,7 @@ async fn relay_forwards_the_most_tolerant_subscriber_max_age() { budget(&mut source, 10, version).await; // A quiet catalog survives far longer than the budget because it has no successor. kio::clock::sleep(Duration::from_secs(1)).await; - assert_eq!(snapshot.sequence, 0, "{version}"); + assert_eq!(snapshot.start, 0, "{version}"); assert_eq!( snapshot.read_frame().await.unwrap().unwrap().payload, b"catalog".as_slice() @@ -124,9 +126,9 @@ async fn relay_forwards_the_smallest_stated_min_delay() { track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = source.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"catalog").unwrap(); - group.finish().unwrap(); + let mut group = source.append_next().unwrap(); + group.write_frame(group.at_start(), b"catalog").unwrap(); + group.close().unwrap(); broadcast.announce().unwrap(); let cache = |origin: &origin::Producer| { diff --git a/rs/moq-net/tests/integration/subscription_end_integrity.rs b/rs/moq-net/tests/integration/subscription_end_integrity.rs index fb043e4b25b..31912620a1c 100644 --- a/rs/moq-net/tests/integration/subscription_end_integrity.rs +++ b/rs/moq-net/tests/integration/subscription_end_integrity.rs @@ -20,10 +20,12 @@ //! send" step is an idle-advance rather than a race. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Timestamp, Version}; +use moq_net::Version; use support::harness::{MockConnectOptions, MockPair, connect_mock}; /// A millisecond track: the frames written here carry timestamps. @@ -83,7 +85,7 @@ async fn round(drop_session: bool) -> (Vec>, Option) { let (go_tx, go_rx) = futures::channel::oneshot::channel::<()>(); let reader = kio::clock::sim::spawn(async move { - let subscription = moq_net::track::Subscription::default().with_range(0..); + let subscription = moq_net::track::Subscription::default().with_ranges([support::keys(0..)]); let mut sub = remote .track("video") .unwrap() @@ -118,11 +120,11 @@ async fn round(drop_session: bool) -> (Vec>, Option) { .expect("no subscriber appeared") .unwrap(); - let mut head = track.append_group().unwrap(); + let mut head = track.append_next().unwrap(); for payload in HEAD { - head.write_frame(Timestamp::ZERO, payload).unwrap(); + head.write_frame(head.at_start(), payload).unwrap(); } - head.finish().unwrap(); + head.close().unwrap(); kio::clock::timeout(TIMEOUT, drained_rx) .await .expect("the head group never arrived") @@ -131,15 +133,15 @@ async fn round(drop_session: bool) -> (Vec>, Option) { // The final group opens and its first frame is sent, then the track's end is // declared: SUBSCRIBE_FINISH reaches the subscriber with the group still open, as it // does whenever a finish races a group still in flight. - let mut tail = track.append_group().unwrap(); - tail.write_frame(Timestamp::ZERO, TAIL[0]).unwrap(); + let mut tail = track.append_next().unwrap(); + tail.write_frame(tail.at_start(), TAIL[0]).unwrap(); let track = track.finish().unwrap(); settle().await; // The rest of the group, its clean end, and the disconnect happen with no await in // between, so the session has sent none of it when it closes. - tail.write_frame(Timestamp::ZERO, TAIL[1]).unwrap(); - tail.finish().unwrap(); + tail.write_frame(tail.at_start(), TAIL[1]).unwrap(); + tail.close().unwrap(); drop(track); let server = if drop_session { drop(server); @@ -240,7 +242,7 @@ async fn killed(version: &str, kill: Kill) -> (Option, Option (Option, Option moq_net::Result; +} + +impl AppendNext for track::Producer { + fn append_next(&self) -> moq_net::Result { + self.append_group(self.largest().map_or(0, |largest| largest.group + 1)) + } +} + +/// A frame time for tests whose frames carry no time of their own. +#[allow(dead_code, reason = "shared by several test binaries, each using only part of it")] +pub trait AtStart { + /// Where the group starts, as a timestamp in its timed track's timescale. + fn at_start(&self) -> Timestamp; +} + +impl AtStart for group::Producer { + fn at_start(&self) -> Timestamp { + Timestamp::new(self.start, self.timescale().expect("a timed track")).unwrap() + } +} + +/// The groups from the one containing `keys`' start through the one containing its end, as +/// time anchors: on a track whose groups start at consecutive keys, the groups with those keys. +/// +/// An omitted start is [`track::Range::oldest`]'s. `..0` holds nothing, so it panics. +#[allow(dead_code, reason = "shared by several test binaries, each using only part of it")] +pub fn keys(keys: impl RangeBounds) -> track::Range { + let start = match keys.start_bound() { + Bound::Included(&start) => track::Anchor::Time(start), + Bound::Excluded(&start) => track::Anchor::Time(start + 1), + Bound::Unbounded => track::Range::oldest().start, + }; + let end = match keys.end_bound() { + Bound::Included(&end) => Some(track::Anchor::Time(end)), + Bound::Excluded(&end) => Some(track::Anchor::Time(end.checked_sub(1).expect("an empty range"))), + Bound::Unbounded => None, + }; + track::Range { start, end } +} /// Wait until anyone wants the track, returning the group they want. #[allow(dead_code, reason = "shared by several test binaries, each using only part of it")] diff --git a/rs/moq-net/tests/integration/track_abort_over_session.rs b/rs/moq-net/tests/integration/track_abort_over_session.rs index 3041443a262..2a173063982 100644 --- a/rs/moq-net/tests/integration/track_abort_over_session.rs +++ b/rs/moq-net/tests/integration/track_abort_over_session.rs @@ -4,6 +4,8 @@ //! subscriber on the far side of one session. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; @@ -49,8 +51,8 @@ async fn after_abort(version: &str, hops: u32) { .unwrap() .unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); - let mut open = track.append_group().unwrap(); - open.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut open = track.append_next().unwrap(); + open.write_frame(open.at_start(), b"head".as_ref()).unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); assert_eq!(group.read_frame().await.unwrap().unwrap().payload, b"head".as_ref()); track.abort(moq_net::Error::Cancel).unwrap(); @@ -90,14 +92,14 @@ async fn recreate(finish: bool) { let track = broadcast.broadcast().create_track("video", timed()).unwrap(); let remote = origin.consume().request("bench", None).await.unwrap(); let mut sub = remote.track("video").unwrap().subscribe(None).await.unwrap(); - let mut open = track.create_group(moq_net::group::Info { sequence: 2 }).unwrap(); - open.write_frame(Timestamp::ZERO, b"head".as_ref()).unwrap(); + let mut open = track.create_group(moq_net::group::Info { start: 2 }).unwrap(); + open.write_frame(open.at_start(), b"head".as_ref()).unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 2); + assert_eq!(group.start, 2); group.read_frame().await.unwrap().unwrap(); track.abort(moq_net::Error::Cancel).unwrap(); let fresh = broadcast.broadcast().create_track("video", timed()).unwrap(); - let mut next = fresh.create_group(moq_net::group::Info { sequence: 3 }).unwrap(); + let mut next = fresh.create_group(moq_net::group::Info { start: 3 }).unwrap(); next.write_frame(Timestamp::from_millis(1).unwrap(), b"next".as_ref()) .unwrap(); drop(open); @@ -112,7 +114,7 @@ async fn recreate(finish: bool) { .expect("group-only failover hung"); assert!(result.is_err()); let next = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(next.sequence, 3); + assert_eq!(next.start, 3); } #[kio::test] diff --git a/rs/moq-net/tests/integration/track_stream_demand.rs b/rs/moq-net/tests/integration/track_stream_demand.rs index a40bfb75120..9e8c64b3bdc 100644 --- a/rs/moq-net/tests/integration/track_stream_demand.rs +++ b/rs/moq-net/tests/integration/track_stream_demand.rs @@ -9,10 +9,12 @@ //! `used` with a round trip per hop in between. use crate::support; +use support::AppendNext; +use support::AtStart; use std::{cell::RefCell, rc::Rc, time::Duration}; -use moq_net::{Timestamp, Version, origin}; +use moq_net::{Version, origin}; use support::{ harness::{MockConnectOptions, MockPair, connect_mock}, mock::MockSession, @@ -96,8 +98,8 @@ async fn one_used_edge(version: &str, relays: usize, reorder: bool) { .get_or_insert_with(|| record_track!(request.demand())); kio::clock::sleep(STARTUP).await; let track = request.accept(timed()); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"key".as_ref()).unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"key".as_ref()).unwrap(); served.0.borrow_mut().push((track, group)); } }); @@ -189,8 +191,8 @@ async fn bare_handle_is_lazy(version: &str, relays: usize) { let publisher = kio::clock::sim::spawn(async move { while let Ok(request) = dynamic.requested_track().await { let track = request.accept(timed()); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, b"key".as_ref()).unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), b"key".as_ref()).unwrap(); served.borrow_mut().push((track, group)); } }); diff --git a/rs/moq-net/tests/integration/track_tail.rs b/rs/moq-net/tests/integration/track_tail.rs index 80aa0f6e7d2..ffab22ecc21 100644 --- a/rs/moq-net/tests/integration/track_tail.rs +++ b/rs/moq-net/tests/integration/track_tail.rs @@ -12,10 +12,12 @@ //! Time is paused, so the one-second grace for a group that never arrives is free. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; -use moq_net::{Timestamp, Version}; +use moq_net::Version; use support::harness::{MockConnectOptions, connect_mock}; /// A millisecond track with no publisher max age: the frames written here carry timestamps, @@ -114,7 +116,7 @@ async fn connect(version: &str) -> Pair { /// Publish a group below the declared end (or none for an empty track), hold its stream /// back until the subscription ends, then deliver or lose it. The subscriber reads with /// `max_delay`. -async fn round(version: &str, late: Late, final_sequence: u64, max_delay: Option) -> Outcome { +async fn round(version: &str, late: Late, final_key: u64, max_delay: Option) -> Outcome { let Pair { pair, mut track, @@ -124,7 +126,7 @@ async fn round(version: &str, late: Late, final_sequence: u64, max_delay: Option let reader = kio::clock::sim::spawn(async move { let subscription = moq_net::track::Subscription::default() - .with_range(0..) + .with_ranges([support::keys(0..)]) .with_delay(moq_net::track::Delay::default().with_max(max_delay)); let mut sub = remote .track("video") @@ -160,17 +162,17 @@ async fn round(version: &str, late: Late, final_sequence: u64, max_delay: Option kio::clock::sleep(Duration::from_millis(1)).await; pair.server_transport.hold_unis(); - if final_sequence > 0 { - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); - group.finish().unwrap(); + if final_key > 0 { + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), PAYLOAD).unwrap(); + group.close().unwrap(); } track - .finish_at(moq_net::track::Position::group(final_sequence), None) + .finish_at(moq_net::track::Position::group(final_key), None) .unwrap(); drop(track); - if final_sequence > 0 { + if final_key > 0 { // Paused time advances only when every task is idle, after the publisher and // subscriber have processed the subscription's end. kio::clock::sleep(GRACE / 10).await; @@ -271,7 +273,7 @@ async fn a_session_death_before_the_tail_aborts_the_track() { } } -/// Skipped sequences have no stream, so lite-05+ publishers name them with SUBSCRIBE_DROP. +/// Skipped keys have no stream, so lite-05+ publishers name them with SUBSCRIBE_DROP. #[kio::test] async fn skipped_groups_end_without_the_grace() { for version in ["moq-lite-05", "moq-lite-06", "moq-lite-07-wip"] { @@ -310,15 +312,15 @@ async fn draft18_a_join_reaching_a_finished_track_keeps_its_head() { _keep, } = connect("moq-transport-18").await; pair.client_transport.set_latency(RTT / 2); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), PAYLOAD).unwrap(); + group.close().unwrap(); let reader = kio::clock::sim::spawn(async move { let mut sub = remote .track("video") .unwrap() - .subscribe(moq_net::track::Subscription::default().with_range(0..)) + .subscribe(moq_net::track::Subscription::default().with_ranges([support::keys(0..)])) .await .expect("subscribe"); let mut frames = Vec::new(); @@ -367,7 +369,7 @@ async fn ietf_leaving_cancels_a_blocked_end_of_track() { _keep, } = connect(version).await; - let subscription = moq_net::track::Subscription::default().with_range(0..); + let subscription = moq_net::track::Subscription::default().with_ranges([support::keys(0..)]); let mut sub = remote .track("video") .unwrap() @@ -378,9 +380,9 @@ async fn ietf_leaving_cancels_a_blocked_end_of_track() { .await .expect("no subscriber appeared"); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), PAYLOAD).unwrap(); + group.close().unwrap(); let mut group = kio::clock::timeout(TIMEOUT, sub.recv_group()) .await .expect("group timeout") @@ -425,7 +427,7 @@ async fn a_group_whose_first_frame_trails_its_header_is_delivered() { let reader = kio::clock::sim::spawn(async move { let subscription = moq_net::track::Subscription::default() - .with_range(0..) + .with_ranges([support::keys(0..)]) .with_delay(moq_net::track::Delay::default().with_max(TIMEOUT)); let mut sub = remote .track("video") @@ -437,7 +439,7 @@ async fn a_group_whose_first_frame_trails_its_header_is_delivered() { while let Some(mut group) = sub.recv_group().await.expect("track aborted") { let frame = group.read_frame().await.expect("group aborted").expect("a frame"); assert_eq!(&frame.payload[..], PAYLOAD); - groups.push(group.sequence); + groups.push(group.start); } groups }); @@ -446,7 +448,7 @@ async fn a_group_whose_first_frame_trails_its_header_is_delivered() { .expect("no subscriber appeared"); track.finish_at(moq_net::track::Position::group(2), None).unwrap(); - let groups = [track.append_group().unwrap(), track.append_group().unwrap()]; + let groups = [track.append_next().unwrap(), track.append_next().unwrap()]; // Paused time advances only once every task is idle: both headers and the end // have reached the subscriber, ahead of any frame. kio::clock::sleep(GRACE / 10).await; @@ -456,8 +458,8 @@ async fn a_group_whose_first_frame_trails_its_header_is_delivered() { ); for mut group in groups { - group.write_frame(Timestamp::ZERO, PAYLOAD).unwrap(); - group.finish().unwrap(); + group.write_frame(group.at_start(), PAYLOAD).unwrap(); + group.close().unwrap(); } let mut groups = kio::clock::timeout(TIMEOUT, reader) .await diff --git a/rs/moq-net/tests/integration/tracks.rs b/rs/moq-net/tests/integration/tracks.rs index c8c68cd6f47..9dd576830f8 100644 --- a/rs/moq-net/tests/integration/tracks.rs +++ b/rs/moq-net/tests/integration/tracks.rs @@ -5,6 +5,7 @@ //! after the sync is new at the source. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -140,9 +141,11 @@ async fn a_listed_track_serves_its_cached_group() { assert_eq!(next(&mut tracks).await, "synced"); let late = broadcast.broadcast().create_track("late", None).unwrap(); - let mut group = late.append_group().unwrap(); - group.write_frame(None, &b"first"[..]).unwrap(); - group.finish().unwrap(); + let mut group = late.append_next().unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), &b"first"[..]) + .unwrap(); + group.close().unwrap(); assert_eq!(next(&mut tracks).await, "+late"); let mut sub = kio::clock::timeout(TIMEOUT, remote.track("late").unwrap().subscribe(None)) @@ -154,7 +157,7 @@ async fn a_listed_track_serves_its_cached_group() { .expect("group timeout") .unwrap() .expect("the group"); - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); assert_eq!(&group.read_frame().await.unwrap().unwrap().payload[..], b"first"); } diff --git a/rs/moq-net/tests/integration/tracks_ietf.rs b/rs/moq-net/tests/integration/tracks_ietf.rs index a8c2040c506..368b4a3c54f 100644 --- a/rs/moq-net/tests/integration/tracks_ietf.rs +++ b/rs/moq-net/tests/integration/tracks_ietf.rs @@ -8,6 +8,7 @@ //! Deterministic: paused time and a single-threaded runtime over the mock transport. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -90,7 +91,7 @@ async fn read(sub: &mut moq_net::track::Subscriber) -> Option<(u64, Vec)> { .await .expect("frame timeout") .ok()??; - Some((group.sequence, frame.payload.to_vec())) + Some((group.start, frame.payload.to_vec())) } /// SUBSCRIBE_TRACKS goes out only for a listing, since many peers do not answer it: @@ -116,9 +117,11 @@ async fn a_listing_of_our_own_peer_fails_alone() { .await .expect("never subscribed") .unwrap(); - let mut group = video.append_group().unwrap(); - group.write_frame(None, &b"before"[..]).unwrap(); - group.finish().unwrap(); + let mut group = video.append_next().unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), &b"before"[..]) + .unwrap(); + group.close().unwrap(); assert_eq!(read(&mut sub).await, Some((0, b"before".to_vec())), "{version}"); assert_eq!(opened(&pair, SUBSCRIBE_TRACKS), 0, "{version}: listed unasked"); @@ -128,9 +131,11 @@ async fn a_listing_of_our_own_peer_fails_alone() { assert!(failed.starts_with("error:"), "{version}: {failed}"); assert_eq!(opened(&pair, SUBSCRIBE_TRACKS), 1, "{version}"); - let mut group = video.append_group().unwrap(); - group.write_frame(None, &b"after"[..]).unwrap(); - group.finish().unwrap(); + let mut group = video.append_next().unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), &b"after"[..]) + .unwrap(); + group.close().unwrap(); assert_eq!(read(&mut sub).await, Some((1, b"after".to_vec())), "{version}"); assert_eq!(subscribes(&pair), 1, "{version}"); } diff --git a/rs/moq-net/tests/integration/untimed.rs b/rs/moq-net/tests/integration/untimed.rs index 1a862b836d6..e4358c73251 100644 --- a/rs/moq-net/tests/integration/untimed.rs +++ b/rs/moq-net/tests/integration/untimed.rs @@ -4,10 +4,11 @@ //! that can say so, and a timed track keeps its timestamps. //! //! No receiver fills in arrival time. Lite-05 and lite-06 cannot encode absence, so their -//! encoder writes its send time instead, which their receivers pass through as the track's -//! time. +//! encoder writes each frame's group key instead, which their receivers pass through as the +//! track's time. use crate::support; +use support::AppendNext; use std::time::Duration; @@ -28,8 +29,8 @@ enum Arrives { /// Both keep what the publisher wrote. Faithful, /// The timed track keeps its timestamps, but the untimed one arrives timed in - /// milliseconds with its send times. - SendTimes, + /// milliseconds with its group keys. + Keys, /// The wire carries no timescale, so even the timed track arrives untimed. Untimed, } @@ -37,7 +38,7 @@ enum Arrives { fn expected(version: &str) -> Arrives { match version { "moq-lite-03" | "moq-lite-04" => Arrives::Untimed, - "moq-lite-05" | "moq-lite-06" => Arrives::SendTimes, + "moq-lite-05" | "moq-lite-06" => Arrives::Keys, _ => Arrives::Faithful, } } @@ -97,7 +98,7 @@ impl Chain { } /// The first frame's timestamp of the next group. -async fn next_frame(subscriber: &mut track::Subscriber) -> Option { +async fn next_frame(subscriber: &mut track::Subscriber) -> Timestamp { let mut group = kio::clock::timeout(TIMEOUT, subscriber.recv_group()) .await .expect("group timeout") @@ -106,10 +107,10 @@ async fn next_frame(subscriber: &mut track::Subscriber) -> Option { group.read_frame().await.unwrap().expect("a frame").timestamp } -/// Publish a single-frame group on a track with `info` through `hops`, returning what the -/// last node reads for it and the track info it received. -async fn frame_through(hops: &[&str], info: track::Info) -> (Option, track::Info) { - let timestamp = info.timescale.map(|_| millis(7)); +/// Publish a single-frame group at key 7 on a track with `info` through `hops`, returning +/// what the last node reads for it and the track info it received. +async fn frame_through(hops: &[&str], info: track::Info) -> (Timestamp, track::Info) { + let timed = info.timescale.is_some(); let chain = Chain::new(hops).await; let broadcast = chain.nodes[0] .route("room", moq_net::origin::RouteOptions::hidden()) @@ -119,12 +120,27 @@ async fn frame_through(hops: &[&str], info: track::Info) -> (Option, kio::clock::sleep(Duration::from_secs(1)).await; let mut subscriber = chain.subscribe("data").await; - producer.write_frame(timestamp, &b"frame"[..]).unwrap(); + match timed { + true => producer.write_frame(millis(7), &b"frame"[..]).unwrap(), + false => { + let mut group = producer.append_group(7).unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), &b"frame"[..]) + .unwrap(); + group.close().unwrap(); + } + } let received = next_frame(&mut subscriber).await; - let received = received.map(|timestamp| timestamp.convert(Timescale::MILLI).unwrap()); + let received = match received.scale() { + Some(_) => received.convert(Timescale::MILLI).unwrap(), + None => received, + }; (received, subscriber.info().clone()) } +/// The key [`frame_through`] publishes its group at, which a frame with no time carries. +const KEY: moq_net::Timestamp = moq_net::Timestamp::unscaled(7); + #[kio::test] async fn frames_arrive_as_published() { for version in VERSIONS { @@ -132,19 +148,19 @@ async fn frames_arrive_as_published() { let (untimed, untimed_info) = frame_through(&[version], untimed()).await; match expected(version) { Arrives::Faithful => { - assert_eq!(timed, Some(millis(7)), "{version}: the timestamp survives"); - assert_eq!(untimed, None, "{version}: absence survives"); + assert_eq!(timed, millis(7), "{version}: the timestamp survives"); + assert_eq!(untimed, KEY, "{version}: absence survives"); assert_eq!(untimed_info.timescale, None, "{version}: no timeline is declared"); } - Arrives::SendTimes => { - assert_eq!(timed, Some(millis(7)), "{version}: the timestamp survives"); - assert!(untimed.is_some(), "{version}: the send time arrives"); + Arrives::Keys => { + assert_eq!(timed, millis(7), "{version}: the timestamp survives"); + assert_eq!(untimed, millis(7), "{version}: the group key arrives"); assert_eq!(untimed_info.timescale, Some(Timescale::MILLI), "{version}"); } Arrives::Untimed => { - assert_eq!(timed, None, "{version}: no units, so no timestamp"); + assert_eq!(timed, KEY, "{version}: no units, so no timestamp"); assert_eq!(timed_info.timescale, None, "{version}: no timeline is declared"); - assert_eq!(untimed, None, "{version}: absence survives"); + assert_eq!(untimed, KEY, "{version}: absence survives"); } } } @@ -159,13 +175,13 @@ async fn a_relay_forwards_an_untimed_track_untimed() { &["moq-lite-07-wip", "moq-lite-07-wip"][..], ] { let (frame, info) = frame_through(hops, untimed()).await; - assert_eq!(frame, None, "{hops:?}: the relay did not stamp it"); + assert_eq!(frame, KEY, "{hops:?}: the relay did not stamp it"); assert_eq!(info.timescale, None, "{hops:?}: the relay must not claim a timeline"); } let hops = ["moq-lite-03", "moq-transport-18"]; let (frame, info) = frame_through(&hops, timed()).await; - assert_eq!(frame, None, "{hops:?}: the first hop has no timestamps to forward"); + assert_eq!(frame, KEY, "{hops:?}: the first hop has no timestamps to forward"); assert_eq!(info.timescale, None, "{hops:?}: the relay must not claim a timeline"); } @@ -182,16 +198,20 @@ async fn an_explicit_start_holds_on_an_untimed_track() { let broadcast = chain.nodes[0] .route("room", moq_net::origin::RouteOptions::hidden()) .unwrap(); - let mut producer = broadcast.broadcast().create_track("data", untimed()).unwrap(); + let producer = broadcast.broadcast().create_track("data", untimed()).unwrap(); broadcast.announce().unwrap(); for _ in 0..5 { - producer.write_frame(None, &b"untimed"[..]).unwrap(); + let mut group = producer.append_next().unwrap(); + group + .write_frame(moq_net::Timestamp::unscaled(group.start), &b"untimed"[..]) + .unwrap(); + group.close().unwrap(); } kio::clock::sleep(Duration::from_secs(1)).await; let consumer = chain.nodes[1].consume().request("room", None).await.unwrap(); let subscription = track::Subscription::default() - .with_range(2..) + .with_ranges([support::keys(2..)]) .with_delay(track::Delay::default().with_max(Duration::from_secs(30))); let mut subscriber = kio::clock::timeout(TIMEOUT, consumer.track("data").unwrap().subscribe(subscription)) .await @@ -203,7 +223,7 @@ async fn an_explicit_start_holds_on_an_untimed_track() { let Ok(next) = kio::clock::timeout(TIMEOUT, subscriber.recv_group()).await else { break; }; - got.push(next.unwrap().expect("a group").sequence); + got.push(next.unwrap().expect("a group").start); } got.sort(); assert_eq!(got, vec![2, 3, 4], "{version}"); @@ -211,7 +231,7 @@ async fn an_explicit_start_holds_on_an_untimed_track() { } /// An untimed datagram track's datagram arrives untimed, except that lite-05 and lite-06 -/// carry its send time on a millisecond track. +/// carry its sequence on a millisecond track. #[kio::test] async fn datagrams_arrive_as_published() { for version in ["moq-lite-05", "moq-lite-07-wip"] { @@ -238,20 +258,27 @@ async fn datagrams_arrive_as_published() { // Datagrams count once the answer is back. let timescale = consumer.info().await.unwrap().timescale; - producer.append(None, &b"untimed"[..]).unwrap(); + producer + .append(moq_net::Timestamp::unscaled(0), &b"untimed"[..]) + .unwrap(); let datagram = kio::clock::timeout(TIMEOUT, consumer.recv()) .await .expect("datagram timeout") .unwrap() .expect("a datagram"); - let sent = expected(version) == Arrives::SendTimes; - assert_eq!(datagram.timestamp.is_some(), sent, "{version}"); - assert_eq!(timescale, sent.then_some(Timescale::MILLI), "{version}"); + let keyed = expected(version) == Arrives::Keys; + // A scale-less datagram's time is its sequence; lite-05/06 declare milliseconds. + let expected = match keyed { + true => millis(datagram.sequence), + false => moq_net::Timestamp::unscaled(datagram.sequence), + }; + assert_eq!(datagram.timestamp, expected, "{version}"); + assert_eq!(timescale, keyed.then_some(Timescale::MILLI), "{version}"); } } /// A standalone FETCH of an untimed track answers with untimed objects, except that lite-05 -/// and lite-06 carry each one's send time. +/// and lite-06 carry each one's group key. #[kio::test] async fn a_fetched_untimed_frame_arrives_untimed() { for version in ["moq-lite-06", "moq-lite-07-wip", "moq-transport-18"] { @@ -264,7 +291,7 @@ async fn a_fetched_untimed_frame_arrives_untimed() { kio::clock::sleep(Duration::from_secs(1)).await; let consumer = chain.nodes[1].consume().request("room", None).await.unwrap(); - let mut waiting = Box::pin(consumer.track("video").unwrap().fetch_group(0, None)); + let mut waiting = Box::pin(consumer.track("video").unwrap().fetch_group(3, None)); let request = match futures::future::select(std::pin::pin!(dynamic.requested_track()), &mut waiting).await { futures::future::Either::Left((request, _)) => request.expect("the track request reaches the publisher"), futures::future::Either::Right(_) => panic!("nothing answered the track"), @@ -275,13 +302,22 @@ async fn a_fetched_untimed_frame_arrives_untimed() { futures::future::Either::Left((request, _)) => request.expect("the fetch reaches the publisher"), futures::future::Either::Right(_) => panic!("nothing answered the fetch"), }; - let mut group = request.accept(0, timed()).unwrap(); - group.write_frame(None, Bytes::from_static(b"segment")).unwrap(); - group.finish().unwrap(); + let mut group = request.accept(3, timed()).unwrap(); + group + .write_frame( + moq_net::Timestamp::unscaled(group.start), + Bytes::from_static(b"segment"), + ) + .unwrap(); + group.close().unwrap(); let mut fetched = waiting.await.unwrap(); let frame = fetched.read_frame().await.unwrap().unwrap(); - let sent = expected(version) == Arrives::SendTimes; - assert_eq!(frame.timestamp.is_some(), sent, "{version}"); + let keyed = expected(version) == Arrives::Keys; + let expected = match keyed { + true => millis(3), + false => moq_net::Timestamp::unscaled(3), + }; + assert_eq!(frame.timestamp, expected, "{version}"); } } diff --git a/rs/moq-net/tests/integration/warm_backfill.rs b/rs/moq-net/tests/integration/warm_backfill.rs index 2edf94242ed..de452a21369 100644 --- a/rs/moq-net/tests/integration/warm_backfill.rs +++ b/rs/moq-net/tests/integration/warm_backfill.rs @@ -49,7 +49,7 @@ async fn subscribe(relay: &origin::Producer) -> track::Subscriber { .subscribe( Subscription::default() .with_delay(Delay::default().with_max(BUDGET)) - .with_range(0..), + .with_ranges([crate::support::keys(0..)]), ) .await .unwrap() @@ -60,7 +60,7 @@ async fn received(subscriber: &mut track::Subscriber) -> BTreeSet { let mut got = BTreeSet::new(); while let Ok(group) = kio::clock::timeout(Duration::from_secs(1), subscriber.recv_group()).await { let group = group.unwrap().expect("the track is live"); - assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + assert!(got.insert(group.start), "group {} delivered twice", group.start); } got } @@ -81,12 +81,12 @@ async fn a_newcomer_on_a_warm_relay_gets_the_backfill() { Info::default().with_timescale(Timescale::MILLI).with_max_age(None), ) .unwrap(); - for sequence in 0..GROUPS { - let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in 0..GROUPS { + let mut group = source.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key * 1000).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } broadcast.announce().unwrap(); @@ -115,12 +115,12 @@ async fn a_refused_transport_backfill_keeps_delivering_live() { .broadcast() .create_track("video", Info::default().with_timescale(Timescale::MILLI)) .unwrap(); - for sequence in 0..GROUPS { - let mut group = source.create_group(sequence.into()).unwrap(); + for key in 0..GROUPS { + let mut group = source.create_group(key.into()).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key * 1000).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } broadcast.announce().unwrap(); let mut subscriber = kio::clock::timeout(TIMEOUT, subscribe(&relay)).await.unwrap(); @@ -130,12 +130,12 @@ async fn a_refused_transport_backfill_keeps_delivering_live() { group .write_frame(Timestamp::from_millis(GROUPS * 1000).unwrap(), &b"live"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let got = kio::clock::timeout(TIMEOUT, subscriber.recv_group()) .await .unwrap() .unwrap() .unwrap(); - assert_eq!(got.sequence, GROUPS, "{version}: still live after the refused backfill"); + assert_eq!(got.start, GROUPS, "{version}: still live after the refused backfill"); } } diff --git a/rs/moq-net/tests/integration/wide_sequence.rs b/rs/moq-net/tests/integration/wide_key.rs similarity index 79% rename from rs/moq-net/tests/integration/wide_sequence.rs rename to rs/moq-net/tests/integration/wide_key.rs index 71e26d1f5f9..c582bea81c6 100644 --- a/rs/moq-net/tests/integration/wide_sequence.rs +++ b/rs/moq-net/tests/integration/wide_key.rs @@ -1,18 +1,19 @@ #![cfg(feature = "async")] -//! Group sequence boundaries, directly and through a forwarding origin. +//! Group key boundaries, directly and through a forwarding origin. use crate::support; +use support::AppendNext; use std::time::Duration; use moq_net::{Timestamp, Version}; use support::harness::{MockConnectOptions, connect_mock}; -/// Published QUIC-varint sequences keep all their bits on the live feed, directly +/// Published QUIC-varint keys keep all their bits on the live feed, directly /// and through a forwarding origin, even beyond JavaScript's exact number range. #[kio::test] -async fn largest_quic_sequence_reaches_the_subscriber() { +async fn largest_quic_key_reaches_the_subscriber() { for alpn in ["moq-lite-02", "moq-lite-06", "moq-transport-18"] { for relayed in [false, true] { let version: Version = alpn.parse().unwrap(); @@ -46,23 +47,26 @@ async fn largest_quic_sequence_reaches_the_subscriber() { .unwrap() .unwrap(); let mut sub = remote.track("data").unwrap().subscribe(None).await.unwrap(); - for sequence in [5, 1 << 32, (1 << 53) - 1, (1 << 53) + 1, (1 << 62) - 1] { + for key in [5, 1 << 32, (1 << 53) - 1, (1 << 53) + 1, (1 << 62) - 1] { // Timestamps stop at 2^61 - 1, the most lite-05/06 can zigzag as a first frame. - let timestamp = Timestamp::from_micros(sequence.min((1 << 61) - 1)).unwrap(); - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let timestamp = Timestamp::from_micros(key.min((1 << 61) - 1)).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: key }).unwrap(); group.write_frame(timestamp, b"ladder".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut got = kio::clock::timeout(Duration::from_secs(5), sub.recv_group()) .await - .unwrap_or_else(|_| panic!("{alpn} relayed={relayed} sequence={sequence} hung")) + .unwrap_or_else(|_| panic!("{alpn} relayed={relayed} key={key} hung")) .unwrap() .unwrap(); - assert_eq!(got.sequence, sequence, "{alpn} relayed={relayed}"); + assert_eq!(got.start, key, "{alpn} relayed={relayed}"); let frame = got.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload, b"ladder".as_ref()); - // Lite-02 carries no timestamps. - let timed = alpn != "moq-lite-02"; - assert_eq!(frame.timestamp, timed.then_some(timestamp), "{alpn} relayed={relayed}"); + // Lite-02 carries no timestamps, so its frames carry their key. + let expected = match alpn { + "moq-lite-02" => Timestamp::unscaled(key), + _ => timestamp, + }; + assert_eq!(frame.timestamp, expected, "{alpn} relayed={relayed}"); } } } @@ -73,6 +77,12 @@ fn timed() -> moq_net::track::Info { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI) } +/// The latest frame time a millisecond track can carry, short of a wide key, which no +/// timestamp reaches. +fn latest() -> Timestamp { + Timestamp::new((1 << 61) - 1, moq_net::Timescale::MILLI).unwrap() +} + fn produce_origin(hop: u64) -> moq_net::origin::Producer { let (producer, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::new(std::num::NonZeroU64::new(hop).unwrap())); @@ -80,10 +90,10 @@ fn produce_origin(hop: u64) -> moq_net::origin::Producer { producer.with_proxy(true) } -/// The relay accepts the wide sequence from lite-07, refuses that subscription to the +/// The relay accepts the wide key from lite-07, refuses that subscription to the /// lite-06 reader that cannot encode it, and keeps the session serving other tracks. #[kio::test] -async fn wide_sequence_fails_only_its_subscription() { +async fn wide_key_fails_only_its_subscription() { const WIDE: u64 = 1 << 62; let publisher = produce_origin(1); @@ -96,9 +106,9 @@ async fn wide_sequence_fails_only_its_subscription() { let wide = broadcast.broadcast().create_track("wide", timed()).unwrap(); let narrow = broadcast.broadcast().create_track("narrow", timed()).unwrap(); broadcast.announce().unwrap(); - let mut group = wide.create_group(moq_net::group::Info { sequence: WIDE }).unwrap(); - group.write_frame(Timestamp::ZERO, b"wide".as_ref()).unwrap(); - let mut group = narrow.append_group().unwrap(); + let mut group = wide.create_group(moq_net::group::Info { start: WIDE }).unwrap(); + group.write_frame(latest(), b"wide".as_ref()).unwrap(); + let mut group = narrow.append_next().unwrap(); group.write_frame(Timestamp::ZERO, b"narrow".as_ref()).unwrap(); let mut options = MockConnectOptions::new("moq-lite-07-wip".parse::().unwrap()); @@ -120,7 +130,7 @@ async fn wide_sequence_fails_only_its_subscription() { .unwrap() .unwrap(); let mut sub = cached.track("wide").unwrap().subscribe(None).await.unwrap(); - assert_eq!(sub.recv_group().await.unwrap().unwrap().sequence, WIDE); + assert_eq!(sub.recv_group().await.unwrap().unwrap().start, WIDE); let consumer = client.consume(); let remote = consumer @@ -133,7 +143,7 @@ async fn wide_sequence_fails_only_its_subscription() { let result = kio::clock::timeout(Duration::from_secs(5), async { let mut sub = remote.track("wide").unwrap().subscribe(None).await?; - sub.recv_group().await.map(|group| group.map(|group| group.sequence)) + sub.recv_group().await.map(|group| group.map(|group| group.start)) }) .await .expect("the wide subscription hung instead of failing"); @@ -165,9 +175,9 @@ async fn a_later_wide_group_fails_only_that_subscription() { let wide = broadcast.broadcast().create_track("wide", timed()).unwrap(); let narrow = broadcast.broadcast().create_track("narrow", timed()).unwrap(); broadcast.announce().unwrap(); - let mut group = wide.append_group().unwrap(); + let mut group = wide.append_next().unwrap(); group.write_frame(Timestamp::ZERO, b"first".as_ref()).unwrap(); - let mut group = narrow.append_group().unwrap(); + let mut group = narrow.append_next().unwrap(); group.write_frame(Timestamp::ZERO, b"narrow".as_ref()).unwrap(); let mut options = MockConnectOptions::new("moq-lite-07-wip".parse::().unwrap()); @@ -190,11 +200,11 @@ async fn a_later_wide_group_fails_only_that_subscription() { let mut sub = remote.track("wide").unwrap().subscribe(None).await.unwrap(); let mut group = sub.recv_group().await.unwrap().unwrap(); - assert_eq!(group.sequence, 0); + assert_eq!(group.start, 0); assert_eq!(group.read_frame().await.unwrap().unwrap().payload, b"first".as_ref()); - let mut group = wide.create_group(moq_net::group::Info { sequence: WIDE }).unwrap(); - group.write_frame(Timestamp::ZERO, b"wide".as_ref()).unwrap(); + let mut group = wide.create_group(moq_net::group::Info { start: WIDE }).unwrap(); + group.write_frame(latest(), b"wide".as_ref()).unwrap(); let result = kio::clock::timeout(Duration::from_secs(5), sub.recv_group()) .await diff --git a/rs/moq-net/tests/integration/widened_max_age.rs b/rs/moq-net/tests/integration/widened_max_age.rs index 30654aceca6..e9ee5ef36fa 100644 --- a/rs/moq-net/tests/integration/widened_max_age.rs +++ b/rs/moq-net/tests/integration/widened_max_age.rs @@ -93,12 +93,12 @@ impl Chain { } fn populate(&self) { - for sequence in 0..GROUPS { - let mut group = self.source.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in 0..GROUPS { + let mut group = self.source.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key * 1000).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } } @@ -134,7 +134,7 @@ async fn received(subscriber: &mut track::Subscriber) -> BTreeSet { let mut got = BTreeSet::new(); while let Ok(group) = kio::clock::timeout(Duration::from_secs(1), subscriber.recv_group()).await { let group = group.unwrap().expect("the track is live"); - assert!(got.insert(group.sequence), "group {} delivered twice", group.sequence); + assert!(got.insert(group.start), "group {} delivered twice", group.start); } got } @@ -194,7 +194,7 @@ async fn a_tightened_start_leaves_history_fillable_by_an_intermediate_reader() { let reading = kio::clock::sim::spawn(async move { let mut got = BTreeSet::new(); while got.len() < 6 { - got.insert(intermediate.recv_group().await.unwrap().unwrap().sequence); + got.insert(intermediate.recv_group().await.unwrap().unwrap().start); } got }); @@ -313,12 +313,12 @@ async fn wider_newcomers_after_the_first_pull_nothing_upstream() { .route("live", moq_net::origin::RouteOptions::hidden()) .unwrap(); let source = broadcast.broadcast().create_track("video", timed()).unwrap(); - for sequence in 0..GROUPS { - let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in 0..GROUPS { + let mut group = source.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key * 1000).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } broadcast.announce().unwrap(); @@ -374,12 +374,12 @@ async fn a_legacy_viewer_beating_a_relay_copy_joins_at_the_live_edge() { .unwrap(); let source = route.broadcast().create_track("video", timed()).unwrap(); route.announce().unwrap(); - for sequence in 0..GROUPS { - let mut group = source.create_group(moq_net::group::Info { sequence }).unwrap(); + for key in 0..GROUPS { + let mut group = source.create_group(moq_net::group::Info { start: key }).unwrap(); group - .write_frame(Timestamp::from_millis(sequence * 1000).unwrap(), &b"x"[..]) + .write_frame(Timestamp::from_millis(key * 1000).unwrap(), &b"x"[..]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let consumer = viewer.consume(); diff --git a/rs/moq-net/tests/integration/zero_copy.rs b/rs/moq-net/tests/integration/zero_copy.rs index dfab49eacab..16a7993f6e1 100644 --- a/rs/moq-net/tests/integration/zero_copy.rs +++ b/rs/moq-net/tests/integration/zero_copy.rs @@ -6,11 +6,13 @@ //! frame never keeps a larger receive allocation alive. use crate::support; +use support::AppendNext; +use support::AtStart; use std::time::Duration; use bytes::Bytes; -use moq_net::{Client, Server, Timestamp, Version, origin, transport::memory}; +use moq_net::{Client, Server, Version, origin, transport::memory}; use support::harness::{now, spawn, subscribed}; const TIMEOUT: Duration = Duration::from_secs(10); @@ -71,9 +73,9 @@ async fn frames_cross_without_a_copy() { subscribed(&track).await; let payload = Bytes::from(vec![7u8; 64 * 1024]); - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - group.finish().unwrap(); + let mut group = track.append_next().unwrap(); + group.write_frame(group.at_start(), payload.clone()).unwrap(); + group.close().unwrap(); let mut group = kio::clock::timeout(TIMEOUT, sub.recv_group()) .await diff --git a/rs/moq-net/verify/resume/ResumeReader.tla b/rs/moq-net/verify/resume/ResumeReader.tla index a58b67472b8..b26ec19746a 100644 --- a/rs/moq-net/verify/resume/ResumeReader.tla +++ b/rs/moq-net/verify/resume/ResumeReader.tla @@ -52,7 +52,7 @@ vars == <> Landed(t) == {trk[t].arr[i] : i \in 1 .. Len(trk[t].arr)} Live(t) == trk[t].made /\ ~trk[t].failed -\* `latest_sequence`: the newest group the track holds. +\* `max_key`: the newest group the track holds. Latest(t) == IF Landed(t) = {} THEN None ELSE Max(Landed(t)) \* `TrackState::is_complete`: the end is declared and the tail holds every group diff --git a/rs/moq-relay/src/fetch.rs b/rs/moq-relay/src/fetch.rs index f7cacd67c3a..165edef0689 100644 --- a/rs/moq-relay/src/fetch.rs +++ b/rs/moq-relay/src/fetch.rs @@ -1,17 +1,17 @@ -/// Fetch one group of `track`: the one at `sequence`, or the newest when `None`. +/// Fetch one group of `track`: the one that starts at `start`, or the newest when `None`. /// -/// The newest needs a live subscription to learn its sequence, since a fetch can -/// only retrieve a sequence already known. Once known it is fetched rather than +/// The newest needs a live subscription to learn its start, since a fetch can +/// only retrieve a start already known. Once known it is fetched rather than /// read off the subscription, so an evicted group is retrieved from upstream /// instead of waited on forever. Fails before returning when the group can never /// be served: [`moq_net::Error::NotFound`] locally, including a track that ends /// before any group, or [`moq_net::StreamError::NotFound`] from upstream. pub async fn fetch_group( track: &moq_net::track::Consumer, - sequence: Option, + start: Option, ) -> moq_net::Result { - if let Some(sequence) = sequence { - return track.fetch_group(sequence, None).await; + if let Some(start) = start { + return track.fetch_group(start, None).await; } // The largest names the newest group, even an empty one, possibly before it arrives. @@ -32,16 +32,22 @@ mod test { fn the_newest_group_can_be_empty() { let broadcast = moq_net::broadcast::Info::default().produce(); let track = broadcast.create_track("t", moq_net::track::Info::default()).unwrap(); - let mut first = track.append_group().unwrap(); - first.write_frame(None, &b"a"[..]).unwrap(); - first.finish().unwrap(); - let _open = track.append_group().unwrap(); + let mut first = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); + first + .write_frame(moq_net::Timestamp::unscaled(first.start), &b"a"[..]) + .unwrap(); + first.close().unwrap(); + let _open = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); let handle = broadcast.consume().track("t").unwrap(); let group = fetch_group(&handle, None) .now_or_never() .expect("the newest group is cached") .unwrap(); - assert_eq!(group.sequence, 1); + assert_eq!(group.start, 1); } } diff --git a/rs/moq-relay/src/internal.rs b/rs/moq-relay/src/internal.rs index 7b3d25cbbf2..fa900fa8177 100644 --- a/rs/moq-relay/src/internal.rs +++ b/rs/moq-relay/src/internal.rs @@ -1831,26 +1831,38 @@ mod tests { let bc = egress.request(update.prefix.clone(), None).await.unwrap(); let mut egress_sub = bc.track("video").unwrap().subscribe(None).await.unwrap(); { - let mut group = pub_track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, vec![0u8; 12]).unwrap(); - group.write_frame(Timestamp::ZERO, vec![0u8; 34]).unwrap(); - group.finish().unwrap(); + let mut group = pub_track + .append_group(pub_track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); + group + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), vec![0u8; 12]) + .unwrap(); + group + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), vec![0u8; 34]) + .unwrap(); + group.close().unwrap(); } { - let mut group = pub_track.append_group().unwrap(); + let mut group = pub_track + .append_group(pub_track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); group .write_frame(Timestamp::from_millis(10_000).unwrap(), vec![0u8; 1234]) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let mut group = egress_sub.recv_group().await.unwrap().unwrap(); while group.read_frame().await.unwrap().is_some() {} // Write 56 ingress bytes into the regional-tier broadcast. { - let mut group = sub_track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, vec![0u8; 56]).unwrap(); - group.finish().unwrap(); + let mut group = sub_track + .append_group(sub_track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); + group + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), vec![0u8; 56]) + .unwrap(); + group.close().unwrap(); } let body = render_metrics(&stats.snapshot(), &[], &[]); @@ -1885,7 +1897,6 @@ mod tests { /// Two plain egress sessions reading one compressed group share a single inflate. #[tokio::test(start_paused = true)] async fn metrics_count_one_inflate_per_group() { - use moq_net::Timestamp; use moq_net::stats::{Registry, Tier}; let stats = Registry::new(Default::default()); @@ -1911,9 +1922,13 @@ mod tests { let bc = egress.request("demo/x", None).await.unwrap(); readers.push(bc.track("catalog.json").unwrap().subscribe(None).await.unwrap()); } - let mut group = track.append_group().unwrap(); - group.write_frame(Timestamp::ZERO, payload.clone()).unwrap(); - group.finish().unwrap(); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); + group + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), payload.clone()) + .unwrap(); + group.close().unwrap(); for reader in &mut readers { let mut group = reader.recv_group().await.unwrap().unwrap(); assert_eq!(group.read_frame().await.unwrap().unwrap().payload, payload.as_bytes()); diff --git a/rs/moq-relay/src/web.rs b/rs/moq-relay/src/web.rs index df7c0403cd9..005bac72ebc 100644 --- a/rs/moq-relay/src/web.rs +++ b/rs/moq-relay/src/web.rs @@ -914,7 +914,7 @@ struct FetchParams { #[derive(Debug, Default)] enum FetchGroup { - // Return the group at the given sequence number. + // Return the group that starts at the given key, in the track's timescale. Num(u64), // Return the latest group. @@ -1104,11 +1104,11 @@ async fn serve_fetch( .ok() .flatten() .ok_or(StatusCode::NOT_FOUND)?; - let sequence = match params.group { - FetchGroup::Num(sequence) => Some(sequence), + let start = match params.group { + FetchGroup::Num(start) => Some(start), FetchGroup::Latest => None, }; - let group = match async { crate::fetch_group(&broadcast.track(&track)?, sequence).await }.await { + let group = match async { crate::fetch_group(&broadcast.track(&track)?, start).await }.await { Ok(group) => group, // A miss upstream arrives as the stream reset that refused the FETCH. A group past // the track's max age is gone, so it is a miss too. @@ -1122,7 +1122,7 @@ async fn serve_fetch( Err(_) => return Err(StatusCode::INTERNAL_SERVER_ERROR), }; - tracing::info!(%track, group = %group.sequence, "serving group"); + tracing::info!(%track, group = %group.start, "serving group"); Ok(group) }) @@ -1267,10 +1267,14 @@ mod tests { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = track.append_group().unwrap(); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); const PAYLOAD: &[u8] = b""; - group.write_frame(moq_net::Timestamp::ZERO, PAYLOAD).unwrap(); - group.finish().unwrap(); + group + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), PAYLOAD) + .unwrap(); + group.close().unwrap(); let mut refused = Web::new( auth::Auth::refuse("test"), cluster.clone(), @@ -1505,9 +1509,13 @@ mod tests { moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI), ) .unwrap(); - let mut group = track.append_group().unwrap(); - group.write_frame(moq_net::Timestamp::ZERO, b"hello".as_ref()).unwrap(); - group.finish().unwrap(); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); + group + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"hello".as_ref()) + .unwrap(); + group.close().unwrap(); let state = Web::new(auth, cluster, certificates, Config::default()).state; for tier in ["first", "second"] { for selector in [FetchGroup::Num(0), FetchGroup::Latest] { @@ -1617,11 +1625,11 @@ mod tests { // Group 0 can only present until group 1 starts at 1s, a full second behind the 2s // edge: past the 500ms max age, so it is no longer served. for millis in [0, 1000, 2000] { - let mut group = track.append_group().unwrap(); + let mut group = track.append_group(millis).unwrap(); group .write_frame(moq_net::Timestamp::from_millis(millis).unwrap(), b"frame".as_ref()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); } let state = Web::new(auth, cluster, certificates, Config::default()).state; let status = serve_fetch( diff --git a/rs/moq-relay/tests/drills.rs b/rs/moq-relay/tests/drills.rs index 14f53048c8e..7fc2f762200 100644 --- a/rs/moq-relay/tests/drills.rs +++ b/rs/moq-relay/tests/drills.rs @@ -346,11 +346,13 @@ fn dial() -> moq_tokio::connect::Config { /// Append one finished group carrying `payload`. fn write_group(track: &mut moq_net::track::Producer, payload: &[u8]) { - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, payload) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), payload) .expect("write frame"); - group.finish().expect("finish group"); + group.close().expect("finish group"); } /// Read the next group's first frame, failing the drill on a timeout, a clean @@ -361,7 +363,7 @@ async fn read_group(reader: &mut Reader, what: &str) -> (u64, Vec) { .unwrap_or_else(|_| panic!("{what}: no group within {TIMEOUT:?}")) .unwrap_or_else(|err| panic!("{what}: track aborted: {err}")) .unwrap_or_else(|| panic!("{what}: track finished instead of delivering a group")); - let sequence = group.sequence; + let sequence = group.start; let mut group = group; let frame = kio::clock::timeout(TIMEOUT, group.read_frame()) @@ -609,8 +611,10 @@ async fn relay_killed_mid_group_aborts_then_resumes(lane: Lane) { // a second: the kill lands mid-group rather than in the quiet between them. // Nothing more is written, so what the reader is waiting for cannot arrive // ahead of the kill and turn this into a race. - let mut open = track.append_group().expect("append group"); - open.write_frame(moq_net::Timestamp::ZERO, b"read".as_ref()) + let mut open = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); + open.write_frame(moq_net::Timestamp::from_millis(open.start).unwrap(), b"read".as_ref()) .expect("write frame"); let mut open_reader = kio::clock::timeout(TIMEOUT, reader.groups.recv_group()) .await @@ -660,7 +664,7 @@ async fn relay_killed_mid_group_aborts_then_resumes(lane: Lane) { // Resumed delivery through the same subscriber origin: the reconnected // publisher re-announced, and a fresh subscribe rides the new session. - open.finish().expect("finish the interrupted group"); + open.close().expect("finish the interrupted group"); write_group(&mut track, b"after"); let mut reader = subscribe(&subscribed, "live").await; read_until(&mut reader, b"after", "after the restart").await; @@ -1258,7 +1262,7 @@ async fn cross_cluster(lane: Lane, link: PeerLink) { Err(err) => panic!("the subscription aborted: {err}"), Ok(None) => panic!("the subscription finished"), }; - let sequence = group.sequence; + let sequence = group.start; // Every unclaimed group below this one is a gap the subscription // skipped: FETCH it now rather than wait. @@ -1529,18 +1533,22 @@ async fn subscriber_max_age_skips_bufferbloat(version: &str) { assert!(path.overflowed() > 0, "the finite queue did not drop overflow"); // 5.12Mb/s of media behind a 64kb/s link, for a second. for sequence in 2..22 { - let mut group = track.append_group().unwrap(); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); group .write_frame(moq_net::Timestamp::from_millis(sequence * 50).unwrap(), payload.clone()) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); kio::clock::sleep(Duration::from_millis(50)).await; } - let mut newest = track.append_group().unwrap(); + let mut newest = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); newest .write_frame(moq_net::Timestamp::from_millis(1100).unwrap(), b"snapshot") .unwrap(); - newest.finish().unwrap(); + newest.close().unwrap(); // No successor means this snapshot must survive even after its budget elapsed. kio::clock::sleep(Duration::from_millis(500)).await; let sequence = kio::clock::timeout(TIMEOUT, async { @@ -1549,7 +1557,7 @@ async fn subscriber_max_age_skips_bufferbloat(version: &str) { if let Ok(Some(frame)) = group.read_frame().await && frame.payload == b"snapshot".as_slice() { - break group.sequence; + break group.start; } } }) diff --git a/rs/moq-relay/tests/integration/auth_lifetime.rs b/rs/moq-relay/tests/integration/auth_lifetime.rs index 15bb4a49de7..9f2aac15885 100644 --- a/rs/moq-relay/tests/integration/auth_lifetime.rs +++ b/rs/moq-relay/tests/integration/auth_lifetime.rs @@ -312,11 +312,13 @@ async fn connect_and_round_trip(url: &url::Url) -> (moq_tokio::Connection, moq_t .broadcast() .create_track("video", timed()) .expect("create track"); - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"hello".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"hello".as_ref()) .expect("write frame"); - group.finish().expect("finish group"); + group.close().expect("finish group"); let pub_session = kio::clock::timeout( TIMEOUT, @@ -761,11 +763,13 @@ async fn a_moved_tier_retags_the_live_session() { let mut track_sub = bc.track("video").unwrap().subscribe(None).await.expect("subscribe"); let mut round_trip = async |payload: &'static [u8]| { - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, payload) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), payload) .expect("write frame"); - group.finish().expect("finish group"); + group.close().expect("finish group"); let mut group = kio::clock::timeout(TIMEOUT, track_sub.recv_group()) .await .expect("recv_group timeout") @@ -903,12 +907,15 @@ async fn a_rechecked_grant_resizes_live_sessions() { ); } let send = |index: usize, sequence: u64| { - let mut group = tracks[index].1.append_group().expect("append group"); + let mut group = tracks[index] + .1 + .append_group(tracks[index].1.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"media".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"media".as_ref()) .expect("write frame"); - let group = group.finish().expect("finish group"); - assert_eq!(group.sequence, sequence); + let group = group.close().expect("finish group"); + assert_eq!(group.start, sequence); }; for (index, sub) in subs.iter_mut().enumerate() { send(index, 0); @@ -949,7 +956,7 @@ async fn a_rechecked_grant_resizes_live_sessions() { .expect("recv_group timeout") .expect("recv_group failed") .expect("the sibling track closed"); - assert_eq!(group.sequence, 1, "{scheme}"); + assert_eq!(group.start, 1, "{scheme}"); // Undeafen: the path is announced again and a new subscription flows. script.on_revalidate(Answer::Grant(grant(Duration::from_secs(3600)))); @@ -973,7 +980,7 @@ async fn a_rechecked_grant_resizes_live_sessions() { .expect("recv_group timeout") .expect("recv_group failed") .expect("the undeafened track closed"); - if group.sequence >= 1 { + if group.start >= 1 { break; } } @@ -1055,9 +1062,11 @@ async fn http_routes_hold_a_lease() { .broadcast() .create_track("video", timed()) .expect("create track"); - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"hello".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"hello".as_ref()) .expect("write frame"); let pub_session = kio::clock::timeout( TIMEOUT, @@ -1967,11 +1976,16 @@ async fn in_band_tokens_widen_and_withdraw_live_prefixes() { .unwrap() .unwrap(); let mut b_sub = b_read.track("video").unwrap().subscribe(None).await.unwrap(); - let mut group = b_track.append_group().unwrap(); + let mut group = b_track + .append_group(b_track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); group - .write_frame(moq_net::Timestamp::ZERO, b"new prefix".as_ref()) + .write_frame( + moq_net::Timestamp::from_millis(group.start).unwrap(), + b"new prefix".as_ref(), + ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert!( kio::clock::timeout(TIMEOUT, b_sub.recv_group()) .await @@ -1991,11 +2005,16 @@ async fn in_band_tokens_widen_and_withdraw_live_prefixes() { "{scheme}: {:?}", ended.as_ref().err() ); - let mut group = a_track.append_group().unwrap(); + let mut group = a_track + .append_group(a_track.largest().map_or(0, |largest| largest.group + 1)) + .unwrap(); group - .write_frame(moq_net::Timestamp::ZERO, b"original prefix".as_ref()) + .write_frame( + moq_net::Timestamp::from_millis(group.start).unwrap(), + b"original prefix".as_ref(), + ) .unwrap(); - group.finish().unwrap(); + group.close().unwrap(); assert!( kio::clock::timeout(TIMEOUT, a_sub.recv_group()) .await diff --git a/rs/moq-relay/tests/integration/bitrate_caps.rs b/rs/moq-relay/tests/integration/bitrate_caps.rs index 1ab7e59d08f..fef7b1e1292 100644 --- a/rs/moq-relay/tests/integration/bitrate_caps.rs +++ b/rs/moq-relay/tests/integration/bitrate_caps.rs @@ -100,11 +100,13 @@ async fn transfer( .expect("subscribe"); let start = kio::clock::now(); - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, vec![7u8; FRAME]) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), vec![7u8; FRAME]) .expect("write frame"); - group.finish().expect("finish group"); + group.close().expect("finish group"); let mut group = kio::clock::timeout(TIMEOUT, subscription.recv_group()) .await .expect("recv_group timeout") diff --git a/rs/moq-relay/tests/integration/cluster_unknown.rs b/rs/moq-relay/tests/integration/cluster_unknown.rs index 1d1c510a640..c39b1462d9e 100644 --- a/rs/moq-relay/tests/integration/cluster_unknown.rs +++ b/rs/moq-relay/tests/integration/cluster_unknown.rs @@ -95,11 +95,13 @@ async fn publish_version(relay: &cluster::Cluster, auth: &auth::Auth, version: & let running = streaming.clone(); let streamer = kio::clock::sim::spawn(async move { while running.load(std::sync::atomic::Ordering::Relaxed) { - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"hello".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"hello".as_ref()) .expect("write frame"); - group.finish().expect("finish group"); + group.close().expect("finish group"); kio::clock::sleep(Duration::from_millis(100)).await; } let _track = track; diff --git a/rs/moq-relay/tests/integration/connection.rs b/rs/moq-relay/tests/integration/connection.rs index a5a00a600f3..46e5c0121f4 100644 --- a/rs/moq-relay/tests/integration/connection.rs +++ b/rs/moq-relay/tests/integration/connection.rs @@ -54,20 +54,23 @@ async fn largest_quic_sequence_reaches_the_subscriber() { for sequence in [5, 1 << 32, (1 << 53) - 1, (1 << 53) + 1, (1 << 62) - 1] { // Timestamps stop at 2^61 - 1, the most lite-05/06 can zigzag as a first frame. let timestamp = moq_net::Timestamp::from_micros(sequence.min((1 << 61) - 1)).unwrap(); - let mut group = track.create_group(moq_net::group::Info { sequence }).unwrap(); + let mut group = track.create_group(moq_net::group::Info { start: sequence }).unwrap(); group.write_frame(timestamp, b"ladder".as_ref()).unwrap(); - group.finish().unwrap(); + group.close().unwrap(); let mut got = kio::clock::timeout(TIMEOUT, sub.recv_group()) .await .unwrap_or_else(|_| panic!("{alpn} sequence={sequence} hung")) .unwrap() .unwrap(); - assert_eq!(got.sequence, sequence, "{alpn}"); + assert_eq!(got.start, sequence, "{alpn}"); let frame = got.read_frame().await.unwrap().unwrap(); assert_eq!(frame.payload, b"ladder".as_ref()); - // Lite-02 carries no timestamps. - let timed = alpn != "moq-lite-02"; - assert_eq!(frame.timestamp, timed.then_some(timestamp), "{alpn}"); + // Lite-02 carries no timestamps, so its frames arrive scale-less at their group key. + let expected = match alpn { + "moq-lite-02" => moq_net::Timestamp::unscaled(sequence), + _ => timestamp, + }; + assert_eq!(frame.timestamp, expected, "{alpn}"); } } } @@ -86,11 +89,13 @@ fn publish( .expect("create broadcast"); let info = moq_net::track::Info::default().with_timescale(moq_net::Timescale::MILLI); let track = broadcast.broadcast().create_track("video", info).expect("create track"); - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, name.as_bytes()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), name.as_bytes()) .expect("write frame"); - group.finish().expect("finish group"); + group.close().expect("finish group"); (origin, broadcast, track) } diff --git a/rs/moq-relay/tests/integration/embed.rs b/rs/moq-relay/tests/integration/embed.rs index 6b832694d06..1d0eb43350b 100644 --- a/rs/moq-relay/tests/integration/embed.rs +++ b/rs/moq-relay/tests/integration/embed.rs @@ -281,11 +281,13 @@ async fn embed_and_stop(mut config: Config) { .broadcast() .create_track("video", timed()) .expect("create track"); - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"hello".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"hello".as_ref()) .expect("write frame"); - group.finish().expect("finish group"); + group.close().expect("finish group"); let url: url::Url = format!("https://{quic}/").parse().expect("parse url"); let subscriber_origin = moq_tokio::origin::spawn(); diff --git a/rs/moq-relay/tests/integration/goaway_cluster.rs b/rs/moq-relay/tests/integration/goaway_cluster.rs index c17dcc8f4c2..7f33d51aff8 100644 --- a/rs/moq-relay/tests/integration/goaway_cluster.rs +++ b/rs/moq-relay/tests/integration/goaway_cluster.rs @@ -160,17 +160,22 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { let write = |group: &mut moq_net::group::Producer, payload: &'static str| { group - .write_frame(moq_net::Timestamp::ZERO, payload.as_bytes()) + .write_frame( + moq_net::Timestamp::from_millis(group.start).unwrap(), + payload.as_bytes(), + ) .expect("write frame"); }; // A group's sequence and its next frame, or `None` at its end. async fn read(group: &mut moq_net::group::Consumer) -> (u64, Option) { let frame = within("read frame", group.read_frame()).await.expect("read"); - (group.sequence, frame.map(|frame| frame.payload)) + (group.start, frame.map(|frame| frame.payload)) } // The first group is open, its first frame read, when A redirects to B. - let mut split = track.append_group().expect("append group"); + let mut split = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); write(&mut split, "a"); let mut reading = within("recv the split group", sub.recv_group()) .await @@ -212,13 +217,15 @@ async fn cluster_continues_a_group_split_by_goaway_inner() { .await; write(&mut split, "b"); - split.finish().expect("finish"); + split.close().expect("finish"); assert_eq!(read(&mut reading).await, (0, Some("b".into()))); assert_eq!(read(&mut reading).await, (0, None)); - let mut after = track.append_group().expect("append group"); + let mut after = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); write(&mut after, "c"); - after.finish().expect("finish"); + after.close().expect("finish"); let mut next = within("recv the next group", sub.recv_group()) .await .expect("recv") @@ -265,11 +272,13 @@ async fn cluster_migrates_on_upstream_goaway_inner() { .await .expect("subscribe"); - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"g0".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"g0".as_ref()) .expect("write frame"); - group.finish().expect("finish"); + group.close().expect("finish"); let mut g0 = within("recv g0", sub.recv_group()) .await @@ -300,18 +309,20 @@ async fn cluster_migrates_on_upstream_goaway_inner() { // New content lands after the swap (both siblings serve the same // origin, so B has it). - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"g1".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"g1".as_ref()) .expect("write frame"); - group.finish().expect("finish"); + group.close().expect("finish"); let mut g1 = within("recv g1", sub.recv_group()) .await .expect("recv g1") .expect("track ended early"); assert_eq!( - g1.sequence, 1, + g1.start, 1, "delivery must resume contiguously at the next group after the swap" ); assert_eq!( @@ -449,13 +460,15 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { // ── group 0 flows through the MID-A leg (all frames verified) ── const FRAMES_PER_GROUP: u64 = 3; - let mut g = track.append_group().expect("append group"); + let mut g = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); for f in 0..FRAMES_PER_GROUP { let payload = format!("diamond_g0_f{f}"); - g.write_frame(moq_net::Timestamp::ZERO, payload.as_bytes()) + g.write_frame(moq_net::Timestamp::from_millis(g.start).unwrap(), payload.as_bytes()) .expect("write frame"); } - g.finish().expect("finish"); + g.close().expect("finish"); let mut seen = BTreeSet::new(); collect_group(&mut sub, &mut seen, FRAMES_PER_GROUP, "pre-failover (via MID-A)").await; @@ -467,13 +480,15 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { const LAST_GROUP: u64 = 20; let publisher = kio::clock::sim::spawn(async move { for seq in 1..=LAST_GROUP { - let mut g = track.append_group().expect("append group"); + let mut g = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); for f in 0..FRAMES_PER_GROUP { let payload = format!("diamond_g{seq}_f{f}"); - g.write_frame(moq_net::Timestamp::ZERO, payload.as_bytes()) + g.write_frame(moq_net::Timestamp::from_millis(g.start).unwrap(), payload.as_bytes()) .expect("write frame"); } - g.finish().expect("finish"); + g.close().expect("finish"); kio::clock::sleep(Duration::from_millis(50)).await; } // Hand the track back so the post-drain phase can keep publishing. @@ -522,13 +537,15 @@ async fn cluster_diamond_goaway_seamless_failover_inner() { const POST_DRAIN_LAST: u64 = LAST_GROUP + 40; let post_publisher = kio::clock::sim::spawn(async move { for seq in (LAST_GROUP + 1)..=POST_DRAIN_LAST { - let mut g = track.append_group().expect("append group"); + let mut g = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); for f in 0..FRAMES_PER_GROUP { let payload = format!("diamond_g{seq}_f{f}"); - g.write_frame(moq_net::Timestamp::ZERO, payload.as_bytes()) + g.write_frame(moq_net::Timestamp::from_millis(g.start).unwrap(), payload.as_bytes()) .expect("write frame"); } - g.finish().expect("finish"); + g.close().expect("finish"); kio::clock::sleep(Duration::from_millis(50)).await; } }); @@ -582,7 +599,7 @@ async fn collect_group(sub: &mut moq_net::track::Subscriber, seen: &mut BTreeSet .await .unwrap_or_else(|err| panic!("subscription errored {stage}: {err}")) .unwrap_or_else(|| panic!("track ended early {stage}")); - let expected_seq = group.sequence; + let expected_seq = group.start; assert!( seen.insert(expected_seq), "group {expected_seq} arrived twice ({stage})" diff --git a/rs/moq-relay/tests/integration/public_rules.rs b/rs/moq-relay/tests/integration/public_rules.rs index 1480ea6a0b6..387be32e1da 100644 --- a/rs/moq-relay/tests/integration/public_rules.rs +++ b/rs/moq-relay/tests/integration/public_rules.rs @@ -68,9 +68,11 @@ async fn publish(url: url::Url, name: &str) -> Box { .broadcast() .create_track("video", timed()) .expect("create track"); - let mut group = track.append_group().expect("append group"); + let mut group = track + .append_group(track.largest().map_or(0, |largest| largest.group + 1)) + .expect("append group"); group - .write_frame(moq_net::Timestamp::ZERO, b"hello".as_ref()) + .write_frame(moq_net::Timestamp::from_millis(group.start).unwrap(), b"hello".as_ref()) .expect("write frame"); let session = kio::clock::timeout( TIMEOUT, diff --git a/rs/moq-relay/tests/integration/quic_sim.rs b/rs/moq-relay/tests/integration/quic_sim.rs index b0da89ecf9b..2d85559c4f8 100644 --- a/rs/moq-relay/tests/integration/quic_sim.rs +++ b/rs/moq-relay/tests/integration/quic_sim.rs @@ -147,10 +147,10 @@ async fn deliver(network: &Network, publish: &Relay, subscribe: &Relay) -> Vec Vec