diff --git a/Cargo.lock b/Cargo.lock index ecc6f8a8d14..e53df4d2167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,15 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "366ffbaa4442f4684d91e2cd7c5ea7c4ed8add41959a31447066e279e432b618" +[[package]] +name = "addr2line" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -557,6 +566,15 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "atoi" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -742,6 +760,21 @@ dependencies = [ "tokio", ] +[[package]] +name = "backtrace" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.8.9", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base16ct" version = "0.2.0" @@ -766,6 +799,16 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "base64ct" version = "1.8.3" @@ -1347,6 +1390,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "combine" version = "4.6.8" @@ -1405,6 +1457,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-str" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18f12cc9948ed9604230cdddc7c86e270f9401ccbe3c2e98a4378c5e7632212f" +dependencies = [ + "const-str-proc-macro", +] + +[[package]] +name = "const-str-proc-macro" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7e7913ec01ed98b697e62f8d3fd63c86dc6cccaf983c7eebc64d0e563b0ad9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1616,6 +1688,15 @@ dependencies = [ "spin 0.10.1", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -2145,6 +2226,12 @@ dependencies = [ "litrs", ] +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "downcast-rs" version = "1.2.1" @@ -2445,7 +2532,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.9.1", "zlib-rs", ] @@ -2749,6 +2836,12 @@ dependencies = [ "polyval", ] +[[package]] +name = "gimli" +version = "0.32.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" + [[package]] name = "gio-sys" version = "0.20.10" @@ -3074,6 +3167,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e712f64ec3850b98572bffac52e2c6f282b29fe6c5fa6d42334b30be438d95c1" +[[package]] +name = "hex-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7685beb53fc20efc2605f32f5d51e9ba18b8ef237961d1760169d2290d3bee" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "hex-slice" version = "0.1.4" @@ -4304,6 +4407,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -4351,6 +4463,7 @@ name = "moq-archive" version = "0.0.11" dependencies = [ "async-trait", + "axum", "bytes", "criterion", "futures", @@ -4362,6 +4475,8 @@ dependencies = [ "object_store", "percent-encoding", "quick_cache", + "s3s", + "s3s-fs", "serde", "serde_json", "tempfile", @@ -5937,6 +6052,12 @@ dependencies = [ "libc", ] +[[package]] +name = "numeric_cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3c00a0c9600379bd32f8972de90676a7672cba3bf4886986bc05902afc1e093" + [[package]] name = "nutype-enum" version = "0.1.5" @@ -6479,6 +6600,15 @@ dependencies = [ "objc2-metal 0.3.2", ] +[[package]] +name = "object" +version = "0.37.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" +dependencies = [ + "memchr", +] + [[package]] name = "object_store" version = "0.14.2" @@ -6620,6 +6750,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + [[package]] name = "owned_ttf_parser" version = "0.25.1" @@ -6717,6 +6853,21 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4" +[[package]] +name = "path-absolutize" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f808742975794703469f67a28dd14b1d1009a1743c18b0353b4b951dbb0068ad" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03351d0f1c066c114015408dc6a3e101f080fc03ef9d8d799aa58ec760cac5a6" + [[package]] name = "pbkdf2" version = "0.12.2" @@ -6790,6 +6941,25 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -7240,6 +7410,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ "memchr", + "serde", ] [[package]] @@ -7735,6 +7906,12 @@ dependencies = [ "windowfunctions", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "1.1.0" @@ -7906,6 +8083,174 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "s3s" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "030a43692a1fd97bbd2c594eeaad8f5c1e452a9d29ad6a3eebce1650937c4a77" +dependencies = [ + "arc-swap", + "arrayvec", + "async-trait", + "atoi", + "base64-simd", + "bytes", + "bytestring", + "cfg-if", + "chrono", + "crc-fast", + "futures", + "hex-simd", + "hmac 0.13.0", + "http", + "http-body", + "http-body-util", + "hyper", + "itoa", + "jiff", + "md-5 0.11.0", + "memchr", + "mime", + "nom 8.0.0", + "numeric_cast", + "phf", + "pin-project-lite", + "quick-xml 0.42.0", + "regex", + "s3s-multipart", + "s3s-rfc2047", + "s3s-sigv2", + "s3s-sigv4", + "serde", + "serde_json", + "serde_urlencoded", + "sha1 0.11.0", + "sha2 0.11.0", + "smallvec", + "std-next", + "subtle", + "sync_wrapper", + "thiserror 2.0.21", + "time", + "tokio", + "tower", + "tracing", + "transform-stream", + "url", + "urlencoding", + "xxhash-rust", + "zeroize", +] + +[[package]] +name = "s3s-fs" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42664753862217444ecae595cb6b2746faf1b8c2d3c104d696aa10592eb042d8" +dependencies = [ + "async-trait", + "base64-simd", + "bytes", + "chrono", + "crc32c", + "futures", + "hex-simd", + "mime", + "numeric_cast", + "path-absolutize", + "s3s", + "s3s-test", + "serde", + "serde_json", + "std-next", + "thiserror 2.0.21", + "time", + "tokio", + "tokio-util", + "tracing", + "tracing-error", + "transform-stream", + "uuid", +] + +[[package]] +name = "s3s-multipart" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a543079d42c20c81781b7e31704ecacd64b0e3819b2384db5bf4869abf6c6f01" +dependencies = [ + "arrayvec", + "bytes", + "futures-core", + "httparse", + "memchr", + "thiserror 2.0.21", +] + +[[package]] +name = "s3s-rfc2047" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918b207c39474baef4774355100e712dfea46020e3289ddd5b4bc6e21927b826" +dependencies = [ + "base64-simd", + "thiserror 2.0.21", +] + +[[package]] +name = "s3s-sigv2" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0979baf8cdf30e38bdfba8e58ee2776d78e55bb3bd3d1f3502f917829da79933" +dependencies = [ + "base64-simd", + "hmac 0.13.0", + "jiff", + "sha1 0.11.0", + "smallvec", + "thiserror 2.0.21", +] + +[[package]] +name = "s3s-sigv4" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330996b9a24804b3df4ad37e62e7817968857d99cf6029c21512b6b64b97f645" +dependencies = [ + "arrayvec", + "base64-simd", + "hex-simd", + "hmac 0.13.0", + "jiff", + "nom 8.0.0", + "serde", + "sha2 0.11.0", + "smallvec", + "std-next", + "thiserror 2.0.21", + "zeroize", +] + +[[package]] +name = "s3s-test" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b9ba61de2952acfe762d56c3f4743324465be961c685eb26c3ed49ef93d13f" +dependencies = [ + "backtrace", + "clap", + "colored", + "const-str", + "dotenvy", + "indexmap 2.14.2", + "regex", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "safe_arch" version = "1.2.0" @@ -8770,6 +9115,16 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c0e04424e733e69714ca1bbb9204c1a57f09f5493439520f9f68c132ad25eec" +[[package]] +name = "std-next" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04082e93ed1a06debd9148c928234b46d2cf260bc65f44e1d1d3fa594c5beebc" +dependencies = [ + "simdutf8", + "thiserror 2.0.21", +] + [[package]] name = "str0m" version = "0.24.1" @@ -9613,6 +9968,16 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-error" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1581020d7a273442f5b45074a6a57d5757ad0a47dac0e9f0bd57b81936f3db" +dependencies = [ + "tracing", + "tracing-subscriber", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -9675,6 +10040,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "transform-stream" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1a814d25437963577f6221d33a2aaa60bfb44acc3330cdc7c334644e9832022" +dependencies = [ + "futures-core", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -10059,6 +10433,12 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "usage-argv" version = "6.11.1" @@ -10156,6 +10536,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "vt100" version = "0.16.2" @@ -11425,6 +11811,12 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "xxhash-rust" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "550a2b930b62486a393c52d5c3b84bff264b28aa437ed64694d31e93b1757af7" + [[package]] name = "yasna" version = "0.5.2" diff --git a/quest/m2/README.md b/quest/m2/README.md index 8acc5a2e83d..95790af5157 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -128,7 +128,6 @@ ranked just below deterministic TS legs, which it requires. - [mTLS on tls://](/quest/m2/tls-listener-mtls.md) - a `tls://` listener can identify a cluster peer by its client certificate - [Link quality](/quest/m2/link-quality.md) - a radio link's cost follows its measured quality without flapping routes - [One transport adapter](/quest/m2/transport-adapter-dedup.md) - the poll transport adapter exists once, and the 64 KiB cap in moq-net's default `poll_read_buf` (not in the adapter) has a test -- [Archive S3 wire proof](/quest/m2/archive-s3.md) - the archive proof also runs through the S3 client against an in-process S3-compatible server - [Archive recovery listing](/quest/m2/archive-recovery-listing.md) - a resumed DVR lists what changed since its checkpoint, not every stored group - [IETF on the ring](/quest/m2/uring-ietf.md) - the io_uring workers serve moq-transport sessions too, so a uring relay drops no client protocol - [Dropped uring session closes](/quest/m2/uring-drop-close.md) - a moq-uring session dropped without close() closes its connection diff --git a/quest/m2/archive-s3.md b/quest/m2/archive-s3.md deleted file mode 100644 index d190490d8c4..00000000000 --- a/quest/m2/archive-s3.md +++ /dev/null @@ -1,22 +0,0 @@ -# [S] Archive S3 wire proof - -## Goal - -The `moq-archive` proof (`rs/moq-archive/src/proof.rs`) also runs through -`object_store`'s S3 client against an in-process S3-compatible server, in CI -with no external network. - -## Plan - -The proof covers memory, local disk, and an S3-style listing fake -(`rs/moq-archive/src/mock.rs`), but not the S3 client: percent-encoded keys -such as `catalog%2Ejson` inside request URLs, `PutMode::Create` through -`If-None-Match`, `start-after` listing, and continuation tokens. Evaluate a -maintained in-process server (for example `s3s` with `s3s-fs`) on loopback; -refuse one that ignores conditional creates, since collisions would pass -silently. Gate it behind a dev-dependency feature if the build cost is large, -and wire it into at least the nightly workflow. - -## Related - -- [Timeline-indexed MoQ archives](/quest/m1/archive/README.md) diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index f216b52fe08..11847c512ff 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -31,10 +31,13 @@ tracing = { workspace = true } [dev-dependencies] async-trait = { workspace = true } +axum = { workspace = true } criterion = { workspace = true } -object_store = { workspace = true, features = ["fs"] } +object_store = { workspace = true, features = ["fs", "aws"] } +s3s = "0.17" +s3s-fs = "0.17" tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt", "test-util"] } +tokio = { workspace = true, features = ["macros", "net", "rt", "test-util"] } [[bench]] name = "writer" diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs index a6e848dfcbb..d72b8ab66c8 100644 --- a/rs/moq-archive/src/proof.rs +++ b/rs/moq-archive/src/proof.rs @@ -402,3 +402,176 @@ async fn an_offline_reader_follows_dvr_expiry() { source.close(); run.await.unwrap().unwrap(); } + +/// Exercise the S3 HTTP adapter, rather than an ObjectStore listing fake. +mod s3 { + use std::num::NonZeroUsize; + use std::sync::{Arc, Mutex}; + + use axum::body::Body; + use axum::http::{Method, Uri}; + use object_store::aws::{AmazonS3, AmazonS3Builder, S3ConditionalPut}; + use object_store::{PutMode, PutPayload}; + use s3s::service::S3ServiceBuilder; + + use super::*; + + struct Request { + method: Method, + uri: Uri, + conditional: bool, + } + + struct Server { + store: Store, + requests: Arc>>, + task: tokio::task::JoinHandle<()>, + _dir: tempfile::TempDir, + } + + impl Server { + async fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("archive-proof")).unwrap(); + let service = S3ServiceBuilder::new(s3s_fs::FileSystem::new(dir.path()).unwrap()).build(); + let requests = Arc::new(Mutex::new(Vec::new())); + let seen = requests.clone(); + let app = axum::Router::new().fallback(move |request: axum::extract::Request| { + let service = service.clone(); + let seen = seen.clone(); + async move { + seen.lock().unwrap().push(Request { + method: request.method().clone(), + uri: request.uri().clone(), + conditional: request.headers().get("if-none-match").is_some_and(|value| value == "*"), + }); + service + .call(request.map(s3s::Body::http_body_unsync)) + .await + .unwrap() + .map(Body::new) + } + }); + let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0)) + .await + .unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let store = AmazonS3Builder::new() + .with_bucket_name("archive-proof") + .with_region("us-east-1") + .with_endpoint(endpoint) + .with_allow_http(true) + .with_skip_signature(true) + .with_conditional_put(S3ConditionalPut::ETagMatch) + .build() + .unwrap(); + Self { + store: Store::new(store, "rec"), + requests, + task, + _dir: dir, + } + } + } + + impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } + } + + #[tokio::test] + async fn archive_proof_crosses_s3_http() { + let memory = Store::new(Mock::memory(), "rec"); + record(&memory).await; + let expected = objects(&memory).await; + let server = Server::new().await; + record(&server.store).await; + assert_eq!(objects(&server.store).await, expected); + let broadcast = broadcast::Info::new().produce(); + let reader = Reader::open(server.store.clone(), &broadcast, config()).await.unwrap(); + let task = tokio::spawn(reader.serve()); + for segment in 0..SEGMENTS { + for (name, sequence, frames) in plan(segment) { + assert_eq!(fetch(&broadcast, name, sequence).await.unwrap(), frames); + } + } + task.abort(); + assert!( + server + .requests + .lock() + .unwrap() + .iter() + .any(|request| request.uri.path().contains("catalog%252Ejson")), + "a percent-encoded recording key must survive URL encoding" + ); + } + + #[tokio::test] + async fn s3_create_offset_and_continuation_are_enforced() { + let server = Server::new().await; + record(&server.store).await; + let expected = objects(&server.store).await; + let (path, original) = expected.first_key_value().unwrap(); + let path = Path::parse(path).unwrap(); + let collision = server + .store + .inner() + .put_opts(&path, PutPayload::from_static(b"different"), PutMode::Create.into()) + .await; + assert!( + matches!(collision, Err(object_store::Error::AlreadyExists { .. })), + "{collision:?}" + ); + assert_eq!( + server.store.inner().get(&path).await.unwrap().bytes().await.unwrap(), + *original + ); + + let mut query = crate::store::list::Query::new().page_size(NonZeroUsize::new(2).unwrap()); + let mut listed = Vec::new(); + loop { + let page = server.store.list_paginated(&query).await.unwrap(); + assert!(page.entries.len() <= 2); + listed.extend( + page.entries + .iter() + .map(|entry| server.store.path(&entry.key).unwrap().to_string()), + ); + let Some(next) = page.next else { break }; + query = next; + } + listed.sort(); + assert_eq!(listed, expected.keys().cloned().collect::>()); + + let after = crate::Key::segments("video/1080p", 0).unwrap(); + let query = crate::store::list::Query::segments("video/1080p") + .unwrap() + .after(&after) + .unwrap(); + let later: Vec<_> = server.store.list(&query).try_collect().await.unwrap(); + assert_eq!(later.len(), (SEGMENTS - 1) as usize); + assert!(later.iter().all( + |entry| matches!(&entry.key, crate::Key::Segments { track, segment } if track == "video/1080p" && *segment > 0) + )); + let requests = server.requests.lock().unwrap(); + assert!( + requests + .iter() + .any(|request| request.method == Method::PUT && request.conditional) + ); + assert!(requests.iter().any(|request| { + request + .uri + .query() + .is_some_and(|query| query.contains("continuation-token=")) + })); + assert!( + requests + .iter() + .any(|request| request.uri.query().is_some_and(|query| query.contains("start-after="))) + ); + } +}