From 8f70ad36b39014ea17e66a4adc7cb81b473be415 Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 14:09:10 +0100 Subject: [PATCH 01/10] Add Azure Cosmos DB for NoSQL vector benchmark engine --- AGENTS.md | 8 +- Cargo.lock | 428 ++++++++++++++- Cargo.toml | 2 + Makefile | 6 + README.md | 40 +- experiments/configurations/cosmosdb.json | 22 + src/bin/vector_db_benchmark/config.rs | 1 + .../vector_db_benchmark/effective_config.rs | 2 + .../vector_db_benchmark/engine/cosmosdb.rs | 517 ++++++++++++++++++ .../engine/filter_guard.rs | 3 +- src/bin/vector_db_benchmark/engine/mod.rs | 12 +- src/bin/vector_db_benchmark/experiment.rs | 7 +- src/cosmosdb.rs | 213 ++++++++ src/lib.rs | 1 + tests/harness_invariants.rs | 1 + tests/integration_cosmosdb.rs | 84 +++ 16 files changed, 1315 insertions(+), 32 deletions(-) create mode 100644 experiments/configurations/cosmosdb.json create mode 100644 src/bin/vector_db_benchmark/engine/cosmosdb.rs create mode 100644 src/cosmosdb.rs create mode 100644 tests/integration_cosmosdb.rs diff --git a/AGENTS.md b/AGENTS.md index e2f8bb662..873ea923c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,10 +2,10 @@ ## Project Overview -Rust implementation of a vector database benchmarking tool. **15 engines**: +Rust implementation of a vector database benchmarking tool. **16 engines**: Redis (RediSearch), VectorSets, Valkey, Dragonfly, KiviDB, Elasticsearch, OpenSearch, Qdrant, pgvector, Weaviate, Milvus, MongoDB, Chroma, Turbopuffer, -Vertex AI. +Vertex AI, Azure Cosmos DB for NoSQL. The thing this repo optimises for is **not being wrong**. Its dominant historical bug class is a run that completes, reports a plausible number, and measured @@ -62,6 +62,7 @@ make integration-test-{valkey,dragonfly,kividb,chroma} make integration-test-{pgvector,qdrant,elasticsearch,opensearch} make integration-test-{weaviate,milvus,mongodb} make integration-test-vertex # cloud-only, needs real GCP credentials +make integration-test-cosmosdb # cloud-only, needs a Cosmos DB account key make integration-test-no-docker ``` @@ -107,7 +108,7 @@ src/ bin/ vector_db_benchmark/ main.rs cli.rs config.rs dataset.rs download.rs experiment.rs ground_truth.rs - engine/ # 15 engines + shared: mod.rs redis_utils.rs filter_guard.rs + engine/ # 16 engines + shared: mod.rs redis_utils.rs filter_guard.rs # index_naming.rs geo.rs generate_dataset.rs # -> `generate-dataset`, writes the synthetic-* fixtures prepare_msmarco.rs # -> `prepare-msmarco`, builds/verifies MS MARCO corpora @@ -454,6 +455,7 @@ now covers more engines than `v0/` did; the table below is the mapping. | Dragonfly | — | `dragonfly` | `redis` 0.27 (RESP) | | KiviDB | — | `kividb` | `redis` 0.27 (RESP) | | Chroma | — | `chroma` | `reqwest` (REST v2) | +| Azure Cosmos DB for NoSQL | — | `cosmosdb` | `azure_data_cosmos` 1.0 | | Vertex AI | — | `vertex` | `tonic`/`prost` (gRPC) + `reqwest` — cloud-only | \* Valkey GLIDE has no Rust crate ([valkey-io/valkey-glide#828](https://github.com/valkey-io/valkey-glide/issues/828), closed NOT_PLANNED). GLIDE maintainers recommend `redis-rs` for Rust. diff --git a/Cargo.lock b/Cargo.lock index 7aa4006ce..60dedcbed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e567177890eb1617b1f774005b66b26b2377afd138a2ca37aae7d8f0c81429d4" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -104,6 +113,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b5d51209f6de83d37a0b33d320acb8da88823e42a94c075140708f032a5" +dependencies = [ + "rustversion", +] + [[package]] name = "arcstr" version = "1.2.0" @@ -128,6 +146,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -220,6 +249,119 @@ dependencies = [ "tower-service", ] +[[package]] +name = "azure_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e41cbd819986ba41904c207d8ffc4106f8f8352a548d773e9554906379bb2fb" +dependencies = [ + "async-lock", + "async-trait", + "azure_core_macros", + "bytes", + "futures", + "hmac 0.12.1", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_core_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "tracing", +] + +[[package]] +name = "azure_data_cosmos" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa75d1999be85b351fb1ace2f48d8df5f13c428fb3c437e0c97214ef2c9a7e9b" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "azure_data_cosmos_driver", + "base64", + "futures", + "pin-project", + "reqwest 0.13.4", + "serde", + "serde_json", + "time", + "tracing", + "url", +] + +[[package]] +name = "azure_data_cosmos_driver" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926a4817a6a75095120f444d71ed59c79edc6178845d7955f4170146ae91d433" +dependencies = [ + "arc-swap", + "async-lock", + "async-trait", + "azure_core", + "azure_data_cosmos_macros", + "backtrace", + "base64", + "bytes", + "crossbeam-epoch", + "futures", + "getrandom 0.4.3", + "h2", + "rand 0.10.2", + "reqwest 0.13.4", + "rustc_version", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "url", + "uuid", + "windows", +] + +[[package]] +name = "azure_data_cosmos_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bedd703baf5f2026093d87de0f79dbd43b1f1e067cb005f1cc643a618295882" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "backtrace" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f4220217f3288537b83684130f42fad43320b1c3b805f1910d683d52ded24f" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.9.1", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.22.1" @@ -250,6 +392,15 @@ dependencies = [ "wyz", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -357,7 +508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -536,6 +687,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -606,6 +766,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -795,15 +965,26 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", "ctutils", ] @@ -882,6 +1063,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -917,7 +1118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.9", ] [[package]] @@ -1044,6 +1245,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1085,6 +1296,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1033caf0b349c518623b5396bfb2cf0bddf44f0306d543a250e5743297aafd10" + [[package]] name = "glob" version = "0.3.3" @@ -1093,9 +1310,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.15" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -1297,13 +1514,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -1879,7 +2105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] @@ -1904,6 +2130,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.2.1" @@ -1968,7 +2203,7 @@ dependencies = [ "hickory-net", "hickory-proto", "hickory-resolver", - "hmac", + "hmac 0.13.0", "macro_magic", "md-5", "mongocrypt", @@ -1982,7 +2217,7 @@ dependencies = [ "serde_bytes", "serde_with", "sha1", - "sha2", + "sha2 0.11.0", "socket2 0.6.4", "stringprep", "strsim", @@ -2153,6 +2388,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2242,6 +2486,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.11.2" @@ -2302,7 +2552,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -2453,11 +2703,11 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator", - "hmac", + "hmac 0.13.0", "md-5", "memchr", "rand 0.10.2", - "sha2", + "sha2 0.11.0", "stringprep", ] @@ -2907,7 +3157,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] @@ -2921,10 +3171,13 @@ dependencies = [ "base64", "bytes", "futures-core", + "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "js-sys", @@ -2939,12 +3192,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-util", "tower 0.5.3", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -2968,6 +3223,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3257,8 +3518,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3267,6 +3528,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -3274,8 +3546,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3871,6 +4143,57 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typespec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "753a2fe021e407d4fc9ee6f4f0a33403cc306d5c54c4e4ebe1b8cbde0ca052b9" +dependencies = [ + "base64", + "bytes", + "futures", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0373af0f9d4f580b3a1a9d9639cedaabe015ed262b35bfbe13941bfb14fe1ea6" +dependencies = [ + "async-trait", + "base64", + "bytes", + "dyn-clone", + "futures", + "pin-project", + "rand 0.10.2", + "reqwest 0.13.4", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c608f4427943f8adb211abc95c87672b1b98847152783507d54e3246e502f60" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.118", +] + [[package]] name = "ucd-trie" version = "0.1.7" @@ -3966,6 +4289,7 @@ checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ "getrandom 0.4.3", "js-sys", + "rand 0.10.2", "serde_core", "wasm-bindgen", ] @@ -3980,6 +4304,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" name = "vector_db_benchmark" version = "0.2.0" dependencies = [ + "azure_data_cosmos", "base64", "bzip2", "chrono", @@ -3987,6 +4312,7 @@ dependencies = [ "crc32fast", "elasticsearch", "flate2", + "futures", "glob", "half", "hdf5", @@ -4010,7 +4336,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.11.0", "tar", "tempfile", "tokio", @@ -4151,6 +4477,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.103" @@ -4230,6 +4569,27 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -4243,6 +4603,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -4271,6 +4642,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + [[package]] name = "windows-registry" version = "0.6.1" @@ -4334,6 +4715,15 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 8d352bdaf..d9466e9e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,8 @@ sha2 = "0.11" crc32fast = "1.5" hdrhistogram = "7.5" base64 = "0.22" +azure_data_cosmos = { version = "1.0", default-features = false, features = ["control_plane", "hmac_rust", "key_auth", "native_tls"] } +futures = "0.3" # Only for `--features release-static` (the prebuilt release archives, #366): # compile HDF5, zlib and OpenSSL from source and link them statically, so the diff --git a/Makefile b/Makefile index 09aac5705..9736033b6 100644 --- a/Makefile +++ b/Makefile @@ -323,6 +323,11 @@ integration-test-vertex: @echo "=== Vertex AI integration tests (cloud-only, needs GCP credentials) ===" cargo test --test integration_vertex --release -- --test-threads=1 +.PHONY: integration-test-cosmosdb +integration-test-cosmosdb: + @echo "=== Azure Cosmos DB for NoSQL integration tests (cloud-only) ===" + cargo test --test integration_cosmosdb --release -- --nocapture --test-threads=1 + .PHONY: integration-test-no-docker integration-test-no-docker: @echo "=== Running integration tests (assumes redis on port 6399) ===" @@ -572,6 +577,7 @@ help: @echo " make integration-test-kividb - KiviDB 1.1.2" @echo " make integration-test-chroma - Chroma" @echo " make integration-test-vertex - Vertex AI (cloud-only, needs GCP creds)" + @echo " make integration-test-cosmosdb - Azure Cosmos DB for NoSQL (cloud-only)" @echo "" @echo " Docker:" @echo " make docker-build - Build Docker image (IMAGE_TAG=latest)" diff --git a/README.md b/README.md index 0a5ce5f03..900694522 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ host's Redis on Linux, and on Docker Desktop only with host networking enabled): | **Vertex AI** (Vector Search) | `reqwest` (Vertex AI REST v1) | HTTP/REST (cloud) | L2, Cosine, Dot | Yes [\*\*\*\*](#vertex-note) | | **Chroma** | `reqwest` (Chroma v2 REST API) | HTTP/REST | L2, Cosine, IP | Yes [\*\*\*\*\*](#chroma-note) | | **KiviDB** | `redis` 1.8.1 | RESP protocol | L2, Cosine, IP | Yes [\*\*\*\*\*\*](#kividb-note) | +| **Azure Cosmos DB for NoSQL** | `azure_data_cosmos` 1.0 | Azure SDK (cloud) | L2, Cosine, Dot | No | \* **Valkey client note:** Valkey GLIDE has no published Rust crate ([valkey-io/valkey-glide#828](https://github.com/valkey-io/valkey-glide/issues/828), closed NOT_PLANNED). The GLIDE maintainers recommend using `redis-rs` for Rust and upstream their improvements to it. The `redis` crate works with Valkey since it speaks the same RESP protocol. @@ -183,6 +184,40 @@ host's Redis on Linux, and on Docker Desktop only with host networking enabled): \*\*\*\*\* **Chroma note:** Uses **Chroma** (OSS) via its **v2 REST API** — a collection of records (`ids` + `embeddings` + scalar `metadatas`) queried with `query` + a `where` document. **Metadata filters** map directly onto the canonical model: `match` → `$eq`, `match_any` → `$in`, `range` → `$gte`/`$gt`/`$lte`/`$lt`, and **AND / OR / nested boolean** to Chroma's native `$and` / `$or` (which nest arbitrarily). Supported datatypes: **keyword, int, float, bool, uuid, datetime** (stored as epoch-seconds int, like Milvus, so numeric range operators apply). **Full-text** (`{match:{text}}`) is supported via `where_document` `$contains` — the `text`-typed field's value is uploaded as each record's Chroma `document`. **NOT supported** by Chroma's metadata engine: **geo-radius** and multi-valued **`labels` arrays** (Chroma metadata values are scalar only). Geo is a *permanent* gap, not a to-do: Chroma's `where` is a closed enum of `field OP literal` comparisons with no geospatial operator, no attribute-vs-attribute comparison and no arithmetic, and a spherical cap is not an axis-aligned box in any query-independent coordinate system — so there is no exact encoding, and an inexact bounding box would under-constrain the query while looking filtered (#223). Since #219 these are a hard **error**, not a silent drop — and a condition tree containing a geo leaf is refused *whole*, so a `geo AND keyword` query cannot quietly run as keyword-only. See the note below. Runs over HTTP/REST via Docker; set the host port with `CHROMA_PORT` (default `8000`, test compose maps `8003`), and optionally `CHROMA_COLLECTION` / `CHROMA_TENANT` / `CHROMA_DATABASE`. Distance space (`l2`/`cosine`/`ip`) is set per collection from the dataset metric. +### Azure Cosmos DB for NoSQL + +Enable **Vector Search for NoSQL API** on your Cosmos DB account. Set +`AZURE_COSMOS_ENDPOINT` to the account URL and provide the account key through +the `AZURE_COSMOS_KEY` environment variable. + +```bash +export AZURE_COSMOS_DATABASE=vdb-benchmark +export AZURE_COSMOS_RU_PER_SECOND=400 +vector-db-benchmark --engines cosmosdb-flat --datasets random-100 --skip-if-exists false +make integration-test-cosmosdb +``` + +The benchmark creates the database if needed and provisions a dedicated +container per config and dataset. It uses manual throughput of 400 RU/s by +default. Set `AZURE_COSMOS_RU_PER_SECOND=serverless` for a serverless account, +or set a higher manual throughput for a larger provisioned run. For a +throughput sweep, give each engine config its own name and a +`"collection_params": { "ru_per_second": 1000 }` override. Search and upload +parallelism are independently configurable. Set +`AZURE_COSMOS_REGION` to prefer an Azure region by name. The runner removes +its container after a successful run unless you pass `--keep-data`. The +database remains available for subsequent runs. Use `--skip-upload` to reuse +an existing container. An upload refuses to overwrite it. + +The adapter benchmarks dense, unfiltered top-K queries across the whole +container. It verifies the vector policy and corpus cardinality. The +`cosmosdb-flat` configuration supports up to 505 dimensions. The +`cosmosdb-diskann` configuration supports up to 4,096 dimensions and requires +at least 1,000 vectors. Cosmos DB can scan while an index builds or while a +physical partition has too few vectors. Results report the realized index +policy and explicitly mark index usage as unverified. See +[Azure Cosmos DB vector indexing](https://learn.microsoft.com/azure/cosmos-db/nosql/vector-search). + ### Filters an engine cannot express are a hard error (#219) @@ -1298,12 +1333,11 @@ vector-db-benchmark compare-results dumps/opensearch-hrrf dumps/redis-hrrf --ref `--skip-upload` asserts *"the server already holds the corpus I want"*. Three rules follow, and all three are enforced: -1. **The configure phase never runs.** `configure()` is destructive on **14 of the - 15 engines** — `FT.DROPINDEX … DD` (Redis), `FT.DROPINDEX` + `SCAN`/`UNLINK` +1. **The configure phase never runs.** `configure()` is destructive on many engines — `FT.DROPINDEX … DD` (Redis), `FT.DROPINDEX` + `SCAN`/`UNLINK` (Valkey / Dragonfly / KiviDB), `collection.drop()` (MongoDB), `DROP TABLE` (pgvector), `DELETE /collections/` (Qdrant / Chroma / Weaviate / Milvus / Turbopuffer), `indices.delete` (Elasticsearch / OpenSearch), `DEL ` - (VectorSets). Only Vertex is non-destructive. Until #238 the + (VectorSets). Vertex and Cosmos DB preserve existing data during configuration. Until #238 the `--skip-upload --skip-vector-index` combination still called it: the flags that mean *"do not upload, do not build an index, just use what is there"* deleted the corpus and then benchmarked the empty index — printing a QPS number and diff --git a/experiments/configurations/cosmosdb.json b/experiments/configurations/cosmosdb.json new file mode 100644 index 000000000..8bedbf419 --- /dev/null +++ b/experiments/configurations/cosmosdb.json @@ -0,0 +1,22 @@ +[ + { + "name": "cosmosdb-flat", + "engine": "cosmosdb", + "algorithm": "flat", + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann", + "engine": "cosmosdb", + "algorithm": "diskANN", + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + } +] diff --git a/src/bin/vector_db_benchmark/config.rs b/src/bin/vector_db_benchmark/config.rs index efe434620..dfefa8cf8 100644 --- a/src/bin/vector_db_benchmark/config.rs +++ b/src/bin/vector_db_benchmark/config.rs @@ -2136,6 +2136,7 @@ mod shipped_config_knob_guard { ("kividb", &["kividb.rs"]), ("vertex", &["vertex.rs", "vertex_grpc.rs"]), ("chroma", &["chroma.rs"]), + ("cosmosdb", &["cosmosdb.rs"]), ]; /// Top-level `search_params[]` knobs consumed by the runner diff --git a/src/bin/vector_db_benchmark/effective_config.rs b/src/bin/vector_db_benchmark/effective_config.rs index 2d260c06e..2458fe721 100644 --- a/src/bin/vector_db_benchmark/effective_config.rs +++ b/src/bin/vector_db_benchmark/effective_config.rs @@ -108,6 +108,7 @@ const SECRET_MARKERS: &[&str] = &[ // and this one did not. "PWD", "API_KEY", + "AZURE_COSMOS_KEY", "APIKEY", "ACCESS_KEY", "PRIVATE_KEY", @@ -208,6 +209,7 @@ const DECLARED_KEY_ALLOWLIST: &[&str] = &[ "default_segment_number", // upload "parallel", + "ru_per_second", "batch_size", "timeout", // a single-letter key that a shipped config genuinely uses diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs new file mode 100644 index 000000000..fadf00734 --- /dev/null +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -0,0 +1,517 @@ +//! Azure Cosmos DB for NoSQL adapter using the official Rust SDK. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Instant; + +use azure_data_cosmos::models::{ContainerProperties, ThroughputProperties, VectorIndexType}; +use azure_data_cosmos::options::{ + ContentResponseOnWrite, CreateContainerOptions, MaxItemCountHint, OperationOptionsBuilder, + QueryOptions, +}; +use azure_data_cosmos::{ + AccountReference, ContainerClient, CosmosClient, CosmosRuntime, FeedScope, Query, + RoutingStrategy, +}; +use futures::{stream, StreamExt, TryStreamExt}; +use serde::{Deserialize, Serialize}; +use vector_db_benchmark::cosmosdb::{ + container_name, container_policy, index_type, knn_query, validate_corpus_size, validate_top, + verify_policy, +}; +use vector_db_benchmark::start_gate::WorkerPool; +use vector_db_benchmark::workload::IndexLayout; + +use super::{compute_search_stats, CorpusCount, Engine, SearchResults, UploadStats}; +use crate::config::{EngineConfig, SearchParams}; +use crate::dataset::Dataset; + +#[derive(Serialize)] +struct Item<'a> { + id: &'a str, + embedding: &'a [f32], +} + +#[derive(Deserialize)] +struct Hit { + id: String, +} + +pub struct CosmosDbEngine { + name: String, + database: String, + container_name: String, + index_type: VectorIndexType, + upload_parallel: usize, + search_params: Vec, + throughput: Option, + client: CosmosClient, + rt: tokio::runtime::Runtime, + container: Option, + created_container: bool, + verified_policy: Option, +} + +impl CosmosDbEngine { + pub fn new(config: &EngineConfig, dataset: &str) -> Result { + let kind = index_type(config.algorithm.as_deref())?; + let search_params = config.search_params.clone().unwrap_or_default(); + for params in &search_params { + if let Some(top) = params.top { + validate_top(usize::try_from(top).map_err(|_| "Cosmos DB top must be positive")?)?; + } + if params.calibration_param.is_some() + || params.num_candidates.is_some() + || params.extra.as_ref().is_some_and(|p| !p.is_empty()) + || params.search_params.as_ref().is_some_and(|p| { + p.ef.is_some() || p.extra.as_ref().is_some_and(|extra| !extra.is_empty()) + }) + { + return Err("Cosmos DB does not support search tuning or calibration knobs".into()); + } + } + let collection = config.collection_params.as_ref(); + if collection.is_some_and(|p| { + p.hnsw_config.is_some() + || p.index_options.is_some() + || p.extra + .as_ref() + .is_some_and(|extra| extra.keys().any(|k| k != "ru_per_second")) + }) { + return Err("Cosmos DB collection_params supports only ru_per_second".into()); + } + let configured_ru = collection + .and_then(|p| p.extra.as_ref()) + .and_then(|options| options.get("ru_per_second")) + .map(|value| { + value.as_u64().filter(|ru| *ru >= 400).ok_or( + "Cosmos DB collection_params.ru_per_second must be an integer of at least 400", + ) + }) + .transpose()?; + let upload_parallel = match config.upload_params.as_ref() { + None => 8, + Some(params) => { + let options = params + .as_object() + .ok_or("Cosmos DB upload_params must be an object")?; + if options.keys().any(|k| k != "parallel") { + return Err( + "Cosmos DB upload_params supports only parallel (one item per request)" + .into(), + ); + } + match options.get("parallel") { + None => 8, + Some(p) => p + .as_u64() + .and_then(|p| usize::try_from(p).ok()) + .filter(|p| *p > 0) + .ok_or("Cosmos DB upload parallel must be positive")?, + } + } + }; + + let endpoint = crate::effective_config::env_opt("AZURE_COSMOS_ENDPOINT") + .ok_or("Set AZURE_COSMOS_ENDPOINT for the Cosmos DB engine")? + .parse() + .map_err(|_| "AZURE_COSMOS_ENDPOINT must be a Cosmos DB account URL")?; + // The configuration recorder redacts this variable. + let key = crate::effective_config::env_opt("AZURE_COSMOS_KEY") + .filter(|key| !key.is_empty()) + .ok_or("Set AZURE_COSMOS_KEY for the Cosmos DB engine")?; + let database = crate::effective_config::env_or("AZURE_COSMOS_DATABASE", "vdb-benchmark"); + let regions = crate::effective_config::env_opt("AZURE_COSMOS_REGION") + .into_iter() + .map(Into::into) + .collect(); + let throughput = if let Some(ru) = configured_ru { + Some(ru) + } else { + match crate::effective_config::env_or("AZURE_COSMOS_RU_PER_SECOND", "400").as_str() { + "serverless" => None, + value => { + Some( + value.parse::().ok().filter(|ru| *ru >= 400).ok_or( + "AZURE_COSMOS_RU_PER_SECOND must be at least 400, or serverless", + )?, + ) + } + } + }; + let rt = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?; + let account = AccountReference::with_authentication_key(endpoint, key); + let client = rt + .block_on(async { + let runtime = CosmosRuntime::builder().build().await?; + CosmosClient::builder() + .with_runtime(runtime) + .with_default_operation_options( + OperationOptionsBuilder::new() + .with_content_response_on_write(ContentResponseOnWrite::Disabled) + .build(), + ) + .build(account, RoutingStrategy::PreferredRegions(regions)) + .await + }) + .map_err(|e| format!("Cosmos DB connection failed: {e}"))?; + + Ok(Self { + name: config.name.clone(), + database, + container_name: container_name(&config.name, dataset), + index_type: kind, + upload_parallel, + search_params, + throughput, + client, + rt, + container: None, + created_container: false, + verified_policy: None, + }) + } + + fn open_container(&mut self) -> Result<&ContainerClient, String> { + if self.container.is_none() { + self.container = Some( + self.rt + .block_on( + self.client + .database_client(&self.database) + .container_client(&self.container_name, None), + ) + .map_err(|e| { + format!( + "Could not open Cosmos DB container '{}': {e}", + self.container_name + ) + })?, + ); + } + self.container + .as_ref() + .ok_or_else(|| "Cosmos DB container is unavailable".into()) + } + + fn policy(&self, dataset: &Dataset) -> Result { + container_policy( + self.container_name.clone(), + self.index_type.clone(), + dataset.distance(), + dataset.vector_size(), + ) + } +} + +impl Engine for CosmosDbEngine { + fn name(&self) -> &str { + &self.name + } + + fn search_params(&self) -> &[SearchParams] { + &self.search_params + } + + fn configure(&mut self, dataset: &Dataset) -> Result<(), String> { + let policy = self.policy(dataset)?; + if let Some(rows) = dataset + .config + .vector_count + .and_then(|n| u64::try_from(n).ok()) + { + validate_corpus_size(&self.index_type, rows)?; + } + // Reject filter conditions before provisioning or uploading a corpus we + // cannot search correctly. Use the same guard in search-only runs. + let (_, _, conditions) = dataset.read_queries()?; + conditions.try_resolve_all("Cosmos DB", refuse_filter)?; + let database = self.client.database_client(&self.database); + self.rt.block_on(async { + match database.read(None).await { + Ok(_) => {} + Err(e) if u16::from(e.status().status_code()) == 404 => { + match self.client.create_database(&self.database, None).await { + Ok(_) => {} + Err(e) if u16::from(e.status().status_code()) == 409 => {} + Err(e) => return Err(format!("Could not create Cosmos DB database: {e}")), + } + } + Err(e) => return Err(format!("Could not read Cosmos DB database: {e}")), + } + let options = self.throughput.map(|rus| { + CreateContainerOptions::default().with_throughput(ThroughputProperties::manual(rus)) + }); + match database.create_container(policy, options).await { + Ok(_) => Ok(()), + Err(e) if u16::from(e.status().status_code()) == 409 => Err(format!( + "Cosmos DB container '{}' already exists. Use --skip-upload to reuse it, or remove it explicitly before uploading.", + self.container_name + )), + Err(e) => Err(format!("Could not create Cosmos DB container: {e}")), + } + })?; + self.created_container = true; + println!("Created Cosmos DB container '{}'", self.container_name); + self.open_container()?; + Ok(()) + } + + fn upload(&mut self, dataset: &Dataset) -> Result { + let total_start = Instant::now(); + let (ids, vectors, _) = dataset.read_vectors(dataset.needs_normalization())?; + validate_corpus_size(&self.index_type, ids.len() as u64)?; + let dimensions = dataset.vector_size() as usize; + if vectors + .iter() + .any(|vector| vector.len() != dimensions || vector.iter().any(|v| !v.is_finite())) + { + return Err( + "Cosmos DB corpus contains a vector with invalid dimensions or values".into(), + ); + } + let container = self + .container + .as_ref() + .ok_or("Cosmos DB container is unavailable")?; + let upload_start = Instant::now(); + self.rt.block_on(async { + stream::iter(ids.iter().zip(&vectors)) + .map(|(id, vector)| async move { + let id = id.to_string(); + let item = Item { + id: &id, + embedding: vector, + }; + container + .upsert_item(id.clone(), &id, item, None) + .await + .map_err(|e| format!("Cosmos DB upload failed for row {id}: {e}")) + }) + .buffer_unordered(self.upload_parallel) + .try_for_each(|_| async { Ok(()) }) + .await + })?; + let upload_time = upload_start.elapsed().as_secs_f64(); + let count = self + .corpus_row_count()? + .ok_or("Cosmos DB did not return a row count")?; + if count.rows != ids.len() as u64 { + return Err(format!( + "Cosmos DB stored {} rows of {} uploaded", + count.rows, + ids.len() + )); + } + Ok(UploadStats { + upload_time, + total_time: total_start.elapsed().as_secs_f64(), + upload_count: ids.len(), + parallel: self.upload_parallel, + batch_size: 1, + ..UploadStats::default() + }) + } + + fn corpus_row_count(&mut self) -> Result, String> { + let container = self.open_container()?.clone(); + // SDK 1.0 does not implement cross-partition aggregate queries on Azure. + // Count the ID projection page by page so reuse has an exact count with + // bounded memory, without reading the stored vectors. + let count = self + .rt + .block_on(async { + let options = + QueryOptions::default().with_max_item_count(MaxItemCountHint::ServerDecides); + container + .query_items::( + "SELECT VALUE c.id FROM c", + FeedScope::full_container(), + Some(options), + ) + .await? + .into_pages() + .try_fold(0_u64, |count, page| async move { + Ok(count + page.items().len() as u64) + }) + .await + }) + .map_err(|e| format!("Cosmos DB row count failed: {e}"))?; + validate_corpus_size(&self.index_type, count)?; + Ok(Some(CorpusCount::exact(count))) + } + + fn check_index_layout(&mut self, dataset: &Dataset, layout: IndexLayout) -> Result<(), String> { + if layout != IndexLayout::Vector { + return Err("Cosmos DB requires the vector index layout".into()); + } + let expected = self.policy(dataset)?; + let container = self.open_container()?.clone(); + let actual = self + .rt + .block_on(container.read(None)) + .and_then(|r| r.into_model()) + .map_err(|e| format!("Could not read Cosmos DB container policy: {e}"))?; + verify_policy(&actual, &expected)?; + if let Some(rus) = self.throughput { + let offer = self + .rt + .block_on(container.read_throughput(None)) + .map_err(|e| format!("Could not read Cosmos DB container throughput: {e}"))?; + if !offer + .is_some_and(|t| t.throughput() == Some(rus) && t.autoscale_maximum().is_none()) + { + return Err(format!( + "Cosmos DB container does not have {rus} manual RU/s" + )); + } + } + self.verified_policy = Some(actual); + if self.index_type == VectorIndexType::DiskANN { + eprintln!("Cosmos DB DiskANN policy verified. Index use is unverified; Azure may fall back to a scan."); + } + Ok(()) + } + + fn search( + &mut self, + dataset: &Dataset, + params: &SearchParams, + num_queries: i64, + ) -> Result { + let (vectors, neighbors, conditions) = dataset.read_queries()?; + conditions.try_resolve_all("Cosmos DB", refuse_filter)?; + let limit = if num_queries > 0 { + (num_queries as usize).min(vectors.len()) + } else { + vectors.len() + }; + if limit == 0 || vectors.len() != neighbors.len() { + return Err("Cosmos DB needs a nonempty query set with aligned ground truth".into()); + } + let parallel = params.parallel.unwrap_or(1).max(1) as usize; + let top = match params.top { + Some(t) => usize::try_from(t).map_err(|_| "Cosmos DB top must be positive")?, + None => neighbors[0].len(), + }; + + let queries = vectors[..limit] + .iter() + .map(|vector| { + if vector.len() != dataset.vector_size() as usize { + return Err("Cosmos DB query vector has the wrong number of dimensions".into()); + } + knn_query(vector, top) + }) + .collect::, String>>()?; + let container = self + .container + .as_ref() + .ok_or("Cosmos DB container is unavailable")?; + let rt = &self.rt; + let next = AtomicUsize::new(0); + let (samples, elapsed) = std::thread::scope(|scope| { + let mut pool = WorkerPool::new(scope, "cosmosdb-search", parallel); + for worker in 0..parallel { + let queries = &queries; + let neighbors = &neighbors; + let next = &next; + pool.spawn(move |ticket| { + let mut samples = Vec::new(); + // Warm the worker's first request outside the measured window. + if let Err(error) = rt.block_on(query_hits(container, &queries[worker % limit])) + { + ticket.fail(format!("Cosmos DB worker {worker} warm-up failed: {error}")); + return Ok(samples); + } + if ticket.arrive_and_wait().is_none() { + return Ok(samples); + } + loop { + let idx = next.fetch_add(1, Ordering::Relaxed); + let Some(query) = queries.get(idx) else { break }; + let started = Instant::now(); + let hits = rt + .block_on(query_hits(container, query)) + .map_err(|e| format!("Cosmos DB query {idx} failed: {e}"))?; + let latency = started.elapsed().as_secs_f64(); + let ids = hits + .iter() + .map(|hit| hit.id.parse::()) + .collect::, _>>() + .map_err(|e| { + format!("Cosmos DB query {idx} returned a non-integer id: {e}") + })?; + let metrics = vector_db_benchmark::metrics::compute_metrics( + &ids, + &neighbors[idx], + top, + ); + samples.push((latency, metrics)); + } + Ok::<_, String>(samples) + })?; + } + let (batches, start) = pool.start()?; + Ok::<_, String>((batches, start.elapsed().as_secs_f64())) + })?; + let (mut times, mut precision, mut recall, mut mrr, mut ndcg) = + (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new()); + for batch in samples { + for (latency, m) in batch? { + times.push(latency); + precision.push(m.precision); + recall.push(m.recall); + mrr.push(m.mrr); + ndcg.push(m.ndcg); + } + } + compute_search_stats( + ×, &precision, &recall, &mrr, &ndcg, elapsed, top, parallel, limit, + ) + } + + fn server_metadata(&mut self) -> Option { + let policy = self.verified_policy.as_ref()?; + Some(serde_json::json!({ + "cosmosdb": { + "database": self.database, + "container": self.container_name, + "partition_key": policy.partition_key, + "vector_embedding_policy": policy.vector_embedding_policy, + "vector_indexing_policy": policy.indexing_policy, + "manual_ru_per_second": self.throughput, + "vector_index_use_verified": false, + } + })) + } + + fn delete(&mut self) -> Result<(), String> { + if self.created_container { + let container = self + .container + .as_ref() + .ok_or("Cosmos DB container is unavailable")?; + self.rt + .block_on(container.delete(None)) + .map_err(|e| format!("Could not delete Cosmos DB container: {e}"))?; + self.created_container = false; + self.container = None; + } + Ok(()) + } +} + +async fn query_hits( + container: &ContainerClient, + query: &Query, +) -> azure_data_cosmos::Result> { + container + .query_items::(query.clone(), FeedScope::full_container(), None) + .await? + .try_collect() + .await +} + +fn refuse_filter(_: &serde_json::Value) -> Result, String> { + Err("filtered vector search is unsupported".into()) +} diff --git a/src/bin/vector_db_benchmark/engine/filter_guard.rs b/src/bin/vector_db_benchmark/engine/filter_guard.rs index b4ac51536..d72df9b8c 100644 --- a/src/bin/vector_db_benchmark/engine/filter_guard.rs +++ b/src/bin/vector_db_benchmark/engine/filter_guard.rs @@ -1064,7 +1064,8 @@ fn every_engine_read_of_query_conditions_is_paired_with_a_resolver() { // and OpenSearch hybrid arms' `read_hybrid_query_set`; +2: the Redis // and OpenSearch hybrid-rrf arms' `read_queries`, each resolved with // `hybrid_rrf::refuse_conditions` (a declared condition is an error). - (28, 28), + // +2: Cosmos DB refuses declared filters before provisioning and search. + (30, 30), "the read/resolve census changed; confirm the new site is guarded" ); } diff --git a/src/bin/vector_db_benchmark/engine/mod.rs b/src/bin/vector_db_benchmark/engine/mod.rs index 30cc028dc..94fff00f0 100644 --- a/src/bin/vector_db_benchmark/engine/mod.rs +++ b/src/bin/vector_db_benchmark/engine/mod.rs @@ -7,6 +7,7 @@ //! - `Searcher` trait = BaseSearcher mod chroma; +mod cosmosdb; mod dragonfly; mod elasticsearch; #[cfg(test)] @@ -1173,7 +1174,11 @@ pub fn resolved_number_of_shards( } /// Create an engine based on config -pub fn create_engine(engine_config: &EngineConfig, host: &str) -> Result, String> { +pub fn create_engine( + engine_config: &EngineConfig, + host: &str, + dataset: &str, +) -> Result, String> { let engine_type = engine_config.engine.as_deref().unwrap_or("unknown"); // HNSW SQ8 keys are typed on the shared HnswConfig and read only by Redis; @@ -1204,8 +1209,9 @@ pub fn create_engine(engine_config: &EngineConfig, host: &str) -> Result Ok(Box::new(KividbEngine::new(engine_config, host)?)), "vertex" => Ok(Box::new(VertexEngine::new(engine_config, host)?)), "chroma" => Ok(Box::new(ChromaEngine::new(engine_config, host)?)), + "cosmosdb" => Ok(Box::new(cosmosdb::CosmosDbEngine::new(engine_config, dataset)?)), other => Err(format!( - "Unsupported engine type: '{}'. Supported: 'redis', 'vectorsets', 'elasticsearch', 'opensearch', 'qdrant', 'weaviate', 'pgvector', 'milvus', 'mongodb', 'valkey', 'turbopuffer', 'dragonfly', 'kividb', 'vertex', 'chroma'.", + "Unsupported engine type: '{}'. Supported: 'redis', 'vectorsets', 'elasticsearch', 'opensearch', 'qdrant', 'weaviate', 'pgvector', 'milvus', 'mongodb', 'valkey', 'turbopuffer', 'dragonfly', 'kividb', 'vertex', 'chroma', 'cosmosdb'.", other )), } @@ -2281,7 +2287,7 @@ mod sq8_engine_guard_tests { "collection_params": { "hnsw_config": hnsw } })) .unwrap(); - match create_engine(&cfg, "localhost") { + match create_engine(&cfg, "localhost", "test") { Ok(_) => panic!("{engine} accepted HNSW SQ8 keys it does not read"), Err(e) => assert!(e.contains("read only by the redis engine"), "{engine}: {e}"), } diff --git a/src/bin/vector_db_benchmark/experiment.rs b/src/bin/vector_db_benchmark/experiment.rs index f0f5c33bf..bdf2a5f4a 100644 --- a/src/bin/vector_db_benchmark/experiment.rs +++ b/src/bin/vector_db_benchmark/experiment.rs @@ -173,6 +173,7 @@ pub fn run(args: &Args) -> Result<(), String> { "kividb", "vertex", "chroma", + "cosmosdb", ]; let mut engines: Vec<_> = engine_configs .iter() @@ -748,7 +749,7 @@ pub fn run(args: &Args) -> Result<(), String> { ); // Create engine - let mut engine = create_engine(&engine_config, &args.host)?; + let mut engine = create_engine(&engine_config, &args.host, &dataset.config.name)?; // Shard count this run will be measured at, recorded in the result // files so it does not have to be inferred from the config name (#211). @@ -1648,9 +1649,9 @@ fn run_single_experiment( } else { // `--skip-upload` means: the server already holds the corpus I want — // do not create, drop, recreate or otherwise modify it. `configure()` is - // destructive on 14 of the 15 engines (FT.DROPINDEX ... DD, SCAN+UNLINK, + // destructive on many engines (FT.DROPINDEX ... DD, SCAN+UNLINK, // collection.drop(), DROP TABLE, DELETE /collections/, indices.delete, - // DEL — only Vertex is not), so it must NOT run here under any flag + // DEL ), so it must NOT run here under any flag // combination. // // This used to have an `else if args.skip_vector_index` arm that called diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs new file mode 100644 index 000000000..65964dd8e --- /dev/null +++ b/src/cosmosdb.rs @@ -0,0 +1,213 @@ +//! Request construction and policy validation for Azure Cosmos DB for NoSQL. + +use azure_data_cosmos::models::{ + ContainerProperties, IndexingMode, IndexingPolicy, VectorDataType, VectorDistanceFunction, + VectorEmbedding, VectorEmbeddingPolicy, VectorIndex, VectorIndexType, +}; +use azure_data_cosmos::Query; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use sha2::{Digest, Sha256}; + +const VECTOR_PATH: &str = "/embedding"; +const VECTOR_EXCLUSION: &str = "/embedding/*"; + +/// Resolve a Cosmos index type. Config names use the service's spelling. +pub fn index_type(algorithm: Option<&str>) -> Result { + match algorithm { + Some("flat") => Ok(VectorIndexType::Flat), + Some("diskANN") => Ok(VectorIndexType::DiskANN), + _ => Err("Cosmos DB algorithm must be flat or diskANN".into()), + } +} + +/// Separate containers by both benchmark config and dataset, even when names +/// contain punctuation that Cosmos forbids in resource IDs. +pub fn container_name(config: &str, dataset: &str) -> String { + let mut hash = Sha256::new(); + hash.update((config.len() as u64).to_le_bytes()); + hash.update(config); + hash.update(dataset); + format!("vdb-benchmark-{}", URL_SAFE_NO_PAD.encode(hash.finalize())) +} + +/// Use the dataset metric and dimension in the server-side policy. +pub fn container_policy( + name: String, + index_type: VectorIndexType, + distance: &str, + dimensions: i64, +) -> Result { + let distance = match distance.to_ascii_lowercase().as_str() { + "cosine" | "angular" => VectorDistanceFunction::Cosine, + "l2" | "euclidean" => VectorDistanceFunction::Euclidean, + "dot" | "ip" => VectorDistanceFunction::DotProduct, + _ => { + return Err(format!( + "Cosmos DB does not support dataset distance '{distance}'" + )) + } + }; + let max_dimensions = if index_type == VectorIndexType::Flat { + 505 + } else { + 4096 + }; + if !(1..=max_dimensions).contains(&dimensions) { + return Err(format!( + "Cosmos DB {index_type:?} requires 1..={max_dimensions} dimensions, got {dimensions}" + )); + } + + let mut indexing = IndexingPolicy::default() + .with_indexing_mode(IndexingMode::Consistent) + .with_included_path("/*") + .with_excluded_path(VECTOR_EXCLUSION) + .with_vector_index(VectorIndex::new(VECTOR_PATH, index_type)); + indexing.automatic = true; + let mut properties = ContainerProperties::new(name, "/id".into()); + properties.indexing_policy = Some(indexing); + properties.vector_embedding_policy = Some(VectorEmbeddingPolicy::default().with_embedding( + VectorEmbedding::new( + VECTOR_PATH, + VectorDataType::Float32, + dimensions as u32, + distance, + ), + )); + Ok(properties) +} + +/// Check the server's realized vector policy, including before reuse. +pub fn verify_policy( + actual: &ContainerProperties, + expected: &ContainerProperties, +) -> Result<(), String> { + let actual_embedding = actual + .vector_embedding_policy + .as_ref() + .map(|p| &p.embeddings); + let expected_embedding = expected + .vector_embedding_policy + .as_ref() + .map(|p| &p.embeddings); + let actual_indexing = actual.indexing_policy.as_ref(); + let expected_index = expected + .indexing_policy + .as_ref() + .and_then(|p| p.vector_indexes.first()); + if actual.partition_key != expected.partition_key + || actual_embedding != expected_embedding + || !actual_indexing.is_some_and(|p| { + p.automatic + && p.indexing_mode == Some(IndexingMode::Consistent) + && p.excluded_paths + .iter() + .any(|path| path.path == VECTOR_EXCLUSION) + // Azure fills in service-selected DiskANN defaults on readback. + // Compare the fields we set and record the full realized policy. + && matches!(p.vector_indexes.as_slice(), [index] if expected_index.is_some_and(|expected| { + index.path == expected.path && index.index_type == expected.index_type + })) + }) + { + return Err(format!( + "Cosmos DB container '{}' has an incompatible partition key, embedding policy or vector index", + actual.id + )); + } + Ok(()) +} + +/// The SDK's buffered cross-partition query window defaults to 1,000. +pub fn validate_top(top: usize) -> Result<(), String> { + if !(1..=1000).contains(&top) { + return Err(format!( + "Cosmos DB top must be between 1 and 1000, got {top}" + )); + } + Ok(()) +} + +/// Fail on corpora too small to qualify for the service's DiskANN threshold. +pub fn validate_corpus_size(kind: &VectorIndexType, count: u64) -> Result<(), String> { + if *kind == VectorIndexType::DiskANN && count < 1000 { + return Err(format!( + "Cosmos DB DiskANN needs at least 1000 vectors to use its index; got {count}. Use cosmosdb-flat for this corpus." + )); + } + Ok(()) +} + +/// Build a server-side TOP K query. The service uses the policy's distance +/// function. TOP is a validated integer literal and the vector is bound. +pub fn knn_query(vector: &[f32], top: usize) -> Result { + validate_top(top)?; + if vector.iter().any(|v| !v.is_finite()) { + return Err("Cosmos DB query vector contains a non-finite value".into()); + } + Query::from(format!( + "SELECT TOP {top} c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" + )) + .with_parameter("@vector", vector) + .map_err(|e| format!("Could not bind Cosmos DB query vector: {e}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, to_value}; + + #[test] + fn distance_and_diskann_policy_are_in_the_wire_request() { + let p = container_policy("bench".into(), VectorIndexType::DiskANN, "dot", 1024).unwrap(); + let wire = to_value(&p).unwrap(); + assert_eq!(wire["partitionKey"]["paths"], json!(["/id"])); + assert_eq!( + wire["vectorEmbeddingPolicy"]["vectorEmbeddings"][0], + json!({ + "path": "/embedding", "dataType": "float32", "dimensions": 1024, + "distanceFunction": "dotproduct" + }) + ); + assert_eq!( + wire["indexingPolicy"]["vectorIndexes"][0]["type"], + "diskANN" + ); + let mut realized = wire; + let vector_index = &mut realized["indexingPolicy"]["vectorIndexes"][0]; + vector_index["quantizerType"] = json!("product"); + vector_index["quantizationByteSize"] = json!(16); + vector_index["indexingSearchListSize"] = json!(100); + let actual = serde_json::from_value(realized.clone()).unwrap(); + assert!(verify_policy(&actual, &p).is_ok()); + realized["indexingPolicy"]["vectorIndexes"][0]["type"] = json!("flat"); + let wrong_index = serde_json::from_value(realized).unwrap(); + assert!(verify_policy(&wrong_index, &p).is_err()); + let other = container_policy("bench".into(), VectorIndexType::DiskANN, "l2", 1024).unwrap(); + assert!(verify_policy(&p, &other).is_err()); + assert!(verify_policy(&p, &p).is_ok()); + assert!(container_policy("bench".into(), VectorIndexType::Flat, "cosine", 506).is_err()); + } + + #[test] + fn vector_queries_request_indexed_top_k_and_bind_the_vector() { + let query = to_value(knn_query(&[0.5, -0.25], 10).unwrap()).unwrap(); + assert_eq!( + query["query"], + "SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" + ); + assert_eq!( + query["parameters"], + json!([{"name": "@vector", "value": [0.5, -0.25]}]) + ); + assert!(knn_query(&[0.5], 0).is_err()); + assert!(knn_query(&[0.5], 1001).is_err()); + } + + #[test] + fn diskann_rejects_the_documented_full_scan_case() { + assert!(validate_corpus_size(&VectorIndexType::DiskANN, 999).is_err()); + assert!(validate_corpus_size(&VectorIndexType::DiskANN, 1000).is_ok()); + assert!(validate_corpus_size(&VectorIndexType::Flat, 999).is_ok()); + } +} diff --git a/src/lib.rs b/src/lib.rs index a600db407..8445d3936 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub mod build_info; pub mod compare_results; pub mod config; +pub mod cosmosdb; pub mod duration_mode; pub mod filter_only; pub mod fulltext; diff --git a/tests/harness_invariants.rs b/tests/harness_invariants.rs index 75d3798ab..84267eaaa 100644 --- a/tests/harness_invariants.rs +++ b/tests/harness_invariants.rs @@ -755,6 +755,7 @@ fn inv_p7_the_scanned_test_target_set_is_pinned() { "harness_invariants.rs", "integration_chroma.rs", "integration_cli.rs", + "integration_cosmosdb.rs", "integration_dragonfly.rs", "integration_elasticsearch.rs", "integration_kividb.rs", diff --git a/tests/integration_cosmosdb.rs b/tests/integration_cosmosdb.rs new file mode 100644 index 000000000..aeafbe27e --- /dev/null +++ b/tests/integration_cosmosdb.rs @@ -0,0 +1,84 @@ +//! Cloud-backed CLI smoke test. Set AZURE_COSMOS_ENDPOINT and AZURE_COSMOS_KEY +//! and run `make integration-test-cosmosdb`. + +mod common; + +use std::fs; + +use serde_json::json; + +#[test] +fn flat_search_matches_the_fixture_and_cleans_up_the_container() { + if std::env::var_os("AZURE_COSMOS_ENDPOINT").is_none() { + eprintln!("SKIP integration_cosmosdb: set AZURE_COSMOS_ENDPOINT and AZURE_COSMOS_KEY"); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let engine_name = format!( + "cosmosdb-it-{}", + root.file_name().unwrap().to_string_lossy() + ); + let fixture = root.join("datasets/random-100"); + fs::create_dir_all(&fixture).unwrap(); + fs::create_dir_all(root.join("experiments/configurations")).unwrap(); + fs::create_dir_all(root.join("results")).unwrap(); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("datasets/random-100"); + for file in ["vectors.jsonl", "queries.jsonl", "neighbours.jsonl"] { + fs::copy(source.join(file), fixture.join(file)).unwrap(); + } + let datasets = json!([{ + "name": "random-100", + "type": "jsonl", + "path": "random-100/", + "distance": "cosine", + "vector_size": 100, + "vector_count": 100, + }]); + fs::write(root.join("datasets/datasets.json"), datasets.to_string()).unwrap(); + let config = json!([{ + "name": engine_name, + "engine": "cosmosdb", + "algorithm": "flat", + "upload_params": { "parallel": 4 }, + "search_params": [{ "parallel": 2, "top": 1 }], + }]); + fs::write( + root.join("experiments/configurations/cosmosdb.json"), + config.to_string(), + ) + .unwrap(); + + assert!(common::run_binary( + root, + &engine_name, + "random-100", + "localhost", + &[] + )); + let uploaded = common::read_result_doc(root, &engine_name); + let recall = uploaded["results"]["mean_recall"].as_f64().unwrap(); + println!("Cosmos DB Flat random-100 recall@1: {recall}"); + assert_eq!(recall, 1.0); + assert_eq!(uploaded["results"]["failed_queries"], 0); + let metadata = &uploaded["server_metadata"]["after"]["cosmosdb"]; + assert_eq!( + metadata["vector_indexing_policy"]["vectorIndexes"][0]["type"], + "flat" + ); + assert_eq!( + metadata["vector_embedding_policy"]["vectorEmbeddings"][0]["dimensions"], + 100 + ); + + // The normal run cleaned up the benchmark container. A second upload with + // the same name can therefore create a fresh corpus. + assert!(common::run_binary( + root, + &engine_name, + "random-100", + "localhost", + &[] + )); +} From b1b33a80b36cb77e27b464dbdde7d355650c813f Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 16:08:01 +0100 Subject: [PATCH 02/10] Validate Cosmos DB top-K and record search throttling --- README.md | 17 +++- .../vector_db_benchmark/effective_config.rs | 10 ++- .../vector_db_benchmark/engine/cosmosdb.rs | 73 +++++++++++++---- src/bin/vector_db_benchmark/engine/mod.rs | 2 + src/bin/vector_db_benchmark/experiment.rs | 5 +- src/cosmosdb.rs | 80 +++++++++++++++++++ tests/integration_cosmosdb.rs | 7 ++ 7 files changed, 175 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 900694522..7de6df6d8 100644 --- a/README.md +++ b/README.md @@ -203,11 +203,22 @@ default. Set `AZURE_COSMOS_RU_PER_SECOND=serverless` for a serverless account, or set a higher manual throughput for a larger provisioned run. For a throughput sweep, give each engine config its own name and a `"collection_params": { "ru_per_second": 1000 }` override. Search and upload -parallelism are independently configurable. Set +parallelism are independently configurable. The 400 RU/s default is suitable +for smoke tests. If the SDK exhausts its HTTP 429 retry budget during upload, +reduce `upload_params.parallel` or set the SDK's +`AZURE_COSMOS_MAX_THROTTLE_RETRY_COUNT` for the upload run. Set `AZURE_COSMOS_REGION` to prefer an Azure region by name. The runner removes its container after a successful run unless you pass `--keep-data`. The database remains available for subsequent runs. Use `--skip-upload` to reuse -an existing container. An upload refuses to overwrite it. +an existing container with a complete corpus. A failed run can leave a +provisioned container that continues billing until you remove it. + +Each Cosmos search result carries `results.cosmosdb_search_diagnostics` with +the SDK's request charge, request attempt count, and observed HTTP 429 count +for measured query pages. Warm-ups and verification queries are outside this +scope. SDK backoff and retries are part of the measured search latency. +Check `attempts_without_status` and `queries_without_request_diagnostics` +when interpreting the 429 count. The adapter benchmarks dense, unfiltered top-K queries across the whole container. It verifies the vector policy and corpus cardinality. The @@ -1333,7 +1344,7 @@ vector-db-benchmark compare-results dumps/opensearch-hrrf dumps/redis-hrrf --ref `--skip-upload` asserts *"the server already holds the corpus I want"*. Three rules follow, and all three are enforced: -1. **The configure phase never runs.** `configure()` is destructive on many engines — `FT.DROPINDEX … DD` (Redis), `FT.DROPINDEX` + `SCAN`/`UNLINK` +1. **The configure phase never runs.** `configure()` is destructive on 14 of the 16 engines — `FT.DROPINDEX … DD` (Redis), `FT.DROPINDEX` + `SCAN`/`UNLINK` (Valkey / Dragonfly / KiviDB), `collection.drop()` (MongoDB), `DROP TABLE` (pgvector), `DELETE /collections/` (Qdrant / Chroma / Weaviate / Milvus / Turbopuffer), `indices.delete` (Elasticsearch / OpenSearch), `DEL ` diff --git a/src/bin/vector_db_benchmark/effective_config.rs b/src/bin/vector_db_benchmark/effective_config.rs index 2458fe721..4f68bd663 100644 --- a/src/bin/vector_db_benchmark/effective_config.rs +++ b/src/bin/vector_db_benchmark/effective_config.rs @@ -2374,12 +2374,19 @@ mod tests { ("REDIS_AUTH", "s3cret"), ("VERTEX_ACCESS_TOKEN", "ya29.xyz"), ("MONGODB_PASSWORD", "pw"), + ("AZURE_COSMOS_KEY", "synthetic-cosmos-key"), ] { r.observe_env(name, Some(raw)); } let s = r.snapshot(); let serialized = serde_json::to_string(&s).unwrap(); - for leak in ["hunter2", "sk-abc", "s3cret", "ya29.xyz"] { + for leak in [ + "hunter2", + "sk-abc", + "s3cret", + "ya29.xyz", + "synthetic-cosmos-key", + ] { assert!( !serialized.contains(leak), "{leak} leaked into the artifact" @@ -2387,6 +2394,7 @@ mod tests { } // …but the fact that authentication was configured survives. assert_eq!(s["env"]["ELASTIC_PASSWORD"], REDACTED); + assert_eq!(s["env"]["AZURE_COSMOS_KEY"], REDACTED); } #[test] diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index fadf00734..a7aa3ea29 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -15,8 +15,8 @@ use azure_data_cosmos::{ use futures::{stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use vector_db_benchmark::cosmosdb::{ - container_name, container_policy, index_type, knn_query, validate_corpus_size, validate_top, - verify_policy, + container_name, container_policy, index_type, knn_query, validate_corpus_size, + validate_hit_count, validate_top, verify_policy, SearchDiagnostics, }; use vector_db_benchmark::start_gate::WorkerPool; use vector_db_benchmark::workload::IndexLayout; @@ -124,6 +124,8 @@ impl CosmosDbEngine { .into_iter() .map(Into::into) .collect(); + // The SDK consumes this environment setting when the runtime starts. + let _ = crate::effective_config::env_opt("AZURE_COSMOS_MAX_THROTTLE_RETRY_COUNT"); let throughput = if let Some(ru) = configured_ru { Some(ru) } else { @@ -244,7 +246,7 @@ impl Engine for CosmosDbEngine { match database.create_container(policy, options).await { Ok(_) => Ok(()), Err(e) if u16::from(e.status().status_code()) == 409 => Err(format!( - "Cosmos DB container '{}' already exists. Use --skip-upload to reuse it, or remove it explicitly before uploading.", + "Cosmos DB container '{}' already exists. An earlier failed run may have left it provisioned. Use --skip-upload for a complete corpus, or remove the container explicitly before uploading.", self.container_name )), Err(e) => Err(format!("Could not create Cosmos DB container: {e}")), @@ -418,7 +420,8 @@ impl Engine for CosmosDbEngine { pool.spawn(move |ticket| { let mut samples = Vec::new(); // Warm the worker's first request outside the measured window. - if let Err(error) = rt.block_on(query_hits(container, &queries[worker % limit])) + if let Err(error) = + rt.block_on(query_hits(container, &queries[worker % limit], top)) { ticket.fail(format!("Cosmos DB worker {worker} warm-up failed: {error}")); return Ok(samples); @@ -430,8 +433,8 @@ impl Engine for CosmosDbEngine { let idx = next.fetch_add(1, Ordering::Relaxed); let Some(query) = queries.get(idx) else { break }; let started = Instant::now(); - let hits = rt - .block_on(query_hits(container, query)) + let (hits, diagnostics) = rt + .block_on(query_hits(container, query, top)) .map_err(|e| format!("Cosmos DB query {idx} failed: {e}"))?; let latency = started.elapsed().as_secs_f64(); let ids = hits @@ -446,7 +449,7 @@ impl Engine for CosmosDbEngine { &neighbors[idx], top, ); - samples.push((latency, metrics)); + samples.push((latency, metrics, diagnostics)); } Ok::<_, String>(samples) })?; @@ -456,18 +459,28 @@ impl Engine for CosmosDbEngine { })?; let (mut times, mut precision, mut recall, mut mrr, mut ndcg) = (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new()); + let mut diagnostics = SearchDiagnostics::default(); for batch in samples { - for (latency, m) in batch? { + for (latency, m, query_diagnostics) in batch? { times.push(latency); precision.push(m.precision); recall.push(m.recall); mrr.push(m.mrr); ndcg.push(m.ndcg); + diagnostics.add_query(query_diagnostics); } } - compute_search_stats( + if diagnostics.http_429_attempts > 0 { + eprintln!( + "Cosmos DB search at parallel={parallel} observed {} HTTP 429 responses. Latency includes SDK backoff and retries.", + diagnostics.http_429_attempts + ); + } + let mut results = compute_search_stats( ×, &precision, &recall, &mrr, &ndcg, elapsed, top, parallel, limit, - ) + )?; + results.cosmosdb_search_diagnostics = Some(diagnostics); + Ok(results) } fn server_metadata(&mut self) -> Option { @@ -504,12 +517,44 @@ impl Engine for CosmosDbEngine { async fn query_hits( container: &ContainerClient, query: &Query, -) -> azure_data_cosmos::Result> { - container + top: usize, +) -> Result<(Vec, SearchDiagnostics), String> { + let mut pages = container .query_items::(query.clone(), FeedScope::full_container(), None) - .await? - .try_collect() .await + .map_err(|error| error.to_string())? + .into_pages(); + let mut hits = Vec::new(); + let mut combined = SearchDiagnostics::default(); + // The SDK merges ORDER BY across partitions. Check the resulting global + // limit anyway, including pages where the server sends no hits. + while let Some(page) = pages.try_next().await.map_err(|error| error.to_string())? { + validate_hit_count(hits.len() + page.items().len(), top)?; + let diagnostics = page.diagnostics(); + let charge = diagnostics.total_request_charge().value(); + let attempts = diagnostics.request_count(); + if let Some(compacted) = diagnostics.compaction() { + combined.observe_page( + charge, + attempts, + compacted + .runs + .iter() + .map(|run| (u16::from(run.status.status_code()), run.count)), + ); + } else { + combined.observe_page( + charge, + attempts, + diagnostics + .requests() + .iter() + .map(|request| (u16::from(request.status().status_code()), 1)), + ); + } + hits.extend(page.into_items()); + } + Ok((hits, combined)) } fn refuse_filter(_: &serde_json::Value) -> Result, String> { diff --git a/src/bin/vector_db_benchmark/engine/mod.rs b/src/bin/vector_db_benchmark/engine/mod.rs index 94fff00f0..e65758479 100644 --- a/src/bin/vector_db_benchmark/engine/mod.rs +++ b/src/bin/vector_db_benchmark/engine/mod.rs @@ -115,6 +115,8 @@ pub struct UpdateSearchRatio { #[allow(dead_code)] #[derive(Debug, Clone, Default)] pub struct SearchResults { + /// Charges and HTTP attempts reported by the Cosmos SDK for this search. + pub cosmosdb_search_diagnostics: Option, /// Metadata-only result contract and count checksums; absent on KNN runs. pub filter_only: Option, pub total_time: f64, diff --git a/src/bin/vector_db_benchmark/experiment.rs b/src/bin/vector_db_benchmark/experiment.rs index bdf2a5f4a..bbce2005e 100644 --- a/src/bin/vector_db_benchmark/experiment.rs +++ b/src/bin/vector_db_benchmark/experiment.rs @@ -1649,7 +1649,7 @@ fn run_single_experiment( } else { // `--skip-upload` means: the server already holds the corpus I want — // do not create, drop, recreate or otherwise modify it. `configure()` is - // destructive on many engines (FT.DROPINDEX ... DD, SCAN+UNLINK, + // destructive on 14 of the 16 engines (FT.DROPINDEX ... DD, SCAN+UNLINK, // collection.drop(), DROP TABLE, DELETE /collections/, indices.delete, // DEL ), so it must NOT run here under any flag // combination. @@ -3109,6 +3109,9 @@ fn build_search_result_json( result["results"]["p90_time"] = json!(results.p90_time); result["results"]["schedule_delay_p90_time"] = json!(results.schedule_delay_p90_time); result["results"]["end_to_end_p90_time"] = json!(results.end_to_end_p90_time); + if let Some(diagnostics) = results.cosmosdb_search_diagnostics { + result["results"]["cosmosdb_search_diagnostics"] = json!(diagnostics); + } let gt_stats = ground_truth.and_then(|gt| gt.stats(results.top)); result["metrics_schema"] = metrics_schema(gt_stats.as_ref()); diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index 65964dd8e..d54cd28f2 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -6,6 +6,7 @@ use azure_data_cosmos::models::{ }; use azure_data_cosmos::Query; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use serde::Serialize; use sha2::{Digest, Sha256}; const VECTOR_PATH: &str = "/embedding"; @@ -128,6 +129,58 @@ pub fn validate_top(top: usize) -> Result<(), String> { Ok(()) } +/// Reject an SDK response that exceeds the global TOP K across all its pages. +pub fn validate_hit_count(hits: usize, top: usize) -> Result<(), String> { + if hits > top { + return Err(format!( + "Cosmos DB returned {hits} hits for global TOP {top}" + )); + } + Ok(()) +} + +/// SDK diagnostics accumulated over the pages of measured queries. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub struct SearchDiagnostics { + pub request_charge_ru: f64, + pub request_attempts: usize, + pub http_429_attempts: usize, + pub attempts_without_status: usize, + pub queries_without_request_diagnostics: usize, +} + +impl SearchDiagnostics { + /// Use the exact SDK totals and count HTTP statuses from either individual + /// attempts or their compacted rollup. Omitted statuses remain unknown. + pub fn observe_page( + &mut self, + request_charge_ru: f64, + request_attempts: usize, + attempt_statuses: impl IntoIterator, + ) { + self.request_charge_ru += request_charge_ru; + self.request_attempts += request_attempts; + let mut classified = 0; + for (status, count) in attempt_statuses { + if (100..600).contains(&status) { + classified += count; + } + if status == 429 { + self.http_429_attempts += count; + } + } + self.attempts_without_status += request_attempts.saturating_sub(classified); + } + + pub fn add_query(&mut self, query: Self) { + self.request_charge_ru += query.request_charge_ru; + self.request_attempts += query.request_attempts; + self.http_429_attempts += query.http_429_attempts; + self.attempts_without_status += query.attempts_without_status; + self.queries_without_request_diagnostics += usize::from(query.request_attempts == 0); + } +} + /// Fail on corpora too small to qualify for the service's DiskANN threshold. pub fn validate_corpus_size(kind: &VectorIndexType, count: u64) -> Result<(), String> { if *kind == VectorIndexType::DiskANN && count < 1000 { @@ -210,4 +263,31 @@ mod tests { assert!(validate_corpus_size(&VectorIndexType::DiskANN, 1000).is_ok()); assert!(validate_corpus_size(&VectorIndexType::Flat, 999).is_ok()); } + + #[test] + fn hit_count_rejects_concatenated_partition_top_k_lists() { + assert!(validate_hit_count(2, 3).is_ok()); + assert!(validate_hit_count(3, 3).is_ok()); + assert!(validate_hit_count(6, 3).is_err()); + } + + #[test] + fn search_diagnostics_sum_pages_retries_and_missing_statuses() { + let mut query = SearchDiagnostics::default(); + query.observe_page(4.5, 5, [(429, 3), (200, 1)]); + query.observe_page(2.0, 1, [(200, 1)]); + let mut run = SearchDiagnostics::default(); + run.add_query(query); + run.add_query(SearchDiagnostics::default()); + assert_eq!( + to_value(run).unwrap(), + json!({ + "request_charge_ru": 6.5, + "request_attempts": 6, + "http_429_attempts": 3, + "attempts_without_status": 1, + "queries_without_request_diagnostics": 1, + }) + ); + } } diff --git a/tests/integration_cosmosdb.rs b/tests/integration_cosmosdb.rs index aeafbe27e..f0bc8f2f3 100644 --- a/tests/integration_cosmosdb.rs +++ b/tests/integration_cosmosdb.rs @@ -62,6 +62,13 @@ fn flat_search_matches_the_fixture_and_cleans_up_the_container() { println!("Cosmos DB Flat random-100 recall@1: {recall}"); assert_eq!(recall, 1.0); assert_eq!(uploaded["results"]["failed_queries"], 0); + let diagnostics = &uploaded["results"]["cosmosdb_search_diagnostics"]; + assert!(diagnostics["request_charge_ru"].as_f64().unwrap() > 0.0); + assert!( + diagnostics["request_attempts"].as_u64().unwrap() + >= uploaded["results"]["succeeded_queries"].as_u64().unwrap() + ); + assert_eq!(diagnostics["queries_without_request_diagnostics"], 0); let metadata = &uploaded["server_metadata"]["after"]["cosmosdb"]; assert_eq!( metadata["vector_indexing_policy"]["vectorIndexes"][0]["type"], From 7de62d1f723ec11d1d063670a3ab42f273bb6e3c Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 19:14:03 +0100 Subject: [PATCH 03/10] Support DiskANN query-time benchmark sweeps --- README.md | 17 +++++++ .../vector_db_benchmark/engine/cosmosdb.rs | 47 +++++++++++++++++-- src/cosmosdb.rs | 29 ++++++++++-- 3 files changed, 84 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7de6df6d8..5a2f8b5af 100644 --- a/README.md +++ b/README.md @@ -197,6 +197,15 @@ vector-db-benchmark --engines cosmosdb-flat --datasets random-100 --skip-if-exis make integration-test-cosmosdb ``` +For an Azure-hosted benchmark over Private Link, place the client VM in the +Cosmos DB region. Create a NoSQL private endpoint with subresource `Sql` and +link the `privatelink.documents.azure.com` private DNS zone to the VM's virtual +network. Keep `AZURE_COSMOS_ENDPOINT` set to the standard account URL +(`https://.documents.azure.com:443/`). From the VM, run +`nslookup .documents.azure.com` and confirm the address matches the +private endpoint before running the benchmark. See Microsoft's +[Cosmos DB Private Link setup guide](https://learn.microsoft.com/en-us/azure/cosmos-db/how-to-configure-private-endpoints). + The benchmark creates the database if needed and provisions a dedicated container per config and dataset. It uses manual throughput of 400 RU/s by default. Set `AZURE_COSMOS_RU_PER_SECOND=serverless` for a serverless account, @@ -220,6 +229,14 @@ scope. SDK backoff and retries are part of the measured search latency. Check `attempts_without_status` and `queries_without_request_diagnostics` when interpreting the 429 count. +For DiskANN, each search entry can set positive integer +`search_params.search_list_size_multiplier` and +`search_params.quantized_vector_list_multiplier`. These map to the +`VectorDistance` query options `searchListSizeMultiplier` and +`quantizedVectorListMultiplier`. Leave both unset for the service defaults. +Add multiple search entries under one engine config to sweep query-time +settings on the same index, keeping `top` fixed for recall comparisons. + The adapter benchmarks dense, unfiltered top-K queries across the whole container. It verifies the vector policy and corpus cardinality. The `cosmosdb-flat` configuration supports up to 505 dimensions. The diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index a7aa3ea29..29a56c409 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -1,5 +1,6 @@ //! Azure Cosmos DB for NoSQL adapter using the official Rust SDK. +use std::num::NonZeroU32; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; @@ -36,6 +37,29 @@ struct Hit { id: String, } +const SEARCH_LIST_SIZE: &str = "search_list_size_multiplier"; +const QUANTIZED_VECTOR_LIST: &str = "quantized_vector_list_multiplier"; + +fn diskann_multiplier( + params: &SearchParams, + kind: &VectorIndexType, + key: &str, +) -> Result, String> { + params + .knob(key) + .map(|value| { + if *kind != VectorIndexType::DiskANN { + return Err(format!("Cosmos DB {key} requires DiskANN")); + } + value + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .and_then(NonZeroU32::new) + .ok_or_else(|| format!("Cosmos DB {key} must be a positive 32-bit integer")) + }) + .transpose() +} + pub struct CosmosDbEngine { name: String, database: String, @@ -59,15 +83,27 @@ impl CosmosDbEngine { if let Some(top) = params.top { validate_top(usize::try_from(top).map_err(|_| "Cosmos DB top must be positive")?)?; } + let unsupported = + |key: &String| !matches!(key.as_str(), SEARCH_LIST_SIZE | QUANTIZED_VECTOR_LIST); if params.calibration_param.is_some() || params.num_candidates.is_some() - || params.extra.as_ref().is_some_and(|p| !p.is_empty()) + || params + .extra + .as_ref() + .is_some_and(|p| p.keys().any(unsupported)) || params.search_params.as_ref().is_some_and(|p| { - p.ef.is_some() || p.extra.as_ref().is_some_and(|extra| !extra.is_empty()) + p.ef.is_some() + || p.extra + .as_ref() + .is_some_and(|extra| extra.keys().any(unsupported)) }) { - return Err("Cosmos DB does not support search tuning or calibration knobs".into()); + return Err( + "Cosmos DB supports only DiskANN search multipliers as tuning knobs".into(), + ); } + diskann_multiplier(params, &kind, SEARCH_LIST_SIZE)?; + diskann_multiplier(params, &kind, QUANTIZED_VECTOR_LIST)?; } let collection = config.collection_params.as_ref(); if collection.is_some_and(|p| { @@ -395,6 +431,9 @@ impl Engine for CosmosDbEngine { Some(t) => usize::try_from(t).map_err(|_| "Cosmos DB top must be positive")?, None => neighbors[0].len(), }; + let search_list_size = diskann_multiplier(params, &self.index_type, SEARCH_LIST_SIZE)?; + let quantized_vector_list = + diskann_multiplier(params, &self.index_type, QUANTIZED_VECTOR_LIST)?; let queries = vectors[..limit] .iter() @@ -402,7 +441,7 @@ impl Engine for CosmosDbEngine { if vector.len() != dataset.vector_size() as usize { return Err("Cosmos DB query vector has the wrong number of dimensions".into()); } - knn_query(vector, top) + knn_query(vector, top, search_list_size, quantized_vector_list) }) .collect::, String>>()?; let container = self diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index d54cd28f2..05d76474b 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -193,13 +193,26 @@ pub fn validate_corpus_size(kind: &VectorIndexType, count: u64) -> Result<(), St /// Build a server-side TOP K query. The service uses the policy's distance /// function. TOP is a validated integer literal and the vector is bound. -pub fn knn_query(vector: &[f32], top: usize) -> Result { +pub fn knn_query( + vector: &[f32], + top: usize, + search_list_size_multiplier: Option, + quantized_vector_list_multiplier: Option, +) -> Result { validate_top(top)?; if vector.iter().any(|v| !v.is_finite()) { return Err("Cosmos DB query vector contains a non-finite value".into()); } + let options = match (search_list_size_multiplier, quantized_vector_list_multiplier) { + (None, None) => String::new(), + (Some(search), None) => format!(", {{\"searchListSizeMultiplier\": {search}}}"), + (None, Some(quantized)) => format!(", {{\"quantizedVectorListMultiplier\": {quantized}}}"), + (Some(search), Some(quantized)) => format!( + ", {{\"searchListSizeMultiplier\": {search}, \"quantizedVectorListMultiplier\": {quantized}}}" + ), + }; Query::from(format!( - "SELECT TOP {top} c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" + "SELECT TOP {top} c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false{options})" )) .with_parameter("@vector", vector) .map_err(|e| format!("Could not bind Cosmos DB query vector: {e}")) @@ -244,7 +257,7 @@ mod tests { #[test] fn vector_queries_request_indexed_top_k_and_bind_the_vector() { - let query = to_value(knn_query(&[0.5, -0.25], 10).unwrap()).unwrap(); + let query = to_value(knn_query(&[0.5, -0.25], 10, None, None).unwrap()).unwrap(); assert_eq!( query["query"], "SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" @@ -253,8 +266,14 @@ mod tests { query["parameters"], json!([{"name": "@vector", "value": [0.5, -0.25]}]) ); - assert!(knn_query(&[0.5], 0).is_err()); - assert!(knn_query(&[0.5], 1001).is_err()); + assert!(knn_query(&[0.5], 0, None, None).is_err()); + assert!(knn_query(&[0.5], 1001, None, None).is_err()); + let tuned = to_value(knn_query(&[0.5], 10, 5.try_into().ok(), 20.try_into().ok()).unwrap()) + .unwrap(); + assert_eq!( + tuned["query"], + r#"SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false, {"searchListSizeMultiplier": 5, "quantizedVectorListMultiplier": 20})"# + ); } #[test] From e8d4370b52578c024dce4c3abf91c34d3c774036 Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 19:19:28 +0100 Subject: [PATCH 04/10] Ship Cosmos DB DiskANN query sweep points --- README.md | 3 +++ experiments/configurations/cosmosdb.json | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5a2f8b5af..5150b9e8d 100644 --- a/README.md +++ b/README.md @@ -236,6 +236,9 @@ For DiskANN, each search entry can set positive integer `quantizedVectorListMultiplier`. Leave both unset for the service defaults. Add multiple search entries under one engine config to sweep query-time settings on the same index, keeping `top` fixed for recall comparisons. +The shipped `cosmosdb-diskann` config compares the service defaults with +search-list multipliers of 1, 5, 10, and 20 at parallelism 1 and 8, all at +top-10. The quantized-vector-list multiplier uses the service default. The adapter benchmarks dense, unfiltered top-K queries across the whole container. It verifies the vector policy and corpus cardinality. The diff --git a/experiments/configurations/cosmosdb.json b/experiments/configurations/cosmosdb.json index 8bedbf419..b9ef216dc 100644 --- a/experiments/configurations/cosmosdb.json +++ b/experiments/configurations/cosmosdb.json @@ -16,7 +16,15 @@ "upload_params": { "parallel": 8 }, "search_params": [ { "top": 10, "parallel": 1 }, - { "top": 10, "parallel": 8 } + { "top": 10, "parallel": 8 }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 1 } }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 5 } }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 10 } }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 20 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 1 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 5 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 10 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 20 } } ] } ] From dc8ee395bfe8093425ca9b7963f6481112984e58 Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 19:26:12 +0100 Subject: [PATCH 05/10] Benchmark Cosmos DB DiskANN index-build settings --- README.md | 24 ++ experiments/configurations/cosmosdb.json | 55 +++++ .../cosmosdb-dbpedia-index-sweep.json | 206 ++++++++++++++++++ .../vector_db_benchmark/effective_config.rs | 3 + .../vector_db_benchmark/engine/cosmosdb.rs | 37 +++- src/cosmosdb.rs | 174 ++++++++++++++- 6 files changed, 485 insertions(+), 14 deletions(-) create mode 100644 experiments/examples/cosmosdb-dbpedia-index-sweep.json diff --git a/README.md b/README.md index 5150b9e8d..e08a93010 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,30 @@ The shipped `cosmosdb-diskann` config compares the service defaults with search-list multipliers of 1, 5, 10, and 20 at parallelism 1 and 8, all at top-10. The quantized-vector-list multiplier uses the service default. +To sweep DiskANN index builds, use `--engines 'cosmosdb-diskann-build-*'` +with a dataset of at least 64 dimensions. Each named config provisions its +own container and uses the same top-10 searches at parallelism 1 and 8 with +service-default query settings. The presets vary +`collection_params.indexing_search_list_size` at 50, 100, and 200 with +quantization fixed at 32 bytes, and vary `collection_params.quantization_byte_size` +at 16, 32, and 64 with the indexing search list fixed at 100. The benchmark +checks the realized values before search, including on container reuse. +With the Rust SDK, the build search list accepts integers from 25 to 500 +and the quantization size accepts integers from 1 to the smaller of 512 +and the vector dimension. Omit a build setting to use the Azure default. +See the [Rust SDK vector index model](https://github.com/Azure/azure-sdk-for-rust/blob/d7e28a1cd76e11d51f005e5d502a6c1c6299f74a/sdk/cosmos/azure_data_cosmos/src/models/indexing_policy.rs#L274-L290). + +The opt-in DBpedia example uses 10,000 RU/s per container. It runs four +index builds and measures top-100 searches at parallelism 1 and 100. The +command uses the registered 975K DBpedia dataset with 100 reference neighbors. +Run it from a repository checkout. + +```bash +vector-db-benchmark \ + --engines-file experiments/examples/cosmosdb-dbpedia-index-sweep.json \ + --datasets dbpedia-openai-975K-1536-angular-100neighbors +``` + The adapter benchmarks dense, unfiltered top-K queries across the whole container. It verifies the vector policy and corpus cardinality. The `cosmosdb-flat` configuration supports up to 505 dimensions. The diff --git a/experiments/configurations/cosmosdb.json b/experiments/configurations/cosmosdb.json index b9ef216dc..a9d06b1c8 100644 --- a/experiments/configurations/cosmosdb.json +++ b/experiments/configurations/cosmosdb.json @@ -26,5 +26,60 @@ { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 10 } }, { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 20 } } ] + }, + { + "name": "cosmosdb-diskann-build-list50-bytes32", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 50, "quantization_byte_size": 32 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list100-bytes32", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 100, "quantization_byte_size": 32 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list200-bytes32", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 200, "quantization_byte_size": 32 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list100-bytes16", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 100, "quantization_byte_size": 16 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list100-bytes64", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 100, "quantization_byte_size": 64 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] } ] diff --git a/experiments/examples/cosmosdb-dbpedia-index-sweep.json b/experiments/examples/cosmosdb-dbpedia-index-sweep.json new file mode 100644 index 000000000..937016dd4 --- /dev/null +++ b/experiments/examples/cosmosdb-dbpedia-index-sweep.json @@ -0,0 +1,206 @@ +[ + { + "name": "cosmosdb-diskann-10000ru-buildlist-50-qbytes-256", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 50, + "quantization_byte_size": 256 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + }, + { + "name": "cosmosdb-diskann-10000ru-buildlist-200-qbytes-256", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 200, + "quantization_byte_size": 256 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + }, + { + "name": "cosmosdb-diskann-10000ru-buildlist-100-qbytes-128", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 100, + "quantization_byte_size": 128 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + }, + { + "name": "cosmosdb-diskann-10000ru-buildlist-100-qbytes-512", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 100, + "quantization_byte_size": 512 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + } +] diff --git a/src/bin/vector_db_benchmark/effective_config.rs b/src/bin/vector_db_benchmark/effective_config.rs index 4f68bd663..956915ddc 100644 --- a/src/bin/vector_db_benchmark/effective_config.rs +++ b/src/bin/vector_db_benchmark/effective_config.rs @@ -188,6 +188,9 @@ const DECLARED_KEY_ALLOWLIST: &[&str] = &[ "type", "number_of_shards", "is_tenant", + // Cosmos DB DiskANN + "indexing_search_list_size", + "quantization_byte_size", // SVS-VAMANA "svs-vamana_config", "GRAPH_MAX_DEGREE", diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index 29a56c409..ab36edfc4 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -17,7 +17,7 @@ use futures::{stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use vector_db_benchmark::cosmosdb::{ container_name, container_policy, index_type, knn_query, validate_corpus_size, - validate_hit_count, validate_top, verify_policy, SearchDiagnostics, + validate_hit_count, validate_top, verify_policy, DiskAnnBuildOptions, SearchDiagnostics, }; use vector_db_benchmark::start_gate::WorkerPool; use vector_db_benchmark::workload::IndexLayout; @@ -65,6 +65,7 @@ pub struct CosmosDbEngine { database: String, container_name: String, index_type: VectorIndexType, + build_options: DiskAnnBuildOptions, upload_parallel: usize, search_params: Vec, throughput: Option, @@ -109,15 +110,33 @@ impl CosmosDbEngine { if collection.is_some_and(|p| { p.hnsw_config.is_some() || p.index_options.is_some() - || p.extra - .as_ref() - .is_some_and(|extra| extra.keys().any(|k| k != "ru_per_second")) + || p.extra.as_ref().is_some_and(|extra| { + extra.keys().any(|key| { + !matches!( + key.as_str(), + "ru_per_second" + | "indexing_search_list_size" + | "quantization_byte_size" + ) + }) + }) }) { - return Err("Cosmos DB collection_params supports only ru_per_second".into()); + return Err( + "Cosmos DB collection_params supports only ru_per_second, indexing_search_list_size and quantization_byte_size" + .into(), + ); } - let configured_ru = collection - .and_then(|p| p.extra.as_ref()) - .and_then(|options| options.get("ru_per_second")) + let collection_option = |key| { + collection + .and_then(|p| p.extra.as_ref()) + .and_then(|p| p.get(key)) + }; + let build_options = DiskAnnBuildOptions::from_config( + &kind, + collection_option("indexing_search_list_size"), + collection_option("quantization_byte_size"), + )?; + let configured_ru = collection_option("ru_per_second") .map(|value| { value.as_u64().filter(|ru| *ru >= 400).ok_or( "Cosmos DB collection_params.ru_per_second must be an integer of at least 400", @@ -198,6 +217,7 @@ impl CosmosDbEngine { database, container_name: container_name(&config.name, dataset), index_type: kind, + build_options, upload_parallel, search_params, throughput, @@ -237,6 +257,7 @@ impl CosmosDbEngine { self.index_type.clone(), dataset.distance(), dataset.vector_size(), + self.build_options, ) } } diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index 05d76474b..852a0d2e0 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -7,6 +7,7 @@ use azure_data_cosmos::models::{ use azure_data_cosmos::Query; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; use serde::Serialize; +use serde_json::Value; use sha2::{Digest, Sha256}; const VECTOR_PATH: &str = "/embedding"; @@ -31,12 +32,57 @@ pub fn container_name(config: &str, dataset: &str) -> String { format!("vdb-benchmark-{}", URL_SAFE_NO_PAD.encode(hash.finalize())) } +/// Optional DiskANN index-build settings. Absent fields leave policy defaults +/// to Azure, including its dimension-dependent quantization size. +#[derive(Debug, Clone, Copy, Default)] +pub struct DiskAnnBuildOptions { + indexing_search_list_size: Option, + quantization_byte_size: Option, +} + +impl DiskAnnBuildOptions { + /// Validate supplied collection parameters against the data-plane SDK's + /// limits. The dimension-dependent cap is checked when building the policy. + pub fn from_config( + kind: &VectorIndexType, + indexing_search_list_size: Option<&Value>, + quantization_byte_size: Option<&Value>, + ) -> Result { + if *kind != VectorIndexType::DiskANN + && (indexing_search_list_size.is_some() || quantization_byte_size.is_some()) + { + return Err("Cosmos DB collection index-build options require DiskANN".into()); + } + + let parse = |value: &Value, key: &str, min: u32, max: u32| { + value + .as_u64() + .and_then(|value| u32::try_from(value).ok()) + .filter(|value| (min..=max).contains(value)) + .ok_or_else(|| { + format!( + "Cosmos DB collection_params.{key} must be an integer from {min} to {max}" + ) + }) + }; + Ok(Self { + indexing_search_list_size: indexing_search_list_size + .map(|value| parse(value, "indexing_search_list_size", 25, 500)) + .transpose()?, + quantization_byte_size: quantization_byte_size + .map(|value| parse(value, "quantization_byte_size", 1, 512)) + .transpose()?, + }) + } +} + /// Use the dataset metric and dimension in the server-side policy. pub fn container_policy( name: String, index_type: VectorIndexType, distance: &str, dimensions: i64, + build: DiskAnnBuildOptions, ) -> Result { let distance = match distance.to_ascii_lowercase().as_str() { "cosine" | "angular" => VectorDistanceFunction::Cosine, @@ -59,11 +105,23 @@ pub fn container_policy( )); } + let mut vector_index = VectorIndex::new(VECTOR_PATH, index_type); + if let Some(size) = build.indexing_search_list_size { + vector_index = vector_index.with_indexing_search_list_size(size); + } + if let Some(size) = build.quantization_byte_size { + if i64::from(size) > dimensions { + return Err(format!( + "Cosmos DB collection_params.quantization_byte_size ({size}) exceeds the vector dimensions ({dimensions})" + )); + } + vector_index = vector_index.with_quantization_byte_size(size); + } let mut indexing = IndexingPolicy::default() .with_indexing_mode(IndexingMode::Consistent) .with_included_path("/*") .with_excluded_path(VECTOR_EXCLUSION) - .with_vector_index(VectorIndex::new(VECTOR_PATH, index_type)); + .with_vector_index(vector_index); indexing.automatic = true; let mut properties = ContainerProperties::new(name, "/id".into()); properties.indexing_policy = Some(indexing); @@ -107,12 +165,19 @@ pub fn verify_policy( // Azure fills in service-selected DiskANN defaults on readback. // Compare the fields we set and record the full realized policy. && matches!(p.vector_indexes.as_slice(), [index] if expected_index.is_some_and(|expected| { - index.path == expected.path && index.index_type == expected.index_type + index.path == expected.path + && index.index_type == expected.index_type + && expected.indexing_search_list_size.is_none_or(|size| { + index.indexing_search_list_size == Some(size) + }) + && expected.quantization_byte_size.is_none_or(|size| { + index.quantization_byte_size == Some(size) + }) })) }) { return Err(format!( - "Cosmos DB container '{}' has an incompatible partition key, embedding policy or vector index", + "Cosmos DB container '{}' has an incompatible partition key, embedding policy, vector index or index-build settings", actual.id )); } @@ -225,7 +290,15 @@ mod tests { #[test] fn distance_and_diskann_policy_are_in_the_wire_request() { - let p = container_policy("bench".into(), VectorIndexType::DiskANN, "dot", 1024).unwrap(); + let defaults = DiskAnnBuildOptions::default(); + let p = container_policy( + "bench".into(), + VectorIndexType::DiskANN, + "dot", + 1024, + defaults, + ) + .unwrap(); let wire = to_value(&p).unwrap(); assert_eq!(wire["partitionKey"]["paths"], json!(["/id"])); assert_eq!( @@ -239,6 +312,9 @@ mod tests { wire["indexingPolicy"]["vectorIndexes"][0]["type"], "diskANN" ); + let default_index = &wire["indexingPolicy"]["vectorIndexes"][0]; + assert!(default_index.get("quantizationByteSize").is_none()); + assert!(default_index.get("indexingSearchListSize").is_none()); let mut realized = wire; let vector_index = &mut realized["indexingPolicy"]["vectorIndexes"][0]; vector_index["quantizerType"] = json!("product"); @@ -249,10 +325,96 @@ mod tests { realized["indexingPolicy"]["vectorIndexes"][0]["type"] = json!("flat"); let wrong_index = serde_json::from_value(realized).unwrap(); assert!(verify_policy(&wrong_index, &p).is_err()); - let other = container_policy("bench".into(), VectorIndexType::DiskANN, "l2", 1024).unwrap(); + let other = container_policy( + "bench".into(), + VectorIndexType::DiskANN, + "l2", + 1024, + defaults, + ) + .unwrap(); assert!(verify_policy(&p, &other).is_err()); assert!(verify_policy(&p, &p).is_ok()); - assert!(container_policy("bench".into(), VectorIndexType::Flat, "cosine", 506).is_err()); + assert!(container_policy( + "bench".into(), + VectorIndexType::Flat, + "cosine", + 506, + defaults + ) + .is_err()); + } + + fn tuned_diskann_policy() -> ContainerProperties { + let kind = VectorIndexType::DiskANN; + let options = + DiskAnnBuildOptions::from_config(&kind, Some(&json!(150)), Some(&json!(32))).unwrap(); + container_policy("bench".into(), kind, "cosine", 100, options).unwrap() + } + + #[test] + fn diskann_build_settings_are_in_wire_policy() { + let wire = to_value(tuned_diskann_policy()).unwrap(); + let index = &wire["indexingPolicy"]["vectorIndexes"][0]; + assert_eq!(index["indexingSearchListSize"], 150); + assert_eq!(index["quantizationByteSize"], 32); + } + + #[test] + fn diskann_build_readback_requires_all_explicit_settings() { + let expected = tuned_diskann_policy(); + let wire = to_value(&expected).unwrap(); + let actual = serde_json::from_value(wire.clone()).unwrap(); + assert!(verify_policy(&actual, &expected).is_ok()); + for key in ["indexingSearchListSize", "quantizationByteSize"] { + let mut changed = wire.clone(); + let index = &mut changed["indexingPolicy"]["vectorIndexes"][0]; + index[key] = json!(100); + let actual = serde_json::from_value(changed.clone()).unwrap(); + assert!( + verify_policy(&actual, &expected).is_err(), + "mismatched {key}" + ); + changed["indexingPolicy"]["vectorIndexes"][0] + .as_object_mut() + .unwrap() + .remove(key); + let actual = serde_json::from_value(changed).unwrap(); + assert!(verify_policy(&actual, &expected).is_err(), "missing {key}"); + } + } + + #[test] + fn diskann_build_settings_validate_types_and_bounds() { + let kind = VectorIndexType::DiskANN; + for (indexing_search_list, min, max) in [(true, 25, 500), (false, 1, 512)] { + let parse = |value: &Value| { + if indexing_search_list { + DiskAnnBuildOptions::from_config(&kind, Some(value), None) + } else { + DiskAnnBuildOptions::from_config(&kind, None, Some(value)) + } + }; + assert!(parse(&json!(min)).is_ok()); + assert!(parse(&json!(max)).is_ok()); + for invalid in [json!(min - 1), json!(max + 1), json!(64.0), json!("64")] { + assert!( + parse(&invalid).is_err(), + "accepted invalid setting {invalid}" + ); + } + } + assert!( + DiskAnnBuildOptions::from_config(&VectorIndexType::Flat, Some(&json!(100)), None) + .is_err() + ); + assert!( + DiskAnnBuildOptions::from_config(&VectorIndexType::Flat, None, Some(&json!(32))) + .is_err() + ); + let options = DiskAnnBuildOptions::from_config(&kind, None, Some(&json!(32))).unwrap(); + assert!(container_policy("bench".into(), kind.clone(), "cosine", 32, options).is_ok()); + assert!(container_policy("bench".into(), kind, "cosine", 31, options).is_err()); } #[test] From 0a0c1b2562db6f871d694db92a46a94ee2dc1512 Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 21:42:20 +0100 Subject: [PATCH 06/10] Add opt-in parallel Cosmos DB vector queries --- README.md | 12 ++ .../vector_db_benchmark/effective_config.rs | 3 + .../vector_db_benchmark/engine/cosmosdb.rs | 166 ++++++++++++++--- src/cosmosdb.rs | 172 ++++++++++++++++-- 4 files changed, 305 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index e08a93010..7c478a341 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,18 @@ scope. SDK backoff and retries are part of the measured search latency. Check `attempts_without_status` and `queries_without_request_diagnostics` when interpreting the 429 count. +By default, the adapter uses the SDK's full-container query, which drains +the partitions sequentially for `ORDER BY VectorDistance`. Set +`"search_params": { "partition_concurrency": 9 }` on a search entry to +query disjoint physical feed ranges concurrently and merge their scored +top-K candidates into one global top K. The effective per-query concurrency +is capped by the number of feed ranges. Across all workers, the maximum +number of in-flight range queries is the greater of the worker count and the +effective per-query concurrency. Before timing, the adapter compares sampled +result IDs with the SDK's full-container query. It aborts if the results +disagree or the feed ranges change during search. Fan-out results record the +effective limits alongside RU charge and HTTP attempts for all ranges. + For DiskANN, each search entry can set positive integer `search_params.search_list_size_multiplier` and `search_params.quantized_vector_list_multiplier`. These map to the diff --git a/src/bin/vector_db_benchmark/effective_config.rs b/src/bin/vector_db_benchmark/effective_config.rs index 956915ddc..9234c0e42 100644 --- a/src/bin/vector_db_benchmark/effective_config.rs +++ b/src/bin/vector_db_benchmark/effective_config.rs @@ -191,6 +191,9 @@ const DECLARED_KEY_ALLOWLIST: &[&str] = &[ // Cosmos DB DiskANN "indexing_search_list_size", "quantization_byte_size", + "search_list_size_multiplier", + "quantized_vector_list_multiplier", + "partition_concurrency", // SVS-VAMANA "svs-vamana_config", "GRAPH_MAX_DEGREE", diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index ab36edfc4..10f250bbf 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -4,20 +4,26 @@ use std::num::NonZeroU32; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Instant; -use azure_data_cosmos::models::{ContainerProperties, ThroughputProperties, VectorIndexType}; +use azure_data_cosmos::feed::FeedRange; +use azure_data_cosmos::models::{ + ContainerProperties, ThroughputProperties, VectorDistanceFunction, VectorIndexType, +}; use azure_data_cosmos::options::{ ContentResponseOnWrite, CreateContainerOptions, MaxItemCountHint, OperationOptionsBuilder, - QueryOptions, + QueryOptions, ReadFeedRangesOptions, }; use azure_data_cosmos::{ AccountReference, ContainerClient, CosmosClient, CosmosRuntime, FeedScope, Query, RoutingStrategy, }; use futures::{stream, StreamExt, TryStreamExt}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; +use tokio::sync::Semaphore; use vector_db_benchmark::cosmosdb::{ - container_name, container_policy, index_type, knn_query, validate_corpus_size, - validate_hit_count, validate_top, verify_policy, DiskAnnBuildOptions, SearchDiagnostics, + container_name, container_policy, index_type, knn_query, merge_partition_hits, + validate_corpus_size, validate_hit_count, validate_top, vector_distance_function, + verify_partition_search, verify_policy, DiskAnnBuildOptions, PartitionFanout, QueryHit, + SearchDiagnostics, }; use vector_db_benchmark::start_gate::WorkerPool; use vector_db_benchmark::workload::IndexLayout; @@ -32,25 +38,14 @@ struct Item<'a> { embedding: &'a [f32], } -#[derive(Deserialize)] -struct Hit { - id: String, -} - const SEARCH_LIST_SIZE: &str = "search_list_size_multiplier"; const QUANTIZED_VECTOR_LIST: &str = "quantized_vector_list_multiplier"; +const PARTITION_CONCURRENCY: &str = "partition_concurrency"; -fn diskann_multiplier( - params: &SearchParams, - kind: &VectorIndexType, - key: &str, -) -> Result, String> { +fn positive_knob(params: &SearchParams, key: &str) -> Result, String> { params .knob(key) .map(|value| { - if *kind != VectorIndexType::DiskANN { - return Err(format!("Cosmos DB {key} requires DiskANN")); - } value .as_u64() .and_then(|n| u32::try_from(n).ok()) @@ -60,6 +55,18 @@ fn diskann_multiplier( .transpose() } +fn diskann_multiplier( + params: &SearchParams, + kind: &VectorIndexType, + key: &str, +) -> Result, String> { + let value = positive_knob(params, key)?; + if value.is_some() && *kind != VectorIndexType::DiskANN { + return Err(format!("Cosmos DB {key} requires DiskANN")); + } + Ok(value) +} + pub struct CosmosDbEngine { name: String, database: String, @@ -84,8 +91,12 @@ impl CosmosDbEngine { if let Some(top) = params.top { validate_top(usize::try_from(top).map_err(|_| "Cosmos DB top must be positive")?)?; } - let unsupported = - |key: &String| !matches!(key.as_str(), SEARCH_LIST_SIZE | QUANTIZED_VECTOR_LIST); + let unsupported = |key: &String| { + !matches!( + key.as_str(), + SEARCH_LIST_SIZE | QUANTIZED_VECTOR_LIST | PARTITION_CONCURRENCY + ) + }; if params.calibration_param.is_some() || params.num_candidates.is_some() || params @@ -100,11 +111,12 @@ impl CosmosDbEngine { }) { return Err( - "Cosmos DB supports only DiskANN search multipliers as tuning knobs".into(), + "Cosmos DB supports only DiskANN search multipliers and partition_concurrency as tuning knobs".into(), ); } diskann_multiplier(params, &kind, SEARCH_LIST_SIZE)?; diskann_multiplier(params, &kind, QUANTIZED_VECTOR_LIST)?; + positive_knob(params, PARTITION_CONCURRENCY)?; } let collection = config.collection_params.as_ref(); if collection.is_some_and(|p| { @@ -455,6 +467,7 @@ impl Engine for CosmosDbEngine { let search_list_size = diskann_multiplier(params, &self.index_type, SEARCH_LIST_SIZE)?; let quantized_vector_list = diskann_multiplier(params, &self.index_type, QUANTIZED_VECTOR_LIST)?; + let partition_concurrency = positive_knob(params, PARTITION_CONCURRENCY)?; let queries = vectors[..limit] .iter() @@ -462,7 +475,13 @@ impl Engine for CosmosDbEngine { if vector.len() != dataset.vector_size() as usize { return Err("Cosmos DB query vector has the wrong number of dimensions".into()); } - knn_query(vector, top, search_list_size, quantized_vector_list) + knn_query( + vector, + top, + search_list_size, + quantized_vector_list, + partition_concurrency.is_some(), + ) }) .collect::, String>>()?; let container = self @@ -470,6 +489,34 @@ impl Engine for CosmosDbEngine { .as_ref() .ok_or("Cosmos DB container is unavailable")?; let rt = &self.rt; + let partition_search = partition_concurrency.map(|concurrency| { + let ranges = rt.block_on(read_search_ranges(container))?; + let per_query_concurrency = (concurrency.get() as usize).min(ranges.len()); + let shared_request_limit = parallel.max(per_query_concurrency); + let summary = PartitionFanout { + feed_ranges: ranges.len(), + per_query_concurrency, + shared_request_limit, + }; + let search = PartitionSearch { + ranges, + summary, + metric: vector_distance_function(dataset.distance())?, + permits: Semaphore::new(shared_request_limit), + }; + for index in (0..limit).step_by((limit / 3).max(1)).take(3) { + let stock_query = knn_query(&vectors[index], top, search_list_size, quantized_vector_list, false)?; + let (stock, _) = rt.block_on(query_hits(container, &stock_query, top, FeedScope::full_container()))?; + let (partitioned, _) = rt.block_on(search_hits(container, &queries[index], top, Some(&search)))?; + verify_partition_search(&stock, &partitioned).map_err(|e| format!("Cosmos DB parity check for query {index}: {e}"))?; + } + println!( + "Cosmos DB feed-range search verified against SDK on sampled queries: {} ranges, {} per query, {} in flight", + summary.feed_ranges, summary.per_query_concurrency, summary.shared_request_limit + ); + Ok::<_, String>(search) + }).transpose()?; + let partition_search = partition_search.as_ref(); let next = AtomicUsize::new(0); let (samples, elapsed) = std::thread::scope(|scope| { let mut pool = WorkerPool::new(scope, "cosmosdb-search", parallel); @@ -480,9 +527,12 @@ impl Engine for CosmosDbEngine { pool.spawn(move |ticket| { let mut samples = Vec::new(); // Warm the worker's first request outside the measured window. - if let Err(error) = - rt.block_on(query_hits(container, &queries[worker % limit], top)) - { + if let Err(error) = rt.block_on(search_hits( + container, + &queries[worker % limit], + top, + partition_search, + )) { ticket.fail(format!("Cosmos DB worker {worker} warm-up failed: {error}")); return Ok(samples); } @@ -494,7 +544,7 @@ impl Engine for CosmosDbEngine { let Some(query) = queries.get(idx) else { break }; let started = Instant::now(); let (hits, diagnostics) = rt - .block_on(query_hits(container, query, top)) + .block_on(search_hits(container, query, top, partition_search)) .map_err(|e| format!("Cosmos DB query {idx} failed: {e}"))?; let latency = started.elapsed().as_secs_f64(); let ids = hits @@ -517,6 +567,11 @@ impl Engine for CosmosDbEngine { let (batches, start) = pool.start()?; Ok::<_, String>((batches, start.elapsed().as_secs_f64())) })?; + if let Some(search) = partition_search { + if rt.block_on(read_search_ranges(container))? != search.ranges { + return Err("Cosmos DB feed ranges changed during the measured search".into()); + } + } let (mut times, mut precision, mut recall, mut mrr, mut ndcg) = (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new()); let mut diagnostics = SearchDiagnostics::default(); @@ -530,6 +585,7 @@ impl Engine for CosmosDbEngine { diagnostics.add_query(query_diagnostics); } } + diagnostics.partition_fanout = partition_search.map(|search| search.summary); if diagnostics.http_429_attempts > 0 { eprintln!( "Cosmos DB search at parallel={parallel} observed {} HTTP 429 responses. Latency includes SDK backoff and retries.", @@ -574,20 +630,70 @@ impl Engine for CosmosDbEngine { } } +struct PartitionSearch { + ranges: Vec, + summary: PartitionFanout, + metric: VectorDistanceFunction, + permits: Semaphore, +} + +async fn read_search_ranges(container: &ContainerClient) -> Result, String> { + let ranges = container + .read_feed_ranges(Some( + ReadFeedRangesOptions::default().with_force_refresh(true), + )) + .await + .map_err(|e| format!("Could not read Cosmos DB feed ranges: {e}"))?; + if ranges.is_empty() { + return Err("Cosmos DB returned no feed ranges".into()); + } + Ok(ranges) +} + +async fn search_hits( + container: &ContainerClient, + query: &Query, + top: usize, + partition_search: Option<&PartitionSearch>, +) -> Result<(Vec, SearchDiagnostics), String> { + let Some(search) = partition_search else { + return query_hits(container, query, top, FeedScope::full_container()).await; + }; + let (hits, diagnostics) = stream::iter(&search.ranges) + .map(|range| async { + let _permit = search.permits.acquire().await.map_err(|e| e.to_string())?; + query_hits(container, query, top, FeedScope::range(range.clone())).await + }) + .buffer_unordered(search.summary.per_query_concurrency) + .try_fold( + (Vec::new(), SearchDiagnostics::default()), + |(mut hits, mut diagnostics), (range_hits, range_diagnostics)| async move { + hits.extend(range_hits); + diagnostics.add_query(range_diagnostics); + Ok((hits, diagnostics)) + }, + ) + .await?; + Ok(( + merge_partition_hits(hits, top, &search.metric)?, + diagnostics, + )) +} + async fn query_hits( container: &ContainerClient, query: &Query, top: usize, -) -> Result<(Vec, SearchDiagnostics), String> { + scope: FeedScope, +) -> Result<(Vec, SearchDiagnostics), String> { let mut pages = container - .query_items::(query.clone(), FeedScope::full_container(), None) + .query_items::(query.clone(), scope, None) .await .map_err(|error| error.to_string())? .into_pages(); let mut hits = Vec::new(); let mut combined = SearchDiagnostics::default(); - // The SDK merges ORDER BY across partitions. Check the resulting global - // limit anyway, including pages where the server sends no hits. + // Enforce TOP K within the given scope across all SDK pages. while let Some(page) = pages.try_next().await.map_err(|error| error.to_string())? { validate_hit_count(hits.len() + page.items().len(), top)?; let diagnostics = page.diagnostics(); diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index 852a0d2e0..80d8d70c7 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -6,9 +6,10 @@ use azure_data_cosmos::models::{ }; use azure_data_cosmos::Query; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; +use std::collections::HashSet; const VECTOR_PATH: &str = "/embedding"; const VECTOR_EXCLUSION: &str = "/embedding/*"; @@ -76,6 +77,18 @@ impl DiskAnnBuildOptions { } } +/// Cosmos DB metric corresponding to a benchmark corpus's distance. +pub fn vector_distance_function(distance: &str) -> Result { + match distance.to_ascii_lowercase().as_str() { + "cosine" | "angular" => Ok(VectorDistanceFunction::Cosine), + "l2" | "euclidean" => Ok(VectorDistanceFunction::Euclidean), + "dot" | "ip" => Ok(VectorDistanceFunction::DotProduct), + _ => Err(format!( + "Cosmos DB does not support dataset distance '{distance}'" + )), + } +} + /// Use the dataset metric and dimension in the server-side policy. pub fn container_policy( name: String, @@ -84,16 +97,7 @@ pub fn container_policy( dimensions: i64, build: DiskAnnBuildOptions, ) -> Result { - let distance = match distance.to_ascii_lowercase().as_str() { - "cosine" | "angular" => VectorDistanceFunction::Cosine, - "l2" | "euclidean" => VectorDistanceFunction::Euclidean, - "dot" | "ip" => VectorDistanceFunction::DotProduct, - _ => { - return Err(format!( - "Cosmos DB does not support dataset distance '{distance}'" - )) - } - }; + let distance = vector_distance_function(distance)?; let max_dimensions = if index_type == VectorIndexType::Flat { 505 } else { @@ -212,6 +216,15 @@ pub struct SearchDiagnostics { pub http_429_attempts: usize, pub attempts_without_status: usize, pub queries_without_request_diagnostics: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub partition_fanout: Option, +} + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct PartitionFanout { + pub feed_ranges: usize, + pub per_query_concurrency: usize, + pub shared_request_limit: usize, } impl SearchDiagnostics { @@ -242,7 +255,9 @@ impl SearchDiagnostics { self.request_attempts += query.request_attempts; self.http_429_attempts += query.http_429_attempts; self.attempts_without_status += query.attempts_without_status; - self.queries_without_request_diagnostics += usize::from(query.request_attempts == 0); + self.queries_without_request_diagnostics += usize::from( + query.request_attempts == 0 || query.queries_without_request_diagnostics > 0, + ); } } @@ -263,6 +278,7 @@ pub fn knn_query( top: usize, search_list_size_multiplier: Option, quantized_vector_list_multiplier: Option, + project_distance: bool, ) -> Result { validate_top(top)?; if vector.iter().any(|v| !v.is_finite()) { @@ -276,13 +292,85 @@ pub fn knn_query( ", {{\"searchListSizeMultiplier\": {search}, \"quantizedVectorListMultiplier\": {quantized}}}" ), }; + let distance = format!("VectorDistance(c.embedding, @vector, false{options})"); + let projection = if project_distance { + format!("c.id, {distance} AS distance") + } else { + "c.id".into() + }; Query::from(format!( - "SELECT TOP {top} c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false{options})" + "SELECT TOP {top} {projection} FROM c ORDER BY {distance}" )) .with_parameter("@vector", vector) .map_err(|e| format!("Could not bind Cosmos DB query vector: {e}")) } +#[derive(Deserialize)] +pub struct QueryHit { + pub id: String, + pub distance: Option, +} + +/// Select the global top K from the scored candidates of disjoint feed ranges. +pub fn merge_partition_hits( + hits: Vec, + top: usize, + metric: &VectorDistanceFunction, +) -> Result, String> { + // The SQL vector ORDER BY is best-first for every metric. Its projected + // value is a similarity for cosine/dot product and a distance for L2. + let higher_is_better = match metric { + VectorDistanceFunction::Cosine | VectorDistanceFunction::DotProduct => true, + VectorDistanceFunction::Euclidean => false, + _ => return Err("Unsupported Cosmos DB vector score ordering".into()), + }; + let mut candidates = hits + .into_iter() + .map(|hit| { + let distance = hit + .distance + .filter(|distance| distance.is_finite()) + .ok_or("Cosmos DB partition query returned a hit without a finite distance")?; + Ok((hit, distance)) + }) + .collect::, String>>()?; + let mut seen = HashSet::with_capacity(candidates.len()); + for (hit, _) in &candidates { + if !seen.insert(hit.id.as_str()) { + return Err(format!( + "Cosmos DB partition queries returned duplicate id {}", + hit.id + )); + } + } + candidates.sort_unstable_by(|(left, a), (right, b)| { + let order = a.total_cmp(b); + (if higher_is_better { + order.reverse() + } else { + order + }) + .then_with(|| left.id.cmp(&right.id)) + }); + candidates.truncate(top); + Ok(candidates.into_iter().map(|(hit, _)| hit).collect()) +} + +/// Verify that the same query covers the same IDs under both routing modes. +pub fn verify_partition_search(stock: &[QueryHit], partitioned: &[QueryHit]) -> Result<(), String> { + let stock_ids: HashSet<_> = stock.iter().map(|hit| &hit.id).collect(); + let partitioned_ids: HashSet<_> = partitioned.iter().map(|hit| &hit.id).collect(); + if stock.len() != partitioned.len() || stock_ids != partitioned_ids { + return Err(format!( + "Cosmos DB partition search disagrees with the SDK's global top K ({} vs {} hits, {} shared IDs)", + stock.len(), + partitioned.len(), + stock_ids.intersection(&partitioned_ids).count(), + )); + } + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -419,7 +507,7 @@ mod tests { #[test] fn vector_queries_request_indexed_top_k_and_bind_the_vector() { - let query = to_value(knn_query(&[0.5, -0.25], 10, None, None).unwrap()).unwrap(); + let query = to_value(knn_query(&[0.5, -0.25], 10, None, None, false).unwrap()).unwrap(); assert_eq!( query["query"], "SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" @@ -428,14 +516,62 @@ mod tests { query["parameters"], json!([{"name": "@vector", "value": [0.5, -0.25]}]) ); - assert!(knn_query(&[0.5], 0, None, None).is_err()); - assert!(knn_query(&[0.5], 1001, None, None).is_err()); - let tuned = to_value(knn_query(&[0.5], 10, 5.try_into().ok(), 20.try_into().ok()).unwrap()) - .unwrap(); + assert!(knn_query(&[0.5], 0, None, None, false).is_err()); + assert!(knn_query(&[0.5], 1001, None, None, false).is_err()); + let tuned = + to_value(knn_query(&[0.5], 10, 5.try_into().ok(), 20.try_into().ok(), false).unwrap()) + .unwrap(); assert_eq!( tuned["query"], r#"SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false, {"searchListSizeMultiplier": 5, "quantizedVectorListMultiplier": 20})"# ); + let scored = to_value(knn_query(&[0.5], 10, None, None, true).unwrap()).unwrap(); + assert_eq!( + scored["query"], + "SELECT TOP 10 c.id, VectorDistance(c.embedding, @vector, false) AS distance FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" + ); + } + + #[test] + fn partition_merge_selects_global_top_k_and_rejects_incomplete_hits() { + let hit = |id: &str, distance| QueryHit { + id: id.into(), + distance, + }; + let candidates = || { + vec![ + hit("0", Some(0.9)), + hit("2", Some(0.3)), + hit("1", Some(0.3)), + hit("3", Some(0.1)), + ] + }; + for (metric, expected) in [ + (VectorDistanceFunction::Euclidean, ["3", "1"]), + (VectorDistanceFunction::Cosine, ["0", "1"]), + (VectorDistanceFunction::DotProduct, ["0", "1"]), + ] { + let merged = merge_partition_hits(candidates(), 2, &metric).unwrap(); + assert_eq!( + merged.iter().map(|hit| hit.id.as_str()).collect::>(), + expected + ); + assert!(verify_partition_search( + &[hit(expected[1], None), hit(expected[0], None)], + &merged + ) + .is_ok()); + assert!( + verify_partition_search(&[hit("2", None), hit(expected[0], None)], &merged) + .is_err() + ); + } + let metric = VectorDistanceFunction::Cosine; + assert!(merge_partition_hits(vec![hit("0", None)], 1, &metric).is_err()); + assert!( + merge_partition_hits(vec![hit("0", Some(0.1)), hit("0", Some(0.2))], 1, &metric) + .is_err() + ); } #[test] From eb5653bfe6eab55fe4aceae4be2fd60aca8d9f2b Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 21:49:46 +0100 Subject: [PATCH 07/10] Discover Cosmos DB partition concurrency automatically --- README.md | 7 ++- .../vector_db_benchmark/engine/cosmosdb.rs | 11 ++-- src/cosmosdb.rs | 57 ++++++++++++++++++- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7c478a341..00a299c38 100644 --- a/README.md +++ b/README.md @@ -231,10 +231,11 @@ when interpreting the 429 count. By default, the adapter uses the SDK's full-container query, which drains the partitions sequentially for `ORDER BY VectorDistance`. Set -`"search_params": { "partition_concurrency": 9 }` on a search entry to +`"search_params": { "partition_concurrency": "auto" }` on a search entry to query disjoint physical feed ranges concurrently and merge their scored -top-K candidates into one global top K. The effective per-query concurrency -is capped by the number of feed ranges. Across all workers, the maximum +top-K candidates into one global top K. The adapter looks up the feed ranges +before timing. `"auto"` uses their count as the per-query concurrency, and +a positive integer caps it at that value. Across all workers, the maximum number of in-flight range queries is the greater of the worker count and the effective per-query concurrency. Before timing, the adapter compares sampled result IDs with the SDK's full-container query. It aborts if the results diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index 10f250bbf..4287d1114 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -22,8 +22,8 @@ use tokio::sync::Semaphore; use vector_db_benchmark::cosmosdb::{ container_name, container_policy, index_type, knn_query, merge_partition_hits, validate_corpus_size, validate_hit_count, validate_top, vector_distance_function, - verify_partition_search, verify_policy, DiskAnnBuildOptions, PartitionFanout, QueryHit, - SearchDiagnostics, + verify_partition_search, verify_policy, DiskAnnBuildOptions, PartitionConcurrency, + PartitionFanout, QueryHit, SearchDiagnostics, }; use vector_db_benchmark::start_gate::WorkerPool; use vector_db_benchmark::workload::IndexLayout; @@ -116,7 +116,7 @@ impl CosmosDbEngine { } diskann_multiplier(params, &kind, SEARCH_LIST_SIZE)?; diskann_multiplier(params, &kind, QUANTIZED_VECTOR_LIST)?; - positive_knob(params, PARTITION_CONCURRENCY)?; + PartitionConcurrency::from_config(params.knob(PARTITION_CONCURRENCY))?; } let collection = config.collection_params.as_ref(); if collection.is_some_and(|p| { @@ -467,7 +467,8 @@ impl Engine for CosmosDbEngine { let search_list_size = diskann_multiplier(params, &self.index_type, SEARCH_LIST_SIZE)?; let quantized_vector_list = diskann_multiplier(params, &self.index_type, QUANTIZED_VECTOR_LIST)?; - let partition_concurrency = positive_knob(params, PARTITION_CONCURRENCY)?; + let partition_concurrency = + PartitionConcurrency::from_config(params.knob(PARTITION_CONCURRENCY))?; let queries = vectors[..limit] .iter() @@ -491,7 +492,7 @@ impl Engine for CosmosDbEngine { let rt = &self.rt; let partition_search = partition_concurrency.map(|concurrency| { let ranges = rt.block_on(read_search_ranges(container))?; - let per_query_concurrency = (concurrency.get() as usize).min(ranges.len()); + let per_query_concurrency = concurrency.resolve(ranges.len()); let shared_request_limit = parallel.max(per_query_concurrency); let summary = PartitionFanout { feed_ranges: ranges.len(), diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index 80d8d70c7..4bc3da5d3 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::HashSet; +use std::num::NonZeroU32; const VECTOR_PATH: &str = "/embedding"; const VECTOR_EXCLUSION: &str = "/embedding/*"; @@ -227,6 +228,40 @@ pub struct PartitionFanout { pub shared_request_limit: usize, } +#[derive(Debug, PartialEq, Eq)] +pub enum PartitionConcurrency { + Auto, + Limit(NonZeroU32), +} + +impl PartitionConcurrency { + /// Missing selects the stock SDK path. Explicit fan-out accepts "auto" or a cap. + pub fn from_config(value: Option<&Value>) -> Result, String> { + value + .map(|value| { + if value.as_str() == Some("auto") { + return Ok(Self::Auto); + } + value + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .and_then(NonZeroU32::new) + .map(Self::Limit) + .ok_or_else(|| { + "Cosmos DB partition_concurrency must be \"auto\" or a positive 32-bit integer".into() + }) + }) + .transpose() + } + + pub fn resolve(&self, feed_ranges: usize) -> usize { + match self { + Self::Auto => feed_ranges, + Self::Limit(cap) => feed_ranges.min(cap.get() as usize), + } + } +} + impl SearchDiagnostics { /// Use the exact SDK totals and count HTTP statuses from either individual /// attempts or their compacted rollup. Omitted statuses remain unknown. @@ -276,8 +311,8 @@ pub fn validate_corpus_size(kind: &VectorIndexType, count: u64) -> Result<(), St pub fn knn_query( vector: &[f32], top: usize, - search_list_size_multiplier: Option, - quantized_vector_list_multiplier: Option, + search_list_size_multiplier: Option, + quantized_vector_list_multiplier: Option, project_distance: bool, ) -> Result { validate_top(top)?; @@ -532,6 +567,24 @@ mod tests { ); } + #[test] + fn partition_concurrency_uses_the_realized_feed_range_count() { + assert_eq!(PartitionConcurrency::from_config(None).unwrap(), None); + let auto = PartitionConcurrency::from_config(Some(&json!("auto"))) + .unwrap() + .unwrap(); + assert_eq!(auto.resolve(5), 5); + assert_eq!(auto.resolve(12), 12); + let limited = PartitionConcurrency::from_config(Some(&json!(9))) + .unwrap() + .unwrap(); + assert_eq!(limited.resolve(5), 5); + assert_eq!(limited.resolve(12), 9); + for invalid in [json!(0), json!(-1), json!(1.5), json!("9"), json!(null)] { + assert!(PartitionConcurrency::from_config(Some(&invalid)).is_err()); + } + } + #[test] fn partition_merge_selects_global_top_k_and_rejects_incomplete_hits() { let hit = |id: &str, distance| QueryHit { From de9bf3c63bdbedf05e9e297c3f372af4047c393f Mon Sep 17 00:00:00 2001 From: fcenedes Date: Wed, 18 Nov 2026 22:26:41 +0100 Subject: [PATCH 08/10] Retry sampled Cosmos ANN parity checks before timing --- README.md | 8 ++-- .../vector_db_benchmark/engine/cosmosdb.rs | 39 ++++++++++++++++--- src/cosmosdb.rs | 1 + 3 files changed, 39 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 00a299c38..8444493b8 100644 --- a/README.md +++ b/README.md @@ -238,9 +238,11 @@ before timing. `"auto"` uses their count as the per-query concurrency, and a positive integer caps it at that value. Across all workers, the maximum number of in-flight range queries is the greater of the worker count and the effective per-query concurrency. Before timing, the adapter compares sampled -result IDs with the SDK's full-container query. It aborts if the results -disagree or the feed ranges change during search. Fan-out results record the -effective limits alongside RU charge and HTTP attempts for all ranges. +result IDs with the SDK's full-container query. If a sampled pair differs, it +can retry the pair up to three times to account for DiskANN result variability. +Persistent mismatches or a feed-range change invalidate the repetition. +Fan-out results record the preflight retry count and effective limits +alongside RU charge and HTTP attempts for all ranges. For DiskANN, each search entry can set positive integer `search_params.search_list_size_multiplier` and diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index 4287d1114..dfe0c3685 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -498,8 +498,9 @@ impl Engine for CosmosDbEngine { feed_ranges: ranges.len(), per_query_concurrency, shared_request_limit, + preflight_retries: 0, }; - let search = PartitionSearch { + let mut search = PartitionSearch { ranges, summary, metric: vector_distance_function(dataset.distance())?, @@ -507,13 +508,15 @@ impl Engine for CosmosDbEngine { }; for index in (0..limit).step_by((limit / 3).max(1)).take(3) { let stock_query = knn_query(&vectors[index], top, search_list_size, quantized_vector_list, false)?; - let (stock, _) = rt.block_on(query_hits(container, &stock_query, top, FeedScope::full_container()))?; - let (partitioned, _) = rt.block_on(search_hits(container, &queries[index], top, Some(&search)))?; - verify_partition_search(&stock, &partitioned).map_err(|e| format!("Cosmos DB parity check for query {index}: {e}"))?; + let retries = rt + .block_on(check_partition_parity(container, &stock_query, &queries[index], top, &search)) + .map_err(|e| format!("Cosmos DB parity check for query {index}: {e}"))?; + search.summary.preflight_retries += retries; } println!( - "Cosmos DB feed-range search verified against SDK on sampled queries: {} ranges, {} per query, {} in flight", - summary.feed_ranges, summary.per_query_concurrency, summary.shared_request_limit + "Cosmos DB feed-range search verified against SDK on sampled queries: {} ranges, {} per query, {} in flight, {} parity retries", + summary.feed_ranges, summary.per_query_concurrency, summary.shared_request_limit, + search.summary.preflight_retries ); Ok::<_, String>(search) }).transpose()?; @@ -638,6 +641,30 @@ struct PartitionSearch { permits: Semaphore, } +async fn check_partition_parity( + container: &ContainerClient, + stock_query: &Query, + partition_query: &Query, + top: usize, + search: &PartitionSearch, +) -> Result { + // DiskANN can return different neighbors at the cutoff for repeated + // requests, including two stock requests. Retry the sampled pair while + // still requiring exact ID agreement before any timing. + const MAX_RETRIES: usize = 3; + let mut retries = 0; + loop { + let (stock, _) = + query_hits(container, stock_query, top, FeedScope::full_container()).await?; + let (partitioned, _) = search_hits(container, partition_query, top, Some(search)).await?; + let verdict = verify_partition_search(&stock, &partitioned); + if verdict.is_ok() || retries == MAX_RETRIES { + return verdict.map(|()| retries); + } + retries += 1; + } +} + async fn read_search_ranges(container: &ContainerClient) -> Result, String> { let ranges = container .read_feed_ranges(Some( diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index 4bc3da5d3..46cdb3c8b 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -226,6 +226,7 @@ pub struct PartitionFanout { pub feed_ranges: usize, pub per_query_concurrency: usize, pub shared_request_limit: usize, + pub preflight_retries: usize, } #[derive(Debug, PartialEq, Eq)] From 7b5a9e7f376a037935a812ae16b6a410c7a38af1 Mon Sep 17 00:00:00 2001 From: fcenedes Date: Thu, 19 Nov 2026 15:09:05 +0100 Subject: [PATCH 09/10] Verify Cosmos DB DiskANN index use before timing Probe every feed range before search: compare the RU charge of an index-requested query with a forced scan and wait, up to AZURE_COSMOS_INDEX_PROBE_SECONDS, until the index serves each range. The run fails if it never does. Upload total_time now covers the upload and the index wait, and server metadata records the probe result. Replace the retry-until-green fan-out parity check with a noise-floor check: each sample runs the stock query twice and the fan-out once, and the fan-out may miss no more IDs than a small multiple of what stock misses against itself. The counts are recorded under partition_fanout.parity (was preflight_retries). Also: validate config keys and values in src/cosmosdb.rs with unit tests, build the query text once per search, count rows per feed range concurrently, record AZURE_COSMOS_* settings the SDK reads, and add an env-gated fan-out integration test. Co-Authored-By: Claude Fable 5.2 --- README.md | 33 +- .../vector_db_benchmark/effective_config.rs | 33 ++ .../vector_db_benchmark/engine/cosmosdb.rs | 462 +++++++++++------- .../engine/filter_guard.rs | 3 +- src/cosmosdb.rs | 455 +++++++++++++++-- tests/integration_cosmosdb.rs | 82 +++- 6 files changed, 829 insertions(+), 239 deletions(-) diff --git a/README.md b/README.md index 8444493b8..67f5f662a 100644 --- a/README.md +++ b/README.md @@ -237,12 +237,19 @@ top-K candidates into one global top K. The adapter looks up the feed ranges before timing. `"auto"` uses their count as the per-query concurrency, and a positive integer caps it at that value. Across all workers, the maximum number of in-flight range queries is the greater of the worker count and the -effective per-query concurrency. Before timing, the adapter compares sampled -result IDs with the SDK's full-container query. If a sampled pair differs, it -can retry the pair up to three times to account for DiskANN result variability. -Persistent mismatches or a feed-range change invalidate the repetition. -Fan-out results record the preflight retry count and effective limits -alongside RU charge and HTTP attempts for all ranges. +effective per-query concurrency. When the worker count exceeds the range +count, range queries wait for a free slot inside the measured latency, so +compare a fan-out row with a stock row on throughput and recall, not on tail +latency. + +Before timing, the adapter runs up to eight sampled queries under both modes, +with the stock query sent twice. Repeated DiskANN requests can return +different neighbors at the cutoff, so the stock query's disagreement with +itself sets the bar. Fan-out may miss at most three times as many stock IDs +as the repeated stock request misses. Flat is exact, so it needs identical +IDs. A larger difference or a feed-range change fails the run. Fan-out +results record both counts under `partition_fanout.parity`, with the +effective limits, RU charge, and HTTP attempts for all ranges. For DiskANN, each search entry can set positive integer `search_params.search_list_size_multiplier` and @@ -284,8 +291,18 @@ container. It verifies the vector policy and corpus cardinality. The `cosmosdb-flat` configuration supports up to 505 dimensions. The `cosmosdb-diskann` configuration supports up to 4,096 dimensions and requires at least 1,000 vectors. Cosmos DB can scan while an index builds or while a -physical partition has too few vectors. Results report the realized index -policy and explicitly mark index usage as unverified. See +physical partition has too few vectors. Before any DiskANN search, the adapter +therefore probes each feed range. It runs one forced scan and then repeats an +index-requested query every 30 seconds until that query costs less than half +the scan's request charge on every range. The run fails if a range does not +get there within 30 minutes. Set `AZURE_COSMOS_INDEX_PROBE_SECONDS` to change +the limit, or to `0` to skip the probe. Results record the per-range charges +and `vector_index_use_verified`, which is `false` when the probe was skipped. +The SDK reads further `AZURE_COSMOS_*` settings from the environment, such +as retry and connection options. Every such variable that is set is recorded +in the result's effective configuration, with the account key redacted. +After an upload, `upload_time` covers the upserts and `total_time` adds the +probe's wait for the index. The row-count check is in neither. See [Azure Cosmos DB vector indexing](https://learn.microsoft.com/azure/cosmos-db/nosql/vector-search). diff --git a/src/bin/vector_db_benchmark/effective_config.rs b/src/bin/vector_db_benchmark/effective_config.rs index 9234c0e42..b73b1a8ba 100644 --- a/src/bin/vector_db_benchmark/effective_config.rs +++ b/src/bin/vector_db_benchmark/effective_config.rs @@ -1047,6 +1047,23 @@ pub fn env_opt(name: &str) -> Option { value } +/// Record every set environment variable whose name starts with `prefix`. +/// +/// For a client library that reads its own settings from the environment +/// (transport, retry and pool options the adapter never names). They change +/// what a run measures, so they belong in the artifact like any other knob. +/// Secrets are redacted by name as usual. +pub fn observe_env_prefix(prefix: &str) { + let mut names: Vec = std::env::vars_os() + .filter_map(|(name, _)| name.into_string().ok()) + .filter(|name| name.starts_with(prefix)) + .collect(); + names.sort(); + for name in names { + let _ = env_opt(&name); + } +} + /// Record a resolved knob that did not come straight from one environment /// variable — a value chosen between the environment, the configuration file /// and a built-in default, or read back off the server. @@ -2208,6 +2225,22 @@ mod tests { } } + #[test] + fn a_prefix_scan_records_settings_the_adapter_never_names() { + let _l = test_lock(); + let _a = EnvGuard::set("VDBB_PREFIX_TEST_POOL_SIZE", Some("32")); + let _b = EnvGuard::set("VDBB_PREFIX_TEST_API_KEY", Some("prefix-secret")); + let _c = EnvGuard::set("VDBB_OTHER_TEST_SETTING", Some("ignored")); + reset(); + observe_env_prefix("VDBB_PREFIX_TEST_"); + let s = snapshot(); + assert_eq!(s["env"]["VDBB_PREFIX_TEST_POOL_SIZE"], "32"); + assert_eq!(s["effective"]["VDBB_PREFIX_TEST_POOL_SIZE"], "32"); + assert_eq!(s["env"]["VDBB_PREFIX_TEST_API_KEY"], REDACTED); + assert!(!serde_json::to_string(&s).unwrap().contains("prefix-secret")); + assert!(s["env"].get("VDBB_OTHER_TEST_SETTING").is_none()); + } + #[test] fn env_parsed_records_the_number_the_caller_received() { let _l = test_lock(); diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index dfe0c3685..de2f7f753 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -2,7 +2,7 @@ use std::num::NonZeroU32; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::time::Instant; +use std::time::{Duration, Instant}; use azure_data_cosmos::feed::FeedRange; use azure_data_cosmos::models::{ @@ -20,10 +20,11 @@ use futures::{stream, StreamExt, TryStreamExt}; use serde::Serialize; use tokio::sync::Semaphore; use vector_db_benchmark::cosmosdb::{ - container_name, container_policy, index_type, knn_query, merge_partition_hits, - validate_corpus_size, validate_hit_count, validate_top, vector_distance_function, - verify_partition_search, verify_policy, DiskAnnBuildOptions, PartitionConcurrency, - PartitionFanout, QueryHit, SearchDiagnostics, + container_name, container_policy, index_probe_deadline, index_type, knn_query, + manual_throughput, merge_partition_hits, positive_u32, reject_unknown_keys, scan_query, + upload_parallel, validate_corpus_size, validate_hit_count, validate_query_vector, validate_top, + vector_distance_function, verify_policy, DiskAnnBuildOptions, IndexProbe, KnnQueryText, + ParityTally, PartitionConcurrency, PartitionFanout, QueryHit, SearchDiagnostics, }; use vector_db_benchmark::start_gate::WorkerPool; use vector_db_benchmark::workload::IndexLayout; @@ -41,17 +42,36 @@ struct Item<'a> { const SEARCH_LIST_SIZE: &str = "search_list_size_multiplier"; const QUANTIZED_VECTOR_LIST: &str = "quantized_vector_list_multiplier"; const PARTITION_CONCURRENCY: &str = "partition_concurrency"; +const SEARCH_KNOBS: [&str; 3] = [ + SEARCH_LIST_SIZE, + QUANTIZED_VECTOR_LIST, + PARTITION_CONCURRENCY, +]; +const COLLECTION_KEYS: [&str; 3] = [ + "ru_per_second", + "indexing_search_list_size", + "quantization_byte_size", +]; +const UNSUPPORTED_SEARCH_KNOB: &str = + "Cosmos DB supports only DiskANN search multipliers and partition_concurrency as tuning knobs"; +const UNSUPPORTED_COLLECTION_KEY: &str = + "Cosmos DB collection_params supports only ru_per_second, indexing_search_list_size and quantization_byte_size"; +/// Sampled queries compared across routing modes before a fan-out search. +const PARITY_SAMPLES: usize = 8; +/// Feed ranges whose IDs are counted at once. +const ROW_COUNT_CONCURRENCY: usize = 16; +/// TOP K of the index-use probe queries. +const INDEX_PROBE_TOP: usize = 10; +const INDEX_PROBE_INTERVAL: Duration = Duration::from_secs(30); +/// Feed ranges probed at once. +const INDEX_PROBE_CONCURRENCY: usize = 16; +/// Sampled parity queries in flight at once. +const PARITY_CONCURRENCY: usize = 4; fn positive_knob(params: &SearchParams, key: &str) -> Result, String> { params .knob(key) - .map(|value| { - value - .as_u64() - .and_then(|n| u32::try_from(n).ok()) - .and_then(NonZeroU32::new) - .ok_or_else(|| format!("Cosmos DB {key} must be a positive 32-bit integer")) - }) + .map(|value| positive_u32(value, key)) .transpose() } @@ -81,6 +101,11 @@ pub struct CosmosDbEngine { container: Option, created_container: bool, verified_policy: Option, + /// `None` skips the DiskANN index-use probe. + index_probe_deadline: Option, + /// Per-range probe results once every range served from its index. + index_probes: Option>, + index_wait_seconds: Option, } impl CosmosDbEngine { @@ -91,52 +116,29 @@ impl CosmosDbEngine { if let Some(top) = params.top { validate_top(usize::try_from(top).map_err(|_| "Cosmos DB top must be positive")?)?; } - let unsupported = |key: &String| { - !matches!( - key.as_str(), - SEARCH_LIST_SIZE | QUANTIZED_VECTOR_LIST | PARTITION_CONCURRENCY - ) - }; if params.calibration_param.is_some() || params.num_candidates.is_some() || params - .extra + .search_params .as_ref() - .is_some_and(|p| p.keys().any(unsupported)) - || params.search_params.as_ref().is_some_and(|p| { - p.ef.is_some() - || p.extra - .as_ref() - .is_some_and(|extra| extra.keys().any(unsupported)) - }) + .is_some_and(|p| p.ef.is_some()) { - return Err( - "Cosmos DB supports only DiskANN search multipliers and partition_concurrency as tuning knobs".into(), - ); + return Err(UNSUPPORTED_SEARCH_KNOB.into()); + } + let nested = params.search_params.as_ref().and_then(|p| p.extra.as_ref()); + for extra in [params.extra.as_ref(), nested].into_iter().flatten() { + reject_unknown_keys(extra.keys(), &SEARCH_KNOBS, UNSUPPORTED_SEARCH_KNOB)?; } diskann_multiplier(params, &kind, SEARCH_LIST_SIZE)?; diskann_multiplier(params, &kind, QUANTIZED_VECTOR_LIST)?; PartitionConcurrency::from_config(params.knob(PARTITION_CONCURRENCY))?; } let collection = config.collection_params.as_ref(); - if collection.is_some_and(|p| { - p.hnsw_config.is_some() - || p.index_options.is_some() - || p.extra.as_ref().is_some_and(|extra| { - extra.keys().any(|key| { - !matches!( - key.as_str(), - "ru_per_second" - | "indexing_search_list_size" - | "quantization_byte_size" - ) - }) - }) - }) { - return Err( - "Cosmos DB collection_params supports only ru_per_second, indexing_search_list_size and quantization_byte_size" - .into(), - ); + if collection.is_some_and(|p| p.hnsw_config.is_some() || p.index_options.is_some()) { + return Err(UNSUPPORTED_COLLECTION_KEY.into()); + } + if let Some(extra) = collection.and_then(|p| p.extra.as_ref()) { + reject_unknown_keys(extra.keys(), &COLLECTION_KEYS, UNSUPPORTED_COLLECTION_KEY)?; } let collection_option = |key| { collection @@ -148,35 +150,10 @@ impl CosmosDbEngine { collection_option("indexing_search_list_size"), collection_option("quantization_byte_size"), )?; - let configured_ru = collection_option("ru_per_second") - .map(|value| { - value.as_u64().filter(|ru| *ru >= 400).ok_or( - "Cosmos DB collection_params.ru_per_second must be an integer of at least 400", - ) - }) - .transpose()?; - let upload_parallel = match config.upload_params.as_ref() { - None => 8, - Some(params) => { - let options = params - .as_object() - .ok_or("Cosmos DB upload_params must be an object")?; - if options.keys().any(|k| k != "parallel") { - return Err( - "Cosmos DB upload_params supports only parallel (one item per request)" - .into(), - ); - } - match options.get("parallel") { - None => 8, - Some(p) => p - .as_u64() - .and_then(|p| usize::try_from(p).ok()) - .filter(|p| *p > 0) - .ok_or("Cosmos DB upload parallel must be positive")?, - } - } - }; + let upload_parallel = upload_parallel(config.upload_params.as_ref())?; + let index_probe_deadline = index_probe_deadline( + crate::effective_config::env_opt("AZURE_COSMOS_INDEX_PROBE_SECONDS").as_deref(), + )?; let endpoint = crate::effective_config::env_opt("AZURE_COSMOS_ENDPOINT") .ok_or("Set AZURE_COSMOS_ENDPOINT for the Cosmos DB engine")? @@ -191,22 +168,13 @@ impl CosmosDbEngine { .into_iter() .map(Into::into) .collect(); - // The SDK consumes this environment setting when the runtime starts. - let _ = crate::effective_config::env_opt("AZURE_COSMOS_MAX_THROTTLE_RETRY_COUNT"); - let throughput = if let Some(ru) = configured_ru { - Some(ru) - } else { - match crate::effective_config::env_or("AZURE_COSMOS_RU_PER_SECOND", "400").as_str() { - "serverless" => None, - value => { - Some( - value.parse::().ok().filter(|ru| *ru >= 400).ok_or( - "AZURE_COSMOS_RU_PER_SECOND must be at least 400, or serverless", - )?, - ) - } - } - }; + // The SDK reads its own retry, transport and pool settings from the + // environment when the runtime starts. Record all of them. + crate::effective_config::observe_env_prefix("AZURE_COSMOS_"); + let throughput = manual_throughput( + collection_option("ru_per_second"), + &crate::effective_config::env_or("AZURE_COSMOS_RU_PER_SECOND", "400"), + )?; let rt = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?; let account = AccountReference::with_authentication_key(endpoint, key); let client = rt @@ -238,6 +206,9 @@ impl CosmosDbEngine { container: None, created_container: false, verified_policy: None, + index_probe_deadline, + index_probes: None, + index_wait_seconds: None, }) } @@ -272,6 +243,85 @@ impl CosmosDbEngine { self.build_options, ) } + + /// Wait until every feed range answers an index-requested query for + /// clearly less than a forced scan of that range costs. Azure scans while + /// a DiskANN index builds or a range holds too few vectors, so a matching + /// row count does not show that searches will use the index. + fn verify_index_use(&mut self, dataset: &Dataset) -> Result<(), String> { + if self.index_type != VectorIndexType::DiskANN || self.index_probes.is_some() { + return Ok(()); + } + let Some(deadline) = self.index_probe_deadline else { + eprintln!("Cosmos DB DiskANN index-use probe skipped (AZURE_COSMOS_INDEX_PROBE_SECONDS=0). Azure may answer from a scan."); + return Ok(()); + }; + let (vectors, _, conditions) = dataset.read_queries()?; + conditions.try_resolve_all("Cosmos DB", refuse_filter)?; + let vector = vectors + .first() + .ok_or("Cosmos DB needs a query vector for the index-use probe")?; + let scan = scan_query(vector, INDEX_PROBE_TOP)?; + let indexed = knn_query(vector, INDEX_PROBE_TOP, None, None, false)?; + let container = self.open_container()?.clone(); + let rt = &self.rt; + let ranges = rt.block_on(read_search_ranges(&container))?; + let started = Instant::now(); + // One forced scan per range is the costly part, so run the ranges + // concurrently. Only ranges still waiting are queried again. + let every_range: Vec = (0..ranges.len()).collect(); + let mut probes: Vec = rt + .block_on(probe_charges(&container, &scan, &ranges, &every_range))? + .into_iter() + .map(|(feed_range, scan_ru)| IndexProbe { + feed_range, + scan_ru, + index_ru: f64::NAN, + }) + .collect(); + probes.sort_by_key(|probe| probe.feed_range); + loop { + let pending: Vec = probes + .iter() + .filter(|probe| !probe.index_in_use()) + .map(|probe| probe.feed_range) + .collect(); + for (feed_range, index_ru) in + rt.block_on(probe_charges(&container, &indexed, &ranges, &pending))? + { + probes[feed_range].index_ru = index_ru; + } + let waiting: Vec<_> = probes.iter().filter(|p| !p.index_in_use()).collect(); + if waiting.is_empty() { + break; + } + let Some(remaining) = deadline.checked_sub(started.elapsed()) else { + return Err(format!( + "Cosmos DB DiskANN index is not serving {} of {} feed ranges after {}s (index-requested query costs as much as a forced scan: {waiting:?}). The index may still be building, or a range may hold too few vectors. Raise AZURE_COSMOS_INDEX_PROBE_SECONDS, or set it to 0 to run unverified.", + waiting.len(), + probes.len(), + deadline.as_secs() + )); + }; + println!( + "Waiting for the Cosmos DB DiskANN index on {} of {} feed ranges", + waiting.len(), + probes.len() + ); + std::thread::sleep(INDEX_PROBE_INTERVAL.min(remaining)); + } + if rt.block_on(read_search_ranges(&container))? != ranges { + return Err("Cosmos DB feed ranges changed during the index-use probe".into()); + } + let waited = started.elapsed().as_secs_f64(); + println!( + "Cosmos DB DiskANN index use verified on {} feed ranges after {waited:.1}s", + probes.len() + ); + self.index_wait_seconds = Some(waited); + self.index_probes = Some(probes); + Ok(()) + } } impl Engine for CosmosDbEngine { @@ -328,7 +378,6 @@ impl Engine for CosmosDbEngine { } fn upload(&mut self, dataset: &Dataset) -> Result { - let total_start = Instant::now(); let (ids, vectors, _) = dataset.read_vectors(dataset.needs_normalization())?; validate_corpus_size(&self.index_type, ids.len() as u64)?; let dimensions = dataset.vector_size() as usize; @@ -363,20 +412,24 @@ impl Engine for CosmosDbEngine { .await })?; let upload_time = upload_start.elapsed().as_secs_f64(); + let upload_count = ids.len(); + drop((ids, vectors)); + // The row-count scan is verification, so it stays out of both times. let count = self .corpus_row_count()? .ok_or("Cosmos DB did not return a row count")?; - if count.rows != ids.len() as u64 { + if count.rows != upload_count as u64 { return Err(format!( - "Cosmos DB stored {} rows of {} uploaded", - count.rows, - ids.len() + "Cosmos DB stored {} rows of {upload_count} uploaded", + count.rows )); } + // For DiskANN, total time adds the wait until the index serves queries. + self.verify_index_use(dataset)?; Ok(UploadStats { upload_time, - total_time: total_start.elapsed().as_secs_f64(), - upload_count: ids.len(), + total_time: upload_time + self.index_wait_seconds.unwrap_or(0.0), + upload_count, parallel: self.upload_parallel, batch_size: 1, ..UploadStats::default() @@ -385,28 +438,24 @@ impl Engine for CosmosDbEngine { fn corpus_row_count(&mut self) -> Result, String> { let container = self.open_container()?.clone(); - // SDK 1.0 does not implement cross-partition aggregate queries on Azure. - // Count the ID projection page by page so reuse has an exact count with - // bounded memory, without reading the stored vectors. - let count = self - .rt - .block_on(async { - let options = - QueryOptions::default().with_max_item_count(MaxItemCountHint::ServerDecides); - container - .query_items::( - "SELECT VALUE c.id FROM c", - FeedScope::full_container(), - Some(options), - ) - .await? - .into_pages() - .try_fold(0_u64, |count, page| async move { - Ok(count + page.items().len() as u64) - }) - .await - }) - .map_err(|e| format!("Cosmos DB row count failed: {e}"))?; + // SDK 1.0 rejects aggregates in every feed-range and full-container + // query, so count the ID projection page by page. That keeps memory + // bounded and never reads the stored vectors. The SDK drains a + // full-container query one range at a time, so count the disjoint + // feed ranges concurrently instead. + let count = self.rt.block_on(async { + let ranges = read_search_ranges(&container).await?; + let count = stream::iter(&ranges) + .map(|range| count_ids(&container, FeedScope::range(range.clone()))) + .buffer_unordered(ROW_COUNT_CONCURRENCY) + .try_fold(0_u64, |total, rows| async move { Ok(total + rows) }) + .await?; + // A split during the count could count a row twice or not at all. + if read_search_ranges(&container).await? != ranges { + return Err("Cosmos DB feed ranges changed during the row count".to_string()); + } + Ok(count) + })?; validate_corpus_size(&self.index_type, count)?; Ok(Some(CorpusCount::exact(count))) } @@ -437,10 +486,7 @@ impl Engine for CosmosDbEngine { } } self.verified_policy = Some(actual); - if self.index_type == VectorIndexType::DiskANN { - eprintln!("Cosmos DB DiskANN policy verified. Index use is unverified; Azure may fall back to a scan."); - } - Ok(()) + self.verify_index_use(dataset) } fn search( @@ -470,21 +516,20 @@ impl Engine for CosmosDbEngine { let partition_concurrency = PartitionConcurrency::from_config(params.knob(PARTITION_CONCURRENCY))?; - let queries = vectors[..limit] - .iter() - .map(|vector| { - if vector.len() != dataset.vector_size() as usize { - return Err("Cosmos DB query vector has the wrong number of dimensions".into()); - } - knn_query( - vector, - top, - search_list_size, - quantized_vector_list, - partition_concurrency.is_some(), - ) - }) - .collect::, String>>()?; + // Validate every vector before timing. Workers bind each one to the + // shared query text as they send it. + let dimensions = dataset.vector_size() as usize; + for vector in &vectors[..limit] { + validate_query_vector(vector, dimensions)?; + } + let stock_text = KnnQueryText::new(top, search_list_size, quantized_vector_list, false)?; + let text = KnnQueryText::new( + top, + search_list_size, + quantized_vector_list, + partition_concurrency.is_some(), + )?; + let text = &text; let container = self .container .as_ref() @@ -498,7 +543,7 @@ impl Engine for CosmosDbEngine { feed_ranges: ranges.len(), per_query_concurrency, shared_request_limit, - preflight_retries: 0, + parity: ParityTally::default(), }; let mut search = PartitionSearch { ranges, @@ -506,17 +551,29 @@ impl Engine for CosmosDbEngine { metric: vector_distance_function(dataset.distance())?, permits: Semaphore::new(shared_request_limit), }; - for index in (0..limit).step_by((limit / 3).max(1)).take(3) { - let stock_query = knn_query(&vectors[index], top, search_list_size, quantized_vector_list, false)?; - let retries = rt - .block_on(check_partition_parity(container, &stock_query, &queries[index], top, &search)) - .map_err(|e| format!("Cosmos DB parity check for query {index}: {e}"))?; - search.summary.preflight_retries += retries; + let samples = (0..limit).step_by((limit / PARITY_SAMPLES).max(1)).take(PARITY_SAMPLES); + let outcomes = rt.block_on( + stream::iter(samples) + .map(|index| { + let (search, stock_text, vector) = (&search, &stock_text, &vectors[index]); + async move { + parity_sample(container, stock_text, text, vector, top, search) + .await + .map_err(|e| format!("Cosmos DB parity check for query {index}: {e}")) + } + }) + .buffer_unordered(PARITY_CONCURRENCY) + .try_collect::>(), + )?; + for (stock, stock_repeat, partitioned) in &outcomes { + search.summary.parity.observe(stock, stock_repeat, partitioned)?; } + let parity = search.summary.parity; + parity.verdict()?; println!( - "Cosmos DB feed-range search verified against SDK on sampled queries: {} ranges, {} per query, {} in flight, {} parity retries", - summary.feed_ranges, summary.per_query_concurrency, summary.shared_request_limit, - search.summary.preflight_retries + "Cosmos DB feed-range search verified against SDK on {} sampled queries: {} ranges, {} per query, {} in flight; fan-out missed {} stock IDs, repeated stock requests missed {}", + parity.samples, summary.feed_ranges, summary.per_query_concurrency, summary.shared_request_limit, + parity.cross_mode_missing_ids, parity.stock_repeat_missing_ids ); Ok::<_, String>(search) }).transpose()?; @@ -525,7 +582,7 @@ impl Engine for CosmosDbEngine { let (samples, elapsed) = std::thread::scope(|scope| { let mut pool = WorkerPool::new(scope, "cosmosdb-search", parallel); for worker in 0..parallel { - let queries = &queries; + let vectors = &vectors[..limit]; let neighbors = &neighbors; let next = &next; pool.spawn(move |ticket| { @@ -533,7 +590,8 @@ impl Engine for CosmosDbEngine { // Warm the worker's first request outside the measured window. if let Err(error) = rt.block_on(search_hits( container, - &queries[worker % limit], + text, + &vectors[worker % limit], top, partition_search, )) { @@ -545,10 +603,12 @@ impl Engine for CosmosDbEngine { } loop { let idx = next.fetch_add(1, Ordering::Relaxed); - let Some(query) = queries.get(idx) else { break }; + let Some(vector) = vectors.get(idx) else { + break; + }; let started = Instant::now(); let (hits, diagnostics) = rt - .block_on(search_hits(container, query, top, partition_search)) + .block_on(search_hits(container, text, vector, top, partition_search)) .map_err(|e| format!("Cosmos DB query {idx} failed: {e}"))?; let latency = started.elapsed().as_secs_f64(); let ids = hits @@ -613,7 +673,9 @@ impl Engine for CosmosDbEngine { "vector_embedding_policy": policy.vector_embedding_policy, "vector_indexing_policy": policy.indexing_policy, "manual_ru_per_second": self.throughput, - "vector_index_use_verified": false, + "vector_index_use_verified": self.index_probes.is_some(), + "vector_index_use_probes": self.index_probes, + "vector_index_wait_seconds": self.index_wait_seconds, } })) } @@ -641,28 +703,57 @@ struct PartitionSearch { permits: Semaphore, } -async fn check_partition_parity( +/// One sampled query under both routing modes. The stock query runs twice +/// because repeated DiskANN requests can differ at the cutoff, and that +/// variation is the bar the fan-out result is held to. +async fn parity_sample( container: &ContainerClient, - stock_query: &Query, - partition_query: &Query, + stock_text: &KnnQueryText, + partition_text: &KnnQueryText, + vector: &[f32], top: usize, search: &PartitionSearch, -) -> Result { - // DiskANN can return different neighbors at the cutoff for repeated - // requests, including two stock requests. Retry the sampled pair while - // still requiring exact ID agreement before any timing. - const MAX_RETRIES: usize = 3; - let mut retries = 0; - loop { - let (stock, _) = - query_hits(container, stock_query, top, FeedScope::full_container()).await?; - let (partitioned, _) = search_hits(container, partition_query, top, Some(search)).await?; - let verdict = verify_partition_search(&stock, &partitioned); - if verdict.is_ok() || retries == MAX_RETRIES { - return verdict.map(|()| retries); - } - retries += 1; - } +) -> Result<(Vec, Vec, Vec), String> { + let ((stock, _), (stock_repeat, _), (partitioned, _)) = futures::try_join!( + search_hits(container, stock_text, vector, top, None), + search_hits(container, stock_text, vector, top, None), + search_hits(container, partition_text, vector, top, Some(search)), + )?; + Ok((stock, stock_repeat, partitioned)) +} + +/// Request charge of `query` on each listed feed range, as (range index, RU). +async fn probe_charges( + container: &ContainerClient, + query: &Query, + ranges: &[FeedRange], + indices: &[usize], +) -> Result, String> { + stream::iter(indices.iter().copied()) + .map(|feed_range| async move { + let scope = FeedScope::range(ranges[feed_range].clone()); + query_hits(container, query.clone(), INDEX_PROBE_TOP, scope) + .await + .map(|(_, diagnostics)| (feed_range, diagnostics.request_charge_ru)) + .map_err(|e| format!("Cosmos DB index-use probe failed: {e}")) + }) + .buffer_unordered(INDEX_PROBE_CONCURRENCY) + .try_collect() + .await +} + +async fn count_ids(container: &ContainerClient, scope: FeedScope) -> Result { + let options = QueryOptions::default().with_max_item_count(MaxItemCountHint::ServerDecides); + container + .query_items::("SELECT VALUE c.id FROM c", scope, Some(options)) + .await + .map_err(|e| format!("Cosmos DB row count failed: {e}"))? + .into_pages() + .try_fold(0_u64, |count, page| async move { + Ok(count + page.items().len() as u64) + }) + .await + .map_err(|e| format!("Cosmos DB row count failed: {e}")) } async fn read_search_ranges(container: &ContainerClient) -> Result, String> { @@ -680,17 +771,30 @@ async fn read_search_ranges(container: &ContainerClient) -> Result, ) -> Result<(Vec, SearchDiagnostics), String> { let Some(search) = partition_search else { - return query_hits(container, query, top, FeedScope::full_container()).await; + return query_hits( + container, + text.bind(vector)?, + top, + FeedScope::full_container(), + ) + .await; }; let (hits, diagnostics) = stream::iter(&search.ranges) .map(|range| async { let _permit = search.permits.acquire().await.map_err(|e| e.to_string())?; - query_hits(container, query, top, FeedScope::range(range.clone())).await + query_hits( + container, + text.bind(vector)?, + top, + FeedScope::range(range.clone()), + ) + .await }) .buffer_unordered(search.summary.per_query_concurrency) .try_fold( @@ -710,12 +814,12 @@ async fn search_hits( async fn query_hits( container: &ContainerClient, - query: &Query, + query: Query, top: usize, scope: FeedScope, ) -> Result<(Vec, SearchDiagnostics), String> { let mut pages = container - .query_items::(query.clone(), scope, None) + .query_items::(query, scope, None) .await .map_err(|error| error.to_string())? .into_pages(); diff --git a/src/bin/vector_db_benchmark/engine/filter_guard.rs b/src/bin/vector_db_benchmark/engine/filter_guard.rs index d72df9b8c..1ff5bd0f3 100644 --- a/src/bin/vector_db_benchmark/engine/filter_guard.rs +++ b/src/bin/vector_db_benchmark/engine/filter_guard.rs @@ -1065,7 +1065,8 @@ fn every_engine_read_of_query_conditions_is_paired_with_a_resolver() { // and OpenSearch hybrid-rrf arms' `read_queries`, each resolved with // `hybrid_rrf::refuse_conditions` (a declared condition is an error). // +2: Cosmos DB refuses declared filters before provisioning and search. - (30, 30), + // +1: the Cosmos DB index-use probe reads a query vector, same refusal. + (31, 31), "the read/resolve census changed; confirm the new site is guarded" ); } diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index 46cdb3c8b..87d651dec 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -11,6 +11,7 @@ use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::HashSet; use std::num::NonZeroU32; +use std::time::Duration; const VECTOR_PATH: &str = "/embedding"; const VECTOR_EXCLUSION: &str = "/embedding/*"; @@ -78,6 +79,75 @@ impl DiskAnnBuildOptions { } } +/// Reject a config key the adapter does not read, so a typo cannot run as a +/// default and be reported under the declared name. +pub fn reject_unknown_keys<'a>( + keys: impl IntoIterator, + allowed: &[&str], + message: &str, +) -> Result<(), String> { + if keys.into_iter().any(|key| !allowed.contains(&key.as_str())) { + return Err(message.into()); + } + Ok(()) +} + +/// A tuning knob that must be a positive 32-bit integer. +pub fn positive_u32(value: &Value, key: &str) -> Result { + value + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .and_then(NonZeroU32::new) + .ok_or_else(|| format!("Cosmos DB {key} must be a positive 32-bit integer")) +} + +/// Manual RU/s for the container. A per-config value wins over the +/// environment setting. `None` means a serverless account. +pub fn manual_throughput(configured: Option<&Value>, env: &str) -> Result, String> { + if let Some(value) = configured { + return value + .as_u64() + .filter(|ru| *ru >= 400) + .map(Some) + .ok_or_else(|| { + "Cosmos DB collection_params.ru_per_second must be an integer of at least 400" + .into() + }); + } + if env == "serverless" { + return Ok(None); + } + env.parse::() + .ok() + .filter(|ru| *ru >= 400) + .map(Some) + .ok_or_else(|| "AZURE_COSMOS_RU_PER_SECOND must be at least 400, or serverless".into()) +} + +/// Concurrent single-item upserts. The adapter sends one item per request. +pub fn upload_parallel(params: Option<&Value>) -> Result { + const DEFAULT: usize = 8; + let Some(params) = params else { + return Ok(DEFAULT); + }; + let options = params + .as_object() + .ok_or("Cosmos DB upload_params must be an object")?; + reject_unknown_keys( + options.keys(), + &["parallel"], + "Cosmos DB upload_params supports only parallel (one item per request)", + )?; + match options.get("parallel") { + None => Ok(DEFAULT), + Some(parallel) => parallel + .as_u64() + .and_then(|p| usize::try_from(p).ok()) + .filter(|p| *p > 0) + .ok_or_else(|| "Cosmos DB upload parallel must be positive".into()), + } +} + /// Cosmos DB metric corresponding to a benchmark corpus's distance. pub fn vector_distance_function(distance: &str) -> Result { match distance.to_ascii_lowercase().as_str() { @@ -226,7 +296,7 @@ pub struct PartitionFanout { pub feed_ranges: usize, pub per_query_concurrency: usize, pub shared_request_limit: usize, - pub preflight_retries: usize, + pub parity: ParityTally, } #[derive(Debug, PartialEq, Eq)] @@ -307,8 +377,59 @@ pub fn validate_corpus_size(kind: &VectorIndexType, count: u64) -> Result<(), St Ok(()) } -/// Build a server-side TOP K query. The service uses the policy's distance -/// function. TOP is a validated integer literal and the vector is bound. +/// The text of a server-side TOP K query, built once per search. The service +/// uses the policy's distance function and TOP is a validated integer literal. +/// Only the bound vector differs between requests, so a search keeps one text +/// and binds each vector when it sends the request. Pre-building every query +/// would hold each vector a second time as JSON values. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KnnQueryText(String); + +impl KnnQueryText { + pub fn new( + top: usize, + search_list_size_multiplier: Option, + quantized_vector_list_multiplier: Option, + project_distance: bool, + ) -> Result { + validate_top(top)?; + let options = match (search_list_size_multiplier, quantized_vector_list_multiplier) { + (None, None) => String::new(), + (Some(search), None) => format!(", {{\"searchListSizeMultiplier\": {search}}}"), + (None, Some(quantized)) => { + format!(", {{\"quantizedVectorListMultiplier\": {quantized}}}") + } + (Some(search), Some(quantized)) => format!( + ", {{\"searchListSizeMultiplier\": {search}, \"quantizedVectorListMultiplier\": {quantized}}}" + ), + }; + Ok(Self(query_text( + top, + &format!("false{options}"), + project_distance, + ))) + } + + /// Bind one query vector. The caller checks it with + /// [`validate_query_vector`] before timing, so this does no validation. + pub fn bind(&self, vector: &[f32]) -> Result { + bind_vector(self.0.clone(), vector) + } +} + +/// A query vector must match the container's dimensions and be finite. JSON +/// has no encoding for NaN or infinity, so one would be sent as `null`. +pub fn validate_query_vector(vector: &[f32], dimensions: usize) -> Result<(), String> { + if vector.len() != dimensions { + return Err("Cosmos DB query vector has the wrong number of dimensions".into()); + } + if vector.iter().any(|v| !v.is_finite()) { + return Err("Cosmos DB query vector contains a non-finite value".into()); + } + Ok(()) +} + +/// Build and bind one validated TOP K query. pub fn knn_query( vector: &[f32], top: usize, @@ -316,29 +437,75 @@ pub fn knn_query( quantized_vector_list_multiplier: Option, project_distance: bool, ) -> Result { + validate_query_vector(vector, vector.len())?; + KnnQueryText::new( + top, + search_list_size_multiplier, + quantized_vector_list_multiplier, + project_distance, + )? + .bind(vector) +} + +/// The same TOP K query with the service's brute-force flag set, so it scans +/// every vector in scope. Its request charge is the baseline that the +/// index-use probe compares an index-requested query against. +pub fn scan_query(vector: &[f32], top: usize) -> Result { validate_top(top)?; - if vector.iter().any(|v| !v.is_finite()) { - return Err("Cosmos DB query vector contains a non-finite value".into()); - } - let options = match (search_list_size_multiplier, quantized_vector_list_multiplier) { - (None, None) => String::new(), - (Some(search), None) => format!(", {{\"searchListSizeMultiplier\": {search}}}"), - (None, Some(quantized)) => format!(", {{\"quantizedVectorListMultiplier\": {quantized}}}"), - (Some(search), Some(quantized)) => format!( - ", {{\"searchListSizeMultiplier\": {search}, \"quantizedVectorListMultiplier\": {quantized}}}" - ), - }; - let distance = format!("VectorDistance(c.embedding, @vector, false{options})"); + validate_query_vector(vector, vector.len())?; + bind_vector(query_text(top, "true", false), vector) +} + +fn query_text(top: usize, distance_arguments: &str, project_distance: bool) -> String { + let distance = format!("VectorDistance(c.embedding, @vector, {distance_arguments})"); let projection = if project_distance { format!("c.id, {distance} AS distance") } else { "c.id".into() }; - Query::from(format!( - "SELECT TOP {top} {projection} FROM c ORDER BY {distance}" - )) - .with_parameter("@vector", vector) - .map_err(|e| format!("Could not bind Cosmos DB query vector: {e}")) + format!("SELECT TOP {top} {projection} FROM c ORDER BY {distance}") +} + +fn bind_vector(text: String, vector: &[f32]) -> Result { + Query::from(text) + .with_parameter("@vector", vector) + .map_err(|e| format!("Could not bind Cosmos DB query vector: {e}")) +} + +/// An index-requested query must cost less than this share of a forced scan +/// of the same feed range before the range counts as served by DiskANN. +pub const INDEX_PROBE_MAX_RU_RATIO: f64 = 0.5; + +/// Request charges of one feed range's forced scan and index-requested query. +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub struct IndexProbe { + pub feed_range: usize, + pub scan_ru: f64, + pub index_ru: f64, +} + +impl IndexProbe { + /// Azure answers an index-requested query with a scan while the index is + /// building or the range holds too few vectors. A scan costs what the + /// forced scan costs, so only a clearly cheaper query shows index use. + pub fn index_in_use(&self) -> bool { + self.scan_ru > 0.0 + && self.index_ru > 0.0 + && self.index_ru < self.scan_ru * INDEX_PROBE_MAX_RU_RATIO + } +} + +/// How long to wait for every feed range to serve from its DiskANN index. +/// Unset uses 30 minutes. `0` skips the probe and leaves index use unverified. +pub fn index_probe_deadline(value: Option<&str>) -> Result, String> { + const DEFAULT_SECONDS: u64 = 1800; + let seconds = match value { + None => DEFAULT_SECONDS, + Some(value) => value.parse::().map_err(|_| { + "AZURE_COSMOS_INDEX_PROBE_SECONDS must be a whole number of seconds, or 0 to skip the probe" + })?, + }; + Ok((seconds > 0).then(|| Duration::from_secs(seconds))) } #[derive(Deserialize)] @@ -392,19 +559,61 @@ pub fn merge_partition_hits( Ok(candidates.into_iter().map(|(hit, _)| hit).collect()) } -/// Verify that the same query covers the same IDs under both routing modes. -pub fn verify_partition_search(stock: &[QueryHit], partitioned: &[QueryHit]) -> Result<(), String> { - let stock_ids: HashSet<_> = stock.iter().map(|hit| &hit.id).collect(); - let partitioned_ids: HashSet<_> = partitioned.iter().map(|hit| &hit.id).collect(); - if stock.len() != partitioned.len() || stock_ids != partitioned_ids { - return Err(format!( - "Cosmos DB partition search disagrees with the SDK's global top K ({} vs {} hits, {} shared IDs)", - stock.len(), - partitioned.len(), - stock_ids.intersection(&partitioned_ids).count(), - )); +/// Fan-out may miss this many times the IDs that a repeated stock request +/// misses. Two stock comparisons are counted per sample, so 2 is the expected +/// ratio when both routes behave alike. 3 leaves headroom for sampling noise. +pub const PARITY_CROSS_MODE_ALLOWANCE: usize = 3; + +/// Agreement between the stock SDK query and the feed-range fan-out on +/// sampled queries, measured against the stock query's own repeatability. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct ParityTally { + pub samples: usize, + /// IDs of one stock response that a repeated stock request did not return. + pub stock_repeat_missing_ids: usize, + /// IDs of the two stock responses that the fan-out response did not return. + pub cross_mode_missing_ids: usize, +} + +impl ParityTally { + /// Record one sampled query: two stock responses and one fan-out response. + pub fn observe( + &mut self, + stock: &[QueryHit], + stock_repeat: &[QueryHit], + partitioned: &[QueryHit], + ) -> Result<(), String> { + if stock.len() != stock_repeat.len() || stock.len() != partitioned.len() { + return Err(format!( + "Cosmos DB routing modes returned different hit counts ({} and {} stock, {} fan-out)", + stock.len(), + stock_repeat.len(), + partitioned.len() + )); + } + let ids = |hits: &[QueryHit]| -> HashSet { + hits.iter().map(|hit| hit.id.clone()).collect() + }; + let (stock, stock_repeat, partitioned) = (ids(stock), ids(stock_repeat), ids(partitioned)); + self.samples += 1; + self.stock_repeat_missing_ids += stock.difference(&stock_repeat).count(); + self.cross_mode_missing_ids += + stock.difference(&partitioned).count() + stock_repeat.difference(&partitioned).count(); + Ok(()) + } + + /// Fan-out must agree with the stock query about as well as the stock + /// query agrees with itself. An exact engine therefore needs exact parity. + pub fn verdict(&self) -> Result<(), String> { + let allowed = self.stock_repeat_missing_ids * PARITY_CROSS_MODE_ALLOWANCE; + if self.samples == 0 || self.cross_mode_missing_ids > allowed { + return Err(format!( + "Cosmos DB partition search disagrees with the SDK's global top K: fan-out missed {} stock IDs over {} sampled queries, and repeated stock requests missed {} (allowance {allowed})", + self.cross_mode_missing_ids, self.samples, self.stock_repeat_missing_ids + )); + } + Ok(()) } - Ok(()) } #[cfg(test)] @@ -568,6 +777,27 @@ mod tests { ); } + #[test] + fn a_bound_query_text_is_the_query_knn_query_builds() { + let multiplier = 5.try_into().ok(); + for project_distance in [false, true] { + let text = KnnQueryText::new(10, multiplier, None, project_distance).unwrap(); + for vector in [[0.5_f32, -0.25], [1.0, 2.0]] { + assert_eq!( + to_value(text.bind(&vector).unwrap()).unwrap(), + to_value(knn_query(&vector, 10, multiplier, None, project_distance).unwrap()) + .unwrap() + ); + } + } + assert!(KnnQueryText::new(0, None, None, false).is_err()); + assert!(KnnQueryText::new(1001, None, None, false).is_err()); + assert!(validate_query_vector(&[0.5, 0.25], 2).is_ok()); + assert!(validate_query_vector(&[0.5], 2).is_err()); + assert!(validate_query_vector(&[0.5, f32::INFINITY], 2).is_err()); + assert!(knn_query(&[f32::NAN], 10, None, None, false).is_err()); + } + #[test] fn partition_concurrency_uses_the_realized_feed_range_count() { assert_eq!(PartitionConcurrency::from_config(None).unwrap(), None); @@ -610,15 +840,6 @@ mod tests { merged.iter().map(|hit| hit.id.as_str()).collect::>(), expected ); - assert!(verify_partition_search( - &[hit(expected[1], None), hit(expected[0], None)], - &merged - ) - .is_ok()); - assert!( - verify_partition_search(&[hit("2", None), hit(expected[0], None)], &merged) - .is_err() - ); } let metric = VectorDistanceFunction::Cosine; assert!(merge_partition_hits(vec![hit("0", None)], 1, &metric).is_err()); @@ -628,6 +849,158 @@ mod tests { ); } + fn hits(ids: &[&str]) -> Vec { + ids.iter() + .map(|id| QueryHit { + id: (*id).into(), + distance: None, + }) + .collect() + } + + #[test] + fn parity_requires_exact_agreement_when_stock_repeats_exactly() { + let mut exact = ParityTally::default(); + exact + .observe(&hits(&["1", "2"]), &hits(&["2", "1"]), &hits(&["1", "2"])) + .unwrap(); + assert_eq!( + exact, + ParityTally { + samples: 1, + stock_repeat_missing_ids: 0, + cross_mode_missing_ids: 0 + } + ); + assert!(exact.verdict().is_ok()); + + let mut wrong = ParityTally::default(); + wrong + .observe(&hits(&["1", "2"]), &hits(&["1", "2"]), &hits(&["1", "3"])) + .unwrap(); + assert_eq!(wrong.cross_mode_missing_ids, 2); + assert!(wrong.verdict().is_err()); + assert!(ParityTally::default().verdict().is_err()); + } + + #[test] + fn parity_allows_fanout_the_variation_stock_shows_against_itself() { + // Stock misses one ID against itself. Fan-out may miss up to three. + let mut within = ParityTally::default(); + within + .observe( + &hits(&["1", "2", "3"]), + &hits(&["1", "2", "4"]), + &hits(&["1", "2", "5"]), + ) + .unwrap(); + assert_eq!(within.stock_repeat_missing_ids, 1); + assert_eq!(within.cross_mode_missing_ids, 2); + assert!(within.verdict().is_ok()); + + let mut beyond = ParityTally::default(); + beyond + .observe( + &hits(&["1", "2", "3"]), + &hits(&["1", "2", "4"]), + &hits(&["1", "8", "9"]), + ) + .unwrap(); + assert_eq!(beyond.cross_mode_missing_ids, 4); + assert!(beyond.verdict().is_err()); + + // The tally is cumulative: a later exact sample does not reset it. + beyond + .observe(&hits(&["1"]), &hits(&["1"]), &hits(&["1"])) + .unwrap(); + assert_eq!(beyond.samples, 2); + assert!(beyond.verdict().is_err()); + assert!(ParityTally::default() + .observe(&hits(&["1", "2"]), &hits(&["1", "2"]), &hits(&["1"])) + .is_err()); + } + + #[test] + fn index_probe_needs_a_query_clearly_cheaper_than_a_forced_scan() { + let probe = |scan_ru, index_ru| IndexProbe { + feed_range: 0, + scan_ru, + index_ru, + }; + assert!(probe(1000.0, 499.0).index_in_use()); + assert!(!probe(1000.0, 500.0).index_in_use()); + assert!(!probe(1000.0, 1000.0).index_in_use()); + assert!(!probe(0.0, 0.0).index_in_use()); + assert!(!probe(1000.0, 0.0).index_in_use()); + let scan = to_value(scan_query(&[0.5], 10).unwrap()).unwrap(); + assert_eq!( + scan["query"], + "SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, true)" + ); + assert!(scan_query(&[f32::NAN], 10).is_err()); + assert!(scan_query(&[0.5], 0).is_err()); + } + + #[test] + fn index_probe_deadline_defaults_and_can_be_skipped() { + assert_eq!( + index_probe_deadline(None).unwrap(), + Some(Duration::from_secs(1800)) + ); + assert_eq!( + index_probe_deadline(Some("90")).unwrap(), + Some(Duration::from_secs(90)) + ); + assert_eq!(index_probe_deadline(Some("0")).unwrap(), None); + for invalid in ["", "-1", "1.5", "soon"] { + assert!(index_probe_deadline(Some(invalid)).is_err(), "{invalid}"); + } + } + + #[test] + fn config_values_are_validated_before_any_request() { + let keys = [String::from("parallel"), String::from("batch_size")]; + assert!(reject_unknown_keys(&keys[..1], &["parallel"], "no").is_ok()); + assert_eq!( + reject_unknown_keys(&keys, &["parallel"], "no"), + Err("no".into()) + ); + + assert_eq!(positive_u32(&json!(5), "k").unwrap().get(), 5); + for invalid in [ + json!(0), + json!(-1), + json!(1.5), + json!("5"), + json!(1u64 << 32), + ] { + assert!(positive_u32(&invalid, "k").is_err(), "{invalid}"); + } + + assert_eq!(manual_throughput(None, "400").unwrap(), Some(400)); + assert_eq!(manual_throughput(None, "serverless").unwrap(), None); + assert_eq!( + manual_throughput(Some(&json!(1000)), "serverless").unwrap(), + Some(1000) + ); + assert!(manual_throughput(None, "399").is_err()); + assert!(manual_throughput(None, "auto").is_err()); + assert!(manual_throughput(Some(&json!(399)), "400").is_err()); + assert!(manual_throughput(Some(&json!("1000")), "400").is_err()); + + assert_eq!(upload_parallel(None).unwrap(), 8); + assert_eq!(upload_parallel(Some(&json!({}))).unwrap(), 8); + assert_eq!(upload_parallel(Some(&json!({"parallel": 16}))).unwrap(), 16); + for invalid in [ + json!({"parallel": 0}), + json!({"parallel": "4"}), + json!({"batch_size": 10}), + json!([4]), + ] { + assert!(upload_parallel(Some(&invalid)).is_err(), "{invalid}"); + } + } + #[test] fn diskann_rejects_the_documented_full_scan_case() { assert!(validate_corpus_size(&VectorIndexType::DiskANN, 999).is_err()); diff --git a/tests/integration_cosmosdb.rs b/tests/integration_cosmosdb.rs index f0bc8f2f3..c9cf16c3a 100644 --- a/tests/integration_cosmosdb.rs +++ b/tests/integration_cosmosdb.rs @@ -7,15 +7,13 @@ use std::fs; use serde_json::json; -#[test] -fn flat_search_matches_the_fixture_and_cleans_up_the_container() { - if std::env::var_os("AZURE_COSMOS_ENDPOINT").is_none() { - eprintln!("SKIP integration_cosmosdb: set AZURE_COSMOS_ENDPOINT and AZURE_COSMOS_KEY"); - return; - } - - let temp = tempfile::tempdir().unwrap(); - let root = temp.path(); +/// Write the random-100 fixture and one Flat engine config under `root`. +/// The engine name is unique to the tempdir, so the test owns its container. +fn write_fixture( + root: &std::path::Path, + collection_params: serde_json::Value, + search_params: serde_json::Value, +) -> String { let engine_name = format!( "cosmosdb-it-{}", root.file_name().unwrap().to_string_lossy() @@ -41,14 +39,28 @@ fn flat_search_matches_the_fixture_and_cleans_up_the_container() { "name": engine_name, "engine": "cosmosdb", "algorithm": "flat", + "collection_params": collection_params, "upload_params": { "parallel": 4 }, - "search_params": [{ "parallel": 2, "top": 1 }], + "search_params": [search_params], }]); fs::write( root.join("experiments/configurations/cosmosdb.json"), config.to_string(), ) .unwrap(); + engine_name +} + +#[test] +fn flat_search_matches_the_fixture_and_cleans_up_the_container() { + if std::env::var_os("AZURE_COSMOS_ENDPOINT").is_none() { + eprintln!("SKIP integration_cosmosdb: set AZURE_COSMOS_ENDPOINT and AZURE_COSMOS_KEY"); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let engine_name = write_fixture(root, json!({}), json!({ "parallel": 2, "top": 1 })); assert!(common::run_binary( root, @@ -89,3 +101,53 @@ fn flat_search_matches_the_fixture_and_cleans_up_the_container() { &[] )); } + +/// Feed-range fan-out on a container with several physical partitions. It +/// provisions 12,000 manual RU/s for the duration of the run, so it needs its +/// own opt-in on top of the account credentials. +#[test] +fn flat_fanout_matches_the_stock_query_across_feed_ranges() { + if std::env::var_os("AZURE_COSMOS_ENDPOINT").is_none() + || std::env::var_os("AZURE_COSMOS_TEST_FANOUT").is_none() + { + eprintln!( + "SKIP integration_cosmosdb fan-out: set AZURE_COSMOS_ENDPOINT, AZURE_COSMOS_KEY and AZURE_COSMOS_TEST_FANOUT=1" + ); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let engine_name = write_fixture( + root, + json!({ "ru_per_second": 12000 }), + json!({ + "parallel": 2, + "top": 1, + "search_params": { "partition_concurrency": "auto" }, + }), + ); + + assert!(common::run_binary( + root, + &engine_name, + "random-100", + "localhost", + &[] + )); + let result = common::read_result_doc(root, &engine_name); + assert_eq!(result["results"]["mean_recall"].as_f64().unwrap(), 1.0); + assert_eq!(result["results"]["failed_queries"], 0); + let fanout = &result["results"]["cosmosdb_search_diagnostics"]["partition_fanout"]; + let feed_ranges = fanout["feed_ranges"].as_u64().unwrap(); + println!("Cosmos DB Flat fan-out over {feed_ranges} feed ranges: {fanout}"); + assert!( + feed_ranges >= 2, + "the container has one feed range, so fan-out was not exercised" + ); + assert_eq!(fanout["per_query_concurrency"], feed_ranges); + // Flat is exact, so both routes must return the same IDs. + assert!(fanout["parity"]["samples"].as_u64().unwrap() > 0); + assert_eq!(fanout["parity"]["stock_repeat_missing_ids"], 0); + assert_eq!(fanout["parity"]["cross_mode_missing_ids"], 0); +} From 4316e91b1d79061a4622f97fd19228ff3907ed1d Mon Sep 17 00:00:00 2001 From: fcenedes Date: Thu, 19 Nov 2026 17:39:44 +0100 Subject: [PATCH 10/10] Resend throttled Cosmos DB uploads and clean up failed ones When the SDK exhausts its HTTP 429 retries on an upload write, back off and send the write again, up to 60 times per row, and report how many writes were resent. The shipped presets at the default throughput no longer fail on the first throttled row. Remove the container this run created when the upload fails. A complete corpus is kept if the index-use probe then fails, so --skip-upload can reuse it. Print the index and forced-scan charge of each waiting feed range on every probe poll and in the deadline error. Poll every 2 seconds at first and less often up to every 30, so the recorded index wait is accurate to about a tenth of its length. Move the probe loop, the per-range row count, the fan-out merge and parity sampling into src/cosmosdb.rs behind closures, with unit tests. Co-Authored-By: Claude Fable 5.2 --- README.md | 15 +- .../vector_db_benchmark/engine/cosmosdb.rs | 290 +++++----- src/cosmosdb.rs | 525 ++++++++++++++++++ 3 files changed, 670 insertions(+), 160 deletions(-) diff --git a/README.md b/README.md index 67f5f662a..9cb5e8bb1 100644 --- a/README.md +++ b/README.md @@ -213,13 +213,17 @@ or set a higher manual throughput for a larger provisioned run. For a throughput sweep, give each engine config its own name and a `"collection_params": { "ru_per_second": 1000 }` override. Search and upload parallelism are independently configurable. The 400 RU/s default is suitable -for smoke tests. If the SDK exhausts its HTTP 429 retry budget during upload, -reduce `upload_params.parallel` or set the SDK's -`AZURE_COSMOS_MAX_THROTTLE_RETRY_COUNT` for the upload run. Set +for smoke tests. When the SDK exhausts its HTTP 429 retry budget on an upload +write, the runner backs off and sends the write again, up to 60 times per +row, and reports how many writes it resent. Upload time includes that +backoff, so raise the RU/s or reduce `upload_params.parallel` for a +meaningful upload figure. Set `AZURE_COSMOS_REGION` to prefer an Azure region by name. The runner removes its container after a successful run unless you pass `--keep-data`. The database remains available for subsequent runs. Use `--skip-upload` to reuse -an existing container with a complete corpus. A failed run can leave a +an existing container with a complete corpus. If the upload fails, the +runner removes the container it created. A run that fails later, for example +on the index-use probe or during search, leaves a complete corpus in a provisioned container that continues billing until you remove it. Each Cosmos search result carries `results.cosmosdb_search_diagnostics` with @@ -293,7 +297,8 @@ container. It verifies the vector policy and corpus cardinality. The at least 1,000 vectors. Cosmos DB can scan while an index builds or while a physical partition has too few vectors. Before any DiskANN search, the adapter therefore probes each feed range. It runs one forced scan and then repeats an -index-requested query every 30 seconds until that query costs less than half +index-requested query, at first every 2 seconds and then less often up to +every 30 seconds, until that query costs less than half the scan's request charge on every range. The run fails if a range does not get there within 30 minutes. Set `AZURE_COSMOS_INDEX_PROBE_SECONDS` to change the limit, or to `0` to skip the probe. Results record the per-range charges diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs index de2f7f753..558d99ab9 100644 --- a/src/bin/vector_db_benchmark/engine/cosmosdb.rs +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -20,11 +20,12 @@ use futures::{stream, StreamExt, TryStreamExt}; use serde::Serialize; use tokio::sync::Semaphore; use vector_db_benchmark::cosmosdb::{ - container_name, container_policy, index_probe_deadline, index_type, knn_query, - manual_throughput, merge_partition_hits, positive_u32, reject_unknown_keys, scan_query, - upload_parallel, validate_corpus_size, validate_hit_count, validate_query_vector, validate_top, + await_index_use, container_name, container_policy, count_stable_ranges, fan_out_hits, + index_probe_deadline, index_type, knn_query, manual_throughput, parity_sample_indices, + positive_u32, reject_unknown_keys, retry_throttled, sample_parity, scan_query, upload_parallel, + validate_corpus_size, validate_hit_count, validate_query_vector, validate_top, vector_distance_function, verify_policy, DiskAnnBuildOptions, IndexProbe, KnnQueryText, - ParityTally, PartitionConcurrency, PartitionFanout, QueryHit, SearchDiagnostics, + ParityTally, PartitionConcurrency, PartitionFanout, ProbeQuery, QueryHit, SearchDiagnostics, }; use vector_db_benchmark::start_gate::WorkerPool; use vector_db_benchmark::workload::IndexLayout; @@ -33,7 +34,7 @@ use super::{compute_search_stats, CorpusCount, Engine, SearchResults, UploadStat use crate::config::{EngineConfig, SearchParams}; use crate::dataset::Dataset; -#[derive(Serialize)] +#[derive(Clone, Copy, Serialize)] struct Item<'a> { id: &'a str, embedding: &'a [f32], @@ -62,7 +63,6 @@ const PARITY_SAMPLES: usize = 8; const ROW_COUNT_CONCURRENCY: usize = 16; /// TOP K of the index-use probe queries. const INDEX_PROBE_TOP: usize = 10; -const INDEX_PROBE_INTERVAL: Duration = Duration::from_secs(30); /// Feed ranges probed at once. const INDEX_PROBE_CONCURRENCY: usize = 16; /// Sampled parity queries in flight at once. @@ -244,6 +244,73 @@ impl CosmosDbEngine { ) } + /// Write every row and check the stored row count. Returns the write + /// time and the row count. + fn upload_corpus(&mut self, dataset: &Dataset) -> Result<(f64, usize), String> { + let (ids, vectors, _) = dataset.read_vectors(dataset.needs_normalization())?; + validate_corpus_size(&self.index_type, ids.len() as u64)?; + let dimensions = dataset.vector_size() as usize; + if vectors + .iter() + .any(|vector| vector.len() != dimensions || vector.iter().any(|v| !v.is_finite())) + { + return Err( + "Cosmos DB corpus contains a vector with invalid dimensions or values".into(), + ); + } + let container = self + .container + .as_ref() + .ok_or("Cosmos DB container is unavailable")?; + let throttle_retries = AtomicUsize::new(0); + let upload_start = Instant::now(); + self.rt.block_on(async { + let throttle_retries = &throttle_retries; + stream::iter(ids.iter().zip(&vectors)) + .map(|(id, vector)| async move { + let id = id.to_string(); + let item = Item { + id: &id, + embedding: vector, + }; + // The SDK gives up on a throttled write after its own + // retries. Back off and send it again. + retry_throttled( + || container.upsert_item(id.clone(), &id, item, None), + |error| u16::from(error.status().status_code()) == 429, + tokio::time::sleep, + throttle_retries, + ) + .await + .map_err(|e| format!("Cosmos DB upload failed for row {id}: {e}")) + }) + .buffer_unordered(self.upload_parallel) + .try_for_each(|_| async { Ok(()) }) + .await + })?; + let upload_time = upload_start.elapsed().as_secs_f64(); + let upload_count = ids.len(); + drop((ids, vectors)); + let throttle_retries = throttle_retries.into_inner(); + if throttle_retries > 0 { + eprintln!( + "Cosmos DB upload at parallel={} resent {throttle_retries} throttled writes after the SDK's own HTTP 429 retries. Upload time includes that backoff. Raise the container's RU/s or lower upload_params.parallel to avoid it.", + self.upload_parallel + ); + } + // The row-count scan is verification, so it stays out of both times. + let count = self + .corpus_row_count()? + .ok_or("Cosmos DB did not return a row count")?; + if count.rows != upload_count as u64 { + return Err(format!( + "Cosmos DB stored {} rows of {upload_count} uploaded", + count.rows + )); + } + Ok((upload_time, upload_count)) + } + /// Wait until every feed range answers an index-requested query for /// clearly less than a forced scan of that range costs. Azure scans while /// a DiskANN index builds or a range holds too few vectors, so a matching @@ -267,49 +334,21 @@ impl CosmosDbEngine { let rt = &self.rt; let ranges = rt.block_on(read_search_ranges(&container))?; let started = Instant::now(); - // One forced scan per range is the costly part, so run the ranges - // concurrently. Only ranges still waiting are queried again. - let every_range: Vec = (0..ranges.len()).collect(); - let mut probes: Vec = rt - .block_on(probe_charges(&container, &scan, &ranges, &every_range))? - .into_iter() - .map(|(feed_range, scan_ru)| IndexProbe { - feed_range, - scan_ru, - index_ru: f64::NAN, - }) - .collect(); - probes.sort_by_key(|probe| probe.feed_range); - loop { - let pending: Vec = probes - .iter() - .filter(|probe| !probe.index_in_use()) - .map(|probe| probe.feed_range) - .collect(); - for (feed_range, index_ru) in - rt.block_on(probe_charges(&container, &indexed, &ranges, &pending))? - { - probes[feed_range].index_ru = index_ru; - } - let waiting: Vec<_> = probes.iter().filter(|p| !p.index_in_use()).collect(); - if waiting.is_empty() { - break; - } - let Some(remaining) = deadline.checked_sub(started.elapsed()) else { - return Err(format!( - "Cosmos DB DiskANN index is not serving {} of {} feed ranges after {}s (index-requested query costs as much as a forced scan: {waiting:?}). The index may still be building, or a range may hold too few vectors. Raise AZURE_COSMOS_INDEX_PROBE_SECONDS, or set it to 0 to run unverified.", - waiting.len(), - probes.len(), - deadline.as_secs() - )); - }; - println!( - "Waiting for the Cosmos DB DiskANN index on {} of {} feed ranges", - waiting.len(), - probes.len() - ); - std::thread::sleep(INDEX_PROBE_INTERVAL.min(remaining)); - } + // One forced scan per range is the costly part, so the ranges run + // concurrently. + let probes = await_index_use( + ranges.len(), + deadline, + |query, pending| { + let query = match query { + ProbeQuery::ForcedScan => &scan, + ProbeQuery::IndexRequested => &indexed, + }; + rt.block_on(probe_charges(&container, query, &ranges, pending)) + }, + || started.elapsed(), + std::thread::sleep, + )?; if rt.block_on(read_search_ranges(&container))? != ranges { return Err("Cosmos DB feed ranges changed during the index-use probe".into()); } @@ -378,54 +417,30 @@ impl Engine for CosmosDbEngine { } fn upload(&mut self, dataset: &Dataset) -> Result { - let (ids, vectors, _) = dataset.read_vectors(dataset.needs_normalization())?; - validate_corpus_size(&self.index_type, ids.len() as u64)?; - let dimensions = dataset.vector_size() as usize; - if vectors - .iter() - .any(|vector| vector.len() != dimensions || vector.iter().any(|v| !v.is_finite())) - { - return Err( - "Cosmos DB corpus contains a vector with invalid dimensions or values".into(), - ); - } - let container = self - .container - .as_ref() - .ok_or("Cosmos DB container is unavailable")?; - let upload_start = Instant::now(); - self.rt.block_on(async { - stream::iter(ids.iter().zip(&vectors)) - .map(|(id, vector)| async move { - let id = id.to_string(); - let item = Item { - id: &id, - embedding: vector, - }; - container - .upsert_item(id.clone(), &id, item, None) - .await - .map_err(|e| format!("Cosmos DB upload failed for row {id}: {e}")) - }) - .buffer_unordered(self.upload_parallel) - .try_for_each(|_| async { Ok(()) }) - .await - })?; - let upload_time = upload_start.elapsed().as_secs_f64(); - let upload_count = ids.len(); - drop((ids, vectors)); - // The row-count scan is verification, so it stays out of both times. - let count = self - .corpus_row_count()? - .ok_or("Cosmos DB did not return a row count")?; - if count.rows != upload_count as u64 { - return Err(format!( - "Cosmos DB stored {} rows of {upload_count} uploaded", - count.rows - )); - } + // A partial corpus is of no use to a later run, and the container + // bills until it is removed. A complete corpus is kept below even if + // the index probe then fails, because `--skip-upload` can reuse it. + let (upload_time, upload_count) = match self.upload_corpus(dataset) { + Ok(uploaded) => uploaded, + Err(error) if self.created_container => { + let error = error.trim_end_matches('.'); + return Err(match self.delete() { + Ok(()) => format!( + "{error}. Removed container '{}', which this run created.", + self.container_name + ), + Err(delete) => format!( + "{error}. Container '{}' is still provisioned and billing: {delete}", + self.container_name + ), + }); + } + Err(error) => return Err(error), + }; // For DiskANN, total time adds the wait until the index serves queries. - self.verify_index_use(dataset)?; + self.verify_index_use(dataset).map_err(|error| { + format!("{error} The uploaded corpus is complete and was kept. Rerun with --skip-upload to search it.") + })?; Ok(UploadStats { upload_time, total_time: upload_time + self.index_wait_seconds.unwrap_or(0.0), @@ -443,19 +458,11 @@ impl Engine for CosmosDbEngine { // bounded and never reads the stored vectors. The SDK drains a // full-container query one range at a time, so count the disjoint // feed ranges concurrently instead. - let count = self.rt.block_on(async { - let ranges = read_search_ranges(&container).await?; - let count = stream::iter(&ranges) - .map(|range| count_ids(&container, FeedScope::range(range.clone()))) - .buffer_unordered(ROW_COUNT_CONCURRENCY) - .try_fold(0_u64, |total, rows| async move { Ok(total + rows) }) - .await?; - // A split during the count could count a row twice or not at all. - if read_search_ranges(&container).await? != ranges { - return Err("Cosmos DB feed ranges changed during the row count".to_string()); - } - Ok(count) - })?; + let count = self.rt.block_on(count_stable_ranges( + || read_search_ranges(&container), + ROW_COUNT_CONCURRENCY, + |range| count_ids(&container, FeedScope::range(range)), + ))?; validate_corpus_size(&self.index_type, count)?; Ok(Some(CorpusCount::exact(count))) } @@ -551,25 +558,12 @@ impl Engine for CosmosDbEngine { metric: vector_distance_function(dataset.distance())?, permits: Semaphore::new(shared_request_limit), }; - let samples = (0..limit).step_by((limit / PARITY_SAMPLES).max(1)).take(PARITY_SAMPLES); - let outcomes = rt.block_on( - stream::iter(samples) - .map(|index| { - let (search, stock_text, vector) = (&search, &stock_text, &vectors[index]); - async move { - parity_sample(container, stock_text, text, vector, top, search) - .await - .map_err(|e| format!("Cosmos DB parity check for query {index}: {e}")) - } - }) - .buffer_unordered(PARITY_CONCURRENCY) - .try_collect::>(), - )?; - for (stock, stock_repeat, partitioned) in &outcomes { - search.summary.parity.observe(stock, stock_repeat, partitioned)?; - } - let parity = search.summary.parity; - parity.verdict()?; + let parity = rt.block_on(sample_parity( + parity_sample_indices(limit, PARITY_SAMPLES), + PARITY_CONCURRENCY, + |index| parity_sample(container, &stock_text, text, &vectors[index], top, &search), + ))?; + search.summary.parity = parity; println!( "Cosmos DB feed-range search verified against SDK on {} sampled queries: {} ranges, {} per query, {} in flight; fan-out missed {} stock IDs, repeated stock requests missed {}", parity.samples, summary.feed_ranges, summary.per_query_concurrency, summary.shared_request_limit, @@ -713,13 +707,13 @@ async fn parity_sample( vector: &[f32], top: usize, search: &PartitionSearch, -) -> Result<(Vec, Vec, Vec), String> { +) -> Result<[Vec; 3], String> { let ((stock, _), (stock_repeat, _), (partitioned, _)) = futures::try_join!( search_hits(container, stock_text, vector, top, None), search_hits(container, stock_text, vector, top, None), search_hits(container, partition_text, vector, top, Some(search)), )?; - Ok((stock, stock_repeat, partitioned)) + Ok([stock, stock_repeat, partitioned]) } /// Request charge of `query` on each listed feed range, as (range index, RU). @@ -785,31 +779,17 @@ async fn search_hits( ) .await; }; - let (hits, diagnostics) = stream::iter(&search.ranges) - .map(|range| async { - let _permit = search.permits.acquire().await.map_err(|e| e.to_string())?; - query_hits( - container, - text.bind(vector)?, - top, - FeedScope::range(range.clone()), - ) - .await - }) - .buffer_unordered(search.summary.per_query_concurrency) - .try_fold( - (Vec::new(), SearchDiagnostics::default()), - |(mut hits, mut diagnostics), (range_hits, range_diagnostics)| async move { - hits.extend(range_hits); - diagnostics.add_query(range_diagnostics); - Ok((hits, diagnostics)) - }, - ) - .await?; - Ok(( - merge_partition_hits(hits, top, &search.metric)?, - diagnostics, - )) + fan_out_hits( + &search.ranges, + search.summary.per_query_concurrency, + &search.permits, + top, + &search.metric, + |range| async move { + query_hits(container, text.bind(vector)?, top, FeedScope::range(range)).await + }, + ) + .await } async fn query_hits( diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs index 87d651dec..93a5a9891 100644 --- a/src/cosmosdb.rs +++ b/src/cosmosdb.rs @@ -6,12 +6,16 @@ use azure_data_cosmos::models::{ }; use azure_data_cosmos::Query; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use futures::{stream, StreamExt, TryStreamExt}; use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest, Sha256}; use std::collections::HashSet; +use std::future::Future; use std::num::NonZeroU32; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; +use tokio::sync::Semaphore; const VECTOR_PATH: &str = "/embedding"; const VECTOR_EXCLUSION: &str = "/embedding/*"; @@ -508,6 +512,202 @@ pub fn index_probe_deadline(value: Option<&str>) -> Result, Str Ok((seconds > 0).then(|| Duration::from_secs(seconds))) } +/// Pause between index-use polls: a tenth of the time already waited, within +/// 2 to 30 seconds. The recorded index wait is then at most about a tenth +/// too long, and a long build is not polled with a scan every few seconds. +pub fn index_probe_interval(waited: Duration) -> Duration { + (waited / 10).clamp(Duration::from_secs(2), Duration::from_secs(30)) +} + +/// The charges of the feed ranges that are not served by the index yet. +pub fn describe_waiting(probes: &[IndexProbe]) -> String { + const SHOWN: usize = 8; + let waiting: Vec<_> = probes.iter().filter(|p| !p.index_in_use()).collect(); + let mut text = waiting + .iter() + .take(SHOWN) + .map(|p| { + format!( + "range {}: index query {:.2} RU, forced scan {:.2} RU", + p.feed_range, p.index_ru, p.scan_ru + ) + }) + .collect::>() + .join("; "); + if waiting.len() > SHOWN { + text.push_str(&format!("; and {} more", waiting.len() - SHOWN)); + } + text +} + +/// Which of the two probe queries to run on a feed range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProbeQuery { + ForcedScan, + IndexRequested, +} + +/// Wait until every feed range answers the index-requested query for clearly +/// less than its forced scan. `charges` returns (range index, RU) for the +/// listed ranges. The forced scan runs once per range, and only ranges still +/// waiting are queried again. `waited` and `pause` are the clock. +pub fn await_index_use( + feed_ranges: usize, + deadline: Duration, + mut charges: impl FnMut(ProbeQuery, &[usize]) -> Result, String>, + waited: impl Fn() -> Duration, + mut pause: impl FnMut(Duration), +) -> Result, String> { + let mut probes: Vec = (0..feed_ranges) + .map(|feed_range| IndexProbe { + feed_range, + scan_ru: f64::NAN, + index_ru: f64::NAN, + }) + .collect(); + let mut pending: Vec = (0..feed_ranges).collect(); + let mut query = ProbeQuery::ForcedScan; + loop { + let answers = charges(query, &pending)?; + let mut answered: Vec = answers.iter().map(|(range, _)| *range).collect(); + answered.sort_unstable(); + if answered != pending { + return Err(format!( + "Cosmos DB index-use probe asked feed ranges {pending:?} and got answers for {answered:?}" + )); + } + for (feed_range, ru) in answers { + match query { + ProbeQuery::ForcedScan => probes[feed_range].scan_ru = ru, + ProbeQuery::IndexRequested => probes[feed_range].index_ru = ru, + } + } + if query == ProbeQuery::ForcedScan { + query = ProbeQuery::IndexRequested; + continue; + } + pending.retain(|&feed_range| !probes[feed_range].index_in_use()); + if pending.is_empty() { + return Ok(probes); + } + let Some(remaining) = deadline + .checked_sub(waited()) + .filter(|left| !left.is_zero()) + else { + return Err(format!( + "Cosmos DB DiskANN index is not serving {} of {feed_ranges} feed ranges after {}s. An index-requested query still costs as much as a forced scan ({}). The index may still be building, or a range may hold too few vectors. Raise AZURE_COSMOS_INDEX_PROBE_SECONDS, or set it to 0 to run unverified.", + pending.len(), + deadline.as_secs(), + describe_waiting(&probes) + )); + }; + println!( + "Waiting for the Cosmos DB DiskANN index on {} of {feed_ranges} feed ranges after {:.0}s ({})", + pending.len(), + waited().as_secs_f64(), + describe_waiting(&probes) + ); + pause(index_probe_interval(waited()).min(remaining)); + } +} + +/// Delay before retrying an upload write that Azure throttled, or `None` +/// once the budget is spent. The SDK retries a 429 a few times on its own. +/// This budget sits on top of that, so a corpus larger than a smoke test +/// still uploads at a throughput below what the writers ask for. +pub fn upload_throttle_backoff(retry: u32) -> Option { + const RETRIES: u32 = 60; + (retry < RETRIES) + .then(|| Duration::from_millis(250 << retry.min(5)).min(Duration::from_secs(5))) +} + +/// Run `write` until it succeeds, fails for a reason other than throttling, +/// or exhausts [`upload_throttle_backoff`]. Every retry is counted. +pub async fn retry_throttled( + mut write: impl FnMut() -> Write, + throttled: impl Fn(&E) -> bool, + sleep: impl Fn(Duration) -> Sleep, + retries: &AtomicUsize, +) -> Result +where + Write: Future>, + Sleep: Future, +{ + let mut retry = 0; + loop { + match write().await { + Err(error) if throttled(&error) => { + let Some(delay) = upload_throttle_backoff(retry) else { + return Err(error); + }; + retries.fetch_add(1, Ordering::Relaxed); + retry += 1; + sleep(delay).await; + } + other => return other, + } + } +} + +/// Sum a per-range count over disjoint feed ranges, `concurrency` at a time. +/// A split during the count could count a row twice or not at all, so the +/// ranges are read again afterwards and must be unchanged. +pub async fn count_stable_ranges( + read_ranges: impl Fn() -> Ranges, + concurrency: usize, + count: impl Fn(R) -> Count, +) -> Result +where + R: Clone + PartialEq, + Ranges: Future, String>>, + Count: Future>, +{ + let ranges = read_ranges().await?; + let total = stream::iter(ranges.iter().cloned()) + .map(&count) + .buffer_unordered(concurrency.max(1)) + .try_fold(0_u64, |total, rows| async move { Ok(total + rows) }) + .await?; + if read_ranges().await? != ranges { + return Err("Cosmos DB feed ranges changed during the row count".into()); + } + Ok(total) +} + +/// Query every feed range, at most `per_query` at a time for this query and +/// one `permits` permit per request across all queries, then select the +/// global top K and add up the diagnostics. +pub async fn fan_out_hits( + ranges: &[R], + per_query: usize, + permits: &Semaphore, + top: usize, + metric: &VectorDistanceFunction, + query: impl Fn(R) -> Hits, +) -> Result<(Vec, SearchDiagnostics), String> +where + R: Clone, + Hits: Future, SearchDiagnostics), String>>, +{ + let query = &query; + let (hits, diagnostics) = stream::iter(ranges.iter().cloned()) + .map(|range| async move { + let _permit = permits.acquire().await.map_err(|e| e.to_string())?; + query(range).await + }) + .buffer_unordered(per_query.max(1)) + .try_fold( + (Vec::new(), SearchDiagnostics::default()), + |(mut hits, mut diagnostics), (range_hits, range_diagnostics)| async move { + hits.extend(range_hits); + diagnostics.add_query(range_diagnostics); + Ok((hits, diagnostics)) + }, + ) + .await?; + Ok((merge_partition_hits(hits, top, metric)?, diagnostics)) +} + #[derive(Deserialize)] pub struct QueryHit { pub id: String, @@ -616,6 +816,43 @@ impl ParityTally { } } +/// Query indices spread evenly over the first `limit` queries, at most `samples`. +pub fn parity_sample_indices(limit: usize, samples: usize) -> Vec { + (0..limit) + .step_by((limit / samples.max(1)).max(1)) + .take(samples) + .collect() +} + +/// Run `sample` for each query index, `concurrency` at a time. It returns the +/// two stock responses and the fan-out response. The tally of all samples +/// must then pass [`ParityTally::verdict`]. +pub async fn sample_parity( + indices: Vec, + concurrency: usize, + sample: impl Fn(usize) -> Sample, +) -> Result +where + Sample: Future; 3], String>>, +{ + let sample = &sample; + let outcomes: Vec<_> = stream::iter(indices) + .map(|index| async move { + sample(index) + .await + .map_err(|e| format!("Cosmos DB parity check for query {index}: {e}")) + }) + .buffer_unordered(concurrency.max(1)) + .try_collect() + .await?; + let mut tally = ParityTally::default(); + for [stock, stock_repeat, partitioned] in &outcomes { + tally.observe(stock, stock_repeat, partitioned)?; + } + tally.verdict()?; + Ok(tally) +} + #[cfg(test)] mod tests { use super::*; @@ -1034,4 +1271,292 @@ mod tests { }) ); } + + /// A clock that only `pause` advances, plus a record of every probe call. + struct ProbeRun { + clock: std::cell::Cell, + calls: std::cell::RefCell)>>, + pauses: std::cell::RefCell>, + } + + /// Each range's index starts serving once the clock reaches `ready_at`. + fn run_probe(ready_at: &[u64], deadline: u64) -> (Result, String>, ProbeRun) { + let run = ProbeRun { + clock: Default::default(), + calls: Default::default(), + pauses: Default::default(), + }; + let result = await_index_use( + ready_at.len(), + Duration::from_secs(deadline), + |query, ranges| { + run.calls.borrow_mut().push((query, ranges.to_vec())); + Ok(ranges + .iter() + .map(|&range| { + let ready = run.clock.get() >= Duration::from_secs(ready_at[range]); + let served = query == ProbeQuery::IndexRequested && ready; + (range, if served { 6.0 } else { 300.0 }) + }) + .collect()) + }, + || run.clock.get(), + |pause| { + run.pauses.borrow_mut().push(pause); + run.clock.set(run.clock.get() + pause); + }, + ); + (result, run) + } + + #[test] + fn index_wait_scans_once_and_polls_only_the_ranges_still_waiting() { + let (probes, run) = run_probe(&[0, 3, 0], 600); + let probes = probes.unwrap(); + assert!(probes.iter().all(IndexProbe::index_in_use)); + assert_eq!( + probes[1], + IndexProbe { + feed_range: 1, + scan_ru: 300.0, + index_ru: 6.0 + } + ); + let calls = run.calls.borrow(); + assert_eq!(calls[0], (ProbeQuery::ForcedScan, vec![0, 1, 2])); + assert_eq!(calls[1], (ProbeQuery::IndexRequested, vec![0, 1, 2])); + // Ranges 0 and 2 passed on the first poll. Only range 1 is asked again. + assert!(calls[2..] + .iter() + .all(|call| *call == (ProbeQuery::IndexRequested, vec![1]))); + assert_eq!(calls.len(), 4); + assert_eq!(*run.pauses.borrow(), [Duration::from_secs(2); 2]); + } + + #[test] + fn index_wait_fails_at_the_deadline_and_names_the_charges() { + let (result, run) = run_probe(&[0, u64::MAX], 100); + let error = result.unwrap_err(); + assert!( + error.contains("not serving 1 of 2 feed ranges after 100s"), + "{error}" + ); + assert!( + error.contains("range 1: index query 300.00 RU, forced scan 300.00 RU"), + "{error}" + ); + // Range 0 is served, so it is not listed as waiting. + assert!(!error.contains("range 0:"), "{error}"); + // It never sleeps past the deadline. + assert_eq!( + run.pauses.borrow().iter().sum::(), + Duration::from_secs(100) + ); + // An answer for the wrong ranges is an error, not a panic or a pass. + // The clock advances, so a missing check ends at the deadline. + let clock = std::cell::Cell::new(Duration::ZERO); + let wrong = await_index_use( + 2, + Duration::from_secs(1), + |_, _| Ok(vec![(0, 1.0)]), + || clock.get(), + |pause| clock.set(clock.get() + pause), + ); + assert!(wrong.unwrap_err().contains("got answers for [0]")); + } + + #[test] + fn index_poll_interval_grows_with_the_wait() { + let interval = |seconds| index_probe_interval(Duration::from_secs(seconds)); + assert_eq!(interval(0), Duration::from_secs(2)); + assert_eq!(interval(60), Duration::from_secs(6)); + assert_eq!(interval(3600), Duration::from_secs(30)); + let many: Vec<_> = (0..10) + .map(|feed_range| IndexProbe { + feed_range, + scan_ru: 9.0, + index_ru: 9.0, + }) + .collect(); + let text = describe_waiting(&many); + assert!(text.starts_with("range 0: index query 9.00 RU, forced scan 9.00 RU; range 1")); + assert!(text.ends_with("; and 2 more"), "{text}"); + } + + #[test] + fn throttled_writes_are_retried_until_the_budget_is_spent() { + assert_eq!(upload_throttle_backoff(0), Some(Duration::from_millis(250))); + assert_eq!(upload_throttle_backoff(3), Some(Duration::from_secs(2))); + assert_eq!(upload_throttle_backoff(59), Some(Duration::from_secs(5))); + assert_eq!(upload_throttle_backoff(60), None); + + let run = |outcomes: Vec>| { + let retries = AtomicUsize::new(0); + let slept = std::cell::RefCell::new(Vec::new()); + let mut outcomes = outcomes.into_iter(); + let result = futures::executor::block_on(retry_throttled( + || { + let outcome = outcomes.next().unwrap_or(Err(429)); + async move { outcome } + }, + |status: &u16| *status == 429, + |delay| { + slept.borrow_mut().push(delay); + async {} + }, + &retries, + )); + (result, retries.into_inner(), slept.into_inner()) + }; + let (result, retries, slept) = run(vec![Err(429), Err(429), Ok(7)]); + assert_eq!((result, retries), (Ok(7), 2)); + assert_eq!( + slept, + [Duration::from_millis(250), Duration::from_millis(500)] + ); + // Any other failure is returned at once. + assert_eq!(run(vec![Err(429), Err(503), Ok(7)]).0, Err(503)); + // Endless throttling ends with the 429 after the whole budget. + let (result, retries, _) = run(vec![]); + assert_eq!((result, retries), (Err(429), 60)); + } + + /// Pending once, so `buffer_unordered` has to interleave the futures. + async fn yield_once() { + let mut yielded = false; + std::future::poll_fn(|cx| { + if std::mem::replace(&mut yielded, true) { + return std::task::Poll::Ready(()); + } + cx.waker().wake_by_ref(); + std::task::Poll::Pending + }) + .await + } + + #[test] + fn row_count_sums_ranges_and_rejects_a_split() { + let count = |ranges_after: Vec| { + let reads = std::cell::Cell::new(0); + futures::executor::block_on(count_stable_ranges( + || { + reads.set(reads.get() + 1); + let ranges = if reads.get() == 1 { + vec![10, 20, 30] + } else { + ranges_after.clone() + }; + async move { Ok(ranges) } + }, + 2, + |rows: u64| async move { + yield_once().await; + if rows == 0 { + Err("boom".to_string()) + } else { + Ok(rows) + } + }, + )) + }; + assert_eq!(count(vec![10, 20, 30]), Ok(60)); + assert!(count(vec![10, 20, 15, 15]) + .unwrap_err() + .contains("feed ranges changed")); + } + + #[test] + fn fan_out_merges_the_global_top_k_within_the_request_limit() { + let scored = |id: &str, distance| QueryHit { + id: id.into(), + distance: Some(distance), + }; + let in_flight = AtomicUsize::new(0); + let peak = AtomicUsize::new(0); + let fan_out = |per_query, permits| { + peak.store(0, Ordering::Relaxed); + let permits = Semaphore::new(permits); + futures::executor::block_on(fan_out_hits( + &[0_usize, 1, 2, 3], + per_query, + &permits, + 2, + &VectorDistanceFunction::Cosine, + |range| { + let (in_flight, peak) = (&in_flight, &peak); + async move { + peak.fetch_max( + in_flight.fetch_add(1, Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); + yield_once().await; + in_flight.fetch_sub(1, Ordering::Relaxed); + if range == 3 && per_query == 1 { + return Err("range 3 failed".to_string()); + } + let mut diagnostics = SearchDiagnostics::default(); + diagnostics.observe_page(2.5, 1, [(200, 1)]); + let name = format!("r{range}"); + Ok(( + vec![ + scored(&name, range as f64), + scored(&format!("{name}b"), -1.0), + ], + diagnostics, + )) + } + }, + )) + }; + let (hits, diagnostics) = fan_out(4, 4).unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.id.as_str()).collect::>(), + ["r3", "r2"] + ); + assert_eq!(diagnostics.request_charge_ru, 10.0); + assert_eq!(diagnostics.request_attempts, 4); + assert_eq!(peak.load(Ordering::Relaxed), 4); + // The shared permits cap in-flight requests below the per-query limit. + fan_out(4, 2).unwrap(); + assert_eq!(peak.load(Ordering::Relaxed), 2); + // So does the per-query limit, and a failed range fails the query. + assert_eq!(fan_out(1, 4).err().unwrap(), "range 3 failed"); + assert_eq!(peak.load(Ordering::Relaxed), 1); + } + + #[test] + fn parity_samples_spread_over_the_queries_and_must_pass_the_verdict() { + assert_eq!( + parity_sample_indices(100, 8), + [0, 12, 24, 36, 48, 60, 72, 84] + ); + assert_eq!(parity_sample_indices(3, 8), [0, 1, 2]); + assert!(parity_sample_indices(0, 8).is_empty()); + + let sample = |fan_out_ids: &'static [&'static str]| { + futures::executor::block_on(sample_parity(vec![0, 5], 2, |index| async move { + yield_once().await; + if index == 5 && fan_out_ids.is_empty() { + return Err("timeout".to_string()); + } + Ok([hits(&["a", "b"]), hits(&["a", "b"]), hits(fan_out_ids)]) + })) + }; + let tally = sample(&["b", "a"]).unwrap(); + assert_eq!( + tally, + ParityTally { + samples: 2, + stock_repeat_missing_ids: 0, + cross_mode_missing_ids: 0 + } + ); + assert!(sample(&["a", "c"]) + .unwrap_err() + .contains("fan-out missed 4 stock IDs")); + assert_eq!( + sample(&[]).unwrap_err(), + "Cosmos DB parity check for query 5: timeout" + ); + } }