diff --git a/AGENTS.md b/AGENTS.md index e2f8bb662..873ea923c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,10 +2,10 @@ ## Project Overview -Rust implementation of a vector database benchmarking tool. **15 engines**: +Rust implementation of a vector database benchmarking tool. **16 engines**: Redis (RediSearch), VectorSets, Valkey, Dragonfly, KiviDB, Elasticsearch, OpenSearch, Qdrant, pgvector, Weaviate, Milvus, MongoDB, Chroma, Turbopuffer, -Vertex AI. +Vertex AI, Azure Cosmos DB for NoSQL. The thing this repo optimises for is **not being wrong**. Its dominant historical bug class is a run that completes, reports a plausible number, and measured @@ -62,6 +62,7 @@ make integration-test-{valkey,dragonfly,kividb,chroma} make integration-test-{pgvector,qdrant,elasticsearch,opensearch} make integration-test-{weaviate,milvus,mongodb} make integration-test-vertex # cloud-only, needs real GCP credentials +make integration-test-cosmosdb # cloud-only, needs a Cosmos DB account key make integration-test-no-docker ``` @@ -107,7 +108,7 @@ src/ bin/ vector_db_benchmark/ main.rs cli.rs config.rs dataset.rs download.rs experiment.rs ground_truth.rs - engine/ # 15 engines + shared: mod.rs redis_utils.rs filter_guard.rs + engine/ # 16 engines + shared: mod.rs redis_utils.rs filter_guard.rs # index_naming.rs geo.rs generate_dataset.rs # -> `generate-dataset`, writes the synthetic-* fixtures prepare_msmarco.rs # -> `prepare-msmarco`, builds/verifies MS MARCO corpora @@ -454,6 +455,7 @@ now covers more engines than `v0/` did; the table below is the mapping. | Dragonfly | — | `dragonfly` | `redis` 0.27 (RESP) | | KiviDB | — | `kividb` | `redis` 0.27 (RESP) | | Chroma | — | `chroma` | `reqwest` (REST v2) | +| Azure Cosmos DB for NoSQL | — | `cosmosdb` | `azure_data_cosmos` 1.0 | | Vertex AI | — | `vertex` | `tonic`/`prost` (gRPC) + `reqwest` — cloud-only | \* Valkey GLIDE has no Rust crate ([valkey-io/valkey-glide#828](https://github.com/valkey-io/valkey-glide/issues/828), closed NOT_PLANNED). GLIDE maintainers recommend `redis-rs` for Rust. diff --git a/Cargo.lock b/Cargo.lock index 7aa4006ce..60dedcbed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "addr2line" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e567177890eb1617b1f774005b66b26b2377afd138a2ca37aae7d8f0c81429d4" +dependencies = [ + "gimli", +] + [[package]] name = "adler2" version = "2.0.1" @@ -104,6 +113,15 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84a22b5d51209f6de83d37a0b33d320acb8da88823e42a94c075140708f032a5" +dependencies = [ + "rustversion", +] + [[package]] name = "arcstr" version = "1.2.0" @@ -128,6 +146,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "async-lock" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0876700b0d007ad25d5508865a4259c3af9af6d296bf663a0d586d0bc3369f6f" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -220,6 +249,119 @@ dependencies = [ "tower-service", ] +[[package]] +name = "azure_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e41cbd819986ba41904c207d8ffc4106f8f8352a548d773e9554906379bb2fb" +dependencies = [ + "async-lock", + "async-trait", + "azure_core_macros", + "bytes", + "futures", + "hmac 0.12.1", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "sha2 0.10.9", + "tokio", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_core_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9b52dba6a345f3ad2d42ff8d0d63df9d0994cfa29657bf18ffdbf149f78a4f5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", + "tracing", +] + +[[package]] +name = "azure_data_cosmos" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa75d1999be85b351fb1ace2f48d8df5f13c428fb3c437e0c97214ef2c9a7e9b" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "azure_data_cosmos_driver", + "base64", + "futures", + "pin-project", + "reqwest 0.13.4", + "serde", + "serde_json", + "time", + "tracing", + "url", +] + +[[package]] +name = "azure_data_cosmos_driver" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "926a4817a6a75095120f444d71ed59c79edc6178845d7955f4170146ae91d433" +dependencies = [ + "arc-swap", + "async-lock", + "async-trait", + "azure_core", + "azure_data_cosmos_macros", + "backtrace", + "base64", + "bytes", + "crossbeam-epoch", + "futures", + "getrandom 0.4.3", + "h2", + "rand 0.10.2", + "reqwest 0.13.4", + "rustc_version", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "url", + "uuid", + "windows", +] + +[[package]] +name = "azure_data_cosmos_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bedd703baf5f2026093d87de0f79dbd43b1f1e067cb005f1cc643a618295882" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "backtrace" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f4220217f3288537b83684130f42fad43320b1c3b805f1910d683d52ded24f" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide 0.9.1", + "object", + "rustc-demangle", + "windows-link", +] + [[package]] name = "base64" version = "0.22.1" @@ -250,6 +392,15 @@ dependencies = [ "wyz", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -357,7 +508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.0", "rand_core 0.10.1", ] @@ -536,6 +687,15 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.0" @@ -606,6 +766,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -795,15 +965,26 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", + "block-buffer 0.12.1", "const-oid", - "crypto-common", + "crypto-common 0.2.2", "ctutils", ] @@ -882,6 +1063,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -917,7 +1118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.9", ] [[package]] @@ -1044,6 +1245,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1085,6 +1296,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "gimli" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1033caf0b349c518623b5396bfb2cf0bddf44f0306d543a250e5743297aafd10" + [[package]] name = "glob" version = "0.3.3" @@ -1093,9 +1310,9 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "h2" -version = "0.4.15" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -1297,13 +1514,22 @@ dependencies = [ "tracing", ] +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "hmac" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -1879,7 +2105,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" dependencies = [ "cfg-if", - "digest", + "digest 0.11.3", ] [[package]] @@ -1904,6 +2130,15 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "1.2.1" @@ -1968,7 +2203,7 @@ dependencies = [ "hickory-net", "hickory-proto", "hickory-resolver", - "hmac", + "hmac 0.13.0", "macro_magic", "md-5", "mongocrypt", @@ -1982,7 +2217,7 @@ dependencies = [ "serde_bytes", "serde_with", "sha1", - "sha2", + "sha2 0.11.0", "socket2 0.6.4", "stringprep", "strsim", @@ -2153,6 +2388,15 @@ dependencies = [ "objc2-core-foundation", ] +[[package]] +name = "object" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -2242,6 +2486,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.11.2" @@ -2302,7 +2552,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest", + "digest 0.11.3", ] [[package]] @@ -2453,11 +2703,11 @@ dependencies = [ "byteorder", "bytes", "fallible-iterator", - "hmac", + "hmac 0.13.0", "md-5", "memchr", "rand 0.10.2", - "sha2", + "sha2 0.11.0", "stringprep", ] @@ -2907,7 +3157,7 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams", + "wasm-streams 0.4.2", "web-sys", "webpki-roots", ] @@ -2921,10 +3171,13 @@ dependencies = [ "base64", "bytes", "futures-core", + "futures-util", + "h2", "http", "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "js-sys", @@ -2939,12 +3192,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-util", "tower 0.5.3", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams 0.5.0", "web-sys", ] @@ -2968,6 +3223,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rustc-demangle" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3257,8 +3518,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3267,6 +3528,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest 0.10.7", +] + [[package]] name = "sha2" version = "0.11.0" @@ -3274,8 +3546,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures", - "digest", + "cpufeatures 0.3.0", + "digest 0.11.3", ] [[package]] @@ -3871,6 +4143,57 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "typespec" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "753a2fe021e407d4fc9ee6f4f0a33403cc306d5c54c4e4ebe1b8cbde0ca052b9" +dependencies = [ + "base64", + "bytes", + "futures", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0373af0f9d4f580b3a1a9d9639cedaabe015ed262b35bfbe13941bfb14fe1ea6" +dependencies = [ + "async-trait", + "base64", + "bytes", + "dyn-clone", + "futures", + "pin-project", + "rand 0.10.2", + "reqwest 0.13.4", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid", +] + +[[package]] +name = "typespec_macros" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c608f4427943f8adb211abc95c87672b1b98847152783507d54e3246e502f60" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.118", +] + [[package]] name = "ucd-trie" version = "0.1.7" @@ -3966,6 +4289,7 @@ checksum = "bf80a72845275afea99e7f2b434723d3bc7e38470fcd1c7ed39a599c73319a53" dependencies = [ "getrandom 0.4.3", "js-sys", + "rand 0.10.2", "serde_core", "wasm-bindgen", ] @@ -3980,6 +4304,7 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" name = "vector_db_benchmark" version = "0.2.0" dependencies = [ + "azure_data_cosmos", "base64", "bzip2", "chrono", @@ -3987,6 +4312,7 @@ dependencies = [ "crc32fast", "elasticsearch", "flate2", + "futures", "glob", "half", "hdf5", @@ -4010,7 +4336,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", - "sha2", + "sha2 0.11.0", "tar", "tempfile", "tokio", @@ -4151,6 +4477,19 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.103" @@ -4230,6 +4569,27 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "527fadee13e0c05939a6a05d5bd6eec6cd2e3dbd648b9f8e447c6518133d8580" +dependencies = [ + "windows-collections", + "windows-core", + "windows-future", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b2d95af1a8a14a3c7367e1ed4fc9c20e0a26e79551b1454d72583c97cc6610" +dependencies = [ + "windows-core", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -4243,6 +4603,17 @@ dependencies = [ "windows-strings", ] +[[package]] +name = "windows-future" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d6f90251fe18a279739e78025bd6ddc52a7e22f921070ccdc67dde84c605cb" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.60.2" @@ -4271,6 +4642,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2e40844ac143cdb44aead537bbf727de9b044e107a0f1220392177d15b0f26" +dependencies = [ + "windows-core", + "windows-link", +] + [[package]] name = "windows-registry" version = "0.6.1" @@ -4334,6 +4715,15 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3949bd5b99cafdf1c7ca86b43ca564028dfe27d66958f2470940f73d86d75b37" +dependencies = [ + "windows-link", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" diff --git a/Cargo.toml b/Cargo.toml index 8d352bdaf..d9466e9e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,6 +53,8 @@ sha2 = "0.11" crc32fast = "1.5" hdrhistogram = "7.5" base64 = "0.22" +azure_data_cosmos = { version = "1.0", default-features = false, features = ["control_plane", "hmac_rust", "key_auth", "native_tls"] } +futures = "0.3" # Only for `--features release-static` (the prebuilt release archives, #366): # compile HDF5, zlib and OpenSSL from source and link them statically, so the diff --git a/Makefile b/Makefile index 09aac5705..9736033b6 100644 --- a/Makefile +++ b/Makefile @@ -323,6 +323,11 @@ integration-test-vertex: @echo "=== Vertex AI integration tests (cloud-only, needs GCP credentials) ===" cargo test --test integration_vertex --release -- --test-threads=1 +.PHONY: integration-test-cosmosdb +integration-test-cosmosdb: + @echo "=== Azure Cosmos DB for NoSQL integration tests (cloud-only) ===" + cargo test --test integration_cosmosdb --release -- --nocapture --test-threads=1 + .PHONY: integration-test-no-docker integration-test-no-docker: @echo "=== Running integration tests (assumes redis on port 6399) ===" @@ -572,6 +577,7 @@ help: @echo " make integration-test-kividb - KiviDB 1.1.2" @echo " make integration-test-chroma - Chroma" @echo " make integration-test-vertex - Vertex AI (cloud-only, needs GCP creds)" + @echo " make integration-test-cosmosdb - Azure Cosmos DB for NoSQL (cloud-only)" @echo "" @echo " Docker:" @echo " make docker-build - Build Docker image (IMAGE_TAG=latest)" diff --git a/README.md b/README.md index 0a5ce5f03..9cb5e8bb1 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,7 @@ host's Redis on Linux, and on Docker Desktop only with host networking enabled): | **Vertex AI** (Vector Search) | `reqwest` (Vertex AI REST v1) | HTTP/REST (cloud) | L2, Cosine, Dot | Yes [\*\*\*\*](#vertex-note) | | **Chroma** | `reqwest` (Chroma v2 REST API) | HTTP/REST | L2, Cosine, IP | Yes [\*\*\*\*\*](#chroma-note) | | **KiviDB** | `redis` 1.8.1 | RESP protocol | L2, Cosine, IP | Yes [\*\*\*\*\*\*](#kividb-note) | +| **Azure Cosmos DB for NoSQL** | `azure_data_cosmos` 1.0 | Azure SDK (cloud) | L2, Cosine, Dot | No | \* **Valkey client note:** Valkey GLIDE has no published Rust crate ([valkey-io/valkey-glide#828](https://github.com/valkey-io/valkey-glide/issues/828), closed NOT_PLANNED). The GLIDE maintainers recommend using `redis-rs` for Rust and upstream their improvements to it. The `redis` crate works with Valkey since it speaks the same RESP protocol. @@ -183,6 +184,132 @@ host's Redis on Linux, and on Docker Desktop only with host networking enabled): \*\*\*\*\* **Chroma note:** Uses **Chroma** (OSS) via its **v2 REST API** — a collection of records (`ids` + `embeddings` + scalar `metadatas`) queried with `query` + a `where` document. **Metadata filters** map directly onto the canonical model: `match` → `$eq`, `match_any` → `$in`, `range` → `$gte`/`$gt`/`$lte`/`$lt`, and **AND / OR / nested boolean** to Chroma's native `$and` / `$or` (which nest arbitrarily). Supported datatypes: **keyword, int, float, bool, uuid, datetime** (stored as epoch-seconds int, like Milvus, so numeric range operators apply). **Full-text** (`{match:{text}}`) is supported via `where_document` `$contains` — the `text`-typed field's value is uploaded as each record's Chroma `document`. **NOT supported** by Chroma's metadata engine: **geo-radius** and multi-valued **`labels` arrays** (Chroma metadata values are scalar only). Geo is a *permanent* gap, not a to-do: Chroma's `where` is a closed enum of `field OP literal` comparisons with no geospatial operator, no attribute-vs-attribute comparison and no arithmetic, and a spherical cap is not an axis-aligned box in any query-independent coordinate system — so there is no exact encoding, and an inexact bounding box would under-constrain the query while looking filtered (#223). Since #219 these are a hard **error**, not a silent drop — and a condition tree containing a geo leaf is refused *whole*, so a `geo AND keyword` query cannot quietly run as keyword-only. See the note below. Runs over HTTP/REST via Docker; set the host port with `CHROMA_PORT` (default `8000`, test compose maps `8003`), and optionally `CHROMA_COLLECTION` / `CHROMA_TENANT` / `CHROMA_DATABASE`. Distance space (`l2`/`cosine`/`ip`) is set per collection from the dataset metric. +### Azure Cosmos DB for NoSQL + +Enable **Vector Search for NoSQL API** on your Cosmos DB account. Set +`AZURE_COSMOS_ENDPOINT` to the account URL and provide the account key through +the `AZURE_COSMOS_KEY` environment variable. + +```bash +export AZURE_COSMOS_DATABASE=vdb-benchmark +export AZURE_COSMOS_RU_PER_SECOND=400 +vector-db-benchmark --engines cosmosdb-flat --datasets random-100 --skip-if-exists false +make integration-test-cosmosdb +``` + +For an Azure-hosted benchmark over Private Link, place the client VM in the +Cosmos DB region. Create a NoSQL private endpoint with subresource `Sql` and +link the `privatelink.documents.azure.com` private DNS zone to the VM's virtual +network. Keep `AZURE_COSMOS_ENDPOINT` set to the standard account URL +(`https://.documents.azure.com:443/`). From the VM, run +`nslookup .documents.azure.com` and confirm the address matches the +private endpoint before running the benchmark. See Microsoft's +[Cosmos DB Private Link setup guide](https://learn.microsoft.com/en-us/azure/cosmos-db/how-to-configure-private-endpoints). + +The benchmark creates the database if needed and provisions a dedicated +container per config and dataset. It uses manual throughput of 400 RU/s by +default. Set `AZURE_COSMOS_RU_PER_SECOND=serverless` for a serverless account, +or set a higher manual throughput for a larger provisioned run. For a +throughput sweep, give each engine config its own name and a +`"collection_params": { "ru_per_second": 1000 }` override. Search and upload +parallelism are independently configurable. The 400 RU/s default is suitable +for smoke tests. When the SDK exhausts its HTTP 429 retry budget on an upload +write, the runner backs off and sends the write again, up to 60 times per +row, and reports how many writes it resent. Upload time includes that +backoff, so raise the RU/s or reduce `upload_params.parallel` for a +meaningful upload figure. Set +`AZURE_COSMOS_REGION` to prefer an Azure region by name. The runner removes +its container after a successful run unless you pass `--keep-data`. The +database remains available for subsequent runs. Use `--skip-upload` to reuse +an existing container with a complete corpus. If the upload fails, the +runner removes the container it created. A run that fails later, for example +on the index-use probe or during search, leaves a complete corpus in a +provisioned container that continues billing until you remove it. + +Each Cosmos search result carries `results.cosmosdb_search_diagnostics` with +the SDK's request charge, request attempt count, and observed HTTP 429 count +for measured query pages. Warm-ups and verification queries are outside this +scope. SDK backoff and retries are part of the measured search latency. +Check `attempts_without_status` and `queries_without_request_diagnostics` +when interpreting the 429 count. + +By default, the adapter uses the SDK's full-container query, which drains +the partitions sequentially for `ORDER BY VectorDistance`. Set +`"search_params": { "partition_concurrency": "auto" }` on a search entry to +query disjoint physical feed ranges concurrently and merge their scored +top-K candidates into one global top K. The adapter looks up the feed ranges +before timing. `"auto"` uses their count as the per-query concurrency, and +a positive integer caps it at that value. Across all workers, the maximum +number of in-flight range queries is the greater of the worker count and the +effective per-query concurrency. When the worker count exceeds the range +count, range queries wait for a free slot inside the measured latency, so +compare a fan-out row with a stock row on throughput and recall, not on tail +latency. + +Before timing, the adapter runs up to eight sampled queries under both modes, +with the stock query sent twice. Repeated DiskANN requests can return +different neighbors at the cutoff, so the stock query's disagreement with +itself sets the bar. Fan-out may miss at most three times as many stock IDs +as the repeated stock request misses. Flat is exact, so it needs identical +IDs. A larger difference or a feed-range change fails the run. Fan-out +results record both counts under `partition_fanout.parity`, with the +effective limits, RU charge, and HTTP attempts for all ranges. + +For DiskANN, each search entry can set positive integer +`search_params.search_list_size_multiplier` and +`search_params.quantized_vector_list_multiplier`. These map to the +`VectorDistance` query options `searchListSizeMultiplier` and +`quantizedVectorListMultiplier`. Leave both unset for the service defaults. +Add multiple search entries under one engine config to sweep query-time +settings on the same index, keeping `top` fixed for recall comparisons. +The shipped `cosmosdb-diskann` config compares the service defaults with +search-list multipliers of 1, 5, 10, and 20 at parallelism 1 and 8, all at +top-10. The quantized-vector-list multiplier uses the service default. + +To sweep DiskANN index builds, use `--engines 'cosmosdb-diskann-build-*'` +with a dataset of at least 64 dimensions. Each named config provisions its +own container and uses the same top-10 searches at parallelism 1 and 8 with +service-default query settings. The presets vary +`collection_params.indexing_search_list_size` at 50, 100, and 200 with +quantization fixed at 32 bytes, and vary `collection_params.quantization_byte_size` +at 16, 32, and 64 with the indexing search list fixed at 100. The benchmark +checks the realized values before search, including on container reuse. +With the Rust SDK, the build search list accepts integers from 25 to 500 +and the quantization size accepts integers from 1 to the smaller of 512 +and the vector dimension. Omit a build setting to use the Azure default. +See the [Rust SDK vector index model](https://github.com/Azure/azure-sdk-for-rust/blob/d7e28a1cd76e11d51f005e5d502a6c1c6299f74a/sdk/cosmos/azure_data_cosmos/src/models/indexing_policy.rs#L274-L290). + +The opt-in DBpedia example uses 10,000 RU/s per container. It runs four +index builds and measures top-100 searches at parallelism 1 and 100. The +command uses the registered 975K DBpedia dataset with 100 reference neighbors. +Run it from a repository checkout. + +```bash +vector-db-benchmark \ + --engines-file experiments/examples/cosmosdb-dbpedia-index-sweep.json \ + --datasets dbpedia-openai-975K-1536-angular-100neighbors +``` + +The adapter benchmarks dense, unfiltered top-K queries across the whole +container. It verifies the vector policy and corpus cardinality. The +`cosmosdb-flat` configuration supports up to 505 dimensions. The +`cosmosdb-diskann` configuration supports up to 4,096 dimensions and requires +at least 1,000 vectors. Cosmos DB can scan while an index builds or while a +physical partition has too few vectors. Before any DiskANN search, the adapter +therefore probes each feed range. It runs one forced scan and then repeats an +index-requested query, at first every 2 seconds and then less often up to +every 30 seconds, until that query costs less than half +the scan's request charge on every range. The run fails if a range does not +get there within 30 minutes. Set `AZURE_COSMOS_INDEX_PROBE_SECONDS` to change +the limit, or to `0` to skip the probe. Results record the per-range charges +and `vector_index_use_verified`, which is `false` when the probe was skipped. +The SDK reads further `AZURE_COSMOS_*` settings from the environment, such +as retry and connection options. Every such variable that is set is recorded +in the result's effective configuration, with the account key redacted. +After an upload, `upload_time` covers the upserts and `total_time` adds the +probe's wait for the index. The row-count check is in neither. See +[Azure Cosmos DB vector indexing](https://learn.microsoft.com/azure/cosmos-db/nosql/vector-search). + ### Filters an engine cannot express are a hard error (#219) @@ -1298,12 +1425,11 @@ vector-db-benchmark compare-results dumps/opensearch-hrrf dumps/redis-hrrf --ref `--skip-upload` asserts *"the server already holds the corpus I want"*. Three rules follow, and all three are enforced: -1. **The configure phase never runs.** `configure()` is destructive on **14 of the - 15 engines** — `FT.DROPINDEX … DD` (Redis), `FT.DROPINDEX` + `SCAN`/`UNLINK` +1. **The configure phase never runs.** `configure()` is destructive on 14 of the 16 engines — `FT.DROPINDEX … DD` (Redis), `FT.DROPINDEX` + `SCAN`/`UNLINK` (Valkey / Dragonfly / KiviDB), `collection.drop()` (MongoDB), `DROP TABLE` (pgvector), `DELETE /collections/` (Qdrant / Chroma / Weaviate / Milvus / Turbopuffer), `indices.delete` (Elasticsearch / OpenSearch), `DEL ` - (VectorSets). Only Vertex is non-destructive. Until #238 the + (VectorSets). Vertex and Cosmos DB preserve existing data during configuration. Until #238 the `--skip-upload --skip-vector-index` combination still called it: the flags that mean *"do not upload, do not build an index, just use what is there"* deleted the corpus and then benchmarked the empty index — printing a QPS number and diff --git a/experiments/configurations/cosmosdb.json b/experiments/configurations/cosmosdb.json new file mode 100644 index 000000000..a9d06b1c8 --- /dev/null +++ b/experiments/configurations/cosmosdb.json @@ -0,0 +1,85 @@ +[ + { + "name": "cosmosdb-flat", + "engine": "cosmosdb", + "algorithm": "flat", + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann", + "engine": "cosmosdb", + "algorithm": "diskANN", + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 1 } }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 5 } }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 10 } }, + { "top": 10, "parallel": 1, "search_params": { "search_list_size_multiplier": 20 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 1 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 5 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 10 } }, + { "top": 10, "parallel": 8, "search_params": { "search_list_size_multiplier": 20 } } + ] + }, + { + "name": "cosmosdb-diskann-build-list50-bytes32", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 50, "quantization_byte_size": 32 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list100-bytes32", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 100, "quantization_byte_size": 32 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list200-bytes32", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 200, "quantization_byte_size": 32 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list100-bytes16", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 100, "quantization_byte_size": 16 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + }, + { + "name": "cosmosdb-diskann-build-list100-bytes64", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { "indexing_search_list_size": 100, "quantization_byte_size": 64 }, + "upload_params": { "parallel": 8 }, + "search_params": [ + { "top": 10, "parallel": 1 }, + { "top": 10, "parallel": 8 } + ] + } +] diff --git a/experiments/examples/cosmosdb-dbpedia-index-sweep.json b/experiments/examples/cosmosdb-dbpedia-index-sweep.json new file mode 100644 index 000000000..937016dd4 --- /dev/null +++ b/experiments/examples/cosmosdb-dbpedia-index-sweep.json @@ -0,0 +1,206 @@ +[ + { + "name": "cosmosdb-diskann-10000ru-buildlist-50-qbytes-256", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 50, + "quantization_byte_size": 256 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + }, + { + "name": "cosmosdb-diskann-10000ru-buildlist-200-qbytes-256", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 200, + "quantization_byte_size": 256 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + }, + { + "name": "cosmosdb-diskann-10000ru-buildlist-100-qbytes-128", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 100, + "quantization_byte_size": 128 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + }, + { + "name": "cosmosdb-diskann-10000ru-buildlist-100-qbytes-512", + "engine": "cosmosdb", + "algorithm": "diskANN", + "collection_params": { + "ru_per_second": 10000, + "indexing_search_list_size": 100, + "quantization_byte_size": 512 + }, + "upload_params": { + "parallel": 16 + }, + "search_params": [ + { + "top": 100, + "parallel": 1 + }, + { + "top": 100, + "parallel": 100 + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 1 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 5 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 10 + } + }, + { + "top": 100, + "parallel": 100, + "search_params": { + "search_list_size_multiplier": 20 + } + } + ] + } +] diff --git a/src/bin/vector_db_benchmark/config.rs b/src/bin/vector_db_benchmark/config.rs index efe434620..dfefa8cf8 100644 --- a/src/bin/vector_db_benchmark/config.rs +++ b/src/bin/vector_db_benchmark/config.rs @@ -2136,6 +2136,7 @@ mod shipped_config_knob_guard { ("kividb", &["kividb.rs"]), ("vertex", &["vertex.rs", "vertex_grpc.rs"]), ("chroma", &["chroma.rs"]), + ("cosmosdb", &["cosmosdb.rs"]), ]; /// Top-level `search_params[]` knobs consumed by the runner diff --git a/src/bin/vector_db_benchmark/effective_config.rs b/src/bin/vector_db_benchmark/effective_config.rs index 2d260c06e..b73b1a8ba 100644 --- a/src/bin/vector_db_benchmark/effective_config.rs +++ b/src/bin/vector_db_benchmark/effective_config.rs @@ -108,6 +108,7 @@ const SECRET_MARKERS: &[&str] = &[ // and this one did not. "PWD", "API_KEY", + "AZURE_COSMOS_KEY", "APIKEY", "ACCESS_KEY", "PRIVATE_KEY", @@ -187,6 +188,12 @@ const DECLARED_KEY_ALLOWLIST: &[&str] = &[ "type", "number_of_shards", "is_tenant", + // Cosmos DB DiskANN + "indexing_search_list_size", + "quantization_byte_size", + "search_list_size_multiplier", + "quantized_vector_list_multiplier", + "partition_concurrency", // SVS-VAMANA "svs-vamana_config", "GRAPH_MAX_DEGREE", @@ -208,6 +215,7 @@ const DECLARED_KEY_ALLOWLIST: &[&str] = &[ "default_segment_number", // upload "parallel", + "ru_per_second", "batch_size", "timeout", // a single-letter key that a shipped config genuinely uses @@ -1039,6 +1047,23 @@ pub fn env_opt(name: &str) -> Option { value } +/// Record every set environment variable whose name starts with `prefix`. +/// +/// For a client library that reads its own settings from the environment +/// (transport, retry and pool options the adapter never names). They change +/// what a run measures, so they belong in the artifact like any other knob. +/// Secrets are redacted by name as usual. +pub fn observe_env_prefix(prefix: &str) { + let mut names: Vec = std::env::vars_os() + .filter_map(|(name, _)| name.into_string().ok()) + .filter(|name| name.starts_with(prefix)) + .collect(); + names.sort(); + for name in names { + let _ = env_opt(&name); + } +} + /// Record a resolved knob that did not come straight from one environment /// variable — a value chosen between the environment, the configuration file /// and a built-in default, or read back off the server. @@ -2200,6 +2225,22 @@ mod tests { } } + #[test] + fn a_prefix_scan_records_settings_the_adapter_never_names() { + let _l = test_lock(); + let _a = EnvGuard::set("VDBB_PREFIX_TEST_POOL_SIZE", Some("32")); + let _b = EnvGuard::set("VDBB_PREFIX_TEST_API_KEY", Some("prefix-secret")); + let _c = EnvGuard::set("VDBB_OTHER_TEST_SETTING", Some("ignored")); + reset(); + observe_env_prefix("VDBB_PREFIX_TEST_"); + let s = snapshot(); + assert_eq!(s["env"]["VDBB_PREFIX_TEST_POOL_SIZE"], "32"); + assert_eq!(s["effective"]["VDBB_PREFIX_TEST_POOL_SIZE"], "32"); + assert_eq!(s["env"]["VDBB_PREFIX_TEST_API_KEY"], REDACTED); + assert!(!serde_json::to_string(&s).unwrap().contains("prefix-secret")); + assert!(s["env"].get("VDBB_OTHER_TEST_SETTING").is_none()); + } + #[test] fn env_parsed_records_the_number_the_caller_received() { let _l = test_lock(); @@ -2372,12 +2413,19 @@ mod tests { ("REDIS_AUTH", "s3cret"), ("VERTEX_ACCESS_TOKEN", "ya29.xyz"), ("MONGODB_PASSWORD", "pw"), + ("AZURE_COSMOS_KEY", "synthetic-cosmos-key"), ] { r.observe_env(name, Some(raw)); } let s = r.snapshot(); let serialized = serde_json::to_string(&s).unwrap(); - for leak in ["hunter2", "sk-abc", "s3cret", "ya29.xyz"] { + for leak in [ + "hunter2", + "sk-abc", + "s3cret", + "ya29.xyz", + "synthetic-cosmos-key", + ] { assert!( !serialized.contains(leak), "{leak} leaked into the artifact" @@ -2385,6 +2433,7 @@ mod tests { } // …but the fact that authentication was configured survives. assert_eq!(s["env"]["ELASTIC_PASSWORD"], REDACTED); + assert_eq!(s["env"]["AZURE_COSMOS_KEY"], REDACTED); } #[test] diff --git a/src/bin/vector_db_benchmark/engine/cosmosdb.rs b/src/bin/vector_db_benchmark/engine/cosmosdb.rs new file mode 100644 index 000000000..558d99ab9 --- /dev/null +++ b/src/bin/vector_db_benchmark/engine/cosmosdb.rs @@ -0,0 +1,840 @@ +//! Azure Cosmos DB for NoSQL adapter using the official Rust SDK. + +use std::num::NonZeroU32; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::{Duration, Instant}; + +use azure_data_cosmos::feed::FeedRange; +use azure_data_cosmos::models::{ + ContainerProperties, ThroughputProperties, VectorDistanceFunction, VectorIndexType, +}; +use azure_data_cosmos::options::{ + ContentResponseOnWrite, CreateContainerOptions, MaxItemCountHint, OperationOptionsBuilder, + QueryOptions, ReadFeedRangesOptions, +}; +use azure_data_cosmos::{ + AccountReference, ContainerClient, CosmosClient, CosmosRuntime, FeedScope, Query, + RoutingStrategy, +}; +use futures::{stream, StreamExt, TryStreamExt}; +use serde::Serialize; +use tokio::sync::Semaphore; +use vector_db_benchmark::cosmosdb::{ + await_index_use, container_name, container_policy, count_stable_ranges, fan_out_hits, + index_probe_deadline, index_type, knn_query, manual_throughput, parity_sample_indices, + positive_u32, reject_unknown_keys, retry_throttled, sample_parity, scan_query, upload_parallel, + validate_corpus_size, validate_hit_count, validate_query_vector, validate_top, + vector_distance_function, verify_policy, DiskAnnBuildOptions, IndexProbe, KnnQueryText, + ParityTally, PartitionConcurrency, PartitionFanout, ProbeQuery, QueryHit, SearchDiagnostics, +}; +use vector_db_benchmark::start_gate::WorkerPool; +use vector_db_benchmark::workload::IndexLayout; + +use super::{compute_search_stats, CorpusCount, Engine, SearchResults, UploadStats}; +use crate::config::{EngineConfig, SearchParams}; +use crate::dataset::Dataset; + +#[derive(Clone, Copy, Serialize)] +struct Item<'a> { + id: &'a str, + embedding: &'a [f32], +} + +const SEARCH_LIST_SIZE: &str = "search_list_size_multiplier"; +const QUANTIZED_VECTOR_LIST: &str = "quantized_vector_list_multiplier"; +const PARTITION_CONCURRENCY: &str = "partition_concurrency"; +const SEARCH_KNOBS: [&str; 3] = [ + SEARCH_LIST_SIZE, + QUANTIZED_VECTOR_LIST, + PARTITION_CONCURRENCY, +]; +const COLLECTION_KEYS: [&str; 3] = [ + "ru_per_second", + "indexing_search_list_size", + "quantization_byte_size", +]; +const UNSUPPORTED_SEARCH_KNOB: &str = + "Cosmos DB supports only DiskANN search multipliers and partition_concurrency as tuning knobs"; +const UNSUPPORTED_COLLECTION_KEY: &str = + "Cosmos DB collection_params supports only ru_per_second, indexing_search_list_size and quantization_byte_size"; +/// Sampled queries compared across routing modes before a fan-out search. +const PARITY_SAMPLES: usize = 8; +/// Feed ranges whose IDs are counted at once. +const ROW_COUNT_CONCURRENCY: usize = 16; +/// TOP K of the index-use probe queries. +const INDEX_PROBE_TOP: usize = 10; +/// Feed ranges probed at once. +const INDEX_PROBE_CONCURRENCY: usize = 16; +/// Sampled parity queries in flight at once. +const PARITY_CONCURRENCY: usize = 4; + +fn positive_knob(params: &SearchParams, key: &str) -> Result, String> { + params + .knob(key) + .map(|value| positive_u32(value, key)) + .transpose() +} + +fn diskann_multiplier( + params: &SearchParams, + kind: &VectorIndexType, + key: &str, +) -> Result, String> { + let value = positive_knob(params, key)?; + if value.is_some() && *kind != VectorIndexType::DiskANN { + return Err(format!("Cosmos DB {key} requires DiskANN")); + } + Ok(value) +} + +pub struct CosmosDbEngine { + name: String, + database: String, + container_name: String, + index_type: VectorIndexType, + build_options: DiskAnnBuildOptions, + upload_parallel: usize, + search_params: Vec, + throughput: Option, + client: CosmosClient, + rt: tokio::runtime::Runtime, + container: Option, + created_container: bool, + verified_policy: Option, + /// `None` skips the DiskANN index-use probe. + index_probe_deadline: Option, + /// Per-range probe results once every range served from its index. + index_probes: Option>, + index_wait_seconds: Option, +} + +impl CosmosDbEngine { + pub fn new(config: &EngineConfig, dataset: &str) -> Result { + let kind = index_type(config.algorithm.as_deref())?; + let search_params = config.search_params.clone().unwrap_or_default(); + for params in &search_params { + if let Some(top) = params.top { + validate_top(usize::try_from(top).map_err(|_| "Cosmos DB top must be positive")?)?; + } + if params.calibration_param.is_some() + || params.num_candidates.is_some() + || params + .search_params + .as_ref() + .is_some_and(|p| p.ef.is_some()) + { + return Err(UNSUPPORTED_SEARCH_KNOB.into()); + } + let nested = params.search_params.as_ref().and_then(|p| p.extra.as_ref()); + for extra in [params.extra.as_ref(), nested].into_iter().flatten() { + reject_unknown_keys(extra.keys(), &SEARCH_KNOBS, UNSUPPORTED_SEARCH_KNOB)?; + } + diskann_multiplier(params, &kind, SEARCH_LIST_SIZE)?; + diskann_multiplier(params, &kind, QUANTIZED_VECTOR_LIST)?; + PartitionConcurrency::from_config(params.knob(PARTITION_CONCURRENCY))?; + } + let collection = config.collection_params.as_ref(); + if collection.is_some_and(|p| p.hnsw_config.is_some() || p.index_options.is_some()) { + return Err(UNSUPPORTED_COLLECTION_KEY.into()); + } + if let Some(extra) = collection.and_then(|p| p.extra.as_ref()) { + reject_unknown_keys(extra.keys(), &COLLECTION_KEYS, UNSUPPORTED_COLLECTION_KEY)?; + } + let collection_option = |key| { + collection + .and_then(|p| p.extra.as_ref()) + .and_then(|p| p.get(key)) + }; + let build_options = DiskAnnBuildOptions::from_config( + &kind, + collection_option("indexing_search_list_size"), + collection_option("quantization_byte_size"), + )?; + let upload_parallel = upload_parallel(config.upload_params.as_ref())?; + let index_probe_deadline = index_probe_deadline( + crate::effective_config::env_opt("AZURE_COSMOS_INDEX_PROBE_SECONDS").as_deref(), + )?; + + let endpoint = crate::effective_config::env_opt("AZURE_COSMOS_ENDPOINT") + .ok_or("Set AZURE_COSMOS_ENDPOINT for the Cosmos DB engine")? + .parse() + .map_err(|_| "AZURE_COSMOS_ENDPOINT must be a Cosmos DB account URL")?; + // The configuration recorder redacts this variable. + let key = crate::effective_config::env_opt("AZURE_COSMOS_KEY") + .filter(|key| !key.is_empty()) + .ok_or("Set AZURE_COSMOS_KEY for the Cosmos DB engine")?; + let database = crate::effective_config::env_or("AZURE_COSMOS_DATABASE", "vdb-benchmark"); + let regions = crate::effective_config::env_opt("AZURE_COSMOS_REGION") + .into_iter() + .map(Into::into) + .collect(); + // The SDK reads its own retry, transport and pool settings from the + // environment when the runtime starts. Record all of them. + crate::effective_config::observe_env_prefix("AZURE_COSMOS_"); + let throughput = manual_throughput( + collection_option("ru_per_second"), + &crate::effective_config::env_or("AZURE_COSMOS_RU_PER_SECOND", "400"), + )?; + let rt = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?; + let account = AccountReference::with_authentication_key(endpoint, key); + let client = rt + .block_on(async { + let runtime = CosmosRuntime::builder().build().await?; + CosmosClient::builder() + .with_runtime(runtime) + .with_default_operation_options( + OperationOptionsBuilder::new() + .with_content_response_on_write(ContentResponseOnWrite::Disabled) + .build(), + ) + .build(account, RoutingStrategy::PreferredRegions(regions)) + .await + }) + .map_err(|e| format!("Cosmos DB connection failed: {e}"))?; + + Ok(Self { + name: config.name.clone(), + database, + container_name: container_name(&config.name, dataset), + index_type: kind, + build_options, + upload_parallel, + search_params, + throughput, + client, + rt, + container: None, + created_container: false, + verified_policy: None, + index_probe_deadline, + index_probes: None, + index_wait_seconds: None, + }) + } + + fn open_container(&mut self) -> Result<&ContainerClient, String> { + if self.container.is_none() { + self.container = Some( + self.rt + .block_on( + self.client + .database_client(&self.database) + .container_client(&self.container_name, None), + ) + .map_err(|e| { + format!( + "Could not open Cosmos DB container '{}': {e}", + self.container_name + ) + })?, + ); + } + self.container + .as_ref() + .ok_or_else(|| "Cosmos DB container is unavailable".into()) + } + + fn policy(&self, dataset: &Dataset) -> Result { + container_policy( + self.container_name.clone(), + self.index_type.clone(), + dataset.distance(), + dataset.vector_size(), + self.build_options, + ) + } + + /// Write every row and check the stored row count. Returns the write + /// time and the row count. + fn upload_corpus(&mut self, dataset: &Dataset) -> Result<(f64, usize), String> { + let (ids, vectors, _) = dataset.read_vectors(dataset.needs_normalization())?; + validate_corpus_size(&self.index_type, ids.len() as u64)?; + let dimensions = dataset.vector_size() as usize; + if vectors + .iter() + .any(|vector| vector.len() != dimensions || vector.iter().any(|v| !v.is_finite())) + { + return Err( + "Cosmos DB corpus contains a vector with invalid dimensions or values".into(), + ); + } + let container = self + .container + .as_ref() + .ok_or("Cosmos DB container is unavailable")?; + let throttle_retries = AtomicUsize::new(0); + let upload_start = Instant::now(); + self.rt.block_on(async { + let throttle_retries = &throttle_retries; + stream::iter(ids.iter().zip(&vectors)) + .map(|(id, vector)| async move { + let id = id.to_string(); + let item = Item { + id: &id, + embedding: vector, + }; + // The SDK gives up on a throttled write after its own + // retries. Back off and send it again. + retry_throttled( + || container.upsert_item(id.clone(), &id, item, None), + |error| u16::from(error.status().status_code()) == 429, + tokio::time::sleep, + throttle_retries, + ) + .await + .map_err(|e| format!("Cosmos DB upload failed for row {id}: {e}")) + }) + .buffer_unordered(self.upload_parallel) + .try_for_each(|_| async { Ok(()) }) + .await + })?; + let upload_time = upload_start.elapsed().as_secs_f64(); + let upload_count = ids.len(); + drop((ids, vectors)); + let throttle_retries = throttle_retries.into_inner(); + if throttle_retries > 0 { + eprintln!( + "Cosmos DB upload at parallel={} resent {throttle_retries} throttled writes after the SDK's own HTTP 429 retries. Upload time includes that backoff. Raise the container's RU/s or lower upload_params.parallel to avoid it.", + self.upload_parallel + ); + } + // The row-count scan is verification, so it stays out of both times. + let count = self + .corpus_row_count()? + .ok_or("Cosmos DB did not return a row count")?; + if count.rows != upload_count as u64 { + return Err(format!( + "Cosmos DB stored {} rows of {upload_count} uploaded", + count.rows + )); + } + Ok((upload_time, upload_count)) + } + + /// Wait until every feed range answers an index-requested query for + /// clearly less than a forced scan of that range costs. Azure scans while + /// a DiskANN index builds or a range holds too few vectors, so a matching + /// row count does not show that searches will use the index. + fn verify_index_use(&mut self, dataset: &Dataset) -> Result<(), String> { + if self.index_type != VectorIndexType::DiskANN || self.index_probes.is_some() { + return Ok(()); + } + let Some(deadline) = self.index_probe_deadline else { + eprintln!("Cosmos DB DiskANN index-use probe skipped (AZURE_COSMOS_INDEX_PROBE_SECONDS=0). Azure may answer from a scan."); + return Ok(()); + }; + let (vectors, _, conditions) = dataset.read_queries()?; + conditions.try_resolve_all("Cosmos DB", refuse_filter)?; + let vector = vectors + .first() + .ok_or("Cosmos DB needs a query vector for the index-use probe")?; + let scan = scan_query(vector, INDEX_PROBE_TOP)?; + let indexed = knn_query(vector, INDEX_PROBE_TOP, None, None, false)?; + let container = self.open_container()?.clone(); + let rt = &self.rt; + let ranges = rt.block_on(read_search_ranges(&container))?; + let started = Instant::now(); + // One forced scan per range is the costly part, so the ranges run + // concurrently. + let probes = await_index_use( + ranges.len(), + deadline, + |query, pending| { + let query = match query { + ProbeQuery::ForcedScan => &scan, + ProbeQuery::IndexRequested => &indexed, + }; + rt.block_on(probe_charges(&container, query, &ranges, pending)) + }, + || started.elapsed(), + std::thread::sleep, + )?; + if rt.block_on(read_search_ranges(&container))? != ranges { + return Err("Cosmos DB feed ranges changed during the index-use probe".into()); + } + let waited = started.elapsed().as_secs_f64(); + println!( + "Cosmos DB DiskANN index use verified on {} feed ranges after {waited:.1}s", + probes.len() + ); + self.index_wait_seconds = Some(waited); + self.index_probes = Some(probes); + Ok(()) + } +} + +impl Engine for CosmosDbEngine { + fn name(&self) -> &str { + &self.name + } + + fn search_params(&self) -> &[SearchParams] { + &self.search_params + } + + fn configure(&mut self, dataset: &Dataset) -> Result<(), String> { + let policy = self.policy(dataset)?; + if let Some(rows) = dataset + .config + .vector_count + .and_then(|n| u64::try_from(n).ok()) + { + validate_corpus_size(&self.index_type, rows)?; + } + // Reject filter conditions before provisioning or uploading a corpus we + // cannot search correctly. Use the same guard in search-only runs. + let (_, _, conditions) = dataset.read_queries()?; + conditions.try_resolve_all("Cosmos DB", refuse_filter)?; + let database = self.client.database_client(&self.database); + self.rt.block_on(async { + match database.read(None).await { + Ok(_) => {} + Err(e) if u16::from(e.status().status_code()) == 404 => { + match self.client.create_database(&self.database, None).await { + Ok(_) => {} + Err(e) if u16::from(e.status().status_code()) == 409 => {} + Err(e) => return Err(format!("Could not create Cosmos DB database: {e}")), + } + } + Err(e) => return Err(format!("Could not read Cosmos DB database: {e}")), + } + let options = self.throughput.map(|rus| { + CreateContainerOptions::default().with_throughput(ThroughputProperties::manual(rus)) + }); + match database.create_container(policy, options).await { + Ok(_) => Ok(()), + Err(e) if u16::from(e.status().status_code()) == 409 => Err(format!( + "Cosmos DB container '{}' already exists. An earlier failed run may have left it provisioned. Use --skip-upload for a complete corpus, or remove the container explicitly before uploading.", + self.container_name + )), + Err(e) => Err(format!("Could not create Cosmos DB container: {e}")), + } + })?; + self.created_container = true; + println!("Created Cosmos DB container '{}'", self.container_name); + self.open_container()?; + Ok(()) + } + + fn upload(&mut self, dataset: &Dataset) -> Result { + // A partial corpus is of no use to a later run, and the container + // bills until it is removed. A complete corpus is kept below even if + // the index probe then fails, because `--skip-upload` can reuse it. + let (upload_time, upload_count) = match self.upload_corpus(dataset) { + Ok(uploaded) => uploaded, + Err(error) if self.created_container => { + let error = error.trim_end_matches('.'); + return Err(match self.delete() { + Ok(()) => format!( + "{error}. Removed container '{}', which this run created.", + self.container_name + ), + Err(delete) => format!( + "{error}. Container '{}' is still provisioned and billing: {delete}", + self.container_name + ), + }); + } + Err(error) => return Err(error), + }; + // For DiskANN, total time adds the wait until the index serves queries. + self.verify_index_use(dataset).map_err(|error| { + format!("{error} The uploaded corpus is complete and was kept. Rerun with --skip-upload to search it.") + })?; + Ok(UploadStats { + upload_time, + total_time: upload_time + self.index_wait_seconds.unwrap_or(0.0), + upload_count, + parallel: self.upload_parallel, + batch_size: 1, + ..UploadStats::default() + }) + } + + fn corpus_row_count(&mut self) -> Result, String> { + let container = self.open_container()?.clone(); + // SDK 1.0 rejects aggregates in every feed-range and full-container + // query, so count the ID projection page by page. That keeps memory + // bounded and never reads the stored vectors. The SDK drains a + // full-container query one range at a time, so count the disjoint + // feed ranges concurrently instead. + let count = self.rt.block_on(count_stable_ranges( + || read_search_ranges(&container), + ROW_COUNT_CONCURRENCY, + |range| count_ids(&container, FeedScope::range(range)), + ))?; + validate_corpus_size(&self.index_type, count)?; + Ok(Some(CorpusCount::exact(count))) + } + + fn check_index_layout(&mut self, dataset: &Dataset, layout: IndexLayout) -> Result<(), String> { + if layout != IndexLayout::Vector { + return Err("Cosmos DB requires the vector index layout".into()); + } + let expected = self.policy(dataset)?; + let container = self.open_container()?.clone(); + let actual = self + .rt + .block_on(container.read(None)) + .and_then(|r| r.into_model()) + .map_err(|e| format!("Could not read Cosmos DB container policy: {e}"))?; + verify_policy(&actual, &expected)?; + if let Some(rus) = self.throughput { + let offer = self + .rt + .block_on(container.read_throughput(None)) + .map_err(|e| format!("Could not read Cosmos DB container throughput: {e}"))?; + if !offer + .is_some_and(|t| t.throughput() == Some(rus) && t.autoscale_maximum().is_none()) + { + return Err(format!( + "Cosmos DB container does not have {rus} manual RU/s" + )); + } + } + self.verified_policy = Some(actual); + self.verify_index_use(dataset) + } + + fn search( + &mut self, + dataset: &Dataset, + params: &SearchParams, + num_queries: i64, + ) -> Result { + let (vectors, neighbors, conditions) = dataset.read_queries()?; + conditions.try_resolve_all("Cosmos DB", refuse_filter)?; + let limit = if num_queries > 0 { + (num_queries as usize).min(vectors.len()) + } else { + vectors.len() + }; + if limit == 0 || vectors.len() != neighbors.len() { + return Err("Cosmos DB needs a nonempty query set with aligned ground truth".into()); + } + let parallel = params.parallel.unwrap_or(1).max(1) as usize; + let top = match params.top { + Some(t) => usize::try_from(t).map_err(|_| "Cosmos DB top must be positive")?, + None => neighbors[0].len(), + }; + let search_list_size = diskann_multiplier(params, &self.index_type, SEARCH_LIST_SIZE)?; + let quantized_vector_list = + diskann_multiplier(params, &self.index_type, QUANTIZED_VECTOR_LIST)?; + let partition_concurrency = + PartitionConcurrency::from_config(params.knob(PARTITION_CONCURRENCY))?; + + // Validate every vector before timing. Workers bind each one to the + // shared query text as they send it. + let dimensions = dataset.vector_size() as usize; + for vector in &vectors[..limit] { + validate_query_vector(vector, dimensions)?; + } + let stock_text = KnnQueryText::new(top, search_list_size, quantized_vector_list, false)?; + let text = KnnQueryText::new( + top, + search_list_size, + quantized_vector_list, + partition_concurrency.is_some(), + )?; + let text = &text; + let container = self + .container + .as_ref() + .ok_or("Cosmos DB container is unavailable")?; + let rt = &self.rt; + let partition_search = partition_concurrency.map(|concurrency| { + let ranges = rt.block_on(read_search_ranges(container))?; + let per_query_concurrency = concurrency.resolve(ranges.len()); + let shared_request_limit = parallel.max(per_query_concurrency); + let summary = PartitionFanout { + feed_ranges: ranges.len(), + per_query_concurrency, + shared_request_limit, + parity: ParityTally::default(), + }; + let mut search = PartitionSearch { + ranges, + summary, + metric: vector_distance_function(dataset.distance())?, + permits: Semaphore::new(shared_request_limit), + }; + let parity = rt.block_on(sample_parity( + parity_sample_indices(limit, PARITY_SAMPLES), + PARITY_CONCURRENCY, + |index| parity_sample(container, &stock_text, text, &vectors[index], top, &search), + ))?; + search.summary.parity = parity; + println!( + "Cosmos DB feed-range search verified against SDK on {} sampled queries: {} ranges, {} per query, {} in flight; fan-out missed {} stock IDs, repeated stock requests missed {}", + parity.samples, summary.feed_ranges, summary.per_query_concurrency, summary.shared_request_limit, + parity.cross_mode_missing_ids, parity.stock_repeat_missing_ids + ); + Ok::<_, String>(search) + }).transpose()?; + let partition_search = partition_search.as_ref(); + let next = AtomicUsize::new(0); + let (samples, elapsed) = std::thread::scope(|scope| { + let mut pool = WorkerPool::new(scope, "cosmosdb-search", parallel); + for worker in 0..parallel { + let vectors = &vectors[..limit]; + let neighbors = &neighbors; + let next = &next; + pool.spawn(move |ticket| { + let mut samples = Vec::new(); + // Warm the worker's first request outside the measured window. + if let Err(error) = rt.block_on(search_hits( + container, + text, + &vectors[worker % limit], + top, + partition_search, + )) { + ticket.fail(format!("Cosmos DB worker {worker} warm-up failed: {error}")); + return Ok(samples); + } + if ticket.arrive_and_wait().is_none() { + return Ok(samples); + } + loop { + let idx = next.fetch_add(1, Ordering::Relaxed); + let Some(vector) = vectors.get(idx) else { + break; + }; + let started = Instant::now(); + let (hits, diagnostics) = rt + .block_on(search_hits(container, text, vector, top, partition_search)) + .map_err(|e| format!("Cosmos DB query {idx} failed: {e}"))?; + let latency = started.elapsed().as_secs_f64(); + let ids = hits + .iter() + .map(|hit| hit.id.parse::()) + .collect::, _>>() + .map_err(|e| { + format!("Cosmos DB query {idx} returned a non-integer id: {e}") + })?; + let metrics = vector_db_benchmark::metrics::compute_metrics( + &ids, + &neighbors[idx], + top, + ); + samples.push((latency, metrics, diagnostics)); + } + Ok::<_, String>(samples) + })?; + } + let (batches, start) = pool.start()?; + Ok::<_, String>((batches, start.elapsed().as_secs_f64())) + })?; + if let Some(search) = partition_search { + if rt.block_on(read_search_ranges(container))? != search.ranges { + return Err("Cosmos DB feed ranges changed during the measured search".into()); + } + } + let (mut times, mut precision, mut recall, mut mrr, mut ndcg) = + (Vec::new(), Vec::new(), Vec::new(), Vec::new(), Vec::new()); + let mut diagnostics = SearchDiagnostics::default(); + for batch in samples { + for (latency, m, query_diagnostics) in batch? { + times.push(latency); + precision.push(m.precision); + recall.push(m.recall); + mrr.push(m.mrr); + ndcg.push(m.ndcg); + diagnostics.add_query(query_diagnostics); + } + } + diagnostics.partition_fanout = partition_search.map(|search| search.summary); + if diagnostics.http_429_attempts > 0 { + eprintln!( + "Cosmos DB search at parallel={parallel} observed {} HTTP 429 responses. Latency includes SDK backoff and retries.", + diagnostics.http_429_attempts + ); + } + let mut results = compute_search_stats( + ×, &precision, &recall, &mrr, &ndcg, elapsed, top, parallel, limit, + )?; + results.cosmosdb_search_diagnostics = Some(diagnostics); + Ok(results) + } + + fn server_metadata(&mut self) -> Option { + let policy = self.verified_policy.as_ref()?; + Some(serde_json::json!({ + "cosmosdb": { + "database": self.database, + "container": self.container_name, + "partition_key": policy.partition_key, + "vector_embedding_policy": policy.vector_embedding_policy, + "vector_indexing_policy": policy.indexing_policy, + "manual_ru_per_second": self.throughput, + "vector_index_use_verified": self.index_probes.is_some(), + "vector_index_use_probes": self.index_probes, + "vector_index_wait_seconds": self.index_wait_seconds, + } + })) + } + + fn delete(&mut self) -> Result<(), String> { + if self.created_container { + let container = self + .container + .as_ref() + .ok_or("Cosmos DB container is unavailable")?; + self.rt + .block_on(container.delete(None)) + .map_err(|e| format!("Could not delete Cosmos DB container: {e}"))?; + self.created_container = false; + self.container = None; + } + Ok(()) + } +} + +struct PartitionSearch { + ranges: Vec, + summary: PartitionFanout, + metric: VectorDistanceFunction, + permits: Semaphore, +} + +/// One sampled query under both routing modes. The stock query runs twice +/// because repeated DiskANN requests can differ at the cutoff, and that +/// variation is the bar the fan-out result is held to. +async fn parity_sample( + container: &ContainerClient, + stock_text: &KnnQueryText, + partition_text: &KnnQueryText, + vector: &[f32], + top: usize, + search: &PartitionSearch, +) -> Result<[Vec; 3], String> { + let ((stock, _), (stock_repeat, _), (partitioned, _)) = futures::try_join!( + search_hits(container, stock_text, vector, top, None), + search_hits(container, stock_text, vector, top, None), + search_hits(container, partition_text, vector, top, Some(search)), + )?; + Ok([stock, stock_repeat, partitioned]) +} + +/// Request charge of `query` on each listed feed range, as (range index, RU). +async fn probe_charges( + container: &ContainerClient, + query: &Query, + ranges: &[FeedRange], + indices: &[usize], +) -> Result, String> { + stream::iter(indices.iter().copied()) + .map(|feed_range| async move { + let scope = FeedScope::range(ranges[feed_range].clone()); + query_hits(container, query.clone(), INDEX_PROBE_TOP, scope) + .await + .map(|(_, diagnostics)| (feed_range, diagnostics.request_charge_ru)) + .map_err(|e| format!("Cosmos DB index-use probe failed: {e}")) + }) + .buffer_unordered(INDEX_PROBE_CONCURRENCY) + .try_collect() + .await +} + +async fn count_ids(container: &ContainerClient, scope: FeedScope) -> Result { + let options = QueryOptions::default().with_max_item_count(MaxItemCountHint::ServerDecides); + container + .query_items::("SELECT VALUE c.id FROM c", scope, Some(options)) + .await + .map_err(|e| format!("Cosmos DB row count failed: {e}"))? + .into_pages() + .try_fold(0_u64, |count, page| async move { + Ok(count + page.items().len() as u64) + }) + .await + .map_err(|e| format!("Cosmos DB row count failed: {e}")) +} + +async fn read_search_ranges(container: &ContainerClient) -> Result, String> { + let ranges = container + .read_feed_ranges(Some( + ReadFeedRangesOptions::default().with_force_refresh(true), + )) + .await + .map_err(|e| format!("Could not read Cosmos DB feed ranges: {e}"))?; + if ranges.is_empty() { + return Err("Cosmos DB returned no feed ranges".into()); + } + Ok(ranges) +} + +async fn search_hits( + container: &ContainerClient, + text: &KnnQueryText, + vector: &[f32], + top: usize, + partition_search: Option<&PartitionSearch>, +) -> Result<(Vec, SearchDiagnostics), String> { + let Some(search) = partition_search else { + return query_hits( + container, + text.bind(vector)?, + top, + FeedScope::full_container(), + ) + .await; + }; + fan_out_hits( + &search.ranges, + search.summary.per_query_concurrency, + &search.permits, + top, + &search.metric, + |range| async move { + query_hits(container, text.bind(vector)?, top, FeedScope::range(range)).await + }, + ) + .await +} + +async fn query_hits( + container: &ContainerClient, + query: Query, + top: usize, + scope: FeedScope, +) -> Result<(Vec, SearchDiagnostics), String> { + let mut pages = container + .query_items::(query, scope, None) + .await + .map_err(|error| error.to_string())? + .into_pages(); + let mut hits = Vec::new(); + let mut combined = SearchDiagnostics::default(); + // Enforce TOP K within the given scope across all SDK pages. + while let Some(page) = pages.try_next().await.map_err(|error| error.to_string())? { + validate_hit_count(hits.len() + page.items().len(), top)?; + let diagnostics = page.diagnostics(); + let charge = diagnostics.total_request_charge().value(); + let attempts = diagnostics.request_count(); + if let Some(compacted) = diagnostics.compaction() { + combined.observe_page( + charge, + attempts, + compacted + .runs + .iter() + .map(|run| (u16::from(run.status.status_code()), run.count)), + ); + } else { + combined.observe_page( + charge, + attempts, + diagnostics + .requests() + .iter() + .map(|request| (u16::from(request.status().status_code()), 1)), + ); + } + hits.extend(page.into_items()); + } + Ok((hits, combined)) +} + +fn refuse_filter(_: &serde_json::Value) -> Result, String> { + Err("filtered vector search is unsupported".into()) +} diff --git a/src/bin/vector_db_benchmark/engine/filter_guard.rs b/src/bin/vector_db_benchmark/engine/filter_guard.rs index b4ac51536..1ff5bd0f3 100644 --- a/src/bin/vector_db_benchmark/engine/filter_guard.rs +++ b/src/bin/vector_db_benchmark/engine/filter_guard.rs @@ -1064,7 +1064,9 @@ fn every_engine_read_of_query_conditions_is_paired_with_a_resolver() { // and OpenSearch hybrid arms' `read_hybrid_query_set`; +2: the Redis // and OpenSearch hybrid-rrf arms' `read_queries`, each resolved with // `hybrid_rrf::refuse_conditions` (a declared condition is an error). - (28, 28), + // +2: Cosmos DB refuses declared filters before provisioning and search. + // +1: the Cosmos DB index-use probe reads a query vector, same refusal. + (31, 31), "the read/resolve census changed; confirm the new site is guarded" ); } diff --git a/src/bin/vector_db_benchmark/engine/mod.rs b/src/bin/vector_db_benchmark/engine/mod.rs index 30cc028dc..e65758479 100644 --- a/src/bin/vector_db_benchmark/engine/mod.rs +++ b/src/bin/vector_db_benchmark/engine/mod.rs @@ -7,6 +7,7 @@ //! - `Searcher` trait = BaseSearcher mod chroma; +mod cosmosdb; mod dragonfly; mod elasticsearch; #[cfg(test)] @@ -114,6 +115,8 @@ pub struct UpdateSearchRatio { #[allow(dead_code)] #[derive(Debug, Clone, Default)] pub struct SearchResults { + /// Charges and HTTP attempts reported by the Cosmos SDK for this search. + pub cosmosdb_search_diagnostics: Option, /// Metadata-only result contract and count checksums; absent on KNN runs. pub filter_only: Option, pub total_time: f64, @@ -1173,7 +1176,11 @@ pub fn resolved_number_of_shards( } /// Create an engine based on config -pub fn create_engine(engine_config: &EngineConfig, host: &str) -> Result, String> { +pub fn create_engine( + engine_config: &EngineConfig, + host: &str, + dataset: &str, +) -> Result, String> { let engine_type = engine_config.engine.as_deref().unwrap_or("unknown"); // HNSW SQ8 keys are typed on the shared HnswConfig and read only by Redis; @@ -1204,8 +1211,9 @@ pub fn create_engine(engine_config: &EngineConfig, host: &str) -> Result Ok(Box::new(KividbEngine::new(engine_config, host)?)), "vertex" => Ok(Box::new(VertexEngine::new(engine_config, host)?)), "chroma" => Ok(Box::new(ChromaEngine::new(engine_config, host)?)), + "cosmosdb" => Ok(Box::new(cosmosdb::CosmosDbEngine::new(engine_config, dataset)?)), other => Err(format!( - "Unsupported engine type: '{}'. Supported: 'redis', 'vectorsets', 'elasticsearch', 'opensearch', 'qdrant', 'weaviate', 'pgvector', 'milvus', 'mongodb', 'valkey', 'turbopuffer', 'dragonfly', 'kividb', 'vertex', 'chroma'.", + "Unsupported engine type: '{}'. Supported: 'redis', 'vectorsets', 'elasticsearch', 'opensearch', 'qdrant', 'weaviate', 'pgvector', 'milvus', 'mongodb', 'valkey', 'turbopuffer', 'dragonfly', 'kividb', 'vertex', 'chroma', 'cosmosdb'.", other )), } @@ -2281,7 +2289,7 @@ mod sq8_engine_guard_tests { "collection_params": { "hnsw_config": hnsw } })) .unwrap(); - match create_engine(&cfg, "localhost") { + match create_engine(&cfg, "localhost", "test") { Ok(_) => panic!("{engine} accepted HNSW SQ8 keys it does not read"), Err(e) => assert!(e.contains("read only by the redis engine"), "{engine}: {e}"), } diff --git a/src/bin/vector_db_benchmark/experiment.rs b/src/bin/vector_db_benchmark/experiment.rs index f0f5c33bf..bbce2005e 100644 --- a/src/bin/vector_db_benchmark/experiment.rs +++ b/src/bin/vector_db_benchmark/experiment.rs @@ -173,6 +173,7 @@ pub fn run(args: &Args) -> Result<(), String> { "kividb", "vertex", "chroma", + "cosmosdb", ]; let mut engines: Vec<_> = engine_configs .iter() @@ -748,7 +749,7 @@ pub fn run(args: &Args) -> Result<(), String> { ); // Create engine - let mut engine = create_engine(&engine_config, &args.host)?; + let mut engine = create_engine(&engine_config, &args.host, &dataset.config.name)?; // Shard count this run will be measured at, recorded in the result // files so it does not have to be inferred from the config name (#211). @@ -1648,9 +1649,9 @@ fn run_single_experiment( } else { // `--skip-upload` means: the server already holds the corpus I want — // do not create, drop, recreate or otherwise modify it. `configure()` is - // destructive on 14 of the 15 engines (FT.DROPINDEX ... DD, SCAN+UNLINK, + // destructive on 14 of the 16 engines (FT.DROPINDEX ... DD, SCAN+UNLINK, // collection.drop(), DROP TABLE, DELETE /collections/, indices.delete, - // DEL — only Vertex is not), so it must NOT run here under any flag + // DEL ), so it must NOT run here under any flag // combination. // // This used to have an `else if args.skip_vector_index` arm that called @@ -3108,6 +3109,9 @@ fn build_search_result_json( result["results"]["p90_time"] = json!(results.p90_time); result["results"]["schedule_delay_p90_time"] = json!(results.schedule_delay_p90_time); result["results"]["end_to_end_p90_time"] = json!(results.end_to_end_p90_time); + if let Some(diagnostics) = results.cosmosdb_search_diagnostics { + result["results"]["cosmosdb_search_diagnostics"] = json!(diagnostics); + } let gt_stats = ground_truth.and_then(|gt| gt.stats(results.top)); result["metrics_schema"] = metrics_schema(gt_stats.as_ref()); diff --git a/src/cosmosdb.rs b/src/cosmosdb.rs new file mode 100644 index 000000000..93a5a9891 --- /dev/null +++ b/src/cosmosdb.rs @@ -0,0 +1,1562 @@ +//! Request construction and policy validation for Azure Cosmos DB for NoSQL. + +use azure_data_cosmos::models::{ + ContainerProperties, IndexingMode, IndexingPolicy, VectorDataType, VectorDistanceFunction, + VectorEmbedding, VectorEmbeddingPolicy, VectorIndex, VectorIndexType, +}; +use azure_data_cosmos::Query; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine as _}; +use futures::{stream, StreamExt, TryStreamExt}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::future::Future; +use std::num::NonZeroU32; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; +use tokio::sync::Semaphore; + +const VECTOR_PATH: &str = "/embedding"; +const VECTOR_EXCLUSION: &str = "/embedding/*"; + +/// Resolve a Cosmos index type. Config names use the service's spelling. +pub fn index_type(algorithm: Option<&str>) -> Result { + match algorithm { + Some("flat") => Ok(VectorIndexType::Flat), + Some("diskANN") => Ok(VectorIndexType::DiskANN), + _ => Err("Cosmos DB algorithm must be flat or diskANN".into()), + } +} + +/// Separate containers by both benchmark config and dataset, even when names +/// contain punctuation that Cosmos forbids in resource IDs. +pub fn container_name(config: &str, dataset: &str) -> String { + let mut hash = Sha256::new(); + hash.update((config.len() as u64).to_le_bytes()); + hash.update(config); + hash.update(dataset); + format!("vdb-benchmark-{}", URL_SAFE_NO_PAD.encode(hash.finalize())) +} + +/// Optional DiskANN index-build settings. Absent fields leave policy defaults +/// to Azure, including its dimension-dependent quantization size. +#[derive(Debug, Clone, Copy, Default)] +pub struct DiskAnnBuildOptions { + indexing_search_list_size: Option, + quantization_byte_size: Option, +} + +impl DiskAnnBuildOptions { + /// Validate supplied collection parameters against the data-plane SDK's + /// limits. The dimension-dependent cap is checked when building the policy. + pub fn from_config( + kind: &VectorIndexType, + indexing_search_list_size: Option<&Value>, + quantization_byte_size: Option<&Value>, + ) -> Result { + if *kind != VectorIndexType::DiskANN + && (indexing_search_list_size.is_some() || quantization_byte_size.is_some()) + { + return Err("Cosmos DB collection index-build options require DiskANN".into()); + } + + let parse = |value: &Value, key: &str, min: u32, max: u32| { + value + .as_u64() + .and_then(|value| u32::try_from(value).ok()) + .filter(|value| (min..=max).contains(value)) + .ok_or_else(|| { + format!( + "Cosmos DB collection_params.{key} must be an integer from {min} to {max}" + ) + }) + }; + Ok(Self { + indexing_search_list_size: indexing_search_list_size + .map(|value| parse(value, "indexing_search_list_size", 25, 500)) + .transpose()?, + quantization_byte_size: quantization_byte_size + .map(|value| parse(value, "quantization_byte_size", 1, 512)) + .transpose()?, + }) + } +} + +/// Reject a config key the adapter does not read, so a typo cannot run as a +/// default and be reported under the declared name. +pub fn reject_unknown_keys<'a>( + keys: impl IntoIterator, + allowed: &[&str], + message: &str, +) -> Result<(), String> { + if keys.into_iter().any(|key| !allowed.contains(&key.as_str())) { + return Err(message.into()); + } + Ok(()) +} + +/// A tuning knob that must be a positive 32-bit integer. +pub fn positive_u32(value: &Value, key: &str) -> Result { + value + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .and_then(NonZeroU32::new) + .ok_or_else(|| format!("Cosmos DB {key} must be a positive 32-bit integer")) +} + +/// Manual RU/s for the container. A per-config value wins over the +/// environment setting. `None` means a serverless account. +pub fn manual_throughput(configured: Option<&Value>, env: &str) -> Result, String> { + if let Some(value) = configured { + return value + .as_u64() + .filter(|ru| *ru >= 400) + .map(Some) + .ok_or_else(|| { + "Cosmos DB collection_params.ru_per_second must be an integer of at least 400" + .into() + }); + } + if env == "serverless" { + return Ok(None); + } + env.parse::() + .ok() + .filter(|ru| *ru >= 400) + .map(Some) + .ok_or_else(|| "AZURE_COSMOS_RU_PER_SECOND must be at least 400, or serverless".into()) +} + +/// Concurrent single-item upserts. The adapter sends one item per request. +pub fn upload_parallel(params: Option<&Value>) -> Result { + const DEFAULT: usize = 8; + let Some(params) = params else { + return Ok(DEFAULT); + }; + let options = params + .as_object() + .ok_or("Cosmos DB upload_params must be an object")?; + reject_unknown_keys( + options.keys(), + &["parallel"], + "Cosmos DB upload_params supports only parallel (one item per request)", + )?; + match options.get("parallel") { + None => Ok(DEFAULT), + Some(parallel) => parallel + .as_u64() + .and_then(|p| usize::try_from(p).ok()) + .filter(|p| *p > 0) + .ok_or_else(|| "Cosmos DB upload parallel must be positive".into()), + } +} + +/// Cosmos DB metric corresponding to a benchmark corpus's distance. +pub fn vector_distance_function(distance: &str) -> Result { + match distance.to_ascii_lowercase().as_str() { + "cosine" | "angular" => Ok(VectorDistanceFunction::Cosine), + "l2" | "euclidean" => Ok(VectorDistanceFunction::Euclidean), + "dot" | "ip" => Ok(VectorDistanceFunction::DotProduct), + _ => Err(format!( + "Cosmos DB does not support dataset distance '{distance}'" + )), + } +} + +/// Use the dataset metric and dimension in the server-side policy. +pub fn container_policy( + name: String, + index_type: VectorIndexType, + distance: &str, + dimensions: i64, + build: DiskAnnBuildOptions, +) -> Result { + let distance = vector_distance_function(distance)?; + let max_dimensions = if index_type == VectorIndexType::Flat { + 505 + } else { + 4096 + }; + if !(1..=max_dimensions).contains(&dimensions) { + return Err(format!( + "Cosmos DB {index_type:?} requires 1..={max_dimensions} dimensions, got {dimensions}" + )); + } + + let mut vector_index = VectorIndex::new(VECTOR_PATH, index_type); + if let Some(size) = build.indexing_search_list_size { + vector_index = vector_index.with_indexing_search_list_size(size); + } + if let Some(size) = build.quantization_byte_size { + if i64::from(size) > dimensions { + return Err(format!( + "Cosmos DB collection_params.quantization_byte_size ({size}) exceeds the vector dimensions ({dimensions})" + )); + } + vector_index = vector_index.with_quantization_byte_size(size); + } + let mut indexing = IndexingPolicy::default() + .with_indexing_mode(IndexingMode::Consistent) + .with_included_path("/*") + .with_excluded_path(VECTOR_EXCLUSION) + .with_vector_index(vector_index); + indexing.automatic = true; + let mut properties = ContainerProperties::new(name, "/id".into()); + properties.indexing_policy = Some(indexing); + properties.vector_embedding_policy = Some(VectorEmbeddingPolicy::default().with_embedding( + VectorEmbedding::new( + VECTOR_PATH, + VectorDataType::Float32, + dimensions as u32, + distance, + ), + )); + Ok(properties) +} + +/// Check the server's realized vector policy, including before reuse. +pub fn verify_policy( + actual: &ContainerProperties, + expected: &ContainerProperties, +) -> Result<(), String> { + let actual_embedding = actual + .vector_embedding_policy + .as_ref() + .map(|p| &p.embeddings); + let expected_embedding = expected + .vector_embedding_policy + .as_ref() + .map(|p| &p.embeddings); + let actual_indexing = actual.indexing_policy.as_ref(); + let expected_index = expected + .indexing_policy + .as_ref() + .and_then(|p| p.vector_indexes.first()); + if actual.partition_key != expected.partition_key + || actual_embedding != expected_embedding + || !actual_indexing.is_some_and(|p| { + p.automatic + && p.indexing_mode == Some(IndexingMode::Consistent) + && p.excluded_paths + .iter() + .any(|path| path.path == VECTOR_EXCLUSION) + // Azure fills in service-selected DiskANN defaults on readback. + // Compare the fields we set and record the full realized policy. + && matches!(p.vector_indexes.as_slice(), [index] if expected_index.is_some_and(|expected| { + index.path == expected.path + && index.index_type == expected.index_type + && expected.indexing_search_list_size.is_none_or(|size| { + index.indexing_search_list_size == Some(size) + }) + && expected.quantization_byte_size.is_none_or(|size| { + index.quantization_byte_size == Some(size) + }) + })) + }) + { + return Err(format!( + "Cosmos DB container '{}' has an incompatible partition key, embedding policy, vector index or index-build settings", + actual.id + )); + } + Ok(()) +} + +/// The SDK's buffered cross-partition query window defaults to 1,000. +pub fn validate_top(top: usize) -> Result<(), String> { + if !(1..=1000).contains(&top) { + return Err(format!( + "Cosmos DB top must be between 1 and 1000, got {top}" + )); + } + Ok(()) +} + +/// Reject an SDK response that exceeds the global TOP K across all its pages. +pub fn validate_hit_count(hits: usize, top: usize) -> Result<(), String> { + if hits > top { + return Err(format!( + "Cosmos DB returned {hits} hits for global TOP {top}" + )); + } + Ok(()) +} + +/// SDK diagnostics accumulated over the pages of measured queries. +#[derive(Debug, Default, Clone, Copy, Serialize)] +pub struct SearchDiagnostics { + pub request_charge_ru: f64, + pub request_attempts: usize, + pub http_429_attempts: usize, + pub attempts_without_status: usize, + pub queries_without_request_diagnostics: usize, + #[serde(skip_serializing_if = "Option::is_none")] + pub partition_fanout: Option, +} + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct PartitionFanout { + pub feed_ranges: usize, + pub per_query_concurrency: usize, + pub shared_request_limit: usize, + pub parity: ParityTally, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum PartitionConcurrency { + Auto, + Limit(NonZeroU32), +} + +impl PartitionConcurrency { + /// Missing selects the stock SDK path. Explicit fan-out accepts "auto" or a cap. + pub fn from_config(value: Option<&Value>) -> Result, String> { + value + .map(|value| { + if value.as_str() == Some("auto") { + return Ok(Self::Auto); + } + value + .as_u64() + .and_then(|n| u32::try_from(n).ok()) + .and_then(NonZeroU32::new) + .map(Self::Limit) + .ok_or_else(|| { + "Cosmos DB partition_concurrency must be \"auto\" or a positive 32-bit integer".into() + }) + }) + .transpose() + } + + pub fn resolve(&self, feed_ranges: usize) -> usize { + match self { + Self::Auto => feed_ranges, + Self::Limit(cap) => feed_ranges.min(cap.get() as usize), + } + } +} + +impl SearchDiagnostics { + /// Use the exact SDK totals and count HTTP statuses from either individual + /// attempts or their compacted rollup. Omitted statuses remain unknown. + pub fn observe_page( + &mut self, + request_charge_ru: f64, + request_attempts: usize, + attempt_statuses: impl IntoIterator, + ) { + self.request_charge_ru += request_charge_ru; + self.request_attempts += request_attempts; + let mut classified = 0; + for (status, count) in attempt_statuses { + if (100..600).contains(&status) { + classified += count; + } + if status == 429 { + self.http_429_attempts += count; + } + } + self.attempts_without_status += request_attempts.saturating_sub(classified); + } + + pub fn add_query(&mut self, query: Self) { + self.request_charge_ru += query.request_charge_ru; + self.request_attempts += query.request_attempts; + self.http_429_attempts += query.http_429_attempts; + self.attempts_without_status += query.attempts_without_status; + self.queries_without_request_diagnostics += usize::from( + query.request_attempts == 0 || query.queries_without_request_diagnostics > 0, + ); + } +} + +/// Fail on corpora too small to qualify for the service's DiskANN threshold. +pub fn validate_corpus_size(kind: &VectorIndexType, count: u64) -> Result<(), String> { + if *kind == VectorIndexType::DiskANN && count < 1000 { + return Err(format!( + "Cosmos DB DiskANN needs at least 1000 vectors to use its index; got {count}. Use cosmosdb-flat for this corpus." + )); + } + Ok(()) +} + +/// The text of a server-side TOP K query, built once per search. The service +/// uses the policy's distance function and TOP is a validated integer literal. +/// Only the bound vector differs between requests, so a search keeps one text +/// and binds each vector when it sends the request. Pre-building every query +/// would hold each vector a second time as JSON values. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KnnQueryText(String); + +impl KnnQueryText { + pub fn new( + top: usize, + search_list_size_multiplier: Option, + quantized_vector_list_multiplier: Option, + project_distance: bool, + ) -> Result { + validate_top(top)?; + let options = match (search_list_size_multiplier, quantized_vector_list_multiplier) { + (None, None) => String::new(), + (Some(search), None) => format!(", {{\"searchListSizeMultiplier\": {search}}}"), + (None, Some(quantized)) => { + format!(", {{\"quantizedVectorListMultiplier\": {quantized}}}") + } + (Some(search), Some(quantized)) => format!( + ", {{\"searchListSizeMultiplier\": {search}, \"quantizedVectorListMultiplier\": {quantized}}}" + ), + }; + Ok(Self(query_text( + top, + &format!("false{options}"), + project_distance, + ))) + } + + /// Bind one query vector. The caller checks it with + /// [`validate_query_vector`] before timing, so this does no validation. + pub fn bind(&self, vector: &[f32]) -> Result { + bind_vector(self.0.clone(), vector) + } +} + +/// A query vector must match the container's dimensions and be finite. JSON +/// has no encoding for NaN or infinity, so one would be sent as `null`. +pub fn validate_query_vector(vector: &[f32], dimensions: usize) -> Result<(), String> { + if vector.len() != dimensions { + return Err("Cosmos DB query vector has the wrong number of dimensions".into()); + } + if vector.iter().any(|v| !v.is_finite()) { + return Err("Cosmos DB query vector contains a non-finite value".into()); + } + Ok(()) +} + +/// Build and bind one validated TOP K query. +pub fn knn_query( + vector: &[f32], + top: usize, + search_list_size_multiplier: Option, + quantized_vector_list_multiplier: Option, + project_distance: bool, +) -> Result { + validate_query_vector(vector, vector.len())?; + KnnQueryText::new( + top, + search_list_size_multiplier, + quantized_vector_list_multiplier, + project_distance, + )? + .bind(vector) +} + +/// The same TOP K query with the service's brute-force flag set, so it scans +/// every vector in scope. Its request charge is the baseline that the +/// index-use probe compares an index-requested query against. +pub fn scan_query(vector: &[f32], top: usize) -> Result { + validate_top(top)?; + validate_query_vector(vector, vector.len())?; + bind_vector(query_text(top, "true", false), vector) +} + +fn query_text(top: usize, distance_arguments: &str, project_distance: bool) -> String { + let distance = format!("VectorDistance(c.embedding, @vector, {distance_arguments})"); + let projection = if project_distance { + format!("c.id, {distance} AS distance") + } else { + "c.id".into() + }; + format!("SELECT TOP {top} {projection} FROM c ORDER BY {distance}") +} + +fn bind_vector(text: String, vector: &[f32]) -> Result { + Query::from(text) + .with_parameter("@vector", vector) + .map_err(|e| format!("Could not bind Cosmos DB query vector: {e}")) +} + +/// An index-requested query must cost less than this share of a forced scan +/// of the same feed range before the range counts as served by DiskANN. +pub const INDEX_PROBE_MAX_RU_RATIO: f64 = 0.5; + +/// Request charges of one feed range's forced scan and index-requested query. +#[derive(Debug, Clone, Copy, PartialEq, Serialize)] +pub struct IndexProbe { + pub feed_range: usize, + pub scan_ru: f64, + pub index_ru: f64, +} + +impl IndexProbe { + /// Azure answers an index-requested query with a scan while the index is + /// building or the range holds too few vectors. A scan costs what the + /// forced scan costs, so only a clearly cheaper query shows index use. + pub fn index_in_use(&self) -> bool { + self.scan_ru > 0.0 + && self.index_ru > 0.0 + && self.index_ru < self.scan_ru * INDEX_PROBE_MAX_RU_RATIO + } +} + +/// How long to wait for every feed range to serve from its DiskANN index. +/// Unset uses 30 minutes. `0` skips the probe and leaves index use unverified. +pub fn index_probe_deadline(value: Option<&str>) -> Result, String> { + const DEFAULT_SECONDS: u64 = 1800; + let seconds = match value { + None => DEFAULT_SECONDS, + Some(value) => value.parse::().map_err(|_| { + "AZURE_COSMOS_INDEX_PROBE_SECONDS must be a whole number of seconds, or 0 to skip the probe" + })?, + }; + Ok((seconds > 0).then(|| Duration::from_secs(seconds))) +} + +/// Pause between index-use polls: a tenth of the time already waited, within +/// 2 to 30 seconds. The recorded index wait is then at most about a tenth +/// too long, and a long build is not polled with a scan every few seconds. +pub fn index_probe_interval(waited: Duration) -> Duration { + (waited / 10).clamp(Duration::from_secs(2), Duration::from_secs(30)) +} + +/// The charges of the feed ranges that are not served by the index yet. +pub fn describe_waiting(probes: &[IndexProbe]) -> String { + const SHOWN: usize = 8; + let waiting: Vec<_> = probes.iter().filter(|p| !p.index_in_use()).collect(); + let mut text = waiting + .iter() + .take(SHOWN) + .map(|p| { + format!( + "range {}: index query {:.2} RU, forced scan {:.2} RU", + p.feed_range, p.index_ru, p.scan_ru + ) + }) + .collect::>() + .join("; "); + if waiting.len() > SHOWN { + text.push_str(&format!("; and {} more", waiting.len() - SHOWN)); + } + text +} + +/// Which of the two probe queries to run on a feed range. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProbeQuery { + ForcedScan, + IndexRequested, +} + +/// Wait until every feed range answers the index-requested query for clearly +/// less than its forced scan. `charges` returns (range index, RU) for the +/// listed ranges. The forced scan runs once per range, and only ranges still +/// waiting are queried again. `waited` and `pause` are the clock. +pub fn await_index_use( + feed_ranges: usize, + deadline: Duration, + mut charges: impl FnMut(ProbeQuery, &[usize]) -> Result, String>, + waited: impl Fn() -> Duration, + mut pause: impl FnMut(Duration), +) -> Result, String> { + let mut probes: Vec = (0..feed_ranges) + .map(|feed_range| IndexProbe { + feed_range, + scan_ru: f64::NAN, + index_ru: f64::NAN, + }) + .collect(); + let mut pending: Vec = (0..feed_ranges).collect(); + let mut query = ProbeQuery::ForcedScan; + loop { + let answers = charges(query, &pending)?; + let mut answered: Vec = answers.iter().map(|(range, _)| *range).collect(); + answered.sort_unstable(); + if answered != pending { + return Err(format!( + "Cosmos DB index-use probe asked feed ranges {pending:?} and got answers for {answered:?}" + )); + } + for (feed_range, ru) in answers { + match query { + ProbeQuery::ForcedScan => probes[feed_range].scan_ru = ru, + ProbeQuery::IndexRequested => probes[feed_range].index_ru = ru, + } + } + if query == ProbeQuery::ForcedScan { + query = ProbeQuery::IndexRequested; + continue; + } + pending.retain(|&feed_range| !probes[feed_range].index_in_use()); + if pending.is_empty() { + return Ok(probes); + } + let Some(remaining) = deadline + .checked_sub(waited()) + .filter(|left| !left.is_zero()) + else { + return Err(format!( + "Cosmos DB DiskANN index is not serving {} of {feed_ranges} feed ranges after {}s. An index-requested query still costs as much as a forced scan ({}). The index may still be building, or a range may hold too few vectors. Raise AZURE_COSMOS_INDEX_PROBE_SECONDS, or set it to 0 to run unverified.", + pending.len(), + deadline.as_secs(), + describe_waiting(&probes) + )); + }; + println!( + "Waiting for the Cosmos DB DiskANN index on {} of {feed_ranges} feed ranges after {:.0}s ({})", + pending.len(), + waited().as_secs_f64(), + describe_waiting(&probes) + ); + pause(index_probe_interval(waited()).min(remaining)); + } +} + +/// Delay before retrying an upload write that Azure throttled, or `None` +/// once the budget is spent. The SDK retries a 429 a few times on its own. +/// This budget sits on top of that, so a corpus larger than a smoke test +/// still uploads at a throughput below what the writers ask for. +pub fn upload_throttle_backoff(retry: u32) -> Option { + const RETRIES: u32 = 60; + (retry < RETRIES) + .then(|| Duration::from_millis(250 << retry.min(5)).min(Duration::from_secs(5))) +} + +/// Run `write` until it succeeds, fails for a reason other than throttling, +/// or exhausts [`upload_throttle_backoff`]. Every retry is counted. +pub async fn retry_throttled( + mut write: impl FnMut() -> Write, + throttled: impl Fn(&E) -> bool, + sleep: impl Fn(Duration) -> Sleep, + retries: &AtomicUsize, +) -> Result +where + Write: Future>, + Sleep: Future, +{ + let mut retry = 0; + loop { + match write().await { + Err(error) if throttled(&error) => { + let Some(delay) = upload_throttle_backoff(retry) else { + return Err(error); + }; + retries.fetch_add(1, Ordering::Relaxed); + retry += 1; + sleep(delay).await; + } + other => return other, + } + } +} + +/// Sum a per-range count over disjoint feed ranges, `concurrency` at a time. +/// A split during the count could count a row twice or not at all, so the +/// ranges are read again afterwards and must be unchanged. +pub async fn count_stable_ranges( + read_ranges: impl Fn() -> Ranges, + concurrency: usize, + count: impl Fn(R) -> Count, +) -> Result +where + R: Clone + PartialEq, + Ranges: Future, String>>, + Count: Future>, +{ + let ranges = read_ranges().await?; + let total = stream::iter(ranges.iter().cloned()) + .map(&count) + .buffer_unordered(concurrency.max(1)) + .try_fold(0_u64, |total, rows| async move { Ok(total + rows) }) + .await?; + if read_ranges().await? != ranges { + return Err("Cosmos DB feed ranges changed during the row count".into()); + } + Ok(total) +} + +/// Query every feed range, at most `per_query` at a time for this query and +/// one `permits` permit per request across all queries, then select the +/// global top K and add up the diagnostics. +pub async fn fan_out_hits( + ranges: &[R], + per_query: usize, + permits: &Semaphore, + top: usize, + metric: &VectorDistanceFunction, + query: impl Fn(R) -> Hits, +) -> Result<(Vec, SearchDiagnostics), String> +where + R: Clone, + Hits: Future, SearchDiagnostics), String>>, +{ + let query = &query; + let (hits, diagnostics) = stream::iter(ranges.iter().cloned()) + .map(|range| async move { + let _permit = permits.acquire().await.map_err(|e| e.to_string())?; + query(range).await + }) + .buffer_unordered(per_query.max(1)) + .try_fold( + (Vec::new(), SearchDiagnostics::default()), + |(mut hits, mut diagnostics), (range_hits, range_diagnostics)| async move { + hits.extend(range_hits); + diagnostics.add_query(range_diagnostics); + Ok((hits, diagnostics)) + }, + ) + .await?; + Ok((merge_partition_hits(hits, top, metric)?, diagnostics)) +} + +#[derive(Deserialize)] +pub struct QueryHit { + pub id: String, + pub distance: Option, +} + +/// Select the global top K from the scored candidates of disjoint feed ranges. +pub fn merge_partition_hits( + hits: Vec, + top: usize, + metric: &VectorDistanceFunction, +) -> Result, String> { + // The SQL vector ORDER BY is best-first for every metric. Its projected + // value is a similarity for cosine/dot product and a distance for L2. + let higher_is_better = match metric { + VectorDistanceFunction::Cosine | VectorDistanceFunction::DotProduct => true, + VectorDistanceFunction::Euclidean => false, + _ => return Err("Unsupported Cosmos DB vector score ordering".into()), + }; + let mut candidates = hits + .into_iter() + .map(|hit| { + let distance = hit + .distance + .filter(|distance| distance.is_finite()) + .ok_or("Cosmos DB partition query returned a hit without a finite distance")?; + Ok((hit, distance)) + }) + .collect::, String>>()?; + let mut seen = HashSet::with_capacity(candidates.len()); + for (hit, _) in &candidates { + if !seen.insert(hit.id.as_str()) { + return Err(format!( + "Cosmos DB partition queries returned duplicate id {}", + hit.id + )); + } + } + candidates.sort_unstable_by(|(left, a), (right, b)| { + let order = a.total_cmp(b); + (if higher_is_better { + order.reverse() + } else { + order + }) + .then_with(|| left.id.cmp(&right.id)) + }); + candidates.truncate(top); + Ok(candidates.into_iter().map(|(hit, _)| hit).collect()) +} + +/// Fan-out may miss this many times the IDs that a repeated stock request +/// misses. Two stock comparisons are counted per sample, so 2 is the expected +/// ratio when both routes behave alike. 3 leaves headroom for sampling noise. +pub const PARITY_CROSS_MODE_ALLOWANCE: usize = 3; + +/// Agreement between the stock SDK query and the feed-range fan-out on +/// sampled queries, measured against the stock query's own repeatability. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct ParityTally { + pub samples: usize, + /// IDs of one stock response that a repeated stock request did not return. + pub stock_repeat_missing_ids: usize, + /// IDs of the two stock responses that the fan-out response did not return. + pub cross_mode_missing_ids: usize, +} + +impl ParityTally { + /// Record one sampled query: two stock responses and one fan-out response. + pub fn observe( + &mut self, + stock: &[QueryHit], + stock_repeat: &[QueryHit], + partitioned: &[QueryHit], + ) -> Result<(), String> { + if stock.len() != stock_repeat.len() || stock.len() != partitioned.len() { + return Err(format!( + "Cosmos DB routing modes returned different hit counts ({} and {} stock, {} fan-out)", + stock.len(), + stock_repeat.len(), + partitioned.len() + )); + } + let ids = |hits: &[QueryHit]| -> HashSet { + hits.iter().map(|hit| hit.id.clone()).collect() + }; + let (stock, stock_repeat, partitioned) = (ids(stock), ids(stock_repeat), ids(partitioned)); + self.samples += 1; + self.stock_repeat_missing_ids += stock.difference(&stock_repeat).count(); + self.cross_mode_missing_ids += + stock.difference(&partitioned).count() + stock_repeat.difference(&partitioned).count(); + Ok(()) + } + + /// Fan-out must agree with the stock query about as well as the stock + /// query agrees with itself. An exact engine therefore needs exact parity. + pub fn verdict(&self) -> Result<(), String> { + let allowed = self.stock_repeat_missing_ids * PARITY_CROSS_MODE_ALLOWANCE; + if self.samples == 0 || self.cross_mode_missing_ids > allowed { + return Err(format!( + "Cosmos DB partition search disagrees with the SDK's global top K: fan-out missed {} stock IDs over {} sampled queries, and repeated stock requests missed {} (allowance {allowed})", + self.cross_mode_missing_ids, self.samples, self.stock_repeat_missing_ids + )); + } + Ok(()) + } +} + +/// Query indices spread evenly over the first `limit` queries, at most `samples`. +pub fn parity_sample_indices(limit: usize, samples: usize) -> Vec { + (0..limit) + .step_by((limit / samples.max(1)).max(1)) + .take(samples) + .collect() +} + +/// Run `sample` for each query index, `concurrency` at a time. It returns the +/// two stock responses and the fan-out response. The tally of all samples +/// must then pass [`ParityTally::verdict`]. +pub async fn sample_parity( + indices: Vec, + concurrency: usize, + sample: impl Fn(usize) -> Sample, +) -> Result +where + Sample: Future; 3], String>>, +{ + let sample = &sample; + let outcomes: Vec<_> = stream::iter(indices) + .map(|index| async move { + sample(index) + .await + .map_err(|e| format!("Cosmos DB parity check for query {index}: {e}")) + }) + .buffer_unordered(concurrency.max(1)) + .try_collect() + .await?; + let mut tally = ParityTally::default(); + for [stock, stock_repeat, partitioned] in &outcomes { + tally.observe(stock, stock_repeat, partitioned)?; + } + tally.verdict()?; + Ok(tally) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, to_value}; + + #[test] + fn distance_and_diskann_policy_are_in_the_wire_request() { + let defaults = DiskAnnBuildOptions::default(); + let p = container_policy( + "bench".into(), + VectorIndexType::DiskANN, + "dot", + 1024, + defaults, + ) + .unwrap(); + let wire = to_value(&p).unwrap(); + assert_eq!(wire["partitionKey"]["paths"], json!(["/id"])); + assert_eq!( + wire["vectorEmbeddingPolicy"]["vectorEmbeddings"][0], + json!({ + "path": "/embedding", "dataType": "float32", "dimensions": 1024, + "distanceFunction": "dotproduct" + }) + ); + assert_eq!( + wire["indexingPolicy"]["vectorIndexes"][0]["type"], + "diskANN" + ); + let default_index = &wire["indexingPolicy"]["vectorIndexes"][0]; + assert!(default_index.get("quantizationByteSize").is_none()); + assert!(default_index.get("indexingSearchListSize").is_none()); + let mut realized = wire; + let vector_index = &mut realized["indexingPolicy"]["vectorIndexes"][0]; + vector_index["quantizerType"] = json!("product"); + vector_index["quantizationByteSize"] = json!(16); + vector_index["indexingSearchListSize"] = json!(100); + let actual = serde_json::from_value(realized.clone()).unwrap(); + assert!(verify_policy(&actual, &p).is_ok()); + realized["indexingPolicy"]["vectorIndexes"][0]["type"] = json!("flat"); + let wrong_index = serde_json::from_value(realized).unwrap(); + assert!(verify_policy(&wrong_index, &p).is_err()); + let other = container_policy( + "bench".into(), + VectorIndexType::DiskANN, + "l2", + 1024, + defaults, + ) + .unwrap(); + assert!(verify_policy(&p, &other).is_err()); + assert!(verify_policy(&p, &p).is_ok()); + assert!(container_policy( + "bench".into(), + VectorIndexType::Flat, + "cosine", + 506, + defaults + ) + .is_err()); + } + + fn tuned_diskann_policy() -> ContainerProperties { + let kind = VectorIndexType::DiskANN; + let options = + DiskAnnBuildOptions::from_config(&kind, Some(&json!(150)), Some(&json!(32))).unwrap(); + container_policy("bench".into(), kind, "cosine", 100, options).unwrap() + } + + #[test] + fn diskann_build_settings_are_in_wire_policy() { + let wire = to_value(tuned_diskann_policy()).unwrap(); + let index = &wire["indexingPolicy"]["vectorIndexes"][0]; + assert_eq!(index["indexingSearchListSize"], 150); + assert_eq!(index["quantizationByteSize"], 32); + } + + #[test] + fn diskann_build_readback_requires_all_explicit_settings() { + let expected = tuned_diskann_policy(); + let wire = to_value(&expected).unwrap(); + let actual = serde_json::from_value(wire.clone()).unwrap(); + assert!(verify_policy(&actual, &expected).is_ok()); + for key in ["indexingSearchListSize", "quantizationByteSize"] { + let mut changed = wire.clone(); + let index = &mut changed["indexingPolicy"]["vectorIndexes"][0]; + index[key] = json!(100); + let actual = serde_json::from_value(changed.clone()).unwrap(); + assert!( + verify_policy(&actual, &expected).is_err(), + "mismatched {key}" + ); + changed["indexingPolicy"]["vectorIndexes"][0] + .as_object_mut() + .unwrap() + .remove(key); + let actual = serde_json::from_value(changed).unwrap(); + assert!(verify_policy(&actual, &expected).is_err(), "missing {key}"); + } + } + + #[test] + fn diskann_build_settings_validate_types_and_bounds() { + let kind = VectorIndexType::DiskANN; + for (indexing_search_list, min, max) in [(true, 25, 500), (false, 1, 512)] { + let parse = |value: &Value| { + if indexing_search_list { + DiskAnnBuildOptions::from_config(&kind, Some(value), None) + } else { + DiskAnnBuildOptions::from_config(&kind, None, Some(value)) + } + }; + assert!(parse(&json!(min)).is_ok()); + assert!(parse(&json!(max)).is_ok()); + for invalid in [json!(min - 1), json!(max + 1), json!(64.0), json!("64")] { + assert!( + parse(&invalid).is_err(), + "accepted invalid setting {invalid}" + ); + } + } + assert!( + DiskAnnBuildOptions::from_config(&VectorIndexType::Flat, Some(&json!(100)), None) + .is_err() + ); + assert!( + DiskAnnBuildOptions::from_config(&VectorIndexType::Flat, None, Some(&json!(32))) + .is_err() + ); + let options = DiskAnnBuildOptions::from_config(&kind, None, Some(&json!(32))).unwrap(); + assert!(container_policy("bench".into(), kind.clone(), "cosine", 32, options).is_ok()); + assert!(container_policy("bench".into(), kind, "cosine", 31, options).is_err()); + } + + #[test] + fn vector_queries_request_indexed_top_k_and_bind_the_vector() { + let query = to_value(knn_query(&[0.5, -0.25], 10, None, None, false).unwrap()).unwrap(); + assert_eq!( + query["query"], + "SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" + ); + assert_eq!( + query["parameters"], + json!([{"name": "@vector", "value": [0.5, -0.25]}]) + ); + assert!(knn_query(&[0.5], 0, None, None, false).is_err()); + assert!(knn_query(&[0.5], 1001, None, None, false).is_err()); + let tuned = + to_value(knn_query(&[0.5], 10, 5.try_into().ok(), 20.try_into().ok(), false).unwrap()) + .unwrap(); + assert_eq!( + tuned["query"], + r#"SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, false, {"searchListSizeMultiplier": 5, "quantizedVectorListMultiplier": 20})"# + ); + let scored = to_value(knn_query(&[0.5], 10, None, None, true).unwrap()).unwrap(); + assert_eq!( + scored["query"], + "SELECT TOP 10 c.id, VectorDistance(c.embedding, @vector, false) AS distance FROM c ORDER BY VectorDistance(c.embedding, @vector, false)" + ); + } + + #[test] + fn a_bound_query_text_is_the_query_knn_query_builds() { + let multiplier = 5.try_into().ok(); + for project_distance in [false, true] { + let text = KnnQueryText::new(10, multiplier, None, project_distance).unwrap(); + for vector in [[0.5_f32, -0.25], [1.0, 2.0]] { + assert_eq!( + to_value(text.bind(&vector).unwrap()).unwrap(), + to_value(knn_query(&vector, 10, multiplier, None, project_distance).unwrap()) + .unwrap() + ); + } + } + assert!(KnnQueryText::new(0, None, None, false).is_err()); + assert!(KnnQueryText::new(1001, None, None, false).is_err()); + assert!(validate_query_vector(&[0.5, 0.25], 2).is_ok()); + assert!(validate_query_vector(&[0.5], 2).is_err()); + assert!(validate_query_vector(&[0.5, f32::INFINITY], 2).is_err()); + assert!(knn_query(&[f32::NAN], 10, None, None, false).is_err()); + } + + #[test] + fn partition_concurrency_uses_the_realized_feed_range_count() { + assert_eq!(PartitionConcurrency::from_config(None).unwrap(), None); + let auto = PartitionConcurrency::from_config(Some(&json!("auto"))) + .unwrap() + .unwrap(); + assert_eq!(auto.resolve(5), 5); + assert_eq!(auto.resolve(12), 12); + let limited = PartitionConcurrency::from_config(Some(&json!(9))) + .unwrap() + .unwrap(); + assert_eq!(limited.resolve(5), 5); + assert_eq!(limited.resolve(12), 9); + for invalid in [json!(0), json!(-1), json!(1.5), json!("9"), json!(null)] { + assert!(PartitionConcurrency::from_config(Some(&invalid)).is_err()); + } + } + + #[test] + fn partition_merge_selects_global_top_k_and_rejects_incomplete_hits() { + let hit = |id: &str, distance| QueryHit { + id: id.into(), + distance, + }; + let candidates = || { + vec![ + hit("0", Some(0.9)), + hit("2", Some(0.3)), + hit("1", Some(0.3)), + hit("3", Some(0.1)), + ] + }; + for (metric, expected) in [ + (VectorDistanceFunction::Euclidean, ["3", "1"]), + (VectorDistanceFunction::Cosine, ["0", "1"]), + (VectorDistanceFunction::DotProduct, ["0", "1"]), + ] { + let merged = merge_partition_hits(candidates(), 2, &metric).unwrap(); + assert_eq!( + merged.iter().map(|hit| hit.id.as_str()).collect::>(), + expected + ); + } + let metric = VectorDistanceFunction::Cosine; + assert!(merge_partition_hits(vec![hit("0", None)], 1, &metric).is_err()); + assert!( + merge_partition_hits(vec![hit("0", Some(0.1)), hit("0", Some(0.2))], 1, &metric) + .is_err() + ); + } + + fn hits(ids: &[&str]) -> Vec { + ids.iter() + .map(|id| QueryHit { + id: (*id).into(), + distance: None, + }) + .collect() + } + + #[test] + fn parity_requires_exact_agreement_when_stock_repeats_exactly() { + let mut exact = ParityTally::default(); + exact + .observe(&hits(&["1", "2"]), &hits(&["2", "1"]), &hits(&["1", "2"])) + .unwrap(); + assert_eq!( + exact, + ParityTally { + samples: 1, + stock_repeat_missing_ids: 0, + cross_mode_missing_ids: 0 + } + ); + assert!(exact.verdict().is_ok()); + + let mut wrong = ParityTally::default(); + wrong + .observe(&hits(&["1", "2"]), &hits(&["1", "2"]), &hits(&["1", "3"])) + .unwrap(); + assert_eq!(wrong.cross_mode_missing_ids, 2); + assert!(wrong.verdict().is_err()); + assert!(ParityTally::default().verdict().is_err()); + } + + #[test] + fn parity_allows_fanout_the_variation_stock_shows_against_itself() { + // Stock misses one ID against itself. Fan-out may miss up to three. + let mut within = ParityTally::default(); + within + .observe( + &hits(&["1", "2", "3"]), + &hits(&["1", "2", "4"]), + &hits(&["1", "2", "5"]), + ) + .unwrap(); + assert_eq!(within.stock_repeat_missing_ids, 1); + assert_eq!(within.cross_mode_missing_ids, 2); + assert!(within.verdict().is_ok()); + + let mut beyond = ParityTally::default(); + beyond + .observe( + &hits(&["1", "2", "3"]), + &hits(&["1", "2", "4"]), + &hits(&["1", "8", "9"]), + ) + .unwrap(); + assert_eq!(beyond.cross_mode_missing_ids, 4); + assert!(beyond.verdict().is_err()); + + // The tally is cumulative: a later exact sample does not reset it. + beyond + .observe(&hits(&["1"]), &hits(&["1"]), &hits(&["1"])) + .unwrap(); + assert_eq!(beyond.samples, 2); + assert!(beyond.verdict().is_err()); + assert!(ParityTally::default() + .observe(&hits(&["1", "2"]), &hits(&["1", "2"]), &hits(&["1"])) + .is_err()); + } + + #[test] + fn index_probe_needs_a_query_clearly_cheaper_than_a_forced_scan() { + let probe = |scan_ru, index_ru| IndexProbe { + feed_range: 0, + scan_ru, + index_ru, + }; + assert!(probe(1000.0, 499.0).index_in_use()); + assert!(!probe(1000.0, 500.0).index_in_use()); + assert!(!probe(1000.0, 1000.0).index_in_use()); + assert!(!probe(0.0, 0.0).index_in_use()); + assert!(!probe(1000.0, 0.0).index_in_use()); + let scan = to_value(scan_query(&[0.5], 10).unwrap()).unwrap(); + assert_eq!( + scan["query"], + "SELECT TOP 10 c.id FROM c ORDER BY VectorDistance(c.embedding, @vector, true)" + ); + assert!(scan_query(&[f32::NAN], 10).is_err()); + assert!(scan_query(&[0.5], 0).is_err()); + } + + #[test] + fn index_probe_deadline_defaults_and_can_be_skipped() { + assert_eq!( + index_probe_deadline(None).unwrap(), + Some(Duration::from_secs(1800)) + ); + assert_eq!( + index_probe_deadline(Some("90")).unwrap(), + Some(Duration::from_secs(90)) + ); + assert_eq!(index_probe_deadline(Some("0")).unwrap(), None); + for invalid in ["", "-1", "1.5", "soon"] { + assert!(index_probe_deadline(Some(invalid)).is_err(), "{invalid}"); + } + } + + #[test] + fn config_values_are_validated_before_any_request() { + let keys = [String::from("parallel"), String::from("batch_size")]; + assert!(reject_unknown_keys(&keys[..1], &["parallel"], "no").is_ok()); + assert_eq!( + reject_unknown_keys(&keys, &["parallel"], "no"), + Err("no".into()) + ); + + assert_eq!(positive_u32(&json!(5), "k").unwrap().get(), 5); + for invalid in [ + json!(0), + json!(-1), + json!(1.5), + json!("5"), + json!(1u64 << 32), + ] { + assert!(positive_u32(&invalid, "k").is_err(), "{invalid}"); + } + + assert_eq!(manual_throughput(None, "400").unwrap(), Some(400)); + assert_eq!(manual_throughput(None, "serverless").unwrap(), None); + assert_eq!( + manual_throughput(Some(&json!(1000)), "serverless").unwrap(), + Some(1000) + ); + assert!(manual_throughput(None, "399").is_err()); + assert!(manual_throughput(None, "auto").is_err()); + assert!(manual_throughput(Some(&json!(399)), "400").is_err()); + assert!(manual_throughput(Some(&json!("1000")), "400").is_err()); + + assert_eq!(upload_parallel(None).unwrap(), 8); + assert_eq!(upload_parallel(Some(&json!({}))).unwrap(), 8); + assert_eq!(upload_parallel(Some(&json!({"parallel": 16}))).unwrap(), 16); + for invalid in [ + json!({"parallel": 0}), + json!({"parallel": "4"}), + json!({"batch_size": 10}), + json!([4]), + ] { + assert!(upload_parallel(Some(&invalid)).is_err(), "{invalid}"); + } + } + + #[test] + fn diskann_rejects_the_documented_full_scan_case() { + assert!(validate_corpus_size(&VectorIndexType::DiskANN, 999).is_err()); + assert!(validate_corpus_size(&VectorIndexType::DiskANN, 1000).is_ok()); + assert!(validate_corpus_size(&VectorIndexType::Flat, 999).is_ok()); + } + + #[test] + fn hit_count_rejects_concatenated_partition_top_k_lists() { + assert!(validate_hit_count(2, 3).is_ok()); + assert!(validate_hit_count(3, 3).is_ok()); + assert!(validate_hit_count(6, 3).is_err()); + } + + #[test] + fn search_diagnostics_sum_pages_retries_and_missing_statuses() { + let mut query = SearchDiagnostics::default(); + query.observe_page(4.5, 5, [(429, 3), (200, 1)]); + query.observe_page(2.0, 1, [(200, 1)]); + let mut run = SearchDiagnostics::default(); + run.add_query(query); + run.add_query(SearchDiagnostics::default()); + assert_eq!( + to_value(run).unwrap(), + json!({ + "request_charge_ru": 6.5, + "request_attempts": 6, + "http_429_attempts": 3, + "attempts_without_status": 1, + "queries_without_request_diagnostics": 1, + }) + ); + } + + /// A clock that only `pause` advances, plus a record of every probe call. + struct ProbeRun { + clock: std::cell::Cell, + calls: std::cell::RefCell)>>, + pauses: std::cell::RefCell>, + } + + /// Each range's index starts serving once the clock reaches `ready_at`. + fn run_probe(ready_at: &[u64], deadline: u64) -> (Result, String>, ProbeRun) { + let run = ProbeRun { + clock: Default::default(), + calls: Default::default(), + pauses: Default::default(), + }; + let result = await_index_use( + ready_at.len(), + Duration::from_secs(deadline), + |query, ranges| { + run.calls.borrow_mut().push((query, ranges.to_vec())); + Ok(ranges + .iter() + .map(|&range| { + let ready = run.clock.get() >= Duration::from_secs(ready_at[range]); + let served = query == ProbeQuery::IndexRequested && ready; + (range, if served { 6.0 } else { 300.0 }) + }) + .collect()) + }, + || run.clock.get(), + |pause| { + run.pauses.borrow_mut().push(pause); + run.clock.set(run.clock.get() + pause); + }, + ); + (result, run) + } + + #[test] + fn index_wait_scans_once_and_polls_only_the_ranges_still_waiting() { + let (probes, run) = run_probe(&[0, 3, 0], 600); + let probes = probes.unwrap(); + assert!(probes.iter().all(IndexProbe::index_in_use)); + assert_eq!( + probes[1], + IndexProbe { + feed_range: 1, + scan_ru: 300.0, + index_ru: 6.0 + } + ); + let calls = run.calls.borrow(); + assert_eq!(calls[0], (ProbeQuery::ForcedScan, vec![0, 1, 2])); + assert_eq!(calls[1], (ProbeQuery::IndexRequested, vec![0, 1, 2])); + // Ranges 0 and 2 passed on the first poll. Only range 1 is asked again. + assert!(calls[2..] + .iter() + .all(|call| *call == (ProbeQuery::IndexRequested, vec![1]))); + assert_eq!(calls.len(), 4); + assert_eq!(*run.pauses.borrow(), [Duration::from_secs(2); 2]); + } + + #[test] + fn index_wait_fails_at_the_deadline_and_names_the_charges() { + let (result, run) = run_probe(&[0, u64::MAX], 100); + let error = result.unwrap_err(); + assert!( + error.contains("not serving 1 of 2 feed ranges after 100s"), + "{error}" + ); + assert!( + error.contains("range 1: index query 300.00 RU, forced scan 300.00 RU"), + "{error}" + ); + // Range 0 is served, so it is not listed as waiting. + assert!(!error.contains("range 0:"), "{error}"); + // It never sleeps past the deadline. + assert_eq!( + run.pauses.borrow().iter().sum::(), + Duration::from_secs(100) + ); + // An answer for the wrong ranges is an error, not a panic or a pass. + // The clock advances, so a missing check ends at the deadline. + let clock = std::cell::Cell::new(Duration::ZERO); + let wrong = await_index_use( + 2, + Duration::from_secs(1), + |_, _| Ok(vec![(0, 1.0)]), + || clock.get(), + |pause| clock.set(clock.get() + pause), + ); + assert!(wrong.unwrap_err().contains("got answers for [0]")); + } + + #[test] + fn index_poll_interval_grows_with_the_wait() { + let interval = |seconds| index_probe_interval(Duration::from_secs(seconds)); + assert_eq!(interval(0), Duration::from_secs(2)); + assert_eq!(interval(60), Duration::from_secs(6)); + assert_eq!(interval(3600), Duration::from_secs(30)); + let many: Vec<_> = (0..10) + .map(|feed_range| IndexProbe { + feed_range, + scan_ru: 9.0, + index_ru: 9.0, + }) + .collect(); + let text = describe_waiting(&many); + assert!(text.starts_with("range 0: index query 9.00 RU, forced scan 9.00 RU; range 1")); + assert!(text.ends_with("; and 2 more"), "{text}"); + } + + #[test] + fn throttled_writes_are_retried_until_the_budget_is_spent() { + assert_eq!(upload_throttle_backoff(0), Some(Duration::from_millis(250))); + assert_eq!(upload_throttle_backoff(3), Some(Duration::from_secs(2))); + assert_eq!(upload_throttle_backoff(59), Some(Duration::from_secs(5))); + assert_eq!(upload_throttle_backoff(60), None); + + let run = |outcomes: Vec>| { + let retries = AtomicUsize::new(0); + let slept = std::cell::RefCell::new(Vec::new()); + let mut outcomes = outcomes.into_iter(); + let result = futures::executor::block_on(retry_throttled( + || { + let outcome = outcomes.next().unwrap_or(Err(429)); + async move { outcome } + }, + |status: &u16| *status == 429, + |delay| { + slept.borrow_mut().push(delay); + async {} + }, + &retries, + )); + (result, retries.into_inner(), slept.into_inner()) + }; + let (result, retries, slept) = run(vec![Err(429), Err(429), Ok(7)]); + assert_eq!((result, retries), (Ok(7), 2)); + assert_eq!( + slept, + [Duration::from_millis(250), Duration::from_millis(500)] + ); + // Any other failure is returned at once. + assert_eq!(run(vec![Err(429), Err(503), Ok(7)]).0, Err(503)); + // Endless throttling ends with the 429 after the whole budget. + let (result, retries, _) = run(vec![]); + assert_eq!((result, retries), (Err(429), 60)); + } + + /// Pending once, so `buffer_unordered` has to interleave the futures. + async fn yield_once() { + let mut yielded = false; + std::future::poll_fn(|cx| { + if std::mem::replace(&mut yielded, true) { + return std::task::Poll::Ready(()); + } + cx.waker().wake_by_ref(); + std::task::Poll::Pending + }) + .await + } + + #[test] + fn row_count_sums_ranges_and_rejects_a_split() { + let count = |ranges_after: Vec| { + let reads = std::cell::Cell::new(0); + futures::executor::block_on(count_stable_ranges( + || { + reads.set(reads.get() + 1); + let ranges = if reads.get() == 1 { + vec![10, 20, 30] + } else { + ranges_after.clone() + }; + async move { Ok(ranges) } + }, + 2, + |rows: u64| async move { + yield_once().await; + if rows == 0 { + Err("boom".to_string()) + } else { + Ok(rows) + } + }, + )) + }; + assert_eq!(count(vec![10, 20, 30]), Ok(60)); + assert!(count(vec![10, 20, 15, 15]) + .unwrap_err() + .contains("feed ranges changed")); + } + + #[test] + fn fan_out_merges_the_global_top_k_within_the_request_limit() { + let scored = |id: &str, distance| QueryHit { + id: id.into(), + distance: Some(distance), + }; + let in_flight = AtomicUsize::new(0); + let peak = AtomicUsize::new(0); + let fan_out = |per_query, permits| { + peak.store(0, Ordering::Relaxed); + let permits = Semaphore::new(permits); + futures::executor::block_on(fan_out_hits( + &[0_usize, 1, 2, 3], + per_query, + &permits, + 2, + &VectorDistanceFunction::Cosine, + |range| { + let (in_flight, peak) = (&in_flight, &peak); + async move { + peak.fetch_max( + in_flight.fetch_add(1, Ordering::Relaxed) + 1, + Ordering::Relaxed, + ); + yield_once().await; + in_flight.fetch_sub(1, Ordering::Relaxed); + if range == 3 && per_query == 1 { + return Err("range 3 failed".to_string()); + } + let mut diagnostics = SearchDiagnostics::default(); + diagnostics.observe_page(2.5, 1, [(200, 1)]); + let name = format!("r{range}"); + Ok(( + vec![ + scored(&name, range as f64), + scored(&format!("{name}b"), -1.0), + ], + diagnostics, + )) + } + }, + )) + }; + let (hits, diagnostics) = fan_out(4, 4).unwrap(); + assert_eq!( + hits.iter().map(|hit| hit.id.as_str()).collect::>(), + ["r3", "r2"] + ); + assert_eq!(diagnostics.request_charge_ru, 10.0); + assert_eq!(diagnostics.request_attempts, 4); + assert_eq!(peak.load(Ordering::Relaxed), 4); + // The shared permits cap in-flight requests below the per-query limit. + fan_out(4, 2).unwrap(); + assert_eq!(peak.load(Ordering::Relaxed), 2); + // So does the per-query limit, and a failed range fails the query. + assert_eq!(fan_out(1, 4).err().unwrap(), "range 3 failed"); + assert_eq!(peak.load(Ordering::Relaxed), 1); + } + + #[test] + fn parity_samples_spread_over_the_queries_and_must_pass_the_verdict() { + assert_eq!( + parity_sample_indices(100, 8), + [0, 12, 24, 36, 48, 60, 72, 84] + ); + assert_eq!(parity_sample_indices(3, 8), [0, 1, 2]); + assert!(parity_sample_indices(0, 8).is_empty()); + + let sample = |fan_out_ids: &'static [&'static str]| { + futures::executor::block_on(sample_parity(vec![0, 5], 2, |index| async move { + yield_once().await; + if index == 5 && fan_out_ids.is_empty() { + return Err("timeout".to_string()); + } + Ok([hits(&["a", "b"]), hits(&["a", "b"]), hits(fan_out_ids)]) + })) + }; + let tally = sample(&["b", "a"]).unwrap(); + assert_eq!( + tally, + ParityTally { + samples: 2, + stock_repeat_missing_ids: 0, + cross_mode_missing_ids: 0 + } + ); + assert!(sample(&["a", "c"]) + .unwrap_err() + .contains("fan-out missed 4 stock IDs")); + assert_eq!( + sample(&[]).unwrap_err(), + "Cosmos DB parity check for query 5: timeout" + ); + } +} diff --git a/src/lib.rs b/src/lib.rs index a600db407..8445d3936 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,6 +18,7 @@ pub mod build_info; pub mod compare_results; pub mod config; +pub mod cosmosdb; pub mod duration_mode; pub mod filter_only; pub mod fulltext; diff --git a/tests/harness_invariants.rs b/tests/harness_invariants.rs index 75d3798ab..84267eaaa 100644 --- a/tests/harness_invariants.rs +++ b/tests/harness_invariants.rs @@ -755,6 +755,7 @@ fn inv_p7_the_scanned_test_target_set_is_pinned() { "harness_invariants.rs", "integration_chroma.rs", "integration_cli.rs", + "integration_cosmosdb.rs", "integration_dragonfly.rs", "integration_elasticsearch.rs", "integration_kividb.rs", diff --git a/tests/integration_cosmosdb.rs b/tests/integration_cosmosdb.rs new file mode 100644 index 000000000..c9cf16c3a --- /dev/null +++ b/tests/integration_cosmosdb.rs @@ -0,0 +1,153 @@ +//! Cloud-backed CLI smoke test. Set AZURE_COSMOS_ENDPOINT and AZURE_COSMOS_KEY +//! and run `make integration-test-cosmosdb`. + +mod common; + +use std::fs; + +use serde_json::json; + +/// Write the random-100 fixture and one Flat engine config under `root`. +/// The engine name is unique to the tempdir, so the test owns its container. +fn write_fixture( + root: &std::path::Path, + collection_params: serde_json::Value, + search_params: serde_json::Value, +) -> String { + let engine_name = format!( + "cosmosdb-it-{}", + root.file_name().unwrap().to_string_lossy() + ); + let fixture = root.join("datasets/random-100"); + fs::create_dir_all(&fixture).unwrap(); + fs::create_dir_all(root.join("experiments/configurations")).unwrap(); + fs::create_dir_all(root.join("results")).unwrap(); + let source = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("datasets/random-100"); + for file in ["vectors.jsonl", "queries.jsonl", "neighbours.jsonl"] { + fs::copy(source.join(file), fixture.join(file)).unwrap(); + } + let datasets = json!([{ + "name": "random-100", + "type": "jsonl", + "path": "random-100/", + "distance": "cosine", + "vector_size": 100, + "vector_count": 100, + }]); + fs::write(root.join("datasets/datasets.json"), datasets.to_string()).unwrap(); + let config = json!([{ + "name": engine_name, + "engine": "cosmosdb", + "algorithm": "flat", + "collection_params": collection_params, + "upload_params": { "parallel": 4 }, + "search_params": [search_params], + }]); + fs::write( + root.join("experiments/configurations/cosmosdb.json"), + config.to_string(), + ) + .unwrap(); + engine_name +} + +#[test] +fn flat_search_matches_the_fixture_and_cleans_up_the_container() { + if std::env::var_os("AZURE_COSMOS_ENDPOINT").is_none() { + eprintln!("SKIP integration_cosmosdb: set AZURE_COSMOS_ENDPOINT and AZURE_COSMOS_KEY"); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let engine_name = write_fixture(root, json!({}), json!({ "parallel": 2, "top": 1 })); + + assert!(common::run_binary( + root, + &engine_name, + "random-100", + "localhost", + &[] + )); + let uploaded = common::read_result_doc(root, &engine_name); + let recall = uploaded["results"]["mean_recall"].as_f64().unwrap(); + println!("Cosmos DB Flat random-100 recall@1: {recall}"); + assert_eq!(recall, 1.0); + assert_eq!(uploaded["results"]["failed_queries"], 0); + let diagnostics = &uploaded["results"]["cosmosdb_search_diagnostics"]; + assert!(diagnostics["request_charge_ru"].as_f64().unwrap() > 0.0); + assert!( + diagnostics["request_attempts"].as_u64().unwrap() + >= uploaded["results"]["succeeded_queries"].as_u64().unwrap() + ); + assert_eq!(diagnostics["queries_without_request_diagnostics"], 0); + let metadata = &uploaded["server_metadata"]["after"]["cosmosdb"]; + assert_eq!( + metadata["vector_indexing_policy"]["vectorIndexes"][0]["type"], + "flat" + ); + assert_eq!( + metadata["vector_embedding_policy"]["vectorEmbeddings"][0]["dimensions"], + 100 + ); + + // The normal run cleaned up the benchmark container. A second upload with + // the same name can therefore create a fresh corpus. + assert!(common::run_binary( + root, + &engine_name, + "random-100", + "localhost", + &[] + )); +} + +/// Feed-range fan-out on a container with several physical partitions. It +/// provisions 12,000 manual RU/s for the duration of the run, so it needs its +/// own opt-in on top of the account credentials. +#[test] +fn flat_fanout_matches_the_stock_query_across_feed_ranges() { + if std::env::var_os("AZURE_COSMOS_ENDPOINT").is_none() + || std::env::var_os("AZURE_COSMOS_TEST_FANOUT").is_none() + { + eprintln!( + "SKIP integration_cosmosdb fan-out: set AZURE_COSMOS_ENDPOINT, AZURE_COSMOS_KEY and AZURE_COSMOS_TEST_FANOUT=1" + ); + return; + } + + let temp = tempfile::tempdir().unwrap(); + let root = temp.path(); + let engine_name = write_fixture( + root, + json!({ "ru_per_second": 12000 }), + json!({ + "parallel": 2, + "top": 1, + "search_params": { "partition_concurrency": "auto" }, + }), + ); + + assert!(common::run_binary( + root, + &engine_name, + "random-100", + "localhost", + &[] + )); + let result = common::read_result_doc(root, &engine_name); + assert_eq!(result["results"]["mean_recall"].as_f64().unwrap(), 1.0); + assert_eq!(result["results"]["failed_queries"], 0); + let fanout = &result["results"]["cosmosdb_search_diagnostics"]["partition_fanout"]; + let feed_ranges = fanout["feed_ranges"].as_u64().unwrap(); + println!("Cosmos DB Flat fan-out over {feed_ranges} feed ranges: {fanout}"); + assert!( + feed_ranges >= 2, + "the container has one feed range, so fan-out was not exercised" + ); + assert_eq!(fanout["per_query_concurrency"], feed_ranges); + // Flat is exact, so both routes must return the same IDs. + assert!(fanout["parity"]["samples"].as_u64().unwrap() > 0); + assert_eq!(fanout["parity"]["stock_repeat_missing_ids"], 0); + assert_eq!(fanout["parity"]["cross_mode_missing_ids"], 0); +}