diff --git a/CHANGELOG.md b/CHANGELOG.md index c899c5377..70a37deb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.6.43] - 2026-10-03 + +### Added + +- cli: make --benchmark measure output production; -v shows the rows + +### Changed + +- daemon: never force a tier change on memory pressure + +### Fixed + +- client: report the client's own deadline as Timeout, never as a warm-up retry +- daemon: surface search timeouts as errors and cancel the orphaned scan +- daemon: log the per-tick USN refresh at debug, not info + ## [0.6.42] - 2026-10-03 ### Fixed @@ -2864,7 +2880,8 @@ thin clients over a unified `uffsd` process. ### Fixed - Various MFT parsing edge cases -[Unreleased]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.42...HEAD +[Unreleased]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.43...HEAD +[0.6.43]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.42...v0.6.43 [0.6.42]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.41...v0.6.42 [0.6.41]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.40...v0.6.41 [0.6.40]: https://github.com/skyllc-ai/UltraFastFileSearch/compare/v0.6.38...v0.6.40 diff --git a/Cargo.lock b/Cargo.lock index 0b79deb9f..98e53a39b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4329,7 +4329,7 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "uffs-bench" -version = "0.6.42" +version = "0.6.43" dependencies = [ "chrono", "clap", @@ -4346,7 +4346,7 @@ dependencies = [ [[package]] name = "uffs-broker" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "serde", @@ -4365,14 +4365,14 @@ dependencies = [ [[package]] name = "uffs-broker-protocol" -version = "0.6.42" +version = "0.6.43" dependencies = [ "thiserror 2.0.21", ] [[package]] name = "uffs-ci-pipeline" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "chrono", @@ -4391,7 +4391,7 @@ dependencies = [ [[package]] name = "uffs-cli" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "assert_cmd", @@ -4414,7 +4414,7 @@ dependencies = [ [[package]] name = "uffs-client" -version = "0.6.42" +version = "0.6.43" dependencies = [ "dirs-next", "libc", @@ -4434,7 +4434,7 @@ dependencies = [ [[package]] name = "uffs-core" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "bytemuck", @@ -4465,7 +4465,7 @@ dependencies = [ [[package]] name = "uffs-daemon" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "clap", @@ -4498,7 +4498,7 @@ dependencies = [ [[package]] name = "uffs-diag" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "chrono", @@ -4513,7 +4513,7 @@ dependencies = [ [[package]] name = "uffs-fetch" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "hex", @@ -4524,7 +4524,7 @@ dependencies = [ [[package]] name = "uffs-format" -version = "0.6.42" +version = "0.6.43" dependencies = [ "chrono", "itoa", @@ -4535,7 +4535,7 @@ dependencies = [ [[package]] name = "uffs-gen-hooks" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "clap", @@ -4547,7 +4547,7 @@ dependencies = [ [[package]] name = "uffs-gen-workflow" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "clap", @@ -4560,7 +4560,7 @@ dependencies = [ [[package]] name = "uffs-manifest-audit" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "clap", @@ -4572,7 +4572,7 @@ dependencies = [ [[package]] name = "uffs-mcp" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "axum", @@ -4596,7 +4596,7 @@ dependencies = [ [[package]] name = "uffs-mft" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "bitflags", @@ -4637,14 +4637,14 @@ dependencies = [ [[package]] name = "uffs-polars" -version = "0.6.42" +version = "0.6.43" dependencies = [ "polars", ] [[package]] name = "uffs-security" -version = "0.6.42" +version = "0.6.43" dependencies = [ "aes-gcm", "dirs-next", @@ -4659,22 +4659,22 @@ dependencies = [ [[package]] name = "uffs-statusfmt" -version = "0.6.42" +version = "0.6.43" [[package]] name = "uffs-text" -version = "0.6.42" +version = "0.6.43" dependencies = [ "bytemuck", ] [[package]] name = "uffs-time" -version = "0.6.42" +version = "0.6.43" [[package]] name = "uffs-update" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "dirs-next", @@ -4691,11 +4691,11 @@ dependencies = [ [[package]] name = "uffs-version" -version = "0.6.42" +version = "0.6.43" [[package]] name = "uffs-vss-requestor" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "cc", @@ -4707,7 +4707,7 @@ dependencies = [ [[package]] name = "uffs-watchdog" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "dirs-next", @@ -4716,7 +4716,7 @@ dependencies = [ [[package]] name = "uffs-winsvc" -version = "0.6.42" +version = "0.6.43" dependencies = [ "anyhow", "windows", diff --git a/Cargo.toml b/Cargo.toml index e5afc077a..19384d67a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ members = [ # Workspace Package Metadata (inherited by all crates) # ───────────────────────────────────────────────────────────────────────────── [workspace.package] -version = "0.6.42" +version = "0.6.43" edition = "2024" # No `rust-version` claim: the workspace is structurally nightly-only. # `crates/uffs-polars` enables `polars/nightly` unconditionally, which @@ -135,36 +135,36 @@ publish = false # proposed-plan output for 12 days because `release-plz update` # failed at `cargo package` with this very error. See # `release-automation-baseline.md` §10 for the diagnostic trail. -uffs-polars = { path = "crates/uffs-polars", version = "0.6.42" } -uffs-security = { path = "crates/uffs-security", version = "0.6.42" } -uffs-text = { path = "crates/uffs-text", version = "0.6.42" } -uffs-time = { path = "crates/uffs-time", version = "0.6.42" } -uffs-version = { path = "crates/uffs-version", version = "0.6.42" } -uffs-statusfmt = { path = "crates/uffs-statusfmt", version = "0.6.42" } -uffs-mft = { path = "crates/uffs-mft", version = "0.6.42" } -uffs-format = { path = "crates/uffs-format", version = "0.6.42" } -uffs-core = { path = "crates/uffs-core", version = "0.6.42" } -uffs-client = { path = "crates/uffs-client", version = "0.6.42" } +uffs-polars = { path = "crates/uffs-polars", version = "0.6.43" } +uffs-security = { path = "crates/uffs-security", version = "0.6.43" } +uffs-text = { path = "crates/uffs-text", version = "0.6.43" } +uffs-time = { path = "crates/uffs-time", version = "0.6.43" } +uffs-version = { path = "crates/uffs-version", version = "0.6.43" } +uffs-statusfmt = { path = "crates/uffs-statusfmt", version = "0.6.43" } +uffs-mft = { path = "crates/uffs-mft", version = "0.6.43" } +uffs-format = { path = "crates/uffs-format", version = "0.6.43" } +uffs-core = { path = "crates/uffs-core", version = "0.6.43" } +uffs-client = { path = "crates/uffs-client", version = "0.6.43" } # `uffs-broker-protocol` carries the wire-protocol types shared between # `uffs-broker` (the elevated handle vendor, Windows-only binary) and # `uffs-daemon::broker_client` (the handle consumer). Pure-logic # Layer-0 lib — cross-platform tests run on every CI lane. Added in # F5 (issue #205) so neither side duplicates `BROKER_PIPE_NAME` / # wire-format byte literals. -uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.42" } +uffs-broker-protocol = { path = "crates/uffs-broker-protocol", version = "0.6.43" } # `uffs-winsvc` — native Windows service control (SCM query/start/stop) + # the non-connecting broker-pipe readiness probe. Layer-0 leaf: its only # dependency is the `windows` crate (windows-target), with non-Windows # stubs so cross-platform consumers (uffs-update, uffs-cli) compile. # Single source of truth for the `sc`/SCM mechanics previously duplicated # across uffs-broker, uffs-update, and uffs-cli. -uffs-winsvc = { path = "crates/uffs-winsvc", version = "0.6.42" } +uffs-winsvc = { path = "crates/uffs-winsvc", version = "0.6.43" } # `uffs-fetch` — hardened release-asset transport (blocking reqwest + # rustls with retry/timeout/byte-cap, plus `SHA256SUMS` verification), # extracted from `uffs-update` as a small public lib so external products # can reuse it. Cross-platform pure-logic leaf; keeps the HTTP/TLS stack # out of the lean `uffs` CLI exactly as before. -uffs-fetch = { path = "crates/uffs-fetch", version = "0.6.42" } +uffs-fetch = { path = "crates/uffs-fetch", version = "0.6.43" } # NOTE: no `uffs-broker` workspace dependency alias on purpose — # `uffs-broker` is a binary-only crate (the only `[lib]` it carries is # this protocol module's now-extracted sibling); no other workspace diff --git a/README.md b/README.md index a2ab38010..7532695e9 100644 --- a/README.md +++ b/README.md @@ -265,7 +265,7 @@ uffs --daemon forget C --force # evict + delete on-disk caches ### Memory tiering at a glance -The daemon keeps each drive's compact index in one of four tiers, demoted automatically by an idle TTL ladder + memory-pressure cascade and promoted on first search: +The daemon keeps each drive's compact index in one of four tiers, demoted automatically by an idle TTL ladder and promoted on first search (kernel memory pressure is logged, never acted on): | Tier | RAM cost | Source-of-truth | When | |---|---|---|---| diff --git a/crates/uffs-cli/Cargo.toml b/crates/uffs-cli/Cargo.toml index 334c0ef40..babad84e3 100644 --- a/crates/uffs-cli/Cargo.toml +++ b/crates/uffs-cli/Cargo.toml @@ -70,7 +70,7 @@ path = "src/main.rs" # by the release bump) is required for `cargo package` validation — see # root `Cargo.toml`'s [workspace.dependencies] note for the full # rationale (R6 of `release-automation-plan.md`). -uffs-client = { path = "../uffs-client", version = "0.6.42", default-features = false } +uffs-client = { path = "../uffs-client", version = "0.6.43", default-features = false } # Canonical CSV / parity / legacy-footer writer. Direct dep (not a # re-export chain through `uffs-client`) so the CLI and the daemon @@ -79,7 +79,7 @@ uffs-client = { path = "../uffs-client", version = "0.6.42", default-features = # by the release bump) is required for `cargo package` validation — see # root `Cargo.toml`'s [workspace.dependencies] note for the full # rationale (R6 of `release-automation-plan.md`). -uffs-format = { path = "../uffs-format", version = "0.6.42" } +uffs-format = { path = "../uffs-format", version = "0.6.43" } # Typed drive-letter newtype. Direct dep so the CLI command signatures # (`daemon_load`, `daemon_tiering`, etc.) name `DriveLetter` natively diff --git a/crates/uffs-cli/src/args_help.rs b/crates/uffs-cli/src/args_help.rs index 5f232f0af..9175008ec 100644 --- a/crates/uffs-cli/src/args_help.rs +++ b/crates/uffs-cli/src/args_help.rs @@ -38,7 +38,7 @@ COMMANDS: --status Show combined system status COMMON OPTIONS: - -v, --verbose Verbose output + -v, --verbose Verbose output (with --benchmark: also print the rows) -d, --drive Drive letter (e.g. C or C:) --drives Multiple drive letters --mft-file Raw MFT file(s), comma-separated @@ -61,7 +61,12 @@ COMMON OPTIONS: --min-size Minimum file size (e.g. 100KB, 10MB) --max-size Maximum file size --profile Show timing breakdown - --benchmark Measure only, skip output + --benchmark Time the whole pipeline incl. output formatting; + the rows go to a sink, not the screen (add -v to see + them) + --no-output Match only: the daemon builds no rows, so this times + 'how many files match' and nothing else (auto-set + when stdout is NUL) --help Print this help --version Print version "; diff --git a/crates/uffs-cli/src/client_profile.rs b/crates/uffs-cli/src/client_profile.rs index 4595cdd15..94214f99a 100644 --- a/crates/uffs-cli/src/client_profile.rs +++ b/crates/uffs-cli/src/client_profile.rs @@ -8,6 +8,95 @@ //! and renders it to stderr, with no I/O or daemon knowledge of its //! own. +/// What producing the output cost on the client, measured around the +/// full formatting + write pass. +#[derive(Debug, Clone, Copy)] +pub(crate) struct OutputCost { + /// Wall-clock milliseconds for the whole output pass. + pub(crate) ms: u128, + /// Bytes the pass produced. Exact for the benchmark sink (it + /// counts every byte); for a real stdout the count is what the + /// payload carried, not what the terminal consumed. + pub(crate) bytes: u64, + /// Where the bytes went: `"sink"` under `--benchmark` (formatted and + /// discarded, so only the terminal is excluded), `"stdout"` otherwise. + pub(crate) target: &'static str, +} + +/// Which transport the daemon picked for the payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum PayloadKind { + /// No rows (no match, `--no-output`, or `--out` written by the daemon). + Empty, + /// Typed rows inline in the JSON envelope. + InlineRows, + /// Typed rows in a shared-memory file. + ShmemRows, + /// Pre-rendered text inline in the envelope. + InlineBlob, + /// Pre-rendered bytes in a shared-memory file. + ShmemBlob, +} + +/// What the profile needs to know about the payload, captured **before** +/// the output pass consumes it — the profile prints after the output so +/// it can report the output cost, and cloning a 40 M-row payload to keep +/// it around would be its own benchmark. +#[derive(Debug, Clone, Copy)] +pub(crate) struct PayloadSummary { + /// Transport variant. + pub(crate) kind: PayloadKind, + /// Row count from the cheapest authoritative source for the variant + /// (see [`Self::of`]). + pub(crate) row_count: usize, +} + +impl PayloadSummary { + /// Summarise `payload`, resolving the row count from the cheapest + /// authoritative source per variant: + /// + /// 1. `ShmemBlob` → mmap'd file; counting newlines would read every page + /// just to discard the count, so use the daemon's pre-computed + /// `total_count` instead. + /// 2. `InlineBlob` → inline string already in memory; scanning for `\n` is + /// ~5 GB/s, cheap. + /// 3. Rows variants (`InlineRows`, `ShmemRows`) → `row_count_hint()` is + /// O(1) — `Vec::len` or the daemon's pre-computed count. + /// 4. `Empty` → zero rows, nothing to count. + pub(crate) fn of( + payload: &uffs_client::protocol::response::SearchPayload, + total_count: u64, + ) -> Self { + use uffs_client::protocol::response::SearchPayload; + match payload { + SearchPayload::ShmemBlob(_) => Self { + kind: PayloadKind::ShmemBlob, + // `try_from` instead of `as` to preserve correctness on + // hypothetical 32-bit targets where `u64` would truncate + // (clippy::cast_possible_truncation). `usize::MAX` is a + // strictly larger fallback than any realistic row count. + row_count: usize::try_from(total_count).unwrap_or(usize::MAX), + }, + SearchPayload::InlineBlob(blob) => Self { + kind: PayloadKind::InlineBlob, + row_count: blob.bytes().filter(|byte| *byte == b'\n').count(), + }, + SearchPayload::InlineRows(_) => Self { + kind: PayloadKind::InlineRows, + row_count: payload.row_count_hint().unwrap_or(0), + }, + SearchPayload::ShmemRows { .. } => Self { + kind: PayloadKind::ShmemRows, + row_count: payload.row_count_hint().unwrap_or(0), + }, + SearchPayload::Empty => Self { + kind: PayloadKind::Empty, + row_count: 0, + }, + } + } +} + /// Packaging these into a struct keeps `run_search` under the /// `clippy::too-many-lines` cap and lets the profile helper take one /// argument instead of six. @@ -25,16 +114,11 @@ pub(crate) struct ClientProfile<'a> { /// back in before the scan. `0` on a warm index; tens of seconds /// on a cold one, where it is the entire wall-clock story. pub(crate) promotion_ms: u64, - /// Payload delivery channel the daemon picked for this response. - /// Used by [`print_client_profile`] to show the transport name - /// and to pick the cheapest authoritative row-count source. - pub(crate) payload: &'a uffs_client::protocol::response::SearchPayload, - /// Total row count reported by the daemon, independent of which - /// transport carries the payload. Used to display the "Total - /// matches:" line when the transport is a shmem blob — counting - /// newlines in the mmap would consume the file before the stdout - /// write and double the syscall cost. - pub(crate) total_count: u64, + /// Payload delivery channel the daemon picked for this response + /// plus its row count, captured before the output pass consumed + /// the payload. Used by [`print_client_profile`] to show the + /// transport name and the count. + pub(crate) payload: PayloadSummary, /// Daemon-side `profile` object from the response envelope. When /// populated, its `scan_ms` / `sort_ms` / `path_resolve_ms` / /// `write_ms` fields are rendered as a sub-phase breakdown inside @@ -42,6 +126,10 @@ pub(crate) struct ClientProfile<'a> { /// per-query cost sits (scan vs sort vs path resolution vs disk /// write). pub(crate) daemon_profile: Option<&'a uffs_client::protocol::response::SearchProfile>, + /// Client-side output cost, when an output pass ran (`None` under + /// `--no-output`, where neither the daemon nor the client produces + /// rows). + pub(crate) output: Option, } /// Print the `--profile` / `--benchmark` client-side timing block to @@ -51,8 +139,6 @@ pub(crate) struct ClientProfile<'a> { reason = "intentional --profile output to stderr" )] pub(crate) fn print_client_profile(prof: &ClientProfile<'_>) { - use uffs_client::protocol::response::SearchPayload; - eprintln!("=== PROFILE: Client → Daemon ==="); eprintln!(" Connect: {:>6} ms", prof.connect_ms); eprintln!(" Await ready: {:>6} ms", prof.ready_ms); @@ -62,6 +148,12 @@ pub(crate) fn print_client_profile(prof: &ClientProfile<'_>) { ); // Printed only when it happened: a warm index promotes nothing, and // a zero line every run would train the eye to skip it. + if let Some(output) = prof.output { + eprintln!( + " Output ({:<6}): {:>6} ms ({} bytes formatted)", + output.target, output.ms, output.bytes + ); + } if prof.promotion_ms > 0 { eprintln!( " Index warm-up: {:>6} ms (paged parked/cold drives back in)", @@ -115,55 +207,35 @@ pub(crate) fn print_client_profile(prof: &ClientProfile<'_>) { ); } } - // Row count resolution — pick the cheapest authoritative source - // depending on which payload variant the daemon used: - // 1. `ShmemBlob` → mmap'd file; counting newlines would read every page just to - // discard the count, so use the daemon's pre- computed `total_count` - // instead. - // 2. `InlineBlob` → inline string already in memory; scanning for `\n` is ~5 - // GB/s, cheap. - // 3. Rows variants (`InlineRows`, `ShmemRows`) → `row_count_hint()` is O(1) — - // `Vec::len` or the daemon's pre-computed count. - // 4. `Empty` → zero rows, nothing to count. - let row_count = match prof.payload { - SearchPayload::ShmemBlob(_) => { - // `try_from` instead of `as` to preserve correctness on - // hypothetical 32-bit targets where `u64` would truncate - // (clippy::cast_possible_truncation). `u64::MAX` is a - // strictly larger fallback than any realistic row count. - usize::try_from(prof.total_count).unwrap_or(usize::MAX) - } - SearchPayload::InlineBlob(blob) => blob.bytes().filter(|byte| *byte == b'\n').count(), - SearchPayload::InlineRows(_) | SearchPayload::ShmemRows { .. } | SearchPayload::Empty => { - prof.payload.row_count_hint().unwrap_or(0) - } - }; + // Row count resolution lives in `PayloadSummary::of` (captured before + // the output pass consumed the payload). + let row_count = prof.payload.row_count; // Label the count by what it actually measures per transport: blob // variants carry rendered text (newline count includes header/footer // lines) or the daemon's pre-limit total, NOT the post-`--limit` page // (2026-06-12 dry run: `--limit 5` printed "Rows returned: 7"). - match prof.payload { - SearchPayload::ShmemBlob(_) => { + match prof.payload.kind { + PayloadKind::ShmemBlob => { eprintln!(" Total matches: {row_count:>6}"); } - SearchPayload::InlineBlob(_) => { + PayloadKind::InlineBlob => { eprintln!(" Output lines: {row_count:>6}"); } - SearchPayload::InlineRows(_) | SearchPayload::ShmemRows { .. } | SearchPayload::Empty => { + PayloadKind::InlineRows | PayloadKind::ShmemRows | PayloadKind::Empty => { eprintln!(" Rows returned: {row_count:>6}"); } } - match prof.payload { - SearchPayload::ShmemBlob(_) => { + match prof.payload.kind { + PayloadKind::ShmemBlob => { eprintln!(" Transport: shmem_blob (mmap + write_all, binary)"); } - SearchPayload::InlineBlob(_) => { + PayloadKind::InlineBlob => { eprintln!(" Transport: inline_blob (single write_all)"); } - SearchPayload::ShmemRows { .. } => { + PayloadKind::ShmemRows => { eprintln!(" Transport: shmem_rows (mmap + per-row format)"); } - SearchPayload::InlineRows(_) | SearchPayload::Empty => { + PayloadKind::InlineRows | PayloadKind::Empty => { // inline_rows is the default — no extra line needed. // empty responses skip the transport line entirely. } diff --git a/crates/uffs-cli/src/commands/output/mod.rs b/crates/uffs-cli/src/commands/output/mod.rs index ecaf06192..dab6aed33 100644 --- a/crates/uffs-cli/src/commands/output/mod.rs +++ b/crates/uffs-cli/src/commands/output/mod.rs @@ -8,12 +8,14 @@ mod parity; +mod render_into; use core::time::Duration; use std::fs::File; use std::io::{BufWriter, Write}; use anyhow::{Context as _, Result}; use parity::{write_legacy_drive_footer, write_parity}; +pub use render_into::render_native_results_into; use serde_json::Value; // ── Value extraction helpers ─────────────────────────────────────────── @@ -97,14 +99,7 @@ pub fn write_native_results( row_count: rows.len(), }; - let parity_ctx = ParityContext { - pos, - neg, - tz_offset_secs: tz_offset.map_or_else( - || *LOCAL_TZ_OFFSET_SECS, - |hours| hours.saturating_mul(3_600_i32), - ), - }; + let parity_ctx = ParityContext::new(pos, neg, tz_offset); if is_console { write_to_stdout( @@ -248,6 +243,22 @@ struct ParityContext<'a> { tz_offset_secs: i32, } +impl<'a> ParityContext<'a> { + /// Resolve the parity context from the CLI's `--pos` / `--neg` / + /// `--tz-offset` (hours) settings; an absent offset uses the local + /// zone. + fn new(pos: &'a str, neg: &'a str, tz_offset: Option) -> Self { + Self { + pos, + neg, + tz_offset_secs: tz_offset.map_or_else( + || *LOCAL_TZ_OFFSET_SECS, + |hours| hours.saturating_mul(3_600_i32), + ), + } + } +} + /// Dispatch to the appropriate formatter. #[expect(clippy::too_many_arguments, reason = "output config forwarding")] fn write_formatted( diff --git a/crates/uffs-cli/src/commands/output/render_into.rs b/crates/uffs-cli/src/commands/output/render_into.rs new file mode 100644 index 000000000..b9ece027e --- /dev/null +++ b/crates/uffs-cli/src/commands/output/render_into.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! The `--benchmark` output sink: render search rows with the console +//! formatter into any writer. +//! +//! Split from [`super`] for the 800-LOC file-size policy; shares the +//! parent's private `write_formatted` / context builders so the sink +//! renders byte-for-byte what the console would. + +use std::io::Write; + +use anyhow::Result; +use serde_json::Value; + +use super::{CppFooterContext, ParityContext, write_formatted}; + +/// Render rows exactly as the console path would, into `writer`. +/// +/// The `--benchmark` sink: the full formatting pipeline (column +/// resolution, quoting, header / footer, parity timestamps) runs through +/// this into a byte-counting writer, so the measured cost is everything +/// UFFS does to produce the output with only the terminal left out. +/// Same formatter as [`super::write_native_results`]'s console branch; only the +/// destination differs. +/// +/// # Errors +/// +/// Returns an error if formatting or the write fails. +#[expect(clippy::too_many_arguments, reason = "output config forwarding")] +pub fn render_native_results_into( + writer: &mut W, + rows: &[Value], + format: &str, + columns: &str, + separator: &str, + quote: &str, + header: bool, + pos: &str, + neg: &str, + tz_offset: Option, + output_targets: &[uffs_mft::platform::DriveLetter], + pattern: &str, +) -> Result<()> { + let footer_ctx = CppFooterContext { + output_targets, + pattern, + row_count: rows.len(), + }; + let parity_ctx = ParityContext::new(pos, neg, tz_offset); + write_formatted( + writer, + rows, + format, + columns, + separator, + quote, + header, + &footer_ctx, + &parity_ctx, + )?; + writer.flush()?; + Ok(()) +} diff --git a/crates/uffs-cli/src/commands/search/dispatch.rs b/crates/uffs-cli/src/commands/search/dispatch.rs index 583f11be3..21f5d1b8a 100644 --- a/crates/uffs-cli/src/commands/search/dispatch.rs +++ b/crates/uffs-cli/src/commands/search/dispatch.rs @@ -6,12 +6,12 @@ //! Extracts format/column/separator settings from raw CLI args and //! delegates to the output module for formatting. -use std::io::Write as _; +use std::io::Write; use anyhow::Result; use uffs_client::format::extract_drive_letter; -use super::super::output::write_native_results; +use super::super::output::{render_native_results_into, write_native_results}; // ── Thin-client output helpers ───────────────────────────────────────── // @@ -47,84 +47,161 @@ fn default_format(out_is_console: bool) -> &'static str { } } -/// Write search result rows to console using format extracted from raw -/// CLI args. -/// -/// The daemon already writes to file when `--out` is set (OPT-4), -/// so this only handles console output. -/// -/// # Errors -/// -/// Returns an error if writing fails. -pub fn write_rows(rows: &[serde_json::Value], args: &[String]) -> Result<()> { - let out = arg_val(args, "--out").unwrap_or("console"); - let format = arg_val(args, "--format") - .or_else(|| arg_val(args, "-f")) - .unwrap_or_else(|| default_format(out == "console")); - // --parity-compat implies --columns parity (matches legacy OutputConfig - // behaviour). - let parity_compat = args.iter().any(|arg| arg == "--parity-compat"); - let columns = if parity_compat { - "parity" - } else { - arg_val(args, "--columns").unwrap_or("") - }; - let sep = arg_val(args, "--sep").unwrap_or(","); - let quotes = arg_val(args, "--quotes").unwrap_or("\""); - let header = arg_val(args, "--header").is_none_or(|val| val != "false" && val != "0"); - let pos = arg_val(args, "--pos").unwrap_or("1"); - let neg = arg_val(args, "--neg").unwrap_or("0"); - let tz_offset = arg_val(args, "--tz-offset").and_then(|val| val.parse::().ok()); +/// Output settings resolved from the raw CLI args — the one place +/// `--format` / `--columns` / `--sep` / drive targets are read, shared +/// by the console path ([`write_rows`]) and the benchmark sink +/// ([`write_rows_into`]) so both render byte-for-byte the same output. +struct OutputSettings<'a> { + /// `--out` destination (`console` when absent). + out: &'a str, + /// Resolved `--format`. + format: &'a str, + /// `--columns`, or `parity` when `--parity-compat` is set. + columns: &'a str, + /// `--sep`. + sep: &'a str, + /// `--quotes`. + quotes: &'a str, + /// `--header` (default on). + header: bool, + /// `--pos` parity boolean string. + pos: &'a str, + /// `--neg` parity boolean string. + neg: &'a str, + /// `--tz-offset` in hours. + tz_offset: Option, + /// Drive letters for the footer. + targets: Vec, + /// The search pattern (first positional). + pattern: &'a str, +} - // Extract drive targets for footer. - let drive = arg_val(args, "--drive").or_else(|| arg_val(args, "-d")); - let drives_str = arg_val(args, "--drives"); - let mft_str = arg_val(args, "--mft-file"); - let mut targets: Vec = Vec::new(); - if let Some(drive_val) = drive { - if let Some(letter) = drive_val - .chars() - .next() - .and_then(|ch| uffs_mft::platform::DriveLetter::parse(ch).ok()) - { - targets.push(letter); - } - } else if let Some(drives_val) = drives_str { - for part in drives_val.split(',') { - let trimmed = part.trim(); - let stripped = trimmed.strip_suffix(':').unwrap_or(trimmed); - if let Some(letter) = stripped +impl<'a> OutputSettings<'a> { + /// Read every output-affecting flag from `args`. + fn from_args(args: &'a [String]) -> Self { + let out = arg_val(args, "--out").unwrap_or("console"); + let format = arg_val(args, "--format") + .or_else(|| arg_val(args, "-f")) + .unwrap_or_else(|| default_format(out == "console")); + // --parity-compat implies --columns parity (matches legacy OutputConfig + // behaviour). + let parity_compat = args.iter().any(|arg| arg == "--parity-compat"); + let columns = if parity_compat { + "parity" + } else { + arg_val(args, "--columns").unwrap_or("") + }; + let sep = arg_val(args, "--sep").unwrap_or(","); + let quotes = arg_val(args, "--quotes").unwrap_or("\""); + let header = arg_val(args, "--header").is_none_or(|val| val != "false" && val != "0"); + let pos = arg_val(args, "--pos").unwrap_or("1"); + let neg = arg_val(args, "--neg").unwrap_or("0"); + let tz_offset = arg_val(args, "--tz-offset").and_then(|val| val.parse::().ok()); + + // Extract drive targets for footer. + let drive = arg_val(args, "--drive").or_else(|| arg_val(args, "-d")); + let drives_str = arg_val(args, "--drives"); + let mft_str = arg_val(args, "--mft-file"); + let mut targets: Vec = Vec::new(); + if let Some(drive_val) = drive { + if let Some(letter) = drive_val .chars() .next() .and_then(|ch| uffs_mft::platform::DriveLetter::parse(ch).ok()) { targets.push(letter); } - } - } else if let Some(mft_val) = mft_str { - for part in mft_val.split(',') { - if let Some(letter) = extract_drive_letter(part.trim()) { - targets.push(letter); + } else if let Some(drives_val) = drives_str { + for part in drives_val.split(',') { + let trimmed = part.trim(); + let stripped = trimmed.strip_suffix(':').unwrap_or(trimmed); + if let Some(letter) = stripped + .chars() + .next() + .and_then(|ch| uffs_mft::platform::DriveLetter::parse(ch).ok()) + { + targets.push(letter); + } + } + } else if let Some(mft_val) = mft_str { + for part in mft_val.split(',') { + if let Some(letter) = extract_drive_letter(part.trim()) { + targets.push(letter); + } } } - } - let pattern = args.first().map_or("*", String::as_str); + let pattern = args.first().map_or("*", String::as_str); + Self { + out, + format, + columns, + sep, + quotes, + header, + pos, + neg, + tz_offset, + targets, + pattern, + } + } +} + +/// Write search result rows to console using format extracted from raw +/// CLI args. +/// +/// The daemon already writes to file when `--out` is set (OPT-4), +/// so this only handles console output. +/// +/// # Errors +/// +/// Returns an error if writing fails. +pub fn write_rows(rows: &[serde_json::Value], args: &[String]) -> Result<()> { + let cfg = OutputSettings::from_args(args); write_native_results( rows, - format, - out, - columns, - sep, - quotes, - header, - pos, - neg, - tz_offset, - &targets, + cfg.format, + cfg.out, + cfg.columns, + cfg.sep, + cfg.quotes, + cfg.header, + cfg.pos, + cfg.neg, + cfg.tz_offset, + &cfg.targets, core::time::Duration::ZERO, - pattern, + cfg.pattern, + ) +} + +/// Render search result rows with the same settings as [`write_rows`], +/// but into `writer` instead of the console — the `--benchmark` sink. +/// +/// # Errors +/// +/// Returns an error if formatting or the write fails. +pub(crate) fn write_rows_into( + writer: &mut W, + rows: &[serde_json::Value], + args: &[String], +) -> Result<()> { + let cfg = OutputSettings::from_args(args); + render_native_results_into( + writer, + rows, + cfg.format, + cfg.columns, + cfg.sep, + cfg.quotes, + cfg.header, + cfg.pos, + cfg.neg, + cfg.tz_offset, + &cfg.targets, + cfg.pattern, ) } diff --git a/crates/uffs-cli/src/commands/search/run.rs b/crates/uffs-cli/src/commands/search/run.rs index 4131da313..62ff189f4 100644 --- a/crates/uffs-cli/src/commands/search/run.rs +++ b/crates/uffs-cli/src/commands/search/run.rs @@ -11,11 +11,12 @@ //! the single place where a search leaves the CLI. use anyhow::{Context as _, Result}; +use uffs_client::error::ClientError; use uffs_client::protocol::response::SearchPayload; use super::args::{extract_spawn_args, inject_no_output_for_null_stdout, resolve_out_path}; -use super::dispatch::{write_aggregations, write_rows}; -use crate::client_profile::{ClientProfile, print_client_profile}; +use super::dispatch::{write_aggregations, write_rows, write_rows_into}; +use crate::client_profile::{ClientProfile, OutputCost, PayloadSummary, print_client_profile}; use crate::{args, dispatch, search_retry}; /// Forward raw search args to the daemon via `search_cli` RPC. @@ -83,8 +84,11 @@ pub(crate) fn run_search(args: &[String]) -> Result<()> { // sets that would otherwise push 3.5 MB through the pipe just to // discard the bytes client-side. let args_owned: Vec = inject_no_output_for_null_stdout(resolve_out_path(args)); - let raw_response = search_retry::search_cli_with_warm_retry(&mut client, &args_owned) - .with_context(|| "Daemon search_cli failed")?; + let raw_response = match search_retry::search_cli_with_warm_retry(&mut client, &args_owned) { + Ok(response) => response, + Err(ClientError::Timeout) => anyhow::bail!(client_timeout_message()), + Err(err) => return Err(err).with_context(|| "Daemon search_cli failed"), + }; let ipc_ms = t_search.elapsed().as_millis(); // v0.5.62: deserialise the daemon response into the typed @@ -102,71 +106,191 @@ pub(crate) fn run_search(args: &[String]) -> Result<()> { serde_json::from_value(raw_response) .with_context(|| "Failed to deserialize search response from daemon")?; - if args + let profiling = args .iter() - .any(|arg| arg == "--profile" || arg == "--benchmark") - { + .any(|arg| arg == "--profile" || arg == "--benchmark"); + + // OPT-4: When --out is specified, the daemon writes the file directly + // and returns `SearchPayload::Empty`. Don't overwrite the file. + // Handles both `--out foo.csv` (separate arg) and `--out=foo.csv` (= form). + let has_out = args + .iter() + .any(|arg| arg == "--out" || arg.starts_with("--out=")); + let daemon_wrote_file = has_out && response.payload.is_empty(); + + // The profile block is printed after the output pass so it can report + // what producing the output cost; it goes to stderr, so with `-v` the + // rows and the timings never interleave on one stream. + let payload_summary = PayloadSummary::of(&response.payload, response.total_count); + let payload_bytes = payload_byte_hint(&response.payload); + let output_mode = OutputMode::from_args(&args_owned); + let output_cost = match output_mode { + OutputMode::Skip => None, + OutputMode::Stdout => { + let t_out = std::time::Instant::now(); + if !daemon_wrote_file { + write_search_payload(response.payload, args, &mut OutputTarget::Stdout)?; + } + write_aggregation_values(&response.aggregations, args)?; + Some(OutputCost { + ms: t_out.elapsed().as_millis(), + bytes: payload_bytes, + target: "stdout", + }) + } + OutputMode::Sink => { + let t_out = std::time::Instant::now(); + let mut sink = CountingSink::default(); + if !daemon_wrote_file { + write_search_payload(response.payload, args, &mut OutputTarget::Sink(&mut sink))?; + } + if !response.aggregations.is_empty() { + // Aggregations render as pretty JSON into the sink — the + // table / CSV printers are stdout-bound; `-v` shows them. + let json = serde_json::to_string_pretty(&response.aggregations)?; + std::io::Write::write_all(&mut sink, json.as_bytes())?; + } + Some(OutputCost { + ms: t_out.elapsed().as_millis(), + bytes: sink.bytes, + target: "sink", + }) + } + }; + + if profiling { print_client_profile(&ClientProfile { connect_ms, ready_ms, ipc_ms, duration_ms: response.duration_ms, promotion_ms: response.promotion_ms.unwrap_or(0), - payload: &response.payload, - total_count: response.total_count, + payload: payload_summary, daemon_profile: response.profile.as_ref(), + output: output_cost, }); } - // OPT-4: When --out is specified, the daemon writes the file directly - // and returns `SearchPayload::Empty`. Don't overwrite the file. - // Handles both `--out foo.csv` (separate arg) and `--out=foo.csv` (= form). - let has_out = args - .iter() - .any(|arg| arg == "--out" || arg.starts_with("--out=")); - let daemon_wrote_file = has_out && response.payload.is_empty(); + Ok(()) +} - let suppress_stdout = should_suppress_stdout(&args_owned); +/// Where a search's rows go on the client. +/// +/// * `Stdout` — the normal path, and `--benchmark -v`. +/// * `Sink` — `--benchmark` without `-v`: the full formatting pipeline runs +/// into a byte-counting writer, so the profile measures everything UFFS does +/// to produce the output and excludes only the terminal. Before this the +/// benchmark skipped the write entirely (#626 fix) and so never measured +/// output production at all, which is the thing a benchmark of a file-search +/// tool is for. +/// * `Skip` — `--no-output` (explicit, or auto-injected when stdout is a null +/// device): the daemon does not even build rows, so there is nothing to +/// format. +/// +/// Pure so the decision is unit-testable without a daemon. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OutputMode { + /// Format and write to the real stdout. + Stdout, + /// Format into the counting sink. + Sink, + /// No output pass at all. + Skip, +} - if !daemon_wrote_file && !suppress_stdout { - write_search_payload_to_stdout(response.payload, args)?; +impl OutputMode { + /// Decide the output mode from the raw search args. + fn from_args(args: &[String]) -> Self { + let has = |flag: &str| args.iter().any(|arg| arg == flag); + if has("--no-output") { + Self::Skip + } else if has("--benchmark") && !has("-v") && !has("--verbose") { + Self::Sink + } else { + Self::Stdout + } } +} - if !suppress_stdout && !response.aggregations.is_empty() { - // `write_aggregations` still consumes `&[serde_json::Value]` - // for format flexibility — re-serialise the typed - // `AggregateResultWire` list via `to_value` once up front - // and pass the slice to the helper. Allocation is one per - // aggregation bucket, which is trivial compared to the - // aggregation itself. - let agg_values: Vec = response - .aggregations - .iter() - .filter_map(|agg| serde_json::to_value(agg).ok()) - .collect(); - write_aggregations(&agg_values, args)?; +/// Byte-counting writer backing [`OutputMode::Sink`]. +#[derive(Debug, Default)] +struct CountingSink { + /// Bytes written so far. + bytes: u64, +} + +impl std::io::Write for CountingSink { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.bytes = self.bytes.saturating_add(buf.len() as u64); + Ok(buf.len()) } - Ok(()) + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } } -/// Whether every client-side stdout write for a search should be skipped. -/// -/// Two flags opt out of output: -/// -/// * `--no-output` — the Phase 3.1 NUL fast path, given explicitly or -/// auto-injected when stdout is a null device (`uffs *.dll > NUL`). -/// * `--benchmark` — documented in `args_help.rs` as "Measure only, skip -/// output". Until this helper existed only `--no-output` was honoured, so a -/// benchmark against a wide query printed every matching row and the timing -/// line scrolled straight off the terminal (reported in #626). The profile -/// summary is printed separately and is unaffected: that is the benchmark's -/// actual output. +/// Destination for [`write_search_payload`]. +enum OutputTarget<'a> { + /// The process's stdout. + Stdout, + /// The benchmark sink. + Sink(&'a mut CountingSink), +} + +/// Bytes the payload carries as delivered, for the profile's stdout line +/// (the sink counts exactly; stdout reports what it was handed). +const fn payload_byte_hint(payload: &SearchPayload) -> u64 { + match payload { + SearchPayload::InlineBlob(blob) => blob.len() as u64, + SearchPayload::ShmemBlob(_) + | SearchPayload::ShmemRows { .. } + | SearchPayload::InlineRows(_) + | SearchPayload::Empty => 0, + } +} + +/// Write the aggregation results (if any) to stdout. +fn write_aggregation_values( + aggregations: &[uffs_client::protocol::AggregateResultWire], + args: &[String], +) -> Result<()> { + if aggregations.is_empty() { + return Ok(()); + } + // `write_aggregations` still consumes `&[serde_json::Value]` + // for format flexibility — re-serialise the typed + // `AggregateResultWire` list via `to_value` once up front + // and pass the slice to the helper. Allocation is one per + // aggregation bucket, which is trivial compared to the + // aggregation itself. + let agg_values: Vec = aggregations + .iter() + .filter_map(|agg| serde_json::to_value(agg).ok()) + .collect(); + write_aggregations(&agg_values, args) +} + +/// The message for a search the client stopped waiting for. /// -/// Pure so the decision is unit-testable without a daemon. -fn should_suppress_stdout(args: &[String]) -> bool { - args.iter() - .any(|arg| arg == "--no-output" || arg == "--benchmark") +/// Honest about what happened: the daemon has not failed and is still +/// running the search — in practice it is paging parked drives back in, +/// which re-reads the MFT and can take minutes on a large HDD volume. +/// Before this the client mistook its own deadline for the "index +/// warming" transient, printed a retry note, and re-sent the search up +/// to five times. +fn client_timeout_message() -> String { + let budget = uffs_client::rpc_deadline().map_or_else( + || "the client deadline".to_owned(), + |budget| format!("the client deadline of {}s", budget.as_secs()), + ); + format!( + "the daemon did not answer within {budget} (UFFS_CLIENT_TIMEOUT_SECS).\n\ + The search is still running on the daemon. Most likely it is paging \ + parked drives back in, which re-reads the MFT and can take minutes on a \ + large HDD volume. Re-run once it has finished, or raise \ + UFFS_CLIENT_TIMEOUT_SECS for this query." + ) } /// Write the daemon's search payload to stdout, picking the fastest @@ -190,8 +314,13 @@ fn should_suppress_stdout(args: &[String]) -> bool { /// 5. [`SearchPayload::Empty`] → nothing to write. /// /// Extracted from [`run_search`] to keep that function under the -/// `clippy::too_many_lines` cap. -fn write_search_payload_to_stdout(payload: SearchPayload, args: &[String]) -> Result<()> { +/// `clippy::too_many_lines` cap. `target` picks the real stdout or the +/// `--benchmark` sink; every variant does the same work either way. +fn write_search_payload( + payload: SearchPayload, + args: &[String], + target: &mut OutputTarget<'_>, +) -> Result<()> { match payload { SearchPayload::Empty => { // Nothing to write — no-match query, `--no-output` @@ -205,19 +334,31 @@ fn write_search_payload_to_stdout(payload: SearchPayload, args: &[String]) -> Re // file. No JSON decode, no intermediate allocation, no // UTF-8 re-validation — stdout takes bytes. let shmem_path = std::path::Path::new(&shmem_path_str); - let stdout = std::io::stdout(); - let mut handle = stdout.lock(); - uffs_client::shmem::stream_paths_blob_into(shmem_path, &mut handle) - .with_context(|| format!("Failed to stream shmem_blob from {shmem_path_str}"))?; + match target { + OutputTarget::Stdout => { + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + uffs_client::shmem::stream_paths_blob_into(shmem_path, &mut handle) + } + OutputTarget::Sink(sink) => { + uffs_client::shmem::stream_paths_blob_into(shmem_path, sink) + } + } + .with_context(|| format!("Failed to stream shmem_blob from {shmem_path_str}"))?; } SearchPayload::InlineBlob(blob) => { // Single write_all to stdout — the buffer is one // contiguous slice; the whole point of the blob // inline transport. - let stdout = std::io::stdout(); - let mut handle = stdout.lock(); - std::io::Write::write_all(&mut handle, blob.as_bytes()) - .with_context(|| "Failed to write inline_blob to stdout")?; + match target { + OutputTarget::Stdout => { + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + std::io::Write::write_all(&mut handle, blob.as_bytes()) + } + OutputTarget::Sink(sink) => std::io::Write::write_all(sink, blob.as_bytes()), + } + .with_context(|| "Failed to write inline_blob")?; } SearchPayload::ShmemRows { path, .. } => { // Shmem rows variant: read the file (returns a @@ -236,7 +377,7 @@ fn write_search_payload_to_stdout(payload: SearchPayload, args: &[String]) -> Re .iter() .filter_map(|row| serde_json::to_value(row).ok()) .collect(); - write_rows(&row_values, args)?; + write_row_values(&row_values, args, target)?; } SearchPayload::InlineRows(rows) => { // Traditional per-row format dispatch. `write_rows` @@ -247,40 +388,84 @@ fn write_search_payload_to_stdout(payload: SearchPayload, args: &[String]) -> Re .iter() .filter_map(|row| serde_json::to_value(row).ok()) .collect(); - write_rows(&row_values, args)?; + write_row_values(&row_values, args, target)?; } } Ok(()) } +/// Per-row format dispatch to the chosen target. +fn write_row_values( + rows: &[serde_json::Value], + args: &[String], + target: &mut OutputTarget<'_>, +) -> Result<()> { + match target { + OutputTarget::Stdout => write_rows(rows, args), + OutputTarget::Sink(sink) => write_rows_into(sink, rows, args), + } +} + #[cfg(test)] mod tests { use uffs_client::protocol::SearchParams; - use super::should_suppress_stdout; + use super::OutputMode; fn args(list: &[&str]) -> Vec { list.iter().map(|arg| (*arg).to_owned()).collect() } - /// `--benchmark` is documented as "Measure only, skip output"; before - /// #626 only `--no-output` was honoured, so a benchmark flooded the - /// terminal with every matching row. Both flags must suppress rows, and - /// an ordinary search must not. + /// `--benchmark` must run the output pipeline into the sink (it is + /// what the benchmark measures), `-v` / `--verbose` redirects it to + /// the real stdout, and only `--no-output` skips the pass. A value + /// that merely contains the flag text is not the flag. #[test] - fn benchmark_and_no_output_both_suppress_stdout() { - assert!(should_suppress_stdout(&args(&["*.rs", "--benchmark"]))); - assert!(should_suppress_stdout(&args(&["*.rs", "--no-output"]))); - assert!(should_suppress_stdout(&args(&[ - "--benchmark", - "*.rs", - "--drive", - "C" - ]))); - assert!(!should_suppress_stdout(&args(&["*.rs"]))); - assert!(!should_suppress_stdout(&args(&["*.rs", "--profile"]))); - // A value that merely contains the flag text is not the flag. - assert!(!should_suppress_stdout(&args(&["--benchmark-results.txt"]))); + fn output_mode_resolves_benchmark_verbose_and_no_output() { + assert_eq!(OutputMode::from_args(&args(&["*.rs"])), OutputMode::Stdout); + assert_eq!( + OutputMode::from_args(&args(&["*.rs", "--profile"])), + OutputMode::Stdout + ); + assert_eq!( + OutputMode::from_args(&args(&["*.rs", "--benchmark"])), + OutputMode::Sink + ); + assert_eq!( + OutputMode::from_args(&args(&["--benchmark", "--drive", "C", "*"])), + OutputMode::Sink + ); + assert_eq!( + OutputMode::from_args(&args(&["*.rs", "--benchmark", "-v"])), + OutputMode::Stdout + ); + assert_eq!( + OutputMode::from_args(&args(&["*.rs", "--verbose", "--benchmark"])), + OutputMode::Stdout + ); + assert_eq!( + OutputMode::from_args(&args(&["*.rs", "--no-output"])), + OutputMode::Skip + ); + assert_eq!( + OutputMode::from_args(&args(&["*.rs", "--benchmark", "--no-output"])), + OutputMode::Skip + ); + assert_eq!( + OutputMode::from_args(&args(&["--benchmark-results.txt"])), + OutputMode::Stdout + ); + } + + /// The sink counts every byte it is handed and never fails. + #[test] + fn counting_sink_counts_bytes() { + use std::io::Write as _; + let mut sink = super::CountingSink::default(); + sink.write_all(b"hello\n").expect("sink never fails"); + sink.write_all(b"world").expect("sink never fails"); + sink.flush().expect("sink never fails"); + assert_eq!(sink.bytes, 11); } #[test] diff --git a/crates/uffs-cli/src/search_retry.rs b/crates/uffs-cli/src/search_retry.rs index c4857fc4b..8ba1d75e2 100644 --- a/crates/uffs-cli/src/search_retry.rs +++ b/crates/uffs-cli/src/search_retry.rs @@ -11,6 +11,14 @@ //! primitive ([`uffs-mft`'s `read_handle_at`]) already retries that transient; //! this is the client-side belt-and-suspenders so a user never sees a raw I/O //! error for a recoverable warm-up hiccup — and gets a "warming" note instead. +//! +//! Only an error the **daemon** reported is a candidate for the retry. The +//! client's own per-RPC deadline (`UFFS_CLIENT_TIMEOUT_SECS`, enforced on +//! Windows through `CancelSynchronousIo`, which also yields os error 995 on +//! the cancelled read) surfaces as [`ClientError::Timeout`] and is never +//! retried: the daemon is still executing that search, and re-sending it +//! would only queue another full scan behind the first — the 2026-10-03 +//! benchmark run did exactly that, five `*.*` scans for three invocations. // Each helper is used exactly once on the search path; this is cohesion, not a // smell — the `single_call_fn` restriction lint is relaxed for the module. @@ -28,10 +36,12 @@ const WARM_RETRY_MAX: u32 = 5; const WARM_RETRY_BACKOFF: core::time::Duration = core::time::Duration::from_millis(400); /// Run `search_cli_raw`, transparently retrying the one transient a search can -/// hit while the daemon re-warms parked drives: Windows +/// hit while the daemon re-warms parked drives: a daemon-reported Windows /// `ERROR_OPERATION_ABORTED` (os error 995). /// -/// Bounded + back-off so a genuine, persistent failure still fails fast. +/// Bounded + back-off so a genuine, persistent failure still fails fast. A +/// [`ClientError::Timeout`] is returned as-is on the first occurrence — see the +/// module docs for why the client's own deadline must never trigger a retry. pub(crate) fn search_cli_with_warm_retry( client: &mut UffsClientSync, args: &[String], @@ -50,13 +60,20 @@ pub(crate) fn search_cli_with_warm_retry( } } -/// `true` if `err` is the transient `ERROR_OPERATION_ABORTED` (os error 995) a -/// search hits when it races a parked-drive re-warm. Matched on the stable OS -/// error code in the rendered message — the daemon's typed I/O error is -/// flattened to a string across the JSON-RPC boundary, so the code is the only -/// portable signal left. +/// `true` if `err` is a **daemon-reported** transient `ERROR_OPERATION_ABORTED` +/// (os error 995) — what a search hits when it races a parked-drive re-warm. +/// Matched on the stable OS error code in the rendered message — the daemon's +/// typed I/O error is flattened to a string across the JSON-RPC boundary, so +/// the code is the only portable signal left. +/// +/// A client-side transport error never qualifies, whatever its text: the only +/// client-side source of os error 995 is the deadline watchdog cancelling our +/// own read, and the sync client already reports that as +/// [`ClientError::Timeout`]. fn is_index_warming_abort(err: &ClientError) -> bool { - let message = err.to_string(); + let ClientError::DaemonError { message, .. } = err else { + return false; + }; message.contains("os error 995") || message.contains("operation has been aborted") } @@ -78,24 +95,45 @@ mod tests { use super::is_index_warming_abort; - /// Only a transient `ERROR_OPERATION_ABORTED` (os error 995) — what a - /// search hits while racing a parked-drive re-warm — should be retried; - /// real errors must surface immediately. + /// Only a daemon-reported transient `ERROR_OPERATION_ABORTED` (os error + /// 995) — what a search hits while racing a parked-drive re-warm — should + /// be retried; real errors must surface immediately. #[test] - fn warming_abort_matches_only_995() { - assert!(is_index_warming_abort(&ClientError::Io( - "The I/O operation has been aborted because of either a thread exit \ - or an application request. (os error 995)" - .to_owned() - ))); + fn warming_abort_matches_only_daemon_reported_995() { assert!(is_index_warming_abort(&ClientError::DaemonError { code: -32000, message: "I/O error: ... (os error 995)".to_owned(), })); + assert!(is_index_warming_abort(&ClientError::DaemonError { + code: -32000, + message: "The I/O operation has been aborted because of either a thread exit \ + or an application request." + .to_owned(), + })); // Real failures are NOT retried. + assert!(!is_index_warming_abort(&ClientError::DaemonError { + code: -32000, + message: "permission denied (os error 5)".to_owned(), + })); assert!(!is_index_warming_abort(&ClientError::Io( "permission denied (os error 5)".to_owned() ))); assert!(!is_index_warming_abort(&ClientError::ConnectionClosed)); } + + /// The client's own deadline is not a warm-up transient. On Windows the + /// watchdog's `CancelSynchronousIo` makes our blocked read fail with the + /// same os error 995 the daemon transient carries; the sync client maps + /// that to `Timeout`, and a raw client-side `Io` 995 (a thread-exit + /// cancellation outside the guard) must not be retried either — the + /// daemon is still running the search we just abandoned. + #[test] + fn client_side_cancellation_is_never_retried() { + assert!(!is_index_warming_abort(&ClientError::Timeout)); + assert!(!is_index_warming_abort(&ClientError::Io( + "The I/O operation has been aborted because of either a thread exit \ + or an application request. (os error 995)" + .to_owned() + ))); + } } diff --git a/crates/uffs-client/src/connect_sync.rs b/crates/uffs-client/src/connect_sync.rs index d8a78ec26..fa3740685 100644 --- a/crates/uffs-client/src/connect_sync.rs +++ b/crates/uffs-client/src/connect_sync.rs @@ -15,7 +15,7 @@ //! | Windows | Named pipe via `std::fs::OpenOptions` (no Winsock) | use core::sync::atomic::{AtomicBool, Ordering}; -use std::io::{BufRead as _, BufReader, Read, Write}; +use std::io::{self, BufRead as _, BufReader, Read, Write}; use crate::connect_sync_autostart::auto_start_daemon; use crate::daemon_ctl::{pid_file_path, socket_path}; @@ -392,10 +392,19 @@ impl UffsClientSync { // Arm the Windows deadline guard, if present. `_disarmer` // guarantees disarm on every exit path, including `?`. #[cfg(windows)] - let _disarmer = self.deadline_guard.as_ref().map(|guard| { + let deadline_guard = self.deadline_guard.as_ref(); + #[cfg(windows)] + let _disarmer = deadline_guard.map(|guard| { guard.arm(); DisarmOnDrop { guard } }); + // Classify a failed read/write: the deadline firing is a + // `Timeout`, everything else is a transport `Io` error. + #[cfg(windows)] + let transport_error = + move |err: io::Error| crate::connect_sync_errors::transport_error(deadline_guard, &err); + #[cfg(not(windows))] + let transport_error = |err: io::Error| crate::connect_sync_errors::classify_io_error(&err); let id = self.next_id; self.next_id += 1; @@ -408,13 +417,9 @@ impl UffsClientSync { self.writer .write_all(req.as_bytes()) - .map_err(|err| ClientError::Io(err.to_string()))?; - self.writer - .write_all(b"\n") - .map_err(|err| ClientError::Io(err.to_string()))?; - self.writer - .flush() - .map_err(|err| ClientError::Io(err.to_string()))?; + .map_err(transport_error)?; + self.writer.write_all(b"\n").map_err(transport_error)?; + self.writer.flush().map_err(transport_error)?; // Read lines until we get a response with matching id. // Skip notifications (no `id` field). @@ -423,7 +428,7 @@ impl UffsClientSync { let bytes_read = self .reader .read_line(&mut raw_line) - .map_err(|err| ClientError::Io(err.to_string()))?; + .map_err(transport_error)?; if bytes_read == 0 { return Err(ClientError::ConnectionClosed); } @@ -775,7 +780,3 @@ impl Drop for DisarmOnDrop<'_> { self.guard.disarm(); } } - -// Auto-start daemon helpers (`auto_start_daemon`, `is_process_alive`, -// `is_daemon_process`) live in the sibling [`crate::connect_sync_autostart`] -// module to keep this file under the 800-LOC policy ceiling. diff --git a/crates/uffs-client/src/connect_sync_errors.rs b/crates/uffs-client/src/connect_sync_errors.rs new file mode 100644 index 000000000..7bf71aa8a --- /dev/null +++ b/crates/uffs-client/src/connect_sync_errors.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Transport-error classification for the synchronous client. +//! +//! Split from [`crate::connect_sync`] for the 800-LOC file-size policy. +//! `send_request` maps every failed blocking read / write through +//! `transport_error` (Windows only) or `classify_io_error` (Unix) so the +//! client's own deadline surfaces as [`crate::error::ClientError::Timeout`] +//! and never as a raw transport error. + +use std::io; + +use crate::error::ClientError; + +/// Map a failed blocking read/write to the client error it really is. +/// +/// On Windows the per-RPC deadline is enforced by the watchdog's +/// `CancelSynchronousIo`, which makes the blocked call fail with +/// `ERROR_OPERATION_ABORTED` (os error 995). That code is *also* what +/// a genuine daemon-side transient looks like once flattened across the +/// JSON-RPC boundary, so the raw code alone cannot be trusted: the guard's +/// [`fired`](crate::windows_deadline::WindowsDeadlineGuard::fired) verdict +/// is the discriminator. A cancelled RPC is reported as +/// [`ClientError::Timeout`]; before this the CLI mistook its own deadline +/// for the "index warming" transient and re-sent the search up to five +/// times, orphaning a still-running scan on the daemon each time. +/// +/// A kernel-reported `TimedOut` / `WouldBlock` (the Unix `SO_RCVTIMEO` +/// path, and any Windows transport that enforces its own timeout) is a +/// `Timeout` on every platform. +#[cfg(windows)] +pub(crate) fn transport_error( + guard: Option<&crate::windows_deadline::WindowsDeadlineGuard>, + err: &io::Error, +) -> ClientError { + if guard.is_some_and(crate::windows_deadline::WindowsDeadlineGuard::fired) { + return ClientError::Timeout; + } + classify_io_error(err) +} + +/// Classify a failed blocking read/write by its kind alone — the whole +/// story on Unix, where the kernel enforces the deadline via +/// `SO_RCVTIMEO` / `SO_SNDTIMEO`, and the tail of `transport_error` on +/// Windows: a kernel-reported deadline expiry is a +/// [`ClientError::Timeout`]; everything else is an [`ClientError::Io`]. +pub(crate) fn classify_io_error(err: &io::Error) -> ClientError { + if matches!( + err.kind(), + io::ErrorKind::TimedOut | io::ErrorKind::WouldBlock + ) { + ClientError::Timeout + } else { + ClientError::Io(err.to_string()) + } +} + +// Auto-start daemon helpers (`auto_start_daemon`, `is_process_alive`, +// `is_daemon_process`) live in the sibling [`crate::connect_sync_autostart`] +// module to keep this file under the 800-LOC policy ceiling. + +#[cfg(test)] +mod tests { + use std::io; + + use super::classify_io_error; + use crate::error::ClientError; + + /// A kernel-reported deadline expiry is a `Timeout` on every + /// platform; any other transport failure stays an `Io` carrying the + /// original message. + #[test] + fn kernel_timeouts_classify_as_timeout_everything_else_as_io() { + assert!(matches!( + classify_io_error(&io::Error::from(io::ErrorKind::TimedOut)), + ClientError::Timeout + )); + assert!(matches!( + classify_io_error(&io::Error::from(io::ErrorKind::WouldBlock)), + ClientError::Timeout + )); + let other = classify_io_error(&io::Error::new(io::ErrorKind::BrokenPipe, "pipe gone")); + let ClientError::Io(message) = other else { + panic!("expected Io, got {other:?}"); + }; + assert!( + message.contains("pipe gone"), + "message must be preserved; got {message}" + ); + } +} diff --git a/crates/uffs-client/src/connect_sync_platform.rs b/crates/uffs-client/src/connect_sync_platform.rs index d0996cb5a..0b057f5f9 100644 --- a/crates/uffs-client/src/connect_sync_platform.rs +++ b/crates/uffs-client/src/connect_sync_platform.rs @@ -33,8 +33,9 @@ const DEFAULT_RPC_DEADLINE_SECS: u64 = 60; /// * `UFFS_CLIENT_TIMEOUT_SECS=0` → disables the timeout (useful when attaching /// a debugger). /// * `UFFS_CLIENT_TIMEOUT_SECS=N` → `N`-second deadline. -/// * unset or unparseable → [`DEFAULT_RPC_DEADLINE_SECS`]. -pub(crate) fn rpc_deadline() -> Option { +/// * unset or unparseable → `DEFAULT_RPC_DEADLINE_SECS` (60 s). +#[must_use] +pub fn rpc_deadline() -> Option { let secs = std::env::var("UFFS_CLIENT_TIMEOUT_SECS") .ok() .and_then(|val| val.parse::().ok()) diff --git a/crates/uffs-client/src/lib.rs b/crates/uffs-client/src/lib.rs index 71b842260..8143ad524 100644 --- a/crates/uffs-client/src/lib.rs +++ b/crates/uffs-client/src/lib.rs @@ -132,6 +132,14 @@ pub mod connect_sync; /// `is_daemon_process`) — split off `connect_sync` to keep that file /// under the 800-LOC policy ceiling. pub(crate) mod connect_sync_autostart; +/// Platform-specific `platform_connect` impls and the `rpc_deadline` helper. +/// +/// Split `impl` blocks live on [`connect_sync::UffsClientSync`]; +/// callers see no change. Also hosts the env-override regression +/// tests for `rpc_deadline`. +/// Transport-error classification for the sync client: the client's own +/// deadline surfaces as `ClientError::Timeout`, never as a raw `Io`. +pub(crate) mod connect_sync_errors; /// Memory-tiering RPC helpers (`hibernate`, `preload`). /// /// Phase 8-B / 8-C — split off `connect_sync` so the tiering cluster @@ -139,12 +147,8 @@ pub(crate) mod connect_sync_autostart; /// exception. Same precedent as the daemon-state types in /// [`protocol::response_status`]. pub(crate) mod connect_sync_journal; -/// Platform-specific `platform_connect` impls and the `rpc_deadline` helper. -/// -/// Split `impl` blocks live on [`connect_sync::UffsClientSync`]; -/// callers see no change. Also hosts the env-override regression -/// tests for `rpc_deadline`. pub(crate) mod connect_sync_platform; +pub use connect_sync_platform::rpc_deadline; /// Wire-protocol unit tests for [`connect_sync::UffsClientSync`]. /// /// Exercises the JSON-RPC request/response path via in-memory diff --git a/crates/uffs-client/src/protocol/mod.rs b/crates/uffs-client/src/protocol/mod.rs index 9da027880..bf42795a9 100644 --- a/crates/uffs-client/src/protocol/mod.rs +++ b/crates/uffs-client/src/protocol/mod.rs @@ -231,6 +231,17 @@ pub const ERR_NOT_IMPLEMENTED: i32 = -3; /// delete cache files for a drive whose shard is still warm in RAM. /// Operators must `hibernate` the drive first or pass `force = true`. pub const ERR_DRIVE_BUSY: i32 = -4; +/// The daemon's per-search scan budget expired before the scan finished. +/// +/// The budget is `UFFS_SEARCH_TIMEOUT_SECS` (default 30 s). The daemon +/// cancels the scan and nothing is returned — the client used to receive +/// a success-shaped response with zero rows instead, indistinguishable +/// from "nothing matched". +pub const ERR_SEARCH_TIMEOUT: i32 = -5; +/// Every daemon search slot stayed busy for the whole permit wait: the +/// concurrency cap (`UFFS_SEARCH_MAX_CONCURRENCY`) is saturated. Nothing +/// was scanned; retry shortly. +pub const ERR_SEARCH_BUSY: i32 = -6; // ──────────────────────────────────────────────────────────────────────────── // Method parameters diff --git a/crates/uffs-client/src/protocol/response_tiering.rs b/crates/uffs-client/src/protocol/response_tiering.rs index 1fad6f098..3d61df074 100644 --- a/crates/uffs-client/src/protocol/response_tiering.rs +++ b/crates/uffs-client/src/protocol/response_tiering.rs @@ -29,9 +29,8 @@ use serde::{Deserialize, Serialize}; /// Parameters for the `hibernate` method. /// -/// Hibernating a drive demotes its shard to `Cold` (encrypted cache on -/// disk, zero RAM resident) by walking -/// `cascade_demote_one_step` until the shard reaches the bottom tier. +/// Hibernating a drive demotes its shard straight to `Cold` (encrypted +/// cache on disk, zero RAM resident). /// An empty [`Self::drives`] vector hibernates **every** loaded drive /// — the typical operator action when freeing memory before a long /// idle stretch. @@ -82,7 +81,7 @@ pub const DEFAULT_PRELOAD_PIN_MINUTES: u32 = 30; /// Parameters for the `preload` method. /// /// Preloads one or more drives into the `Hot` tier and pins them -/// against demote (TTL idle and pressure cascades) for +/// against the idle TTL demote for /// [`Self::pin_minutes`] minutes. An empty [`Self::drives`] vector is /// a usage error — preload requires at least one explicit drive /// letter. diff --git a/crates/uffs-client/src/windows_deadline.rs b/crates/uffs-client/src/windows_deadline.rs index e3353e9d3..af8426f97 100644 --- a/crates/uffs-client/src/windows_deadline.rs +++ b/crates/uffs-client/src/windows_deadline.rs @@ -30,8 +30,13 @@ //! fires at most once. //! * `CancelSynchronousIo` causes the blocked `ReadFile` / `WriteFile` on the //! target thread to return `ERROR_OPERATION_ABORTED` (`0x4D3`), which bubbles -//! up through `std::io::Read` / `Write` as a regular I/O error — the caller's -//! existing `ClientError::Io` branch then reports it naturally. +//! up through `std::io::Read` / `Write` as a regular I/O error. The watchdog +//! raises the [`WindowsDeadlineGuard::fired`] flag *before* it cancels, so +//! the owning thread can tell its own deadline cancellation apart from a +//! genuine transport error and report it as `ClientError::Timeout` rather +//! than a raw `os error 995` — the raw code used to be mistaken for the +//! daemon-side "index warming" transient and auto-retried, which orphaned a +//! still-running scan on the daemon for every retry. //! //! # Thread-affinity caveat //! @@ -60,7 +65,7 @@ extern crate alloc; use alloc::sync::Arc; -use core::sync::atomic::{AtomicU64, Ordering}; +use core::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use core::time::Duration; use std::io; use std::sync::mpsc; @@ -121,6 +126,10 @@ pub(crate) struct WindowsDeadlineGuard { /// Absolute tick (`GetTickCount64`) at which the current RPC /// should be aborted. `0` = [`DISARMED`]. deadline_tick_ms: Arc, + /// Set by the watchdog when it cancels the in-flight RPC; cleared + /// by every [`Self::arm`]. Read through [`Self::fired`] by the + /// owning thread after a failed read/write to classify the error. + fired: Arc, /// Channel sender paired with the watchdog's /// [`mpsc::Receiver`]. [`Drop`] sends a single `()` on this /// channel to wake the watchdog immediately; dropping the @@ -164,20 +173,28 @@ impl WindowsDeadlineGuard { pub(crate) fn new(duration: Duration) -> io::Result { let target_thread = SendHandle(duplicate_current_thread()?); let deadline_tick_ms = Arc::new(AtomicU64::new(DISARMED)); + let fired = Arc::new(AtomicBool::new(false)); let (shutdown_tx, shutdown_rx) = mpsc::channel::<()>(); let watchdog_ticks = Arc::clone(&deadline_tick_ms); + let watchdog_fired = Arc::clone(&fired); let watchdog_target = target_thread; let watchdog = thread::Builder::new() .name("uffs-deadline-watchdog".into()) .spawn(move || { - watchdog_loop(&watchdog_ticks, &shutdown_rx, watchdog_target); + watchdog_loop( + &watchdog_ticks, + &watchdog_fired, + &shutdown_rx, + watchdog_target, + ); })?; Ok(Self { duration, deadline_tick_ms, + fired, shutdown_tx, target_thread, watchdog: Some(watchdog), @@ -214,9 +231,26 @@ impl WindowsDeadlineGuard { } else { raw_deadline }; + // A fresh RPC starts with a clean verdict; the previous RPC's + // cancellation must not be attributed to this one. + self.fired.store(false, Ordering::Release); self.deadline_tick_ms.store(deadline, Ordering::Release); } + /// `true` when the watchdog cancelled the most recently armed RPC. + /// + /// The owning thread calls this right after a blocking read or + /// write fails: a `true` means the failure is this guard's own + /// `CancelSynchronousIo` (the RPC exceeded `UFFS_CLIENT_TIMEOUT_SECS`) + /// and should surface as `ClientError::Timeout`; a `false` means the + /// error came from the transport itself. The flag is written with + /// `Release` *before* the cancellation call and read with `Acquire` + /// after the cancelled I/O returns, so the owning thread always + /// observes it set when its I/O was the one cancelled. + pub(crate) fn fired(&self) -> bool { + self.fired.load(Ordering::Acquire) + } + /// Disarm the guard — the current RPC completed in time. /// /// After this call the watchdog will not fire until the next @@ -283,6 +317,7 @@ impl Drop for WindowsDeadlineGuard { /// overshoot. fn watchdog_loop( deadline_tick_ms: &Arc, + fired: &Arc, shutdown_rx: &mpsc::Receiver<()>, target: SendHandle, ) { @@ -309,6 +344,11 @@ fn watchdog_loop( { continue; } + // Publish the verdict before cancelling: the owning thread's + // `ReadFile` returns `ERROR_OPERATION_ABORTED` strictly after + // this call, and it reads the flag strictly after that return, + // so `Release` here + `Acquire` there is enough ordering. + fired.store(true, Ordering::Release); // SAFETY: `target` was produced by `DuplicateHandle` in the // guard's `new`; the guard's `Drop` joins us before closing // the handle, so `target` is live for the whole call. @@ -451,6 +491,10 @@ mod tests { DISARMED, "disarm must restore the sentinel", ); + assert!( + !guard.fired(), + "an RPC that completes in time must not be reported as cancelled", + ); } /// A 1 ms deadline must fire the watchdog within ~100 ms — we @@ -477,6 +521,18 @@ mod tests { "expired deadline must be consumed by the watchdog \ (`compare_exchange` back to 0)", ); + assert!( + guard.fired(), + "the watchdog must record that it cancelled this RPC", + ); + + // The next arm starts clean — a stale verdict must never be + // attributed to a later RPC. + let fresh_guard = + WindowsDeadlineGuard::new(Duration::from_mins(1)).expect("guard construction"); + fresh_guard.arm(); + assert!(!fresh_guard.fired(), "arm must reset the fired flag"); + fresh_guard.disarm(); } /// arm never writes the `DISARMED` sentinel even under weird @@ -500,25 +556,11 @@ mod tests { ); } - /// End-to-end integration test: a **blackhole** named-pipe server - /// accepts a client connection but never writes; the client - /// issues a blocking `ReadFile` under an armed guard; the - /// watchdog fires, `CancelSynchronousIo` unblocks the read, and - /// `ReadFile` returns `ERROR_OPERATION_ABORTED` (995). - /// - /// This is the single most important regression guard for commit - /// D — if the watchdog's cancellation path ever breaks in a real - /// Windows build, this test will fail instead of a user's CLI - /// silently hanging forever. - /// - /// The pipe server lives on a short-lived helper thread that - /// sleeps briefly after accept, then exits; the thread joins at - /// the end of the test so no resources leak. - #[test] - fn watchdog_cancels_blocked_readfile_on_blackhole_pipe() { - use std::io::Read as _; - use std::time::Instant; - + /// Create a named pipe at `server_name`, accept one client and then + /// hold the connection open without ever writing, so the client's + /// blocking `ReadFile` has nothing to return until the watchdog + /// cancels it. Closes the pipe and exits after 2 s. + fn spawn_blackhole_pipe_server(server_name: String) -> JoinHandle<()> { use windows::Win32::Foundation::{CloseHandle, INVALID_HANDLE_VALUE}; use windows::Win32::Storage::FileSystem::PIPE_ACCESS_DUPLEX; use windows::Win32::System::Pipes::{ @@ -526,26 +568,7 @@ mod tests { }; use windows::core::PCWSTR; - // Unique pipe path per-process + per-call so concurrent - // cargo test runs do not collide on the same name. Build it - // through `PipeName::parse` so this test doubles as a - // regression pin: if `PipeName`'s invariants ever drift (e.g. - // a prefix tweak or a length-cap shrink), this fixture starts - // failing here instead of silently producing a path Win32 - // would refuse. - let pipe_name = uffs_security::pipe::PipeName::parse(format!( - "\\\\.\\pipe\\uffs-test-blackhole-{}-{}", - std::process::id(), - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|dur| dur.as_nanos()) - .unwrap_or_default(), - )) - .expect("test-blackhole pipe path is a valid PipeName"); - - // ── Server thread: accept once, never respond ─────────── - let server_name = pipe_name.as_str().to_owned(); - let server = thread::spawn(move || { + thread::spawn(move || { let wide: Vec = format!("{server_name}\0").encode_utf16().collect(); // SAFETY: standard Win32 FFI. The handle is closed // below before the thread exits. `CreateNamedPipeW` @@ -585,7 +608,47 @@ mod tests { #[expect(unsafe_code, reason = "Win32 handle cleanup")] let close = unsafe { CloseHandle(handle) }; drop(close); - }); + }) + } + + /// End-to-end integration test: a **blackhole** named-pipe server + /// accepts a client connection but never writes; the client + /// issues a blocking `ReadFile` under an armed guard; the + /// watchdog fires, `CancelSynchronousIo` unblocks the read, and + /// `ReadFile` returns `ERROR_OPERATION_ABORTED` (995). + /// + /// This is the single most important regression guard for commit + /// D — if the watchdog's cancellation path ever breaks in a real + /// Windows build, this test will fail instead of a user's CLI + /// silently hanging forever. + /// + /// The pipe server lives on a short-lived helper thread that + /// sleeps briefly after accept, then exits; the thread joins at + /// the end of the test so no resources leak. + #[test] + fn watchdog_cancels_blocked_readfile_on_blackhole_pipe() { + use std::io::Read as _; + use std::time::Instant; + + // Unique pipe path per-process + per-call so concurrent + // cargo test runs do not collide on the same name. Build it + // through `PipeName::parse` so this test doubles as a + // regression pin: if `PipeName`'s invariants ever drift (e.g. + // a prefix tweak or a length-cap shrink), this fixture starts + // failing here instead of silently producing a path Win32 + // would refuse. + let pipe_name = uffs_security::pipe::PipeName::parse(format!( + "\\\\.\\pipe\\uffs-test-blackhole-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|dur| dur.as_nanos()) + .unwrap_or_default(), + )) + .expect("test-blackhole pipe path is a valid PipeName"); + + // ── Server thread: accept once, never respond ─────────── + let server = spawn_blackhole_pipe_server(pipe_name.as_str().to_owned()); // Give the server a brief moment to create the pipe before // we try to connect. The retry loop below also covers this @@ -629,6 +692,7 @@ mod tests { let read_result = pipe.read(&mut buf); let elapsed = start.elapsed(); + let fired = guard.fired(); guard.disarm(); // ── Assertions ─────────────────────────────────────────── @@ -637,6 +701,10 @@ mod tests { "read against a blackhole pipe must fail; got Ok with {:?}", read_result.ok(), ); + assert!( + fired, + "the guard must report the cancelled read as its own deadline firing", + ); assert!( elapsed >= Duration::from_millis(400), "read should not fail before the deadline; elapsed = {elapsed:?}", diff --git a/crates/uffs-core/src/search/backend.rs b/crates/uffs-core/src/search/backend.rs index 93e79e485..b388dc782 100644 --- a/crates/uffs-core/src/search/backend.rs +++ b/crates/uffs-core/src/search/backend.rs @@ -14,8 +14,8 @@ use std::time::Instant; use rayon::prelude::*; use super::dispatch::{ - apply_dispatch_safety_nets, dispatch_match_all, dispatch_regex, dispatch_trigram_or_tree, - pick_mode_label, + apply_dispatch_safety_nets, discard_if_cancelled, dispatch_match_all, dispatch_regex, + dispatch_trigram_or_tree, pick_mode_label, }; use crate::compact::DriveCompactIndex; use crate::search::field::FieldId; @@ -713,20 +713,22 @@ pub fn search_index( ) }; + let final_rows = discard_if_cancelled(rows, search_filters, pattern); + let scanned = active_drives.iter().map(|dr| dr.records.len()).sum(); let wall_ms = start.elapsed().as_millis(); let mode = pick_mode_label(is_match_all, is_regex, is_path, is_prefix); tracing::debug!( target: "cache_profile", wall_ms = %wall_ms, - rows = rows.len(), + rows = final_rows.len(), scanned, mode, "search_index_total" ); SearchResult { - rows, + rows: final_rows, duration: start.elapsed(), records_scanned: scanned, phase_timings, diff --git a/crates/uffs-core/src/search/backend_tests.rs b/crates/uffs-core/src/search/backend_tests.rs index 4f3406dea..117309423 100644 --- a/crates/uffs-core/src/search/backend_tests.rs +++ b/crates/uffs-core/src/search/backend_tests.rs @@ -2798,3 +2798,89 @@ fn search_index_bloom_skips_all_drives_when_no_ext_matches_any() { "no drives scanned → no matching rows" ); } + +// ── Cooperative cancellation ──────────────────────────────────────── + +/// A search whose `cancel` flag is raised must come back with **no** +/// rows on every dispatch path — never a partial subset dressed up as +/// the answer. The daemon raises the flag when its scan budget +/// expires and discards whatever comes back; what this pins is that +/// the scan honours the flag and that nothing half-collected leaks. +/// The uncancelled twin of each query is asserted non-empty first so +/// an empty result cannot pass by accident. +#[test] +fn cancelled_search_returns_no_rows_on_every_dispatch_path() { + let index = build_two_drive_index(); + // (pattern, sort) — match-all numeric, match-all tree (Path sort), + // match-all path_only, regex, trigram/substring, prefix. + let cases: [(&str, FieldId); 6] = [ + ("*", FieldId::Modified), + ("*", FieldId::Path), + ("*", FieldId::PathOnly), + (">.*", FieldId::Modified), + ("report", FieldId::Modified), + ("repo*", FieldId::Modified), + ]; + for (pattern, sort) in cases { + let mut live = super::super::filters::SearchFilters::default(); + let live_rows = search_index( + &index, + SearchRequest::new(pattern, &mut live), + sort, + true, + &[], + ) + .rows; + assert!( + !live_rows.is_empty(), + "fixture must match `{pattern}` (sort {sort:?}) when not cancelled" + ); + + let cancel = Arc::new(core::sync::atomic::AtomicBool::new(true)); + let mut cancelled = super::super::filters::SearchFilters { + cancel: Some(cancel), + ..Default::default() + }; + let result = search_index( + &index, + SearchRequest::new(pattern, &mut cancelled), + sort, + true, + &[], + ); + assert!( + result.rows.is_empty(), + "`{pattern}` (sort {sort:?}) must return no rows once cancelled; got {}", + result.rows.len() + ); + } +} + +/// The cancel handle is search-time state, not a filter: a filter set +/// that carries only a cancel flag is still "empty" for every +/// fast-path decision, and the flag survives the per-drive clone the +/// rayon workers take. +#[test] +fn cancel_handle_is_not_a_filter_and_survives_clone() { + let flag = Arc::new(core::sync::atomic::AtomicBool::new(false)); + let filters = super::super::filters::SearchFilters { + cancel: Some(Arc::clone(&flag)), + ..Default::default() + }; + assert!( + filters.is_empty(), + "a cancel handle alone must not count as a filter" + ); + assert!(!filters.is_cancelled()); + let worker_copy = filters.clone(); + flag.store(true, core::sync::atomic::Ordering::Relaxed); + assert!(filters.is_cancelled(), "the original must observe the flag"); + assert!( + worker_copy.is_cancelled(), + "the clone must observe the shared flag" + ); + assert!( + worker_copy.cancelled_at(0) && !worker_copy.cancelled_at(1), + "cancelled_at polls only on the stride boundary" + ); +} diff --git a/crates/uffs-core/src/search/dispatch.rs b/crates/uffs-core/src/search/dispatch.rs index a47dcc028..c4a4f932f 100644 --- a/crates/uffs-core/src/search/dispatch.rs +++ b/crates/uffs-core/src/search/dispatch.rs @@ -340,6 +340,9 @@ pub(super) fn dispatch_regex( let drive_results: Vec> = active_drives .par_iter() .map(|drive| { + if search_filters.is_cancelled() { + return Vec::new(); + } super::query::search_compact_drive_regex(drive, &compiled_re, limit, search_filters) }) .collect(); @@ -378,6 +381,9 @@ pub(super) fn dispatch_trigram_or_tree( let drive_results: Vec> = active_drives .par_iter() .map(|drive| { + if search_filters.is_cancelled() { + return Vec::new(); + } if is_path { super::query::search_compact_drive_tree(drive, needle, limit, search_filters) } else if is_prefix { @@ -427,6 +433,32 @@ pub(super) fn dispatch_trigram_or_tree( rows } +/// Drop the rows of a scan that was cancelled mid-way. +/// +/// A cancelled scan stopped early on some drive: whatever it had +/// collected is an arbitrary subset, and a subset presented as the +/// answer is worse than no answer. The caller that cancelled already +/// knows why, so nothing is returned. +#[expect( + clippy::single_call_fn, + reason = "extracted from search_index to keep it under the cognitive-complexity budget" +)] +pub(super) fn discard_if_cancelled( + rows: Vec, + search_filters: &SearchFilters, + pattern: &str, +) -> Vec { + if !search_filters.is_cancelled() { + return rows; + } + tracing::debug!( + pattern, + partial_rows = rows.len(), + "search cancelled by caller; partial result discarded" + ); + Vec::new() +} + /// Pick the `cache_profile` `mode` tracing label for the chosen /// dispatch branch. Pure function — no side effects. #[expect( diff --git a/crates/uffs-core/src/search/filters/mod.rs b/crates/uffs-core/src/search/filters/mod.rs index 286654d19..25a3182c5 100644 --- a/crates/uffs-core/src/search/filters/mod.rs +++ b/crates/uffs-core/src/search/filters/mod.rs @@ -22,6 +22,9 @@ mod time_parsing; // own doc comment for why a downstream crate needs to call it directly. // The `apply::*` glob below stays `pub(crate)`: everything else in // `apply` (e.g. `row_passes_filters`) is an internal helper. +use alloc::sync::Arc; +use core::sync::atomic::{AtomicBool, Ordering}; + pub use apply::apply_search_filters; pub(crate) use apply::*; pub use attr_parsing::*; @@ -170,8 +173,32 @@ pub struct SearchFilters { /// column so downstream tooling can spot/round-trip corrupt entries. /// `false` = default lossy rendering (matches the reference C++ tool). pub normalize_malformed: bool, + + /// Cooperative cancellation handle for the scan that consumes these + /// filters. `None` (the default) means the scan runs to completion. + /// + /// The daemon sets this before it hands a search to the blocking + /// pool and raises the flag when the search's scan budget expires: + /// every per-drive scan polls it on its hot loop (see + /// [`Self::cancelled_at`]) and stops instead of running the + /// remaining records to the end for a result nobody will read. + /// Before this existed a timed-out `*.*` over 43 M records kept its + /// rayon workers busy for the full scan after the daemon had already + /// answered the client (2026-10-03 benchmark run). + /// + /// Travels with the per-drive `clone()` so rayon workers share the + /// one flag. Not a filter: [`Self::is_empty`] ignores it. + pub cancel: Option>, } +/// Records a hot loop processes between two cancellation polls. +/// +/// A power of two so the stride test is a mask. One relaxed atomic +/// load per 4 096 records is below measurement noise on every scan path +/// while still bounding the overrun after a cancel to a few microseconds +/// per worker. +pub const CANCEL_CHECK_STRIDE: usize = 4096; + impl SearchFilters { /// The [`crate::compact::MalformedRender`] mode implied by /// [`Self::normalize_malformed`]. @@ -183,6 +210,27 @@ impl SearchFilters { crate::compact::MalformedRender::Lossy } } + + /// `true` once the owning search has been cancelled (see + /// [`Self::cancel`]). Cheap enough for per-drive granularity; hot + /// per-record loops use [`Self::cancelled_at`] instead. + #[inline] + #[must_use] + pub fn is_cancelled(&self) -> bool { + self.cancel + .as_ref() + .is_some_and(|flag| flag.load(Ordering::Relaxed)) + } + + /// `true` when `ordinal` sits on a [`CANCEL_CHECK_STRIDE`] boundary + /// *and* the search has been cancelled. Folds the stride test and + /// the atomic load into one `if` so a hot loop pays a single mask + /// compare per record and the load only every 4 096th. + #[inline] + #[must_use] + pub fn cancelled_at(&self, ordinal: usize) -> bool { + ordinal & (CANCEL_CHECK_STRIDE - 1) == 0 && self.is_cancelled() + } } /// Raw parameter inputs for constructing [`SearchFilters`]. @@ -420,6 +468,9 @@ impl SearchFilters { // Display-only; the daemon sets it from the request's // `normalize_malformed` flag, so it defaults off here. normalize_malformed: false, + // Search-time state the daemon attaches per search, never a + // parsed parameter. + cancel: None, } } diff --git a/crates/uffs-core/src/search/query/mod.rs b/crates/uffs-core/src/search/query/mod.rs index 07359a783..0fa71faed 100644 --- a/crates/uffs-core/src/search/query/mod.rs +++ b/crates/uffs-core/src/search/query/mod.rs @@ -193,19 +193,19 @@ pub(crate) fn search_compact_drive_regex( let mut filter_buf: Vec = Vec::with_capacity(256); let t_match = std::time::Instant::now(); - let match_indices: Vec = drive - .records - .iter() - .enumerate() - .filter(|(_, rec)| { - let name = rec.name(&drive.names); - !name.is_empty() - && compiled_re.is_match(name) - && local_filters.matches_record(rec, &drive.names, &mut filter_buf, drive.fold) - }) - .take(limit) - .map(|(idx, _)| uffs_mft::len_to_u32(idx)) - .collect(); + let mut match_indices: Vec = Vec::new(); + for (idx, rec) in drive.records.iter().enumerate() { + if match_indices.len() >= limit || local_filters.cancelled_at(idx) { + break; + } + let name = rec.name(&drive.names); + if !name.is_empty() + && compiled_re.is_match(name) + && local_filters.matches_record(rec, &drive.names, &mut filter_buf, drive.fold) + { + match_indices.push(uffs_mft::len_to_u32(idx)); + } + } let match_ms = t_match.elapsed().as_millis(); let match_count = match_indices.len(); @@ -318,7 +318,7 @@ fn collect_match_indices( None => { let mut out = Vec::new(); for (idx, rec) in drive.records.iter().enumerate() { - if out.len() >= limit { + if out.len() >= limit || filters.cancelled_at(idx) { break; } let name = rec.name(&drive.names); @@ -330,8 +330,8 @@ fn collect_match_indices( } Some(candidate_indices) => { let mut out = Vec::with_capacity(candidate_indices.len().min(limit)); - for &idx in &candidate_indices { - if out.len() >= limit { + for (ordinal, &idx) in candidate_indices.iter().enumerate() { + if out.len() >= limit || filters.cancelled_at(ordinal) { break; } let Some(rec) = drive.records.get(idx as usize) else { @@ -541,6 +541,13 @@ pub(crate) fn search_compact_drive_tree( }; let mut filter_buf: Vec = Vec::with_capacity(256); + // The tree walk is one indivisible traversal; a cancellation that + // lands before it starts skips the drive, one that lands during it + // is honoured by the row-building pass below. + if local_filters.is_cancelled() { + return Vec::new(); + } + let t_tree = std::time::Instant::now(); let match_indices = tree::tree_search(drive, pattern_lower, scan_limit); let tree_ms = t_tree.elapsed().as_millis(); @@ -551,7 +558,9 @@ pub(crate) fn search_compact_drive_tree( let mut mal_cache = tree::malformed_cache_with_capacity(256); let rows: Vec = match_indices .iter() - .filter_map(|&record_idx| { + .enumerate() + .take_while(|&(ordinal, _)| !local_filters.cancelled_at(ordinal)) + .filter_map(|(_, &record_idx)| { let rec = drive.records.get(record_idx as usize)?; let name = rec.name(&drive.names); if name.is_empty() { diff --git a/crates/uffs-core/src/search/query/numeric_top_n.rs b/crates/uffs-core/src/search/query/numeric_top_n.rs index 76ad3a476..10d0eaaf5 100644 --- a/crates/uffs-core/src/search/query/numeric_top_n.rs +++ b/crates/uffs-core/src/search/query/numeric_top_n.rs @@ -214,8 +214,13 @@ fn scan_ext_fast_path( cfg: DriveScanCfg, state: &mut DriveTopN, ) { + let mut visited = 0_usize; for &ext_id in &filters.resolved_ext_ids { for &rec_idx_u32 in drive.records_with_ext(ext_id).iter() { + if filters.cancelled_at(visited) { + return; + } + visited += 1; let rec_idx = rec_idx_u32 as usize; let Some(rec) = drive.records.get(rec_idx) else { continue; @@ -281,6 +286,9 @@ fn scan_full_records( ) -> u64 { let mut filtered = 0_u64; for (rec_idx, rec) in drive.records.iter().enumerate() { + if filters.cancelled_at(rec_idx) { + break; + } if !full_scan_record_passes( rec, drive, @@ -530,6 +538,11 @@ fn scan_all_drives_parallel + Sync>( .enumerate() .map(|(drive_idx, drive_ref)| { let drive = drive_ref.as_ref(); + // A drive whose turn comes after the search was cancelled + // is not scanned at all. + if search_filters.is_cancelled() { + return (Vec::new(), 0_u64); + } let t_drive = std::time::Instant::now(); // Per-worker clone so `resolve_ext_ids_for_drive` writes // into a local copy, never a shared `&mut`. See struct diff --git a/crates/uffs-core/src/search/query/path_only_top_n.rs b/crates/uffs-core/src/search/query/path_only_top_n.rs index 8fd2cf27b..9f8fbfde3 100644 --- a/crates/uffs-core/src/search/query/path_only_top_n.rs +++ b/crates/uffs-core/src/search/query/path_only_top_n.rs @@ -151,7 +151,7 @@ pub(super) fn collect_path_only_sorted_top_n + Sync> let mut fold_buf: Vec = Vec::with_capacity(256); for &drive_idx in &drive_order { - if output.len() >= limit { + if output.len() >= limit || search_filters.is_cancelled() { break; } let Some(drive_ref) = drives.get(drive_idx) else { @@ -263,10 +263,12 @@ fn walk_drive_asc( } } + let mut visited = 0_usize; while let Some(dir_idx) = stack.pop() { - if output.len() >= limit { + if output.len() >= limit || search_filters.cancelled_at(visited) { return; } + visited += 1; let child_slice = drive.children_of(dir_idx); if child_slice.is_empty() { continue; @@ -362,10 +364,12 @@ fn walk_drive_desc( } } + let mut visited = 0_usize; while let Some(task) = stack.pop() { - if output.len() >= limit { + if output.len() >= limit || search_filters.cancelled_at(visited) { return; } + visited += 1; match task { DescTask::Emit(idx) => { emit_if_passes( diff --git a/crates/uffs-core/src/search/query/path_sorted_top_n.rs b/crates/uffs-core/src/search/query/path_sorted_top_n.rs index ecc38e9a6..18f3e14c9 100644 --- a/crates/uffs-core/src/search/query/path_sorted_top_n.rs +++ b/crates/uffs-core/src/search/query/path_sorted_top_n.rs @@ -151,10 +151,12 @@ fn walk_tree_path_sorted>( let mut dir_cache = tree::dir_cache_with_capacity(256); let mut mal_cache = tree::malformed_cache_with_capacity(256); let mut stack: Vec = roots.into_iter().rev().collect(); + let mut visited = 0_usize; while let Some(idx) = stack.pop() { - if path_results.len() >= limit { + if path_results.len() >= limit || search_filters.cancelled_at(visited) { return path_results; } + visited += 1; let Some(rec) = drive.records.get(idx as usize) else { continue; }; diff --git a/crates/uffs-core/src/search/query/prefix_search.rs b/crates/uffs-core/src/search/query/prefix_search.rs index 7de81020c..754856ba1 100644 --- a/crates/uffs-core/src/search/query/prefix_search.rs +++ b/crates/uffs-core/src/search/query/prefix_search.rs @@ -61,7 +61,10 @@ pub(crate) fn search_compact_drive_prefix( drive.fold.fold_into(prefix, &mut fold_buf).to_owned() }; - for rec_idx in candidate_indices { + for (ordinal, rec_idx) in candidate_indices.into_iter().enumerate() { + if local_filters.cancelled_at(ordinal) { + break; + } let Some(rec) = drive.records.get(rec_idx as usize) else { continue; }; diff --git a/crates/uffs-daemon/src/cache/policy.rs b/crates/uffs-daemon/src/cache/policy.rs index c48556ebb..b1c2cfffb 100644 --- a/crates/uffs-daemon/src/cache/policy.rs +++ b/crates/uffs-daemon/src/cache/policy.rs @@ -142,6 +142,21 @@ pub(crate) const PARKED_TO_COLD_IDLE_ENV: &str = "UFFS_PARKED_TO_COLD_IDLE_SECS" /// Env var that overrides [`USN_REFRESH_INTERVAL_SECS`]. pub(crate) const USN_REFRESH_INTERVAL_ENV: &str = "UFFS_USN_REFRESH_INTERVAL_SECS"; +/// Default per-search scan budget, in seconds (see +/// [`search_scan_budget_secs`]). +/// +/// A scan that outlives it is cancelled cooperatively (every per-drive +/// loop polls `SearchFilters::cancel`) and the client gets a JSON-RPC +/// `ERR_SEARCH_TIMEOUT` error — never a success-shaped empty result. +/// 30 s covers a full `*.*` over ~45 M records on the reference box +/// with headroom; the budget is deliberately below the client's own +/// 60 s `UFFS_CLIENT_TIMEOUT_SECS` deadline so the error reaches the +/// client instead of the client giving up first and orphaning the scan. +pub(crate) const SEARCH_SCAN_BUDGET_SECS: u64 = 30; + +/// Env var that overrides [`SEARCH_SCAN_BUDGET_SECS`]. +pub(crate) const SEARCH_SCAN_BUDGET_ENV: &str = "UFFS_SEARCH_TIMEOUT_SECS"; + /// Read a positive `u64` seconds value from `env_name`, falling back /// to `default` on any parse error or non-positive value. Logs a /// single startup line per override so the effective policy is @@ -158,7 +173,7 @@ fn read_env_secs(env_name: &str, default: u64) -> u64 { env_var = env_name, override_secs = effective, default_secs = default, - "idle-threshold override active", + "policy override active", ); } else { tracing::warn!( @@ -166,12 +181,21 @@ fn read_env_secs(env_name: &str, default: u64) -> u64 { env_var = env_name, raw = %raw, default_secs = default, - "idle-threshold env var unparseable; using default", + "policy env var unparseable; using default", ); } effective } +/// Effective per-search scan budget (env override or default), in +/// seconds. Read once; the first search caches it for the daemon's +/// lifetime like the idle thresholds. +#[must_use] +pub(crate) fn search_scan_budget_secs() -> u64 { + static CACHED: OnceLock = OnceLock::new(); + *CACHED.get_or_init(|| read_env_secs(SEARCH_SCAN_BUDGET_ENV, SEARCH_SCAN_BUDGET_SECS)) +} + /// Effective `Hot` → `Warm` idle threshold (env override or default). #[must_use] pub(crate) fn hot_to_warm_idle_secs() -> u64 { diff --git a/crates/uffs-daemon/src/cache/pressure.rs b/crates/uffs-daemon/src/cache/pressure.rs index 1921d815a..17d296664 100644 --- a/crates/uffs-daemon/src/cache/pressure.rs +++ b/crates/uffs-daemon/src/cache/pressure.rs @@ -7,10 +7,11 @@ //! [`MEMORY_RESOURCE_NOTIFICATION_TYPE`][win32-mem]: //! `LowMemoryResourceNotification` fires when free RAM drops below the //! kernel's threshold; `HighMemoryResourceNotification` fires when it -//! rises back above. The daemon's subscriber loop translates `Low` -//! into a cascade demote of LRU Warm shards -//! (see [`crate::index::IndexManager::cascade_demote_one_step`]) -//! until either the registry has no Warm shards left or `High` arrives. +//! rises back above. The daemon's subscriber loop (`lib.rs:: +//! spawn_pressure_subscriber`) logs each transition for operators and +//! takes no action: a shard is promoted when a search needs it and +//! retired by the idle TTL ladder alone (owner ruling 2026-10-03; the +//! former `Low` → cascade-demote behaviour is gone). //! //! On Windows, [`PlatformPressureSignal::new`] spawns a dedicated //! kernel thread (`uffs-pressure`) that owns the two notification @@ -35,8 +36,8 @@ //! `Arc`. Production wires //! [`PlatformPressureSignal`]; the Phase 5 unit tests inject //! `tests::ControllablePressureSignal` so the test can `set(Low)` / -//! `set(High)` and assert the cascade behaviour deterministically -//! without any real OS pressure. +//! `set(High)` and assert the subscriber's observe-only contract +//! deterministically without any real OS pressure. //! //! The signal is delivered as a [`tokio::sync::watch::Receiver`] //! returned by [`PressureSignal::subscribe`]. `watch` is the right @@ -64,21 +65,14 @@ use tokio::sync::watch; /// builds expose only `Normal`: there is no portable process-wide /// memory-resource-notification API on those targets, and the /// kernel handles reclaim itself — demotion is TTL-driven via -/// [`crate::index::IndexManager::demote_idle_shards`] alone. -/// -/// Consumers that want to react to `Low` should use -/// [`Self::requires_cascade_demote`] rather than pattern-matching -/// the variant directly — the method has platform-specific -/// implementations that compile cleanly on Mac/Linux production -/// builds (where `Low` does not exist) and short-circuit to the -/// correct answer (`false`). +/// [`crate::index::IndexManager::demote_idle_shards`] on every +/// platform. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum PressureLevel { - /// No pressure signal yet, or steady state. Subscriber takes no - /// action. + /// No pressure signal yet, or steady state. Normal, /// Free RAM has fallen below the kernel's low-memory threshold. - /// Subscriber cascade-demotes LRU Warm shards. + /// Logged by the subscriber; nothing is demoted. /// /// Only present on Windows production builds (constructed by /// `windows_handles::watcher_loop`) and under `cfg(test)` @@ -86,7 +80,7 @@ pub(crate) enum PressureLevel { #[cfg(any(target_os = "windows", test))] Low, /// Free RAM has risen back above the kernel's high-memory - /// threshold; pressure cleared. Subscriber stops the cascade. + /// threshold; pressure cleared. /// /// Only present on Windows production builds (constructed by /// `windows_handles::watcher_loop`) and under `cfg(test)` @@ -95,30 +89,6 @@ pub(crate) enum PressureLevel { High, } -impl PressureLevel { - /// Returns `true` when this level should drive the daemon's - /// cascade-demote loop. - /// - /// On Windows production / under `cfg(test)`: returns `true` - /// for `Self::Low` and `false` otherwise. On Mac/Linux - /// production builds the only constructible variant is - /// [`Self::Normal`], so this method always returns `false` - /// — the platform-gated `match` arm below evaluates only - /// when `Self::Low` exists. - /// - /// Used by `lib.rs::spawn_pressure_subscriber` so the - /// subscriber loop body stays portable across every target - /// without spreading `#[cfg]` gates through the daemon's main - /// runtime path. - pub(crate) const fn requires_cascade_demote(self) -> bool { - match self { - #[cfg(any(target_os = "windows", test))] - Self::Low => true, - _ => false, - } - } -} - /// Process-level memory-pressure subscriber. /// /// Implementations are held as `Arc` on @@ -150,8 +120,7 @@ pub(crate) trait PressureSignal: Send + Sync + 'static { /// `changed()` never returns. /// /// Phase 5 task 5.3 — paired with the Phase 5 dogfood gate -/// "stress test … daemon log shows `cache.pressure { level: \"Low\" }` -/// and demotion cascade". +/// "stress test … daemon log shows `cache.pressure { level: \"Low\" }`". pub(crate) struct PlatformPressureSignal { /// The watch sender held internally. On Mac/Linux nothing ever /// `send`s on this; on Windows the watcher thread does. @@ -200,9 +169,9 @@ impl PlatformPressureSignal { /// "never-fires" (equivalent to the Mac/Linux stub) and a /// warn-level log line is emitted. This keeps the daemon /// resilient against stripped Windows editions or transient - /// resource exhaustion at startup — the cascade demote is a - /// *best-effort optimisation* on top of the always-available - /// TTL-driven demotion path. + /// resource exhaustion at startup — the signal is observability + /// only; the always-available TTL-driven demotion path does not + /// depend on it. #[must_use] pub(crate) fn new() -> Self { let (sender, _initial_rx) = watch::channel(PressureLevel::Normal); @@ -636,8 +605,8 @@ pub(crate) mod tests { /// Phase 5 task 5.10 fake. Holds the watch sender so tests can /// broadcast pressure transitions deterministically and assert - /// the daemon's cascade-demote behaviour without any real OS - /// pressure. + /// the daemon's observe-only subscriber contract without any real + /// OS pressure. /// /// `set(level)` is a thin wrapper over [`watch::Sender::send_replace`] /// (rather than [`watch::Sender::send`]) so the stored value is diff --git a/crates/uffs-daemon/src/cache/registry.rs b/crates/uffs-daemon/src/cache/registry.rs index d2fd1ff87..9425e0d66 100644 --- a/crates/uffs-daemon/src/cache/registry.rs +++ b/crates/uffs-daemon/src/cache/registry.rs @@ -18,8 +18,8 @@ use super::shard::{ShardEntry, ShardState}; /// /// The registry primitive uses this to populate the `reason` field of /// the canonical `shard.transition` `INFO` event so operators can -/// distinguish TTL-driven idle demotes from kernel-Low pressure -/// cascade demotes by grepping a single field. +/// distinguish TTL-driven idle demotes from operator hibernation by +/// grepping a single field. /// /// Wire format (must stay stable — operator runbooks grep for these /// exact strings): @@ -28,32 +28,24 @@ use super::shard::{ShardEntry, ShardState}; /// rather than `"idle-ttl"` for backwards compatibility with existing /// operator runbooks and the Phase 3 task 3.9 observability contract test /// (`shard_transition_events_emitted_on_demote_and_promote`). -/// * [`Self::PressureCascade`] → `reason="pressure-cascade"`. +/// * [`Self::OperatorHibernate`] → `reason="operator-hibernate"`. /// -/// Phase 5 G4 follow-up — replaces the prior dual-logging pattern -/// where every cascade demote emitted **two** events (the registry -/// primitive's generic `reason="demote"` plus a second -/// cascade-specific event from `cascade_demote_one_step`); the -/// canonical event now carries the discriminator directly so the -/// second event is gone. +/// `reason="pressure-cascade"` existed until 2026-10-03; the kernel-Low +/// pressure cascade that emitted it was removed by owner ruling (memory +/// pressure is logged, never acted on), so no current build emits it. #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub(crate) enum DemoteReason { /// TTL-driven idle demote — the per-tier `_ttl_secs` config has /// elapsed since the last query against this shard. Emitted by /// [`crate::index::IndexManager::demote_idle_shards`]. IdleTtl, - /// Kernel memory-pressure cascade demote — the Windows - /// `LowMemoryResourceNotification` fired and the cascade subscriber - /// loop is draining LRU `Warm` shards. Emitted by - /// [`crate::index::IndexManager::cascade_demote_one_step`]. - PressureCascade, /// Operator-driven `hibernate` RPC. Emitted once per shard /// inside the /// [`crate::index::IndexManager::hibernate_shards`] write-lock /// batch (Phase 8-B). Distinguishable from the controller- - /// driven [`Self::IdleTtl`] / [`Self::PressureCascade`] paths - /// by `reason="operator-hibernate"`, so operator audit logs can - /// separate manual hibernation from automatic demote activity. + /// driven [`Self::IdleTtl`] path by `reason="operator-hibernate"`, + /// so operator audit logs can separate manual hibernation from + /// automatic demote activity. OperatorHibernate, } @@ -65,15 +57,14 @@ impl DemoteReason { /// macro (rather than via `%reason` Display formatting) so the /// `tracing-subscriber` default formatter routes the value /// through `record_str` → Debug-formatted-string → **quoted** - /// output (`reason="pressure-cascade"`). The Display path + /// output (`reason="operator-hibernate"`). The Display path /// (`%`) goes through `record_debug` with `format_args`, which - /// renders the value **unquoted** (`reason=pressure-cascade`) + /// renders the value **unquoted** (`reason=operator-hibernate`) /// — incompatible with the legacy operator runbook regexes /// authored against `reason="demote"` and `reason="usn-refresh"`. pub(crate) const fn as_str(self) -> &'static str { match self { Self::IdleTtl => "demote", - Self::PressureCascade => "pressure-cascade", Self::OperatorHibernate => "operator-hibernate", } } @@ -277,9 +268,9 @@ impl ShardRegistry { /// /// Wired into the production demote path by /// [`crate::index::IndexManager::demote_idle_shards`] (Phase 3 - /// Commit D). The pressure-cascade path uses + /// Commit D). Operator hibernation uses /// [`Self::demote_letter_with_reason`] directly so its events - /// carry `reason="pressure-cascade"` instead of the default + /// carry `reason="operator-hibernate"` instead of the default /// `reason="demote"`. #[must_use] pub(crate) fn demote_letter( @@ -314,17 +305,11 @@ impl ShardRegistry { /// new `Arc` reads the new state forever, and the registry's /// `Vec` swap is the linearisation point. /// - /// **Single canonical event.** Phase 5 G4 follow-up — every - /// demote (TTL idle or pressure cascade) emits exactly one - /// `INFO`-level `shard.transition` event from this method. - /// The cascade path used to emit a second event of its own with - /// `reason="pressure-cascade"`; that event was redundant with the - /// primitive's event and added an artificial 6-836 ms gap (the - /// `WorkingSetTrim::trim` syscall duration) that confused - /// operator log analysis. The discriminator now lives in the - /// `reason` field of the single canonical event, and - /// `last_query_at_ms` (previously cascade-only) is included for - /// every demote so operator runbooks get a uniform schema. + /// **Single canonical event.** Every demote (TTL idle or + /// operator hibernate) emits exactly one `INFO`-level + /// `shard.transition` event from this method; the discriminator + /// lives in its `reason` field and `last_query_at_ms` is included + /// for every demote so operator runbooks get a uniform schema. #[must_use] pub(crate) fn demote_letter_with_reason( &self, @@ -351,7 +336,7 @@ impl ShardRegistry { (body.heap_size_bytes().total / 1_048_576) as u64 }); // Capture the LRU timestamp before we rebuild — useful in the - // canonical event so cascade callers don't need to emit a + // canonical event so callers don't need to emit a // second event of their own just to log this field. let last_query_at_ms = old_arc.stats.last_query_at_ms(); let stats = Arc::clone(&old_arc.stats); @@ -611,7 +596,11 @@ impl ShardRegistry { } }) .collect(); - tracing::info!( + // Warm → Warm with a fresher body is a body swap, not a tier + // transition; it fires on every USN apply tick, so it stays + // below INFO and leaves the `shard.transition` INFO stream to + // real demotes and promotes. + tracing::debug!( target: "shard.transition", letter = %letter, from = %from_state, diff --git a/crates/uffs-daemon/src/cache/shard.rs b/crates/uffs-daemon/src/cache/shard.rs index df815ddb1..9752d88d9 100644 --- a/crates/uffs-daemon/src/cache/shard.rs +++ b/crates/uffs-daemon/src/cache/shard.rs @@ -225,21 +225,19 @@ pub(crate) struct ShardEntry { parked_body: Option>, /// Tier-pin expiry as Unix-millis. /// - /// `0` means "not pinned" (the demote controllers may demote on - /// idle / pressure cascade). Non-zero means "do not demote - /// before this Unix-millis timestamp" — the idle-demote tick - /// (`@/Users/.../uffs-daemon/src/index/transitions.rs::demote_idle_shards`) - /// and the pressure-cascade loop - /// (`@/Users/.../uffs-daemon/src/index/transitions. - /// rs::cascade_demote_one_step`) both consult [`Self::is_pinned`] - /// before taking action. Hibernate (Phase 8-B) explicitly clears the + /// `0` means "not pinned" (the idle-demote controller may demote + /// once the TTL elapses). Non-zero means "do not demote before + /// this Unix-millis timestamp" — the idle-demote tick + /// (`index/transitions.rs::demote_idle_shards`) consults + /// [`Self::is_pinned`] before taking action. Hibernate (Phase 8-B) + /// explicitly clears the /// pin by virtue of rebuilding the shard as `Cold` (the new /// `ShardEntry` starts with `pin_until_ms = 0`). /// /// Phase 8-C — operator-driven `preload ` arms this /// timestamp via [`Self::pin_until`] after the Cold → Warm → Hot - /// promote sequence completes. Atomic so the pressure-cascade - /// subscriber can read it without holding the registry lock. + /// promote sequence completes. Atomic so a reader can check it + /// without holding the registry lock. pin_until_ms: AtomicU64, } diff --git a/crates/uffs-daemon/src/config.rs b/crates/uffs-daemon/src/config.rs index bff427093..133ccbb45 100644 --- a/crates/uffs-daemon/src/config.rs +++ b/crates/uffs-daemon/src/config.rs @@ -118,13 +118,14 @@ pub(crate) struct Config { round-trip and reviewability properties." )] pub(crate) struct MemoryConfig { - /// Global resident-set ceiling, in MiB. Once exceeded, the - /// pressure controller (Phase 5.3) cascades demotes until the - /// total drops back under the cap. + /// Global resident-set ceiling, in MiB. Parsed and reported; + /// no controller enforces it — the idle TTL ladder is the only + /// automatic demote driver (owner ruling 2026-10-03). pub max_total_resident_mb: u64, - /// On Windows, hook the low-memory notification API and treat - /// `LowMemoryResourceNotification` as an immediate demote - /// trigger. Mac is a no-op (no equivalent public API). + /// On Windows, hook the low-memory notification API so + /// `LowMemoryResourceNotification` transitions are logged under + /// `cache.pressure`. They are observed, never acted on. Mac is + /// a no-op (no equivalent public API). pub respect_os_low_memory: bool, /// Call `EmptyWorkingSet` after each demote on Windows so the /// freed pages return to the OS quickly (plan §8.2). diff --git a/crates/uffs-daemon/src/handler.rs b/crates/uffs-daemon/src/handler.rs index fa1640018..7d91a9a80 100644 --- a/crates/uffs-daemon/src/handler.rs +++ b/crates/uffs-daemon/src/handler.rs @@ -429,7 +429,10 @@ impl RequestHandler { ..Default::default() }; - let response = self.index.search(&search_params).await; + let response = match self.index.search(&search_params).await { + Ok(response) => response, + Err(failure) => return diff_handler::search_failure_json(id, &failure), + }; // Extract the first aggregation result. let (values, next_cursor, total_distinct) = response.aggregations.first().map_or_else( diff --git a/crates/uffs-daemon/src/handler_diff.rs b/crates/uffs-daemon/src/handler_diff.rs index 7ebdc86dc..8f6641f94 100644 --- a/crates/uffs-daemon/src/handler_diff.rs +++ b/crates/uffs-daemon/src/handler_diff.rs @@ -22,6 +22,7 @@ use uffs_client::protocol::{ use super::RequestHandler; use crate::index::diff::DiffError; +use crate::index::search::SearchFailure; impl RequestHandler { /// Resolve a search request to its response: a snapshot diff when @@ -35,7 +36,10 @@ impl RequestHandler { if params.diff_baseline.is_some() { self.diff_search_response(id, params).await } else { - Ok(self.index.search(params).await) + self.index + .search(params) + .await + .map_err(|failure| search_failure_json(id, &failure)) } } @@ -132,6 +136,22 @@ impl RequestHandler { )) .unwrap_or_default()) } + Err(DiffError::Search(failure)) => Err(search_failure_json(id, &failure)), } } } + +/// Serialise a [`SearchFailure`] as the JSON-RPC error the client should +/// see: the failure's own code (`ERR_SEARCH_TIMEOUT` / `ERR_SEARCH_BUSY` / +/// `ERR_INTERNAL`) and its operator-facing message. +/// +/// Shared by the live search, the snapshot diff, and `facet_values`, so +/// every path that runs a scan reports a non-completion the same way. +pub(crate) fn search_failure_json(id: u64, failure: &SearchFailure) -> String { + serde_json::to_string(&RpcErrorResponse::error( + Some(id), + failure.rpc_code(), + &failure.to_string(), + )) + .unwrap_or_default() +} diff --git a/crates/uffs-daemon/src/index/constructors.rs b/crates/uffs-daemon/src/index/constructors.rs index 644e238be..fa357f160 100644 --- a/crates/uffs-daemon/src/index/constructors.rs +++ b/crates/uffs-daemon/src/index/constructors.rs @@ -210,7 +210,7 @@ impl IndexManager { /// full memory-tiering lifecycle plus the parsed config. Phase 5 /// tasks 5.8 / 5.9 / 5.10 inject counting / recording / /// controllable fakes here so the demote-batch, promote-on-search, - /// and pressure-cascade assertions can run deterministically + /// and pressure-observation assertions can run deterministically /// without touching the process's actual working set, kernel page /// cache, or OS pressure-notification API; Phase 6 Commit C tests /// pass an explicit `Arc` so per-drive `min_tier` overrides diff --git a/crates/uffs-daemon/src/index/diff.rs b/crates/uffs-daemon/src/index/diff.rs index 610bbdf76..a66d95cc6 100644 --- a/crates/uffs-daemon/src/index/diff.rs +++ b/crates/uffs-daemon/src/index/diff.rs @@ -45,6 +45,10 @@ pub(crate) enum DiffError { /// The underlying load failure. source: anyhow::Error, }, + /// Setup succeeded but the search over the baseline did not complete + /// (scan budget expired, search slots saturated, or the scan task + /// panicked). + Search(super::search::SearchFailure), } impl IndexManager { @@ -55,8 +59,9 @@ impl IndexManager { /// # Errors /// /// [`DiffError::NoDrive`] when no drive is given, - /// [`DiffError::DriveNotLoaded`] when it has no live index, or - /// [`DiffError::BaselineLoad`] when the baseline path cannot be loaded. + /// [`DiffError::DriveNotLoaded`] when it has no live index, + /// [`DiffError::BaselineLoad`] when the baseline path cannot be loaded, + /// or [`DiffError::Search`] when the search itself fails after setup. pub(crate) async fn diff_search( &self, params: &SearchParams, @@ -114,6 +119,8 @@ impl IndexManager { let index = Arc::new(DriveIndex { drives: vec![Arc::new(baseline)], }); - Ok(self.run_search_over(params, Some(index)).await) + self.run_search_over(params, Some(index)) + .await + .map_err(DiffError::Search) } } diff --git a/crates/uffs-daemon/src/index/journal.rs b/crates/uffs-daemon/src/index/journal.rs index 5f703efc4..5177afbaa 100644 --- a/crates/uffs-daemon/src/index/journal.rs +++ b/crates/uffs-daemon/src/index/journal.rs @@ -356,7 +356,11 @@ impl IndexManager { *guard = Arc::new(new_registry); drop(guard); self.bump_index_version(); - tracing::info!( + // Per-tick housekeeping, not a tier change: every 2-3 s per + // warm drive on a busy box (159 lines in one 2026-10-03 + // session). Operators follow transitions at INFO; the tick + // itself is a DEBUG event. + tracing::debug!( target: "shard.journal", drive = %letter, reason, diff --git a/crates/uffs-daemon/src/index/mod.rs b/crates/uffs-daemon/src/index/mod.rs index a966a76bd..3ab4fe8b0 100644 --- a/crates/uffs-daemon/src/index/mod.rs +++ b/crates/uffs-daemon/src/index/mod.rs @@ -25,6 +25,7 @@ mod predicates; mod projection; mod refresh; pub(crate) mod search; +mod search_failure; mod search_filters_build; mod stats; mod status_drives; @@ -227,15 +228,15 @@ pub(crate) struct IndexManager { /// Memory-pressure signal source (Phase 5 task 5.3). Held so /// the daemon's `spawn_pressure_subscriber` (in `lib.rs`) can /// call [`IndexManager::subscribe_pressure`] to obtain a - /// [`tokio::sync::watch::Receiver`] and react to `Low` events - /// by cascade-demoting LRU Warm shards via - /// [`IndexManager::cascade_demote_one_step`] (task 5.6). Production - /// wires [`crate::cache::pressure::PlatformPressureSignal`] - /// (Mac/Linux never-fires, Windows future watcher thread); the - /// Phase 5 task 5.10 tests inject + /// [`tokio::sync::watch::Receiver`] and log every transition. + /// Nothing is demoted on pressure (owner ruling 2026-10-03). + /// Production wires + /// [`crate::cache::pressure::PlatformPressureSignal`] (Mac/Linux + /// never-fires, Windows kernel watcher thread); the lifecycle + /// tests inject /// `crate::cache::pressure::tests::ControllablePressureSignal` - /// to broadcast deterministic transitions and assert the LRU - /// cascade order. + /// to broadcast deterministic transitions and assert the + /// observe-only contract. pressure: Arc, /// Thread-level background-I/O priority hook (Phase 5 task 5.7). /// Held so [`IndexManager::handle_journal_refresh`] can wrap the diff --git a/crates/uffs-daemon/src/index/search.rs b/crates/uffs-daemon/src/index/search.rs index 6174b1eb5..d9c7e6922 100644 --- a/crates/uffs-daemon/src/index/search.rs +++ b/crates/uffs-daemon/src/index/search.rs @@ -12,7 +12,7 @@ //! Search execution: query dispatch, profile construction, and drive info. use alloc::sync::Arc; -use core::sync::atomic::Ordering; +use core::sync::atomic::{AtomicBool, Ordering}; use std::time::Instant; use uffs_client::protocol::response::{ @@ -25,13 +25,23 @@ use uffs_core::search::backend::{ use uffs_core::search::field::FieldId; use super::IndexManager; +pub(crate) use super::search_failure::SearchFailure; impl IndexManager { /// Execute a live search query over the registry snapshot (updates perf /// counters). Snapshot-diff searches (`params.diff_baseline`) are routed by /// the handler to [`Self::diff_search`] instead, so they can surface setup /// errors (missing baseline / unloaded drive) as JSON-RPC errors. - pub(crate) async fn search(&self, params: &SearchParams) -> SearchResponse { + /// + /// # Errors + /// + /// A [`SearchFailure`] when the scan did not complete — budget expired, + /// search slots saturated, or the scan task panicked. Never a + /// success-shaped empty response. + pub(crate) async fn search( + &self, + params: &SearchParams, + ) -> Result { self.run_search_over(params, None).await } @@ -44,6 +54,10 @@ impl IndexManager { /// /// When `params.profile` is `true`, populates `SearchResponse::profile` /// with a per-phase timing breakdown so the CLI can print it. + /// + /// # Errors + /// + /// See [`Self::search`]. #[expect( clippy::too_many_lines, reason = "search orchestration with multi-drive merge, sorting, and response formatting" @@ -56,7 +70,7 @@ impl IndexManager { &self, params: &SearchParams, snapshot_override: Option>, - ) -> SearchResponse { + ) -> Result { let is_diff = snapshot_override.is_some(); // Acquire a concurrency permit — blocks if too many searches // are already in flight. The effective cap is @@ -68,11 +82,9 @@ impl IndexManager { // the `UFFS_SEARCH_MAX_CONCURRENCY` env var. let Some(_permit) = self.acquire_search_permit().await else { // Permit acquisition timed out — the global concurrency - // cap is saturated. Return a no-payload response with - // the remaining metadata fields at their zero defaults - // so the client still sees a valid (if empty) shape. - // Rejected before any promote was attempted. - return empty_response(0, None); + // cap is saturated. Rejected before any promote was + // attempted; the client gets an error, not "0 results". + return Err(SearchFailure::Saturated); }; let query_start = Instant::now(); @@ -240,6 +252,12 @@ impl IndexManager { let match_path = effective_params.match_path; let drives = effective_params.drives.clone(); let agg_snapshot = snapshot.clone(); + // Cooperative cancellation: the flag travels inside `filters` + // into every per-drive scan loop; raised below if the budget + // expires, so the rayon workers stop instead of finishing a + // scan nobody will read. + let cancel = Arc::new(AtomicBool::new(false)); + filters.cancel = Some(Arc::clone(&cancel)); let search_handle = tokio::task::spawn_blocking(move || { search_index( &snapshot, @@ -259,26 +277,36 @@ impl IndexManager { ) }); + let budget_secs = crate::cache::policy::search_scan_budget_secs(); let search_outcome = - tokio::time::timeout(core::time::Duration::from_secs(30), search_handle).await; + tokio::time::timeout(core::time::Duration::from_secs(budget_secs), search_handle).await; let result = match search_outcome { Ok(Ok(res)) => res, Ok(Err(_join_err)) => { - tracing::error!("search task panicked"); - // Promotion already happened; report what it cost even - // though the scan then failed. - return empty_response(0, Some(promotion_ms)); + tracing::error!( + pattern = %effective_params.pattern, + promotion_ms, + "search task panicked" + ); + return Err(SearchFailure::Panicked { promotion_ms }); } Err(_timeout) => { + // Stop the workers; the dropped JoinHandle alone cannot + // (`spawn_blocking` is not abortable). The promotion + // cost is reported alongside because a timeout that + // follows an idle stretch is usually a warm-up story. + cancel.store(true, Ordering::Release); tracing::warn!( pattern = %effective_params.pattern, - "search timed out after 30s" + budget_secs, + promotion_ms, + "search exceeded its scan budget; scan cancelled, error returned to client" ); - // A timeout that spent most of its budget paging the - // index in is a different diagnosis from one that spent - // it scanning — say which. - return empty_response(30_000, Some(promotion_ms)); + return Err(SearchFailure::TimedOut { + budget_secs, + promotion_ms, + }); } }; let search_us = if profiling { @@ -405,7 +433,7 @@ impl IndexManager { } else { None }; - return SearchResponse { + return Ok(SearchResponse { // File-sink path: the daemon already streamed // the rows to `output_path`, so the response // carries no payload — only the `rows_written` @@ -423,7 +451,7 @@ impl IndexManager { response_mode: None, projected_rows: None, aggregations: vec![], - }; + }); } Err(err) => { tracing::error!( @@ -569,7 +597,7 @@ impl IndexManager { SearchPayload::InlineRows(rows) }; - SearchResponse { + Ok(SearchResponse { payload, total_count, records_scanned: result.records_scanned, @@ -582,7 +610,7 @@ impl IndexManager { response_mode: Some(response_mode), projected_rows, aggregations: agg_results, - } + }) } /// Build the `SearchProfile` for `--profile` output. @@ -680,27 +708,6 @@ impl IndexManager { } } -/// A response carrying no rows — the shape every early-out path returns -/// (permit exhaustion, scan panic, scan timeout). Factored out because -/// the three literals differed in two fields, so every new -/// `SearchResponse` field had to be threaded through all of them. -const fn empty_response(duration_ms: u64, promotion_ms: Option) -> SearchResponse { - SearchResponse { - payload: SearchPayload::Empty, - total_count: 0, - records_scanned: 0, - duration_ms, - promotion_ms, - truncated: false, - profile: None, - applied_sorts: Vec::new(), - applied_projection: Vec::new(), - response_mode: None, - projected_rows: None, - aggregations: Vec::new(), - } -} - /// Decide the backend scan limit (the cap applied *before* the daemon's /// final truncate-to-`user_limit`). /// diff --git a/crates/uffs-daemon/src/index/search_failure.rs b/crates/uffs-daemon/src/index/search_failure.rs new file mode 100644 index 000000000..cde0546e1 --- /dev/null +++ b/crates/uffs-daemon/src/index/search_failure.rs @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! The typed non-completion of a daemon search. +//! +//! Split from [`super::search`] for the 800-LOC file-size policy; the +//! only consumer-facing surface is [`SearchFailure`], re-exported there. + +use core::fmt; + +/// Why [`crate::index::IndexManager::run_search_over`] produced no response. +/// +/// Every variant used to come back as a success-shaped +/// [`uffs_client::protocol::response::SearchResponse`] +/// with zero rows and `truncated: false`, indistinguishable from "nothing +/// matched": the 2026-10-03 benchmark run read two scan timeouts as +/// "0 results". Each is now a JSON-RPC error with its own code (see +/// [`Self::rpc_code`]) and an operator-facing message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum SearchFailure { + /// The per-search scan budget expired. The scan was cancelled + /// cooperatively (`SearchFilters::cancel`) and its partial result + /// discarded. + TimedOut { + /// The budget that expired, in seconds + /// (`UFFS_SEARCH_TIMEOUT_SECS`). + budget_secs: u64, + /// Milliseconds spent paging parked / cold drives back in + /// before the scan started — separate from the budget, and the + /// first thing to look at when a timeout follows an idle + /// stretch. + promotion_ms: u64, + }, + /// The blocking scan task panicked; details are in the daemon log. + Panicked { + /// Milliseconds spent on index warm-up before the scan. + promotion_ms: u64, + }, + /// No search permit became available within the wait: the + /// concurrency cap (`UFFS_SEARCH_MAX_CONCURRENCY`) is saturated. + /// Nothing was scanned. + Saturated, +} + +impl SearchFailure { + /// The JSON-RPC error code the client receives for this failure. + pub(crate) const fn rpc_code(&self) -> i32 { + match self { + Self::TimedOut { .. } => uffs_client::protocol::ERR_SEARCH_TIMEOUT, + Self::Panicked { .. } => uffs_client::protocol::ERR_INTERNAL, + Self::Saturated => uffs_client::protocol::ERR_SEARCH_BUSY, + } + } +} + +impl fmt::Display for SearchFailure { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::TimedOut { + budget_secs, + promotion_ms, + } => write!( + f, + "search exceeded the daemon scan budget of {budget_secs}s and was cancelled \ + (index warm-up before the scan: {promotion_ms} ms); narrow the query, or raise \ + UFFS_SEARCH_TIMEOUT_SECS on the daemon" + ), + Self::Panicked { promotion_ms } => write!( + f, + "search task panicked on the daemon (index warm-up before the scan: \ + {promotion_ms} ms); see the daemon log" + ), + Self::Saturated => write!( + f, + "daemon search slots are saturated (UFFS_SEARCH_MAX_CONCURRENCY); nothing was \ + scanned, retry shortly" + ), + } + } +} diff --git a/crates/uffs-daemon/src/index/tests/idle_demote_tracing.rs b/crates/uffs-daemon/src/index/tests/idle_demote_tracing.rs index 4a8419365..765ad39bd 100644 --- a/crates/uffs-daemon/src/index/tests/idle_demote_tracing.rs +++ b/crates/uffs-daemon/src/index/tests/idle_demote_tracing.rs @@ -1,8 +1,7 @@ // SPDX-License-Identifier: MPL-2.0 // Copyright (c) 2025-2026 SKY, LLC. -//! Phase 3 Commit E + Phase 5 G4 — `shard.transition` tracing-event -//! contract tests. +//! Phase 3 Commit E — `shard.transition` tracing-event contract tests. //! //! Split from [`super::idle_demote`] so the state-transition ladder //! tests stay focused on TTL / multi-drive behaviour while this @@ -12,8 +11,6 @@ //! `tracing::event!(target: "shard.transition", ...)` with the `letter` / //! `from` / `to` / `reason` / `freed_mb` / `restored_mb` / `last_query_at_ms` //! field surface. -//! * Phase 5 G4 follow-up — single-canonical-event regression for the -//! pressure-cascade demote path. //! * PR-f — promote refreshes `last_query_at_ms` so the next idle tick doesn't //! immediately re-demote the just-promoted shard (anti-thrash invariant). //! @@ -179,139 +176,6 @@ async fn shard_transition_events_emitted_on_demote_and_promote() { ); } -/// Phase 5 G4 follow-up — the pressure-cascade demote path must -/// emit exactly **one** `INFO`-level `shard.transition` event per -/// shard, with `reason="pressure-cascade"` and `last_query_at_ms` -/// in the field set. -/// -/// Pre-refactor, every cascade demote produced **two** events: the -/// registry primitive's generic `reason="demote"` event followed by -/// a second `reason="pressure-cascade"` event from -/// `cascade_demote_one_step` itself. The two were separated by the -/// `WorkingSetTrim::trim` syscall duration (6-22 ms typically; up -/// to ~1 s on the first cascade demote when the daemon's working -/// set was still large) which confused operator log analysis. -/// -/// This test pins the single-event contract so a future refactor -/// can't reintroduce the dual-event pattern. It also pins the -/// presence of `last_query_at_ms` (formerly cascade-only, now part -/// of the canonical demote event for both TTL and pressure paths). -/// -/// Test topology: 1 Warm drive (`C`) with a known -/// `last_query_at_ms = 1_234` so the assertion can use a literal -/// value instead of `has_field`. `ControllablePressureSignal` is -/// injected for completeness but never driven — the test calls -/// `cascade_demote_one_step` directly, mirroring the contract of -/// the existing -/// `cascade_demote_one_step_picks_lru_warm_and_drains_in_order` -/// test in `lifecycle_hooks.rs` (which pins the demote ordering -/// and trim-call counts but doesn't capture tracing events). -#[tokio::test] -async fn cascade_demote_emits_single_event_with_pressure_cascade_reason() { - use crate::cache::ShardState; - use crate::cache::pressure::tests::ControllablePressureSignal; - use crate::cache::working_set::tests::CountingWorkingSetTrim; - - // Same dummy-Dispatch + thread-local-default + interest-rebuild - // dance as `shard_transition_events_emitted_on_demote_and_promote` - // — see that test's docstring for the rationale. Without this, - // a sibling test on a different thread can pin the - // `shard.transition` callsite's `Interest` cache to `never` - // before our subscriber gets a chance to vote, and the cascade - // event silently disappears. - let log = EventLog::default(); - let _interest_rebuild_dummy = - tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default()); - let _guard = tracing::subscriber::set_default(log.clone()); - tracing::callsite::rebuild_interest_cache(); - - let (tx, _rx) = crate::events::event_channel(); - let counting_trim = Arc::new(CountingWorkingSetTrim::new()); - let pressure_fake = Arc::new(ControllablePressureSignal::new()); - let hooks = crate::index::constructors::LifecycleHooks { - working_set_trim: Arc::clone(&counting_trim) - as Arc, - pressure: Arc::clone(&pressure_fake) as Arc, - ..crate::index::constructors::LifecycleHooks::production() - }; - let mgr = IndexManager::with_lifecycle_hooks_for_test( - None, - tx, - hooks, - Arc::new(crate::config::Config::default()), - ); - mgr.add_drive(build_test_drive()).await; - - // Backdate to a known timestamp so the assertion can use a - // literal value below. `add_drive` already stamped - // `mark_loaded_at(unix_now_ms())`, which would make the assertion - // wall-clock-dependent. - assert!( - mgr.backdate_last_query_at_ms_for_test(uffs_mft::platform::DriveLetter::C, 1_234) - .await - ); - - // Drive the cascade once. With one Warm shard, the LRU pick is - // unambiguous and the call returns `Some((uffs_mft::platform::DriveLetter::C, - // Parked))`. - let result = mgr.cascade_demote_one_step().await; - assert_eq!( - result, - Some((uffs_mft::platform::DriveLetter::C, ShardState::Parked)), - "single-shard cascade demotes C and returns Some", - ); - - // Filter to INFO-level `shard.transition` events whose `reason` - // is in the demote vocabulary. We accept both `"demote"` (the - // legacy generic value) and `"pressure-cascade"` (the new - // discriminator) so this test would still catch a regression - // that flipped the cascade path back to emitting `"demote"` - // — the assertion below pins the EXACT value. - let events = log.events(); - let demotes: Vec<&CapturedEvent> = events - .iter() - .filter(|event| { - event.target == "shard.transition" - && event.level == tracing::Level::INFO - && matches!(event.field("reason"), Some("demote" | "pressure-cascade")) - }) - .collect(); - - assert_eq!( - demotes.len(), - 1, - "G4 follow-up: cascade demote must emit exactly ONE info \ - `shard.transition` event (the registry primitive's canonical \ - event with reason=\"pressure-cascade\"); the legacy second \ - event from `cascade_demote_one_step` is gone. got {}: {:#?}", - demotes.len(), - demotes, - ); - - let cascade = demotes[0]; - assert_eq!(cascade.field("reason"), Some("pressure-cascade")); - assert_eq!(cascade.field("from"), Some("warm")); - assert_eq!(cascade.field("to"), Some("parked")); - assert_eq!(cascade.field("letter"), Some("C")); - assert!( - cascade.has_field("freed_mb"), - "cascade demote event must carry freed_mb field", - ); - assert_eq!( - cascade.field("last_query_at_ms"), - Some("1234"), - "cascade demote event must carry last_query_at_ms (formerly \ - cascade-only; now part of the canonical demote event)", - ); - - // Sanity: trim fired exactly once for the single cascade step. - assert_eq!( - counting_trim.calls(), - 1, - "single cascade step → single trim call", - ); -} - // Phase 6 fix (2026-05-07 24-h soak finding) — `shard.ttl` event // shape regression test extracted to the sibling // [`super::shard_ttl_events`] module to keep this file under the diff --git a/crates/uffs-daemon/src/index/tests/lifecycle_hooks.rs b/crates/uffs-daemon/src/index/tests/lifecycle_hooks.rs index 373fc09b4..8913521dc 100644 --- a/crates/uffs-daemon/src/index/tests/lifecycle_hooks.rs +++ b/crates/uffs-daemon/src/index/tests/lifecycle_hooks.rs @@ -12,9 +12,8 @@ //! batch in `demote_idle_shards`). //! * Plan task 5.9 — `Prefetch::hint()` invocation with the freshly-loaded //! body's records + names regions. -//! * Plan task 5.10 — `cascade_demote_one_step` picks the LRU Warm shard, -//! drains in order, calls `WorkingSetTrim::trim()` exactly once per cascade -//! step. +//! * Owner ruling 2026-10-03 — the pressure subscriber only observes: a `Low` +//! transition demotes nothing and trims nothing. #![expect( clippy::indexing_slicing, @@ -319,366 +318,74 @@ async fn ensure_warm_for_dispatch_invokes_prefetch_with_records_and_names_region ); } -/// Phase 5 task **5.10** — `cascade_demote_one_step` picks the -/// **least-recently-queried** Warm shard, demotes one shard per -/// call (Warm → Parked), invokes [`WorkingSetTrim::trim`] exactly -/// once per cascade step (not coalesced into a batch like the -/// idle-demote controller), and returns `None` once no Warm shards -/// remain so the subscriber loop stops the cascade. +/// Owner ruling 2026-10-03 — the pressure subscriber observes, it does +/// not act. A kernel `Low` transition must leave every shard where it +/// is and never call [`WorkingSetTrim::trim`]: a shard goes Parked → +/// Hot when a search needs it and only the idle TTL ladder retires it. +/// Pins the contract so the forced cascade cannot creep back. /// -/// This pins the LRU contract that closes the deferred Phase 3 -/// task 3.6 — the `last_query_at_ms` timestamp is the LRU key, no -/// separate ordering data structure exists. The Phase 5 docstring -/// on `cascade_demote_one_step` calls this out explicitly. -/// -/// Topology: 3 drives (C, D, E) all Warm, with backdated -/// timestamps that establish a deterministic LRU order: -/// D = 1000 (oldest) → E = 2000 → C = 3000 (newest). The -/// cascade should drain in that order. -/// -/// We inject `ControllablePressureSignal` for completeness even -/// though the test calls `cascade_demote_one_step` directly (per -/// the method docstring contract: "task 5.10 test uses -/// `Self::cascade_demote_one_step` directly without going through -/// the watch channel"). `CountingWorkingSetTrim` asserts the -/// per-step `trim()` invocation count. +/// Drives the timeline through the watch channel against the real +/// [`crate::spawn_pressure_subscriber`] with a `ControllablePressureSignal` +/// fake; a 50 ms quiescent window after each transition is far longer +/// than the subscriber's single `borrow_and_update` + log. /// /// [`WorkingSetTrim::trim`]: crate::cache::working_set::WorkingSetTrim::trim #[tokio::test] -async fn cascade_demote_one_step_picks_lru_warm_and_drains_in_order() { +async fn pressure_subscriber_observes_transitions_without_demoting() { + use core::time::Duration; + use crate::cache::ShardState; + use crate::cache::pressure::PressureLevel; use crate::cache::pressure::tests::ControllablePressureSignal; use crate::cache::working_set::tests::CountingWorkingSetTrim; + use crate::index::constructors::LifecycleHooks; let (tx, _rx) = crate::events::event_channel(); let counting_trim = Arc::new(CountingWorkingSetTrim::new()); let pressure_fake = Arc::new(ControllablePressureSignal::new()); - let hooks = crate::index::constructors::LifecycleHooks { + let hooks = LifecycleHooks { working_set_trim: Arc::clone(&counting_trim) as Arc, pressure: Arc::clone(&pressure_fake) as Arc, - ..crate::index::constructors::LifecycleHooks::production() + ..LifecycleHooks::production() }; - let mgr = IndexManager::with_lifecycle_hooks_for_test( + let mgr = Arc::new(IndexManager::with_lifecycle_hooks_for_test( None, tx, hooks, Arc::new(crate::config::Config::default()), - ); + )); mgr.add_drive(build_test_drive()).await; mgr.add_drive(build_test_drive_d()).await; mgr.add_drive(build_test_drive_e()).await; - // Seed the LRU order: D oldest → E middle → C newest. `add_drive` - // already stamped `mark_loaded_at(unix_now_ms())` on each shard, so - // we backdate to known values to remove wall-clock skew from the - // assertion. - assert!( - mgr.backdate_last_query_at_ms_for_test(uffs_mft::platform::DriveLetter::D, 1_000) - .await - ); - assert!( - mgr.backdate_last_query_at_ms_for_test(uffs_mft::platform::DriveLetter::E, 2_000) - .await - ); - assert!( - mgr.backdate_last_query_at_ms_for_test(uffs_mft::platform::DriveLetter::C, 3_000) - .await - ); - - // Pre-cascade: trim hook never fired. - assert_eq!(counting_trim.calls(), 0, "no cascade yet → no trim"); - - // ── Step 1: pick D (oldest, ts = 1000) ────────────────────── - let step1 = mgr.cascade_demote_one_step().await; - assert_eq!( - step1, - Some((uffs_mft::platform::DriveLetter::D, ShardState::Parked)), - "first cascade step demotes the LRU Warm shard (D, ts=1000)", - ); - assert_eq!( - counting_trim.calls(), - 1, - "trim() fires once per cascade step (not coalesced)", - ); - - // ── Step 2: pick E (next-oldest among Warm, ts = 2000) ───── - let step2 = mgr.cascade_demote_one_step().await; - assert_eq!( - step2, - Some((uffs_mft::platform::DriveLetter::E, ShardState::Parked)), - "second cascade step demotes the next-LRU Warm shard (E, ts=2000)", - ); - assert_eq!(counting_trim.calls(), 2); - - // ── Step 3: pick C (last remaining Warm, ts = 3000) ──────── - let step3 = mgr.cascade_demote_one_step().await; - assert_eq!( - step3, - Some((uffs_mft::platform::DriveLetter::C, ShardState::Parked)), - "third cascade step demotes the last Warm shard (C, ts=3000)", - ); - assert_eq!(counting_trim.calls(), 3); - - // ── Step 4: cascade exhausted ────────────────────────────── - // No Warm shards remain → `None` and `trim()` does NOT fire - // (no syscall when there's no Warm work to consolidate). - let step4 = mgr.cascade_demote_one_step().await; - assert_eq!( - step4, None, - "fourth call exhausts the cascade — no Warm shards, returns None", - ); - assert_eq!( - counting_trim.calls(), - 3, - "exhausted cascade must not re-trim — `pick?` short-circuits", - ); - - // Final state: every shard Parked (in alphabetical-by-letter - // order from `shard_states_for_test`). - let states = mgr.shard_states_for_test().await; - assert_eq!(states, vec![ - (uffs_mft::platform::DriveLetter::C, ShardState::Parked), - (uffs_mft::platform::DriveLetter::D, ShardState::Parked), - (uffs_mft::platform::DriveLetter::E, ShardState::Parked), - ]); - - // The pressure fake was never driven — this test exercises the - // cascade method directly, not the subscriber loop. Asserting - // `receiver_count() == 0` documents that contract: the - // `IndexManager` does NOT auto-subscribe at construction; only - // `spawn_pressure_subscriber` (in `lib.rs`) does. - assert_eq!( - pressure_fake.receiver_count(), - 0, - "IndexManager holds the Arc but does not auto-subscribe", - ); -} - -/// Plan task **5.10 (end-to-end)** + Phase-5 wrap-up regression — the -/// full `spawn_pressure_subscriber` → `cascade_demote_one_step` → -/// preempt loop must: -/// -/// 1. Subscribe to the [`PressureSignal`] (`receiver_count` becomes 1 after -/// spawn). -/// 2. On `Low`, drain every Warm shard one step at a time, calling -/// [`WorkingSetTrim::trim`] exactly once per cascade step. -/// 3. On `High`, become a no-op — the cascade body never runs. -/// 4. On a second `Low` after the first cascade exhausted the Warm set, -/// terminate the inner cascade loop on the first `None` return without -/// firing extra trim calls. -/// -/// The existing [`cascade_demote_one_step_picks_lru_warm_and_drains_in_order`] -/// test pins the cascade method's contract by calling it directly; -/// this test pins the **subscriber wiring** in `lib.rs` so a future -/// refactor of [`crate::spawn_pressure_subscriber`] can't silently -/// drop the cascade-on-Low contract that the Win32 watcher thread -/// depends on. -/// -/// Test architecture mirrors §5.10's direct test (3 shards backdated -/// for a deterministic LRU order) but drives the timeline through -/// the watch channel instead of synchronous calls into the manager. -/// Polling on `shard_states_for_test` plus `receiver_count` keeps -/// the test deterministic without `tokio::time::pause` (which would -/// require a `current_thread` runtime + `start_paused = true`). -/// -/// [`PressureSignal`]: crate::cache::pressure::PressureSignal -/// [`WorkingSetTrim::trim`]: crate::cache::working_set::WorkingSetTrim::trim -#[tokio::test] -async fn pressure_subscriber_drains_warm_cascade_on_low_and_no_ops_on_high() { - use pressure_subscriber_fixtures::{ - CASCADE_DEADLINE, QUIESCENT_WINDOW, build_pressure_subscriber_fixture, poll_until, - }; - - use crate::cache::ShardState; - use crate::cache::pressure::PressureLevel; - - let fixture = build_pressure_subscriber_fixture().await; - - // ── Spawn the subscriber and wait for it to attach ─────────── - let subscriber = crate::spawn_pressure_subscriber(Arc::clone(&fixture.mgr)); - let attach_deadline = std::time::Instant::now() + CASCADE_DEADLINE; - while fixture.pressure_fake.receiver_count() == 0 { + let subscriber = crate::spawn_pressure_subscriber(&mgr); + let attach_deadline = std::time::Instant::now() + Duration::from_secs(2); + while pressure_fake.receiver_count() == 0 { assert!( std::time::Instant::now() < attach_deadline, - "subscriber did not attach to the watch channel within {CASCADE_DEADLINE:?}", + "subscriber did not attach to the watch channel within 2 s", ); tokio::task::yield_now().await; } - assert_eq!( - fixture.pressure_fake.receiver_count(), - 1, - "exactly one subscriber attaches via spawn_pressure_subscriber", - ); - - // ── Step 1: First Low → drains all Warm in LRU order ───────── - assert!( - fixture.pressure_fake.set(PressureLevel::Low), - "broadcast Low must reach the attached subscriber", - ); - poll_until( - &fixture.mgr, - |states| states.iter().all(|(_, s)| *s == ShardState::Parked), - "first Low cascade", - ) - .await; - assert_eq!( - fixture.counting_trim.calls(), - 3, - "trim() fires once per cascade step (3 Warm → 3 calls)", - ); - - // ── Step 2: High → no-op (no additional demotes / trim calls) ─ - assert!(fixture.pressure_fake.set(PressureLevel::High)); - tokio::time::sleep(QUIESCENT_WINDOW).await; - let post_high_states = fixture.mgr.shard_states_for_test().await; - assert!( - post_high_states - .iter() - .all(|(_, s)| *s == ShardState::Parked), - "High transition must not change shard state; got {post_high_states:?}", - ); - assert_eq!( - fixture.counting_trim.calls(), - 3, - "High triggers no additional demotes or trim calls", - ); - - // ── Step 3: Second Low with no Warm left → cascade returns - // None on first call, no extra trim fires ──────────────────── - assert!(fixture.pressure_fake.set(PressureLevel::Low)); - tokio::time::sleep(QUIESCENT_WINDOW).await; - assert_eq!( - fixture.counting_trim.calls(), - 3, - "second Low with no Warm shards must not call trim() again", - ); - - // Clean shutdown — abort the subscriber explicitly so the test - // task tree winds down without waiting on the watch sender's - // own drop (which is racy across the Arc graph). - subscriber.abort(); -} - -/// Test infrastructure for -/// [`pressure_subscriber_drains_warm_cascade_on_low_and_no_ops_on_high`]. -/// -/// Lifted out of the test body so the test stays under clippy's -/// 100-line ceiling without compromising on assertion coverage. -/// Module-private; only the parent test imports its public surface. -mod pressure_subscriber_fixtures { - use core::time::Duration; - use std::sync::Arc; - - use super::{IndexManager, build_test_drive, build_test_drive_d, build_test_drive_e}; - use crate::cache::ShardState; - use crate::cache::pressure::PressureSignal; - use crate::cache::pressure::tests::ControllablePressureSignal; - use crate::cache::working_set::tests::CountingWorkingSetTrim; - use crate::index::constructors::LifecycleHooks; - - /// Polling deadline for cascade-completion observation. Wall-clock - /// bound — generous enough that a busy CI box doesn't false-fail - /// (cascade is microseconds of pure CPU work; 2 s is 1 000× the - /// observed worst case) and short enough that a real bug surfaces - /// fast. - pub(super) const CASCADE_DEADLINE: Duration = Duration::from_secs(2); - - /// Quiescent observation window — after a non-cascade-driving - /// transition (`High`) we wait this long to confirm the - /// subscriber stays idle, then assert no Warm shards demoted and - /// no trim calls fired. Tuned to be > one `tokio::task::yield_now` - /// scheduler pass on every supported runtime. - pub(super) const QUIESCENT_WINDOW: Duration = Duration::from_millis(50); - - /// Bundle of the three handles the test asserts against: - /// the `IndexManager` under test, the controllable pressure - /// fake driving the watch channel, and the trim counter. - pub(super) struct PressureSubscriberFixture { - pub mgr: Arc, - pub pressure_fake: Arc, - pub counting_trim: Arc, - } - /// Build the [`PressureSubscriberFixture`] preconfigured with - /// 3 Warm shards in deterministic LRU order (D = 1000 → - /// E = 2000 → C = 3000), zero trim calls, and zero subscribers - /// — same ordering as the §5.10 direct test so a regression - /// there surfaces in the subscriber test too. Asserts the - /// preconditions before returning so the parent test body - /// can stay focused on the act / observe sequence. - pub(super) async fn build_pressure_subscriber_fixture() -> PressureSubscriberFixture { - let (tx, _rx) = crate::events::event_channel(); - let counting_trim = Arc::new(CountingWorkingSetTrim::new()); - let pressure_fake = Arc::new(ControllablePressureSignal::new()); - let hooks = LifecycleHooks { - working_set_trim: Arc::clone(&counting_trim) - as Arc, - pressure: Arc::clone(&pressure_fake) as Arc, - ..LifecycleHooks::production() - }; - let mgr = Arc::new(IndexManager::with_lifecycle_hooks_for_test( - None, - tx, - hooks, - Arc::new(crate::config::Config::default()), - )); - mgr.add_drive(build_test_drive()).await; - mgr.add_drive(build_test_drive_d()).await; - mgr.add_drive(build_test_drive_e()).await; + for level in [PressureLevel::Low, PressureLevel::High, PressureLevel::Low] { assert!( - mgr.backdate_last_query_at_ms_for_test(uffs_mft::platform::DriveLetter::D, 1_000) - .await + pressure_fake.set(level), + "broadcast {level:?} must reach the attached subscriber", ); + tokio::time::sleep(Duration::from_millis(50)).await; + let states = mgr.shard_states_for_test().await; assert!( - mgr.backdate_last_query_at_ms_for_test(uffs_mft::platform::DriveLetter::E, 2_000) - .await - ); - assert!( - mgr.backdate_last_query_at_ms_for_test(uffs_mft::platform::DriveLetter::C, 3_000) - .await - ); - let initial_states = mgr.shard_states_for_test().await; - assert!( - initial_states.iter().all(|(_, s)| *s == ShardState::Warm), - "preconditions: all 3 shards Warm; got {initial_states:?}", + states.iter().all(|(_, state)| *state == ShardState::Warm), + "{level:?} must not move any shard; got {states:?}", ); assert_eq!( counting_trim.calls(), 0, - "preconditions: no trim calls before subscriber spawn", - ); - assert_eq!( - pressure_fake.receiver_count(), - 0, - "preconditions: no subscribers before spawn", + "{level:?} must not trim the working set", ); - PressureSubscriberFixture { - mgr, - pressure_fake, - counting_trim, - } } - /// Poll [`IndexManager::shard_states_for_test`] until `predicate` - /// holds or the [`CASCADE_DEADLINE`] expires. Panics with a - /// diagnostic message on timeout so a regression surfaces at the - /// failed assertion site, not as a hung test. - pub(super) async fn poll_until(mgr: &IndexManager, predicate: F, label: &str) - where - F: Fn(&[(uffs_mft::platform::DriveLetter, ShardState)]) -> bool, - { - let deadline = std::time::Instant::now() + CASCADE_DEADLINE; - loop { - let states = mgr.shard_states_for_test().await; - if predicate(&states) { - return; - } - assert!( - std::time::Instant::now() < deadline, - "{label} did not converge within {CASCADE_DEADLINE:?}; last states = {states:?}", - ); - tokio::task::yield_now().await; - } - } + subscriber.abort(); } diff --git a/crates/uffs-daemon/src/index/tests/manager.rs b/crates/uffs-daemon/src/index/tests/manager.rs index c8be5d5ae..a49bf1e86 100644 --- a/crates/uffs-daemon/src/index/tests/manager.rs +++ b/crates/uffs-daemon/src/index/tests/manager.rs @@ -47,7 +47,7 @@ async fn search_with_include_rows_false_suppresses_rows_but_counts() { ..uffs_client::protocol::SearchParams::default() }; - let response = mgr.search(¶ms).await; + let response = mgr.search(¶ms).await.expect("search completes"); // `include_rows = false` must leave the payload as `Empty` — // any other variant (InlineRows, blob, shmem) would mean the @@ -81,7 +81,7 @@ async fn search_with_include_rows_true_returns_rows() { ..uffs_client::protocol::SearchParams::default() }; - let response = mgr.search(¶ms).await; + let response = mgr.search(¶ms).await.expect("search completes"); // The happy path delivers `InlineRows` — the small-manager // fixture never breaches the `SHMEM_THRESHOLD` (100 K rows) or diff --git a/crates/uffs-daemon/src/index/tests/mod.rs b/crates/uffs-daemon/src/index/tests/mod.rs index e25c72015..7ff60e7e4 100644 --- a/crates/uffs-daemon/src/index/tests/mod.rs +++ b/crates/uffs-daemon/src/index/tests/mod.rs @@ -19,12 +19,11 @@ //! (Phase 4 task 4.11). //! * [`idle_demote`] — `demote_idle_shards` TTL-driven cascade and round-trip //! query stats (state-ladder behaviour only). -//! * [`idle_demote_tracing`] — `shard.transition` tracing-event contract, -//! pressure-cascade single-event regression, and the PR-f promote anti-thrash -//! invariant. -//! * [`lifecycle_hooks`] — Phase 5 task 5.8 / 5.9 / 5.10 `WorkingSetTrim` + -//! `Prefetch` + `PressureSignal` injection tests, plus the `drives` RPC -//! tier-marker enumeration. +//! * [`idle_demote_tracing`] — `shard.transition` tracing-event contract and +//! the PR-f promote anti-thrash invariant. +//! * [`lifecycle_hooks`] — Phase 5 task 5.8 / 5.9 `WorkingSetTrim` + `Prefetch` +//! injection tests, the observe-only `PressureSignal` subscriber contract, +//! plus the `drives` RPC tier-marker enumeration. //! * [`tracing_capture`] — shared `tracing::Subscriber` scaffold (`EventLog` / //! `CapturedEvent`) used by [`idle_demote_tracing`] and other //! observability-contract tests. diff --git a/crates/uffs-daemon/src/index/tests/registry.rs b/crates/uffs-daemon/src/index/tests/registry.rs index b35c9a179..fb716bf53 100644 --- a/crates/uffs-daemon/src/index/tests/registry.rs +++ b/crates/uffs-daemon/src/index/tests/registry.rs @@ -131,7 +131,7 @@ async fn shard_registry_search_two_drives_returns_rows_from_each() { limit: Some(50), ..Default::default() }; - let resp = mgr.search(¶ms).await; + let resp = mgr.search(¶ms).await.expect("search completes"); assert!( resp.total_count >= 2, "two-drive '*' search must return at least 2 rows; got {}", @@ -181,7 +181,7 @@ async fn search_records_query_on_every_active_shard() { // Three searches. Suffix the loop bound to avoid the implicit i32 // fallback flagged by clippy::default_numeric_fallback. for _ in 0_u32..3_u32 { - drop(mgr.search(¶ms).await); + drop(mgr.search(¶ms).await.expect("search completes")); } let after = mgr.shard_query_totals_for_test().await; diff --git a/crates/uffs-daemon/src/index/tests/tiering_ops.rs b/crates/uffs-daemon/src/index/tests/tiering_ops.rs index 02cce3411..d447d7c08 100644 --- a/crates/uffs-daemon/src/index/tests/tiering_ops.rs +++ b/crates/uffs-daemon/src/index/tests/tiering_ops.rs @@ -298,75 +298,6 @@ async fn preload_pin_blocks_idle_demote() { ); } -/// Phase 8-C pin-contract — the pressure-cascade LRU pick -/// (`cascade_demote_one_step`) excludes pinned shards. When every -/// `Warm` shard is pinned, the cascade returns `None` and the -/// subscriber loop yields. -#[tokio::test] -async fn preload_pin_blocks_cascade_demote() { - let (tx, _rx) = crate::events::event_channel(); - let body = Arc::new(build_test_drive()); - let loader = Arc::new(FixedBodyLoader { - body: Arc::clone(&body), - }); - let mgr = IndexManager::with_body_loader_for_test(None, tx, loader); - mgr.add_drive(build_test_drive()).await; - assert!( - mgr.demote_letter_for_test(uffs_mft::platform::DriveLetter::C, ShardState::Cold) - .await - ); - - let outcome = mgr - .preload_drive(uffs_mft::platform::DriveLetter::C, 30) - .await; - assert!(matches!(outcome, PreloadOutcome::Promoted { .. })); - - // The cascade picks Warm shards only, but the test fixture also - // pins the only loaded shard at Hot. Add a second Warm shard - // that's pinned so the cascade sees a Warm pinned shard - // (worst case for the filter — the policy looks at Warm only, - // and pin must override). Achieve this by demoting C from Hot - // to Warm via the test escape hatch — the pin survives because - // it lives on the same `Arc` that gets demoted. - // - // Wait — that's wrong: the registry rebuild on demote installs - // a fresh `ShardEntry` with `pin_until_ms = 0`. Stick to the - // single-shard Hot case and assert the cascade returns None - // because no Warm shard exists at all (pin_until_ms is 0 only - // matters for Warm-state shards; Hot is filtered out by the - // cascade's `state() == Warm` check separately). Re-add a - // second drive in Warm with no pin and verify the cascade - // picks IT, not C. - mgr.add_drive(build_test_drive_d()).await; - - // Pre-condition assertion: C is Hot (pinned) and D is Warm - // (unpinned). - let pre_states = mgr.shard_states_for_test().await; - assert_eq!(pre_states, vec![ - (uffs_mft::platform::DriveLetter::C, ShardState::Hot), - (uffs_mft::platform::DriveLetter::D, ShardState::Warm) - ]); - - // Cascade: must pick D (Warm, unpinned), not C (Hot, pinned). - let picked = mgr.cascade_demote_one_step().await; - assert_eq!( - picked, - Some((uffs_mft::platform::DriveLetter::D, ShardState::Parked)), - "cascade must pick the unpinned Warm shard D, not the pinned Hot shard C" - ); - - // Post-cascade: C still Hot, D demoted to Parked. - let states = mgr.shard_states_for_test().await; - assert_eq!( - states, - vec![ - (uffs_mft::platform::DriveLetter::C, ShardState::Hot), - (uffs_mft::platform::DriveLetter::D, ShardState::Parked) - ], - "pinned shard C must still be Hot; unpinned D must be Parked" - ); -} - /// Phase 8-B + 8-C — explicit `hibernate` overrides a pin. The /// registry rebuild installs a fresh `ShardEntry` whose /// `pin_until_ms` starts at `0`, so the pin is implicitly cleared diff --git a/crates/uffs-daemon/src/index/tests/tracing_capture.rs b/crates/uffs-daemon/src/index/tests/tracing_capture.rs index d0f0b9972..825d3662a 100644 --- a/crates/uffs-daemon/src/index/tests/tracing_capture.rs +++ b/crates/uffs-daemon/src/index/tests/tracing_capture.rs @@ -18,10 +18,6 @@ //! `reason` / `freed_mb` / `restored_mb` / `last_query_at_ms` //! field contract on the canonical `shard.transition` event for //! the demote-then-promote round-trip. -//! * [`super::idle_demote::cascade_demote_emits_single_event_with_pressure_cascade_reason`] -//! — Phase 5 G4 follow-up — pins the single-canonical-event -//! contract for the pressure-cascade demote path (no second -//! redundant event from `cascade_demote_one_step`). //! * `crate::cache::journal_loop::tests::compact_cache_save_log` — pins the //! literal `"compact-cache save"` message text the Phase 7 24-h soak harness //! greps for (visibility raised to `pub(crate)` in 2026-05-13 to share the diff --git a/crates/uffs-daemon/src/index/tiering_ops.rs b/crates/uffs-daemon/src/index/tiering_ops.rs index 31f592298..ec98a0d2b 100644 --- a/crates/uffs-daemon/src/index/tiering_ops.rs +++ b/crates/uffs-daemon/src/index/tiering_ops.rs @@ -23,9 +23,9 @@ //! pin). //! //! Why a sibling file (instead of folding into -//! [`super::transitions`]): the two background controllers in -//! `transitions.rs` (`demote_idle_shards`, `cascade_demote_one_step`) -//! are policy-driven daemon-internal decisions, while these two +//! [`super::transitions`]): the background controller in +//! `transitions.rs` (`demote_idle_shards`) is a policy-driven +//! daemon-internal decision, while these two //! methods are operator-driven entry points reached over the wire. //! Keeping the two clusters separate makes the audit boundary //! obvious — operator overrides go here, controller automation goes @@ -127,7 +127,7 @@ impl IndexManager { /// The `OperatorHibernate` reason discriminator flows into the /// canonical `shard.transition` event so operators can grep /// `reason="operator-hibernate"` to distinguish manual - /// hibernation from idle-tick or pressure-cascade demotes. + /// hibernation from idle-tick demotes. pub(crate) async fn hibernate_shards( &self, drives: &[uffs_mft::platform::DriveLetter], diff --git a/crates/uffs-daemon/src/index/transitions.rs b/crates/uffs-daemon/src/index/transitions.rs index e552c6d7f..75e2d6120 100644 --- a/crates/uffs-daemon/src/index/transitions.rs +++ b/crates/uffs-daemon/src/index/transitions.rs @@ -18,13 +18,10 @@ //! `[shards.per_drive."X:"].min_tier` floor (plan tasks 6.4, 6.6). Every //! demote evaluation emits a `shard.ttl` tracing event with the chosen TTL, //! the live rate, and a structured reason (plan task 6.7). -//! 2. [`IndexManager::cascade_demote_one_step`] (+ -//! [`IndexManager::subscribe_pressure`]) — Phase 5 task 5.6 -//! pressure-cascade. Picks the LRU Warm shard, demotes it Warm → Parked, -//! and trims the working set; the subscriber loop in `lib.rs` calls this in -//! a tight loop while [`crate::cache::pressure::PressureLevel`] reports -//! `Critical`, yielding between calls so the cascade stops as soon as the -//! pressure clears. +//! 2. [`IndexManager::subscribe_pressure`] — the watch channel the `lib.rs` +//! pressure subscriber logs from. The pressure cascade that used to demote +//! LRU Warm shards from here was removed on 2026-10-03 (owner ruling): the +//! idle ladder above is the only demote driver. //! //! Phase 7 activation moved the third (USN-refresh) controller out //! of this module: the deleted `refresh_usn_for_warm_shards` global @@ -152,14 +149,12 @@ impl IndexManager { } } - /// Subscribe to memory-pressure transitions (Phase 5 task 5.6). + /// Subscribe to memory-pressure transitions. /// /// Returns a [`tokio::sync::watch::Receiver`] carrying the /// current [`PressureLevel`] and waking on every transition. /// The daemon's `spawn_pressure_subscriber` (in `lib.rs`) is - /// the sole production consumer; the Phase 5 task 5.10 test - /// uses [`IndexManager::cascade_demote_one_step`] directly without - /// going through the watch channel. + /// the sole production consumer, and it only logs what it sees. /// /// [`PressureLevel`]: crate::cache::pressure::PressureLevel pub(crate) fn subscribe_pressure( @@ -167,98 +162,6 @@ impl IndexManager { ) -> tokio::sync::watch::Receiver { self.pressure.subscribe() } - - /// Cascade-demote one LRU Warm shard to Parked (Phase 5 task 5.6). - /// - /// Picks the Warm shard with the **oldest** - /// `DriveStats::last_query_at_ms` and demotes it one tier - /// (Warm → Parked). Returns `Some((letter, ShardState::Parked))` - /// when work was done, `None` when no Warm shards remain (the - /// caller stops the cascade). - /// - /// **LRU contract** (closes the deferred Phase 3 task 3.6). The - /// per-shard `last_query_at_ms` already exists from Phase 3; the - /// "LRU bookkeeping" task 3.6 alluded to is just a sort at - /// demote-time — no separate ordering data structure is needed, - /// since the cascade fires rarely (only on Windows pressure - /// transitions) and the Warm subset is small (one shard per - /// loaded drive, capped at the indexed-drive count). - /// - /// **Working-set trim**. Each cascade step calls - /// [`WorkingSetTrim::trim`] once. Unlike the idle-demote - /// batch — where one trim per batch coalesces N shards — the - /// cascade is one shard per call by design (the subscriber - /// loop yields between calls so a `High` transition can stop - /// the cascade promptly), so there's no batch to coalesce. - /// - /// [`WorkingSetTrim::trim`]: crate::cache::working_set::WorkingSetTrim::trim - pub(crate) async fn cascade_demote_one_step( - &self, - ) -> Option<(uffs_mft::platform::DriveLetter, ShardState)> { - // ── Phase 1: read-lock detect (LRU pick) ──────────────────── - // Enumerate Warm shards and keep the one with the oldest - // `last_query_at_ms`. `min_by_key` returns `None` when no - // Warm shards exist; the caller stops the cascade. - // - // Phase 8-C — pinned shards (operator-driven `preload`) - // are excluded from the LRU pick. This means a sustained - // memory-pressure cascade can run out of demote candidates - // even when total RAM remains tight; the pressure - // subscriber loop in `lib.rs` handles that case by yielding - // and waking on the next pressure transition. Operators - // who explicitly pinned a shard accepted that trade-off. - let now_ms = crate::cache::unix_now_ms(); - let pick: Option<(uffs_mft::platform::DriveLetter, u64)> = { - let guard = self.index.read().await; - guard - .iter() - .filter(|shard| shard.state() == ShardState::Warm) - .filter(|shard| !shard.is_pinned(now_ms)) - .map(|shard| (shard.drive, shard.stats.last_query_at_ms())) - .min_by_key(|&(_, ts)| ts) - }; - let (letter, _last_query_at_ms) = pick?; - - // ── Phase 2: write-lock atomic single-shard demote ───────── - // Re-check inside the write lock — a concurrent promote - // could have moved the picked shard back to Hot/Warm - // between the read-lock and the write-lock acquisition. - // `demote_letter_with_reason` returns `None` for an illegal - // transition, in which case we skip and the next cascade - // tick re-picks. The `PressureCascade` reason flows into - // the canonical `shard.transition` event so operators can - // distinguish cascade demotes from TTL idle demotes by - // grepping `reason="pressure-cascade"` (Phase 5 G4 - // follow-up — the cascade no longer emits its own - // duplicate event of the same demote). - let target = ShardState::Parked; - let mut guard = self.index.write().await; - let new_registry = guard.demote_letter_with_reason( - letter, - target, - crate::cache::registry::DemoteReason::PressureCascade, - )?; - *guard = Arc::new(new_registry); - drop(guard); - self.bump_index_version(); - - // ── Phase 3: working-set trim (Phase 5 task 5.4 reuse) ──── - // One trim per cascade step (vs once per idle-demote batch) - // — see method-level docs for the rationale. Best-effort: - // any I/O error is logged at `target: "shard.transition"` - // and the daemon continues. - if let Err(err) = self.working_set_trim.trim() { - tracing::warn!( - target: "shard.transition", - drive = %letter, - error = %err, - reason = "pressure-cascade", - "WorkingSetTrim::trim failed; daemon continues", - ); - } - - Some((letter, target)) - } } // ── Phase 6 Commit C — adaptive idle-demote helpers ────────────────────── diff --git a/crates/uffs-daemon/src/lib.rs b/crates/uffs-daemon/src/lib.rs index 3c23cdd7c..55d7201e9 100644 --- a/crates/uffs-daemon/src/lib.rs +++ b/crates/uffs-daemon/src/lib.rs @@ -50,8 +50,9 @@ //! signal source. //! * `spawn_journal_loops_for_warm_shards` — per-shard USN journal loops, each //! cooperatively cancelled via a dedicated `watch::Sender`. -//! * `spawn_pressure_subscriber` — listens to OS memory-pressure events and -//! drives the demote controller. +//! * `spawn_pressure_subscriber` — logs OS memory-pressure transitions for +//! operators. It takes no action: only the idle TTL ladder retires an index +//! (owner ruling 2026-10-03). //! //! All shutdown coordination flows through the daemon's top-level //! `LifecycleHandle` (`watch::Sender` broadcast + force-exit @@ -254,11 +255,12 @@ pub async fn run_daemon(config: DaemonConfig) -> anyhow::Result<()> { // 5-min global tick with per-letter event-driven refresh — see // `spawn_journal_loops_for_warm_shards` below. - // Phase 5 task 5.6 — memory-pressure subscriber. Cascade- - // demotes LRU Warm shards on `Low` transitions until pressure - // clears (`High`) or no Warm shards remain. No-op on Mac/Linux - // (the platform `PressureSignal` never fires). - let _pressure_task = spawn_pressure_subscriber(Arc::clone(&idx)); + // Memory-pressure subscriber: logs every kernel `Low` / `High` + // transition so an operator can correlate them with the tier + // ladder, and nothing else — a search is what brings a shard back + // to Warm, idle time is what retires it. Nothing fires on + // Mac/Linux (the platform `PressureSignal` never does). + let _pressure_task = spawn_pressure_subscriber(&idx); // Run idle timer (blocks until shutdown or timeout) then tear // everything down. Returns `!` so `force_exit_with_watchdog` @@ -668,53 +670,37 @@ fn make_journal_source( Arc::new(cache::journal_loop::sources::MacStubJournalSource) } -/// Spawn the Phase 5 task 5.6 memory-pressure subscriber. +/// Spawn the memory-pressure subscriber. /// -/// Subscribes to [`IndexManager::subscribe_pressure`] and reacts to -/// transitions: +/// Subscribes to [`IndexManager::subscribe_pressure`] and logs every +/// transition at `INFO` under `target: "cache.pressure"` so an operator +/// reading the daemon log can line kernel pressure up against the tier +/// ladder. That is the whole job: the subscriber **never demotes**. /// -/// * `Low` — enters cascade mode: calls -/// [`IndexManager::cascade_demote_one_step`] in a loop until either no Warm -/// shards remain (the `None` return) or a `High` / `Normal` transition -/// arrives. After every step we `tokio::task::yield_now` and check -/// `rx.has_changed()` so a `High` transition can preempt promptly without -/// waiting for the next demote to finish. -/// * `High` / `Normal` — no-op; the loop returns to `rx.changed().await` for -/// the next transition. -/// -/// The cascade decision is made via -/// [`PressureLevel::requires_cascade_demote`] rather than direct pattern -/// matching on `PressureLevel::Low`, because the `Low` and `High` variants -/// are platform-conditional — they only exist on Windows and under -/// `cfg(test)` (the targets where the watcher / test fake actually -/// constructs them). The method ships a `false` branch on Mac/Linux -/// production builds so this loop body compiles cleanly on every host. +/// Until 2026-10-03 a `Low` transition cascade-parked LRU Warm shards +/// one by one until pressure cleared. On a memory-starved host that +/// fired 2.5 minutes after start, parked all six drives right after a +/// benchmark had warmed them, and turned the next `*.*` into a +/// four-minute MFT re-read (the 2026-10-03 benchmark run). The owner +/// ruled the forced demotion out: a shard goes Parked → Hot when a +/// search needs it and is left alone afterwards; the idle TTL ladder +/// (`spawn_idle_demote_controller`) is the only thing that retires it. /// /// On Mac/Linux the platform [`PressureSignal`] never fires, so this -/// task blocks on `rx.changed().await` forever — TTL-driven demotion -/// via `spawn_idle_demote_controller` is the only demote driver on -/// those targets by design. +/// task blocks on `rx.changed().await` forever. /// /// Returns the [`tokio::task::JoinHandle`] so the caller can `.abort()` /// it during graceful shutdown. When the [`watch::Sender`] inside /// `IndexManager::pressure` is dropped the receiver's `changed()` /// returns `Err`; the loop breaks cleanly without any extra signal. /// -/// `pub(crate)` so the Phase 5 end-to-end integration test in -/// `crate::index::tests::lifecycle_hooks` can drive the full -/// subscribe → cascade → preempt loop against a `ControllablePressureSignal` -/// fake without re-implementing the loop body in test code. Production -/// callers stay limited to [`run_daemon`] which is the only place this -/// runs in the live daemon. +/// `pub(crate)` so `crate::index::tests::lifecycle_hooks` can pin the +/// observe-only contract against a `ControllablePressureSignal` fake. /// -/// [`PressureLevel::requires_cascade_demote`]: crate::cache::pressure::PressureLevel::requires_cascade_demote /// [`PressureSignal`]: crate::cache::pressure::PressureSignal /// [`IndexManager::subscribe_pressure`]: crate::index::IndexManager::subscribe_pressure -/// [`IndexManager::cascade_demote_one_step`]: crate::index::IndexManager::cascade_demote_one_step /// [`watch::Sender`]: tokio::sync::watch::Sender -pub(crate) fn spawn_pressure_subscriber( - idx: Arc, -) -> tokio::task::JoinHandle<()> { +pub(crate) fn spawn_pressure_subscriber(idx: &index::IndexManager) -> tokio::task::JoinHandle<()> { let mut rx = idx.subscribe_pressure(); tokio::spawn(async move { loop { @@ -735,31 +721,6 @@ pub(crate) fn spawn_pressure_subscriber( ?level, "Pressure transition observed", ); - if !level.requires_cascade_demote() { - continue; - } - // Cascade-demote until we run out of Warm shards or - // pressure clears. `cascade_demote_one_step` returns - // `None` when no Warm shards remain. - loop { - let Some(_demoted) = idx.cascade_demote_one_step().await else { - break; // no more Warm shards; cascade exhausted - }; - // Yield so the runtime can deliver a pending - // pressure-clearing transition before we loop. - tokio::task::yield_now().await; - if rx.has_changed().unwrap_or(false) { - let new_level = *rx.borrow_and_update(); - if !new_level.requires_cascade_demote() { - tracing::info!( - target: "cache.pressure", - ?new_level, - "Cascade preempted by transition out of Low", - ); - break; - } - } - } } }) } diff --git a/crates/uffs-mft/fuzz/Cargo.toml b/crates/uffs-mft/fuzz/Cargo.toml index c7a530e15..c61360089 100644 --- a/crates/uffs-mft/fuzz/Cargo.toml +++ b/crates/uffs-mft/fuzz/Cargo.toml @@ -40,4 +40,3 @@ path = "fuzz_targets/fuzz_apply_fixup.rs" test = false doc = false bench = false - diff --git a/docs/architecture/code-quality/concurrency_policy.md b/docs/architecture/code-quality/concurrency_policy.md index b518ce104..b61baa958 100644 --- a/docs/architecture/code-quality/concurrency_policy.md +++ b/docs/architecture/code-quality/concurrency_policy.md @@ -54,15 +54,16 @@ The daemon's index is a per-drive collection of shards, each transitioning betwe |---|---|---|---|---| | `Unknown` | – | – | – | initial discovery | | `Cold` | – | – | – | parked-compactor demote (from `Parked`) | -| `Parked` | – | ✓ | ✓ | `spawn_idle_demote_controller` after TTL OR `spawn_pressure_subscriber` cascade | +| `Parked` | – | ✓ | ✓ | `spawn_idle_demote_controller` after TTL | | `Warm` | mmap | ✓ | ✓ | initial load OR promote from `Parked` | | `Hot` | mmap + prefaulted | ✓ | ✓ | recent search activity | | `Evicting` | (in transit) | – | – | transient — demote in progress | -Legal transitions are pinned in `ShardState::can_transition_to`. The two demote drivers are: +Legal transitions are pinned in `ShardState::can_transition_to`. The automatic demote driver is: * **Idle TTL** — `spawn_idle_demote_controller` runs every 30 s and demotes `Warm`/`Hot` → `Parked` after a configurable idle-since-last-access window. - * **Memory-pressure cascade** — `spawn_pressure_subscriber` listens to the OS memory-pressure watch and cascades `Warm` → `Cold` one step at a time on `Low` transitions, preempted by `High`/`Normal`. No-op on Mac/Linux (the platform `PressureSignal` never fires by design). + + `spawn_pressure_subscriber` still listens to the OS memory-pressure watch but only **logs** `Low`/`High` transitions (`target: cache.pressure`). The former cascade that parked LRU `Warm` shards on `Low` was removed on 2026-10-03 by owner ruling: a shard is promoted when a search needs it and left alone afterwards; only idle time retires it. No-op on Mac/Linux (the platform `PressureSignal` never fires by design). ### 0.3 IPC-request lifecycle diff --git a/docs/architecture/memory-tiering-windows-host-validation.md b/docs/architecture/memory-tiering-windows-host-validation.md index 5a4d00f7d..67626b45e 100644 --- a/docs/architecture/memory-tiering-windows-host-validation.md +++ b/docs/architecture/memory-tiering-windows-host-validation.md @@ -79,6 +79,22 @@ close each gate. ## 1. Phase 5 operator gates — 4 captures, ~75 minutes wall-clock +> **Retired 2026-10-03 — pressure cascade removed.** The kernel-`Low` +> → cascade-demote behaviour that G1 and the G4 cascade assertions +> validate no longer exists. By owner ruling the daemon never forces a +> tier change on memory pressure: `spawn_pressure_subscriber` logs the +> `Pressure transition observed level=…` line and stops there; a shard +> goes Parked → Hot when a search needs it and only the idle TTL ladder +> retires it. `reason="pressure-cascade"`, `cascade_demote_one_step` +> and `PressureLevel::requires_cascade_demote` are gone. The captures +> below are kept as the historical record of v0.5.86–v0.6.42; on a +> current build G1 produces the `Pressure transition observed` line and +> **no** `shard.transition` demote lines, and that is the pass +> criterion. Trigger for the removal: the 2026-10-03 benchmark run on +> a memory-starved host, where the cascade parked all six drives +> 2.5 min after start and turned the next `*.*` into a four-minute MFT +> re-read. + ### G1 — Low-pressure stress: kernel notification → cache.pressure → cascade demote **Duration:** ~5 min wall-clock. diff --git a/docs/user-manual/cli-overview.md b/docs/user-manual/cli-overview.md index 2f2c7b9c3..68ecf4a3b 100644 --- a/docs/user-manual/cli-overview.md +++ b/docs/user-manual/cli-overview.md @@ -311,7 +311,8 @@ These flags are for power users, profiling, and parity testing: |------|-------------| | `-v, --verbose` | Enable verbose output (global) | | `--profile` | Show detailed timing breakdown | -| `--benchmark` | Skip output; measure only MFT reading + filtering | +| `--benchmark` | Time the whole pipeline including output formatting; the rows are rendered into a sink instead of the screen (add `-v` to also print them). The profile block reports the output pass as its own line | +| `--no-output` | Match only: the daemon builds no rows, so this times "how many files match" and nothing else. Auto-set when stdout is a null device | | `--no-bitmap` | Disable MFT bitmap optimisation (read ALL records) | | `--no-cache` | Bypass cache; re-read MFT fresh | | `--query-mode ` | Force query path: `auto`, `index`, `dataframe` | diff --git a/rustfmt.toml b/rustfmt.toml index 02787d558..172637eaf 100644 --- a/rustfmt.toml +++ b/rustfmt.toml @@ -8,14 +8,14 @@ # ───── Nightly & Edition ───── unstable_features = true -edition = "2024" # Parse edition 2024 syntax (let-chains, etc.) -style_edition = "2024" # Use 2024 style guide formatting rules +edition = "2024" # Parse edition 2024 syntax (let-chains, etc.) +style_edition = "2024" # Use 2024 style guide formatting rules # ───── Layout ───── max_width = 100 newline_style = "Unix" use_small_heuristics = "Default" -overflow_delimited_expr = true # Let closures/arrays/blocks overflow the line width +overflow_delimited_expr = true # Let closures/arrays/blocks overflow the line width # ───── Imports ───── imports_granularity = "Module" @@ -27,15 +27,15 @@ reorder_modules = true wrap_comments = true format_code_in_doc_comments = true normalize_comments = true -normalize_doc_attributes = true # Normalize #[doc = "..."] to /// comments +normalize_doc_attributes = true # Normalize #[doc = "..."] to /// comments # ───── Macros ───── -format_macro_matchers = true # Format macro_rules! matcher arms -format_macro_bodies = true # Format macro body expressions +format_macro_matchers = true # Format macro_rules! matcher arms +format_macro_bodies = true # Format macro body expressions # ───── Idiomatic Shorthand ───── -use_field_init_shorthand = true # Foo { x } instead of Foo { x: x } -use_try_shorthand = true # ? instead of try!() +use_field_init_shorthand = true # Foo { x } instead of Foo { x: x } +use_try_shorthand = true # ? instead of try!() # ───── Hex & Literals ───── -hex_literal_case = "Upper" # 0xDEAD_BEEF not 0xdead_beef +hex_literal_case = "Upper" # 0xDEAD_BEEF not 0xdead_beef diff --git a/scripts/tests/definitions/00-warmup.toml b/scripts/tests/definitions/00-warmup.toml index 045ca1065..e0e8bc839 100644 --- a/scripts/tests/definitions/00-warmup.toml +++ b/scripts/tests/definitions/00-warmup.toml @@ -2,13 +2,13 @@ # Auto-split from test-definitions.toml [[test]] -id = "T00" -group = "warmup" -name = "warmup / daemon alive" -title = "Warmup — verify daemon is alive and returning results" -short_desc = "Smoke test: search for anything, expect ≥10 rows" -cli_args = ["*", "--limit", "10"] -rpc_method = "search" +id = "T00" +group = "warmup" +name = "warmup / daemon alive" +title = "Warmup — verify daemon is alive and returning results" +short_desc = "Smoke test: search for anything, expect ≥10 rows" +cli_args = ["*", "--limit", "10"] +rpc_method = "search" expect_min_rows = 10 expect_max_rows = 10 @@ -19,4 +19,3 @@ stdout_contains = ["\"Name\""] [test.api_checks] total_count_min = 1 - diff --git a/scripts/tests/definitions/02-filters.toml b/scripts/tests/definitions/02-filters.toml index 94c2aea12..163153673 100644 --- a/scripts/tests/definitions/02-filters.toml +++ b/scripts/tests/definitions/02-filters.toml @@ -2,297 +2,441 @@ # Auto-split from test-definitions.toml [[test]] -id = "T06" -group = "filters" -name = "--min-size 100MB" -title = "Minimum file size filter (100 MB)" -short_desc = "All files must be ≥ 100 MB" -cli_args = ["*", "--min-size", "104857600", "--files-only", "--limit", "10"] -rpc_method = "search" +id = "T06" +group = "filters" +name = "--min-size 100MB" +title = "Minimum file size filter (100 MB)" +short_desc = "All files must be ≥ 100 MB" +cli_args = ["*", "--min-size", "104857600", "--files-only", "--limit", "10"] +rpc_method = "search" expect_min_rows = 1 expect_max_rows = 10 [[test.column_checks]] column = "Size" -op = "gte" -value = "104857600" +op = "gte" +value = "104857600" [[test]] -id = "T07" -group = "filters" -name = "--max-size 1KB" -title = "Maximum file size filter (1 KB)" -short_desc = "All files must be ≤ 1 KB" -cli_args = ["*", "--max-size", "1024", "--files-only", "--limit", "10"] -rpc_method = "search" +id = "T07" +group = "filters" +name = "--max-size 1KB" +title = "Maximum file size filter (1 KB)" +short_desc = "All files must be ≤ 1 KB" +cli_args = ["*", "--max-size", "1024", "--files-only", "--limit", "10"] +rpc_method = "search" expect_min_rows = 1 expect_max_rows = 10 [[test.column_checks]] column = "Size" -op = "lte" -value = "1024" +op = "lte" +value = "1024" [[test]] -id = "T08" -group = "filters" -name = "--min/max-size 1MB..10MB" -title = "Combined min + max size filter" -short_desc = "All files must be between 1 MB and 10 MB" -cli_args = ["*.pdf", "--min-size", "1048576", "--max-size", "10485760", "--limit", "10"] +id = "T08" +group = "filters" +name = "--min/max-size 1MB..10MB" +title = "Combined min + max size filter" +short_desc = "All files must be between 1 MB and 10 MB" +cli_args = [ + "*.pdf", + "--min-size", + "1048576", + "--max-size", + "10485760", + "--limit", + "10", +] expect_min_rows = 1 -rpc_method = "search" +rpc_method = "search" expect_max_rows = 10 [[test.column_checks]] column = "Size" -op = "gte" -value = "1048576" +op = "gte" +value = "1048576" [[test.column_checks]] column = "Size" -op = "lte" -value = "10485760" +op = "lte" +value = "10485760" [[test]] -id = "T16" -group = "filters" -name = "T16 --exclude backup*" -title = "T16 --exclude backup*" -short_desc = "T16 --exclude backup*" -cli_args = ["*.txt", "--exclude", "backup*", "--limit", "10"] +id = "T16" +group = "filters" +name = "T16 --exclude backup*" +title = "T16 --exclude backup*" +short_desc = "T16 --exclude backup*" +cli_args = ["*.txt", "--exclude", "backup*", "--limit", "10"] expect_min_rows = 1 [[test.column_checks]] column = "Name" -op = "not_starts_with" -value = "backup" -case = "lower" +op = "not_starts_with" +value = "backup" +case = "lower" [[test]] -id = "T23" -group = "filters" -name = "T23 --min-descendants 100" -title = "T23 --min-descendants 100" -short_desc = "T23 --min-descendants 100" -cli_args = ["*", "--dirs-only", "--min-descendants", "100", "--limit", "10", "--columns", "all"] +id = "T23" +group = "filters" +name = "T23 --min-descendants 100" +title = "T23 --min-descendants 100" +short_desc = "T23 --min-descendants 100" +cli_args = [ + "*", + "--dirs-only", + "--min-descendants", + "100", + "--limit", + "10", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Descendants" -op = "gte" -value = "100" +op = "gte" +value = "100" [[test]] -id = "T24" -group = "filters" -name = "T24 --max-descendants 0" -title = "T24 --max-descendants 0" -short_desc = "T24 --max-descendants 0" -cli_args = ["*", "--dirs-only", "--max-descendants", "0", "--limit", "10", "--columns", "all"] +id = "T24" +group = "filters" +name = "T24 --max-descendants 0" +title = "T24 --max-descendants 0" +short_desc = "T24 --max-descendants 0" +cli_args = [ + "*", + "--dirs-only", + "--max-descendants", + "0", + "--limit", + "10", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Descendants" -op = "lte" -value = "0" +op = "lte" +value = "0" [[test]] -id = "T65" -group = "filters" -name = "T65 --sort descendants --sort-desc" -title = "T65 --sort descendants --sort-desc" -short_desc = "T65 --sort descendants --sort-desc" -cli_args = ["*", "--dirs-only", "--sort", "descendants", "--sort-desc", "--limit", "10", "--columns", "all"] +id = "T65" +group = "filters" +name = "T65 --sort descendants --sort-desc" +title = "T65 --sort descendants --sort-desc" +short_desc = "T65 --sort descendants --sort-desc" +cli_args = [ + "*", + "--dirs-only", + "--sort", + "descendants", + "--sort-desc", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T65" +validator = "T65" [[test.sort_checks]] column = "Descendants" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T67c" -group = "filters" -name = "T67c --sort bulkiness" -title = "T67c --sort bulkiness" -short_desc = "T67c --sort bulkiness" -cli_args = ["*", "--files-only", "--min-size", "1024", "--sort", "bulkiness", "--limit", "10"] +id = "T67c" +group = "filters" +name = "T67c --sort bulkiness" +title = "T67c --sort bulkiness" +short_desc = "T67c --sort bulkiness" +cli_args = [ + "*", + "--files-only", + "--min-size", + "1024", + "--sort", + "bulkiness", + "--limit", + "10", +] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Bulkiness" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T67j" -group = "filters" -name = "T67j multi-sort hidden,bulkiness" -title = "T67j multi-sort hidden,bulkiness" -short_desc = "T67j multi-sort hidden,bulkiness" -cli_args = ["*", "--files-only", "--min-size", "1024", "--sort", "hidden,bulkiness", "--limit", "20"] +id = "T67j" +group = "filters" +name = "T67j multi-sort hidden,bulkiness" +title = "T67j multi-sort hidden,bulkiness" +short_desc = "T67j multi-sort hidden,bulkiness" +cli_args = [ + "*", + "--files-only", + "--min-size", + "1024", + "--sort", + "hidden,bulkiness", + "--limit", + "20", +] expect_min_rows = 1 expect_max_rows = 20 [[test.sort_checks]] column = "Hidden" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T78" -group = "filters" -name = "T78 exclude + ext + size" -title = "T78 exclude + ext + size" -short_desc = "T78 exclude + ext + size" -cli_args = ["*.log", "--exclude", "debug*", "--max-size", "1048576", "--files-only", "--limit", "10"] +id = "T78" +group = "filters" +name = "T78 exclude + ext + size" +title = "T78 exclude + ext + size" +short_desc = "T78 exclude + ext + size" +cli_args = [ + "*.log", + "--exclude", + "debug*", + "--max-size", + "1048576", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 -validator = "T78" +validator = "T78" [[test]] -id = "T87" -group = "filters" -name = "T87 ext + sort modified" -title = "T87 ext + sort modified" -short_desc = "T87 ext + sort modified" -cli_args = ["*", "--ext", "txt,log,md", "--sort", "-modified", "--files-only", "--limit", "10"] +id = "T87" +group = "filters" +name = "T87 ext + sort modified" +title = "T87 ext + sort modified" +short_desc = "T87 ext + sort modified" +cli_args = [ + "*", + "--ext", + "txt,log,md", + "--sort", + "-modified", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 -validator = "T87" +validator = "T87" [[test]] -id = "T97" -group = "filters" -name = "T97 --in-path + --exclude" -title = "T97 --in-path + --exclude" -short_desc = "Path contains 'windows', name does not start with 'setup'" -cli_args = ["*.exe", "--in-path", "*windows*", "--exclude", "setup*", "--files-only", "--limit", "10"] +id = "T97" +group = "filters" +name = "T97 --in-path + --exclude" +title = "T97 --in-path + --exclude" +short_desc = "Path contains 'windows', name does not start with 'setup'" +cli_args = [ + "*.exe", + "--in-path", + "*windows*", + "--exclude", + "setup*", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 [[test.column_checks]] column = "Path Only" -op = "contains" -value = "windows" -case = "lower" +op = "contains" +value = "windows" +case = "lower" [[test.column_checks]] column = "Name" -op = "not_starts_with" -value = "setup" -case = "lower" +op = "not_starts_with" +value = "setup" +case = "lower" [[test]] -id = "T99" -group = "filters" -name = "T99 --max-bulkiness 100" -title = "T99 --max-bulkiness 100" -short_desc = "T99 --max-bulkiness 100" -cli_args = ["*", "--max-bulkiness", "100", "--files-only", "--min-size", "1024", "--limit", "10", "--columns", "all"] +id = "T99" +group = "filters" +name = "T99 --max-bulkiness 100" +title = "T99 --max-bulkiness 100" +short_desc = "T99 --max-bulkiness 100" +cli_args = [ + "*", + "--max-bulkiness", + "100", + "--files-only", + "--min-size", + "1024", + "--limit", + "10", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Bulkiness" -op = "lte" -value = "100" +op = "lte" +value = "100" [[test]] -id = "T101" -group = "filters" -name = "T101 --min-treesize 100MB" -title = "T101 --min-treesize 100MB" -short_desc = "T101 --min-treesize 100MB" -cli_args = ["*", "--dirs-only", "--min-treesize", "104857600", "--limit", "10", "--sort", "-size", "--columns", "all"] +id = "T101" +group = "filters" +name = "T101 --min-treesize 100MB" +title = "T101 --min-treesize 100MB" +short_desc = "T101 --min-treesize 100MB" +cli_args = [ + "*", + "--dirs-only", + "--min-treesize", + "104857600", + "--limit", + "10", + "--sort", + "-size", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Tree Size" -op = "gte" -value = "104857600" +op = "gte" +value = "104857600" [[test.sort_checks]] column = "Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T102" -group = "filters" -name = "T102 --max-treesize 1MB" -title = "T102 --max-treesize 1MB" -short_desc = "T102 --max-treesize 1MB" -cli_args = ["*", "--dirs-only", "--max-treesize", "1048576", "--limit", "10", "--columns", "all"] +id = "T102" +group = "filters" +name = "T102 --max-treesize 1MB" +title = "T102 --max-treesize 1MB" +short_desc = "T102 --max-treesize 1MB" +cli_args = [ + "*", + "--dirs-only", + "--max-treesize", + "1048576", + "--limit", + "10", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Tree Size" -op = "lte" -value = "1048576" +op = "lte" +value = "1048576" [[test]] -id = "T108" -group = "filters" -name = "T108 --min-size-on-disk 100MB" -title = "T108 --min-size-on-disk 100MB" -short_desc = "T108 --min-size-on-disk 100MB" -cli_args = ["*", "--min-size-on-disk", "104857600", "--files-only", "--limit", "10", "--columns", "all"] +id = "T108" +group = "filters" +name = "T108 --min-size-on-disk 100MB" +title = "T108 --min-size-on-disk 100MB" +short_desc = "T108 --min-size-on-disk 100MB" +cli_args = [ + "*", + "--min-size-on-disk", + "104857600", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Size on Disk" -op = "gte" -value = "104857600" +op = "gte" +value = "104857600" [[test]] -id = "T109" -group = "filters" -name = "T109 --max-size-on-disk 4096" -title = "T109 --max-size-on-disk 4096" -short_desc = "T109 --max-size-on-disk 4096" -cli_args = ["*", "--max-size-on-disk", "4096", "--files-only", "--min-size", "1", "--limit", "10", "--columns", "all"] +id = "T109" +group = "filters" +name = "T109 --max-size-on-disk 4096" +title = "T109 --max-size-on-disk 4096" +short_desc = "T109 --max-size-on-disk 4096" +cli_args = [ + "*", + "--max-size-on-disk", + "4096", + "--files-only", + "--min-size", + "1", + "--limit", + "10", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Size on Disk" -op = "lte" -value = "4096" +op = "lte" +value = "4096" [[test.column_checks]] column = "Size" -op = "gte" -value = "1" +op = "gte" +value = "1" [[test]] -id = "T118" -group = "filters" -name = "T118 treesize + descendants" -title = "T118 treesize + descendants" -short_desc = "T118 treesize + descendants" -cli_args = ["*", "--dirs-only", "--min-treesize", "10485760", "--min-descendants", "10", "--sort", "-size", "--limit", "10", "--columns", "all"] +id = "T118" +group = "filters" +name = "T118 treesize + descendants" +title = "T118 treesize + descendants" +short_desc = "T118 treesize + descendants" +cli_args = [ + "*", + "--dirs-only", + "--min-treesize", + "10485760", + "--min-descendants", + "10", + "--sort", + "-size", + "--limit", + "10", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.column_checks]] column = "Tree Size" -op = "gte" -value = "10485760" +op = "gte" +value = "10485760" [[test.column_checks]] column = "Descendants" -op = "gte" -value = "10" +op = "gte" +value = "10" [[test.sort_checks]] column = "Size" -order = "desc" -type = "u64" - +order = "desc" +type = "u64" diff --git a/scripts/tests/definitions/03-sort.toml b/scripts/tests/definitions/03-sort.toml index f24d47e70..0755a7994 100644 --- a/scripts/tests/definitions/03-sort.toml +++ b/scripts/tests/definitions/03-sort.toml @@ -2,272 +2,315 @@ # Auto-split from test-definitions.toml [[test]] -id = "T09" -group = "sort" -name = "--sort size (asc)" -title = "Sort by size ascending" -short_desc = "Results sorted by size in ascending order" -cli_args = ["*.exe", "--sort", "size", "--files-only", "--limit", "10"] -rpc_method = "search" +id = "T09" +group = "sort" +name = "--sort size (asc)" +title = "Sort by size ascending" +short_desc = "Results sorted by size in ascending order" +cli_args = ["*.exe", "--sort", "size", "--files-only", "--limit", "10"] +rpc_method = "search" expect_min_rows = 2 expect_max_rows = 10 [[test.sort_checks]] column = "Size" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T10" -group = "sort" -name = "--sort size --sort-desc" -title = "Sort by size descending" -short_desc = "Results sorted by size in descending order" -cli_args = ["*.exe", "--sort", "size", "--sort-desc", "--files-only", "--limit", "10"] -rpc_method = "search" +id = "T10" +group = "sort" +name = "--sort size --sort-desc" +title = "Sort by size descending" +short_desc = "Results sorted by size in descending order" +cli_args = [ + "*.exe", + "--sort", + "size", + "--sort-desc", + "--files-only", + "--limit", + "10", +] +rpc_method = "search" expect_min_rows = 2 expect_max_rows = 10 [[test.sort_checks]] column = "Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T11" -group = "sort" -name = "T11 --sort modified" -title = "T11 --sort modified" -short_desc = "T11 --sort modified" -cli_args = ["*.log", "--sort", "modified", "--limit", "10"] +id = "T11" +group = "sort" +name = "T11 --sort modified" +title = "T11 --sort modified" +short_desc = "T11 --sort modified" +cli_args = ["*.log", "--sort", "modified", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Last Written" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T12" -group = "sort" -name = "T12 --sort size,name" -title = "T12 --sort size,name" -short_desc = "T12 --sort size,name" -cli_args = ["*.dll", "--sort", "size,name", "--limit", "10"] +id = "T12" +group = "sort" +name = "T12 --sort size,name" +title = "T12 --sort size,name" +short_desc = "T12 --sort size,name" +cli_args = ["*.dll", "--sort", "size,name", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Size" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T58" -group = "sort" -name = "T58 --sort name" -title = "T58 --sort name" -short_desc = "T58 --sort name" -cli_args = ["*.txt", "--sort", "name", "--limit", "10"] +id = "T58" +group = "sort" +name = "T58 --sort name" +title = "T58 --sort name" +short_desc = "T58 --sort name" +cli_args = ["*.txt", "--sort", "name", "--limit", "10"] expect_min_rows = 1 [[test.sort_checks]] column = "Name" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T59" -group = "sort" -name = "T59 --sort path" -title = "T59 --sort path" -short_desc = "T59 --sort path" -cli_args = ["*.txt", "--sort", "path", "--limit", "10"] +id = "T59" +group = "sort" +name = "T59 --sort path" +title = "T59 --sort path" +short_desc = "T59 --sort path" +cli_args = ["*.txt", "--sort", "path", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Path" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T60" -group = "sort" -name = "T60 --sort created" -title = "T60 --sort created" -short_desc = "T60 --sort created" -cli_args = ["*.exe", "--sort", "created", "--limit", "10"] +id = "T60" +group = "sort" +name = "T60 --sort created" +title = "T60 --sort created" +short_desc = "T60 --sort created" +cli_args = ["*.exe", "--sort", "created", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Created" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T61" -group = "sort" -name = "T61 --sort accessed" -title = "T61 --sort accessed" -short_desc = "T61 --sort accessed" -cli_args = ["*.exe", "--sort", "accessed", "--limit", "10"] +id = "T61" +group = "sort" +name = "T61 --sort accessed" +title = "T61 --sort accessed" +short_desc = "T61 --sort accessed" +cli_args = ["*.exe", "--sort", "accessed", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Last Accessed" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T62" -group = "sort" -name = "T62 --sort extension" -title = "T62 --sort extension" -short_desc = "T62 --sort extension" -cli_args = ["*.*", "--sort", "extension", "--limit", "10"] +id = "T62" +group = "sort" +name = "T62 --sort extension" +title = "T62 --sort extension" +short_desc = "T62 --sort extension" +cli_args = ["*.*", "--sort", "extension", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Extension" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T63" -group = "sort" -name = "T63 --sort drive" -title = "T63 --sort drive" -short_desc = "T63 --sort drive" -cli_args = ["*.exe", "--sort", "drive", "--limit", "10"] +id = "T63" +group = "sort" +name = "T63 --sort drive" +title = "T63 --sort drive" +short_desc = "T63 --sort drive" +cli_args = ["*.exe", "--sort", "drive", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Drive" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T64" -group = "sort" -name = "T64 --sort allocated" -title = "T64 --sort allocated" -short_desc = "T64 --sort allocated" -cli_args = ["*.exe", "--sort", "allocated", "--files-only", "--limit", "10", "--sort-desc"] +id = "T64" +group = "sort" +name = "T64 --sort allocated" +title = "T64 --sort allocated" +short_desc = "T64 --sort allocated" +cli_args = [ + "*.exe", + "--sort", + "allocated", + "--files-only", + "--limit", + "10", + "--sort-desc", +] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Size on Disk" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T66" -group = "sort" -name = "T66 multi-sort size,-name" -title = "T66 multi-sort size,-name" -short_desc = "T66 multi-sort size,-name" -cli_args = ["*.dll", "--sort", "size,-name", "--files-only", "--limit", "20"] +id = "T66" +group = "sort" +name = "T66 multi-sort size,-name" +title = "T66 multi-sort size,-name" +short_desc = "T66 multi-sort size,-name" +cli_args = ["*.dll", "--sort", "size,-name", "--files-only", "--limit", "20"] expect_min_rows = 1 [[test.sort_checks]] column = "Size" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T67" -group = "sort" -name = "T67 multi-sort -modified,name" -title = "T67 multi-sort -modified,name" -short_desc = "T67 multi-sort -modified,name" -cli_args = ["*.log", "--sort", "-modified,name", "--limit", "10"] +id = "T67" +group = "sort" +name = "T67 multi-sort -modified,name" +title = "T67 multi-sort -modified,name" +short_desc = "T67 multi-sort -modified,name" +cli_args = ["*.log", "--sort", "-modified,name", "--limit", "10"] expect_min_rows = 0 expect_max_rows = 10 [[test.sort_checks]] column = "Last Written" -order = "desc" -type = "string" +order = "desc" +type = "string" [[test]] -id = "T67a" -group = "sort" -name = "T67a --sort treesize" -title = "T67a --sort treesize" -short_desc = "T67a --sort treesize" -cli_args = ["*", "--dirs-only", "--sort", "treesize", "--limit", "10", "--columns", "all"] +id = "T67a" +group = "sort" +name = "T67a --sort treesize" +title = "T67a --sort treesize" +short_desc = "T67a --sort treesize" +cli_args = [ + "*", + "--dirs-only", + "--sort", + "treesize", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T67a" +validator = "T67a" [[test.sort_checks]] column = "Tree Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T67b" -group = "sort" -name = "T67b --sort treeallocated" -title = "T67b --sort treeallocated" -short_desc = "T67b --sort treeallocated" -cli_args = ["*", "--dirs-only", "--sort", "treeallocated", "--limit", "10", "--columns", "all"] +id = "T67b" +group = "sort" +name = "T67b --sort treeallocated" +title = "T67b --sort treeallocated" +short_desc = "T67b --sort treeallocated" +cli_args = [ + "*", + "--dirs-only", + "--sort", + "treeallocated", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T67b" +validator = "T67b" [[test.sort_checks]] column = "Tree Allocated" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T67d" -group = "sort" -name = "T67d --sort namelength" -title = "T67d --sort namelength" -short_desc = "T67d --sort namelength" -cli_args = ["*", "--files-only", "--sort", "namelength", "--limit", "10", "--columns", "all"] +id = "T67d" +group = "sort" +name = "T67d --sort namelength" +title = "T67d --sort namelength" +short_desc = "T67d --sort namelength" +cli_args = [ + "*", + "--files-only", + "--sort", + "namelength", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T67d" +validator = "T67d" [[test]] -id = "T67e" -group = "sort" -name = "T67e --sort pathlength" -title = "T67e --sort pathlength" -short_desc = "T67e --sort pathlength" -cli_args = ["*", "--files-only", "--sort", "pathlength", "--limit", "10"] +id = "T67e" +group = "sort" +name = "T67e --sort pathlength" +title = "T67e --sort pathlength" +short_desc = "T67e --sort pathlength" +cli_args = ["*", "--files-only", "--sort", "pathlength", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Path Length" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T67f" -group = "sort" -name = "T67f --sort path_only" -title = "T67f --sort path_only" -short_desc = "T67f --sort path_only" -cli_args = ["*.exe", "--sort", "path_only", "--limit", "10"] +id = "T67f" +group = "sort" +name = "T67f --sort path_only" +title = "T67f --sort path_only" +short_desc = "T67f --sort path_only" +cli_args = ["*.exe", "--sort", "path_only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.sort_checks]] column = "Path Only" -order = "asc" -type = "string" +order = "asc" +type = "string" # T67f2 pins Windows Explorer's `Folder` column convention: when two # rows share the same parent directory (path_only), the secondary sort @@ -277,150 +320,204 @@ type = "string" # Mirrored by the `search_index_path_only_sort_name_asc_within_same_folder` # Rust unit test in crates/uffs-core/src/search/backend_tests.rs. [[test]] -id = "T67f2" -group = "sort" -name = "T67f2 --sort path_only (name tiebreaker within folder)" -title = "T67f2 --sort path_only (name tiebreaker within folder)" -short_desc = "T67f2 name-ASC tiebreaker within same path_only" -cli_args = ["*.dll", "--sort", "path_only", "--limit", "200"] +id = "T67f2" +group = "sort" +name = "T67f2 --sort path_only (name tiebreaker within folder)" +title = "T67f2 --sort path_only (name tiebreaker within folder)" +short_desc = "T67f2 name-ASC tiebreaker within same path_only" +cli_args = ["*.dll", "--sort", "path_only", "--limit", "200"] expect_min_rows = 2 expect_max_rows = 200 -validator = "T67f2" +validator = "T67f2" [[test]] -id = "T67g" -group = "sort" -name = "T67g --sort type" -title = "T67g --sort type" -short_desc = "T67g --sort type" -cli_args = ["*", "--files-only", "--sort", "type", "--limit", "20"] +id = "T67g" +group = "sort" +name = "T67g --sort type" +title = "T67g --sort type" +short_desc = "T67g --sort type" +cli_args = ["*", "--files-only", "--sort", "type", "--limit", "20"] expect_min_rows = 1 expect_max_rows = 20 [[test.sort_checks]] column = "Type" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T67h" -group = "sort" -name = "T67h multi-sort treesize,name" -title = "T67h multi-sort treesize,name" -short_desc = "T67h multi-sort treesize,name" -cli_args = ["*", "--dirs-only", "--sort", "treesize,name", "--limit", "20", "--columns", "all"] +id = "T67h" +group = "sort" +name = "T67h multi-sort treesize,name" +title = "T67h multi-sort treesize,name" +short_desc = "T67h multi-sort treesize,name" +cli_args = [ + "*", + "--dirs-only", + "--sort", + "treesize,name", + "--limit", + "20", + "--columns", + "all", +] expect_min_rows = 1 expect_max_rows = 20 expect_columns_all = true [[test.sort_checks]] column = "Tree Size" -order = "asc" -type = "u64" +order = "asc" +type = "u64" [[test]] -id = "T67i" -group = "sort" -name = "T67i multi-sort type,-size" -title = "T67i multi-sort type,-size" -short_desc = "T67i multi-sort type,-size" -cli_args = ["*", "--files-only", "--sort", "type,-size", "--limit", "20"] +id = "T67i" +group = "sort" +name = "T67i multi-sort type,-size" +title = "T67i multi-sort type,-size" +short_desc = "T67i multi-sort type,-size" +cli_args = ["*", "--files-only", "--sort", "type,-size", "--limit", "20"] expect_min_rows = 1 expect_max_rows = 20 [[test.sort_checks]] column = "Type" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T76" -group = "sort" -name = "T76 dirs + desc range + sort" -title = "T76 dirs + desc range + sort" -short_desc = "T76 dirs + desc range + sort" -cli_args = ["*", "--dirs-only", "--min-descendants", "10", "--max-descendants", "1000", "--sort", "descendants", "--sort-desc", "--limit", "10", "--columns", "all"] +id = "T76" +group = "sort" +name = "T76 dirs + desc range + sort" +title = "T76 dirs + desc range + sort" +short_desc = "T76 dirs + desc range + sort" +cli_args = [ + "*", + "--dirs-only", + "--min-descendants", + "10", + "--max-descendants", + "1000", + "--sort", + "descendants", + "--sort-desc", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T76" +validator = "T76" [[test.sort_checks]] column = "Descendants" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T83" -group = "sort" -name = "T83 multi-sort drive,ext,-size" -title = "T83 multi-sort drive,ext,-size" -short_desc = "T83 multi-sort drive,ext,-size" -cli_args = ["*.*", "--sort", "drive,extension,-size", "--files-only", "--limit", "20"] +id = "T83" +group = "sort" +name = "T83 multi-sort drive,ext,-size" +title = "T83 multi-sort drive,ext,-size" +short_desc = "T83 multi-sort drive,ext,-size" +cli_args = [ + "*.*", + "--sort", + "drive,extension,-size", + "--files-only", + "--limit", + "20", +] expect_min_rows = 1 expect_max_rows = 20 [[test.sort_checks]] column = "Drive" -order = "asc" -type = "string" +order = "asc" +type = "string" [[test]] -id = "T94" -group = "sort" -name = "T94 --type code + sort size" -title = "T94 --type code + sort size" -short_desc = "T94 --type code + sort size" -cli_args = ["*", "--type", "code", "--files-only", "--sort", "-size", "--limit", "10"] +id = "T94" +group = "sort" +name = "T94 --type code + sort size" +title = "T94 --type code + sort size" +short_desc = "T94 --type code + sort size" +cli_args = [ + "*", + "--type", + "code", + "--files-only", + "--sort", + "-size", + "--limit", + "10", +] expect_min_rows = 1 -validator = "T94" +validator = "T94" [[test.sort_checks]] column = "Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T114" -group = "sort" -name = "T114 --sort hidden:desc" -title = "T114 --sort hidden:desc" -short_desc = "T114 --sort hidden:desc" -cli_args = ["*", "--sort", "hidden:desc", "--limit", "20", "--columns", "all"] +id = "T114" +group = "sort" +name = "T114 --sort hidden:desc" +title = "T114 --sort hidden:desc" +short_desc = "T114 --sort hidden:desc" +cli_args = ["*", "--sort", "hidden:desc", "--limit", "20", "--columns", "all"] expect_columns_all = true expect_min_rows = 1 [[test.sort_checks]] column = "Hidden" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T115" -group = "sort" -name = "T115 --sort compressed:desc" -title = "T115 --sort compressed:desc" -short_desc = "T115 --sort compressed:desc" -cli_args = ["*", "--sort", "compressed:desc", "--limit", "20", "--columns", "all"] +id = "T115" +group = "sort" +name = "T115 --sort compressed:desc" +title = "T115 --sort compressed:desc" +short_desc = "T115 --sort compressed:desc" +cli_args = [ + "*", + "--sort", + "compressed:desc", + "--limit", + "20", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.sort_checks]] column = "Compressed" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T116" -group = "sort" -name = "T116 --sort directory:desc" -title = "T116 --sort directory:desc" -short_desc = "T116 --sort directory:desc" -cli_args = ["*", "--sort", "directory:desc", "--limit", "20", "--columns", "all"] +id = "T116" +group = "sort" +name = "T116 --sort directory:desc" +title = "T116 --sort directory:desc" +short_desc = "T116 --sort directory:desc" +cli_args = [ + "*", + "--sort", + "directory:desc", + "--limit", + "20", + "--columns", + "all", +] expect_columns_all = true expect_min_rows = 1 [[test.sort_checks]] column = "Directory Flag" -order = "desc" -type = "u64" - +order = "desc" +type = "u64" diff --git a/scripts/tests/definitions/04-attributes.toml b/scripts/tests/definitions/04-attributes.toml index a6fde99f5..ce1de1631 100644 --- a/scripts/tests/definitions/04-attributes.toml +++ b/scripts/tests/definitions/04-attributes.toml @@ -2,208 +2,306 @@ # Auto-split from test-definitions.toml [[test]] -id = "T13" -group = "attributes" -name = "T13 --attr hidden" -title = "T13 --attr hidden" -short_desc = "T13 --attr hidden" -cli_args = ["*", "--attr", "hidden", "--files-only", "--limit", "10", "--columns", "all"] +id = "T13" +group = "attributes" +name = "T13 --attr hidden" +title = "T13 --attr hidden" +short_desc = "T13 --attr hidden" +cli_args = [ + "*", + "--attr", + "hidden", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Hidden" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T14" -group = "attributes" -name = "T14 --attr !hidden" -title = "T14 --attr !hidden" -short_desc = "T14 --attr !hidden" -cli_args = ["*", "--attr", "!hidden", "--files-only", "--limit", "10", "--columns", "all"] +id = "T14" +group = "attributes" +name = "T14 --attr !hidden" +title = "T14 --attr !hidden" +short_desc = "T14 --attr !hidden" +cli_args = [ + "*", + "--attr", + "!hidden", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Hidden" -op = "ne" -value = "1" +op = "ne" +value = "1" [[test]] -id = "T15" -group = "attributes" -name = "T15 --attr compressed" -title = "T15 --attr compressed" -short_desc = "T15 --attr compressed" -cli_args = ["*", "--attr", "compressed", "--limit", "10", "--columns", "all"] +id = "T15" +group = "attributes" +name = "T15 --attr compressed" +title = "T15 --attr compressed" +short_desc = "T15 --attr compressed" +cli_args = ["*", "--attr", "compressed", "--limit", "10", "--columns", "all"] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Compressed" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T37" -group = "attributes" -name = "T37 --attr system" -title = "T37 --attr system" -short_desc = "T37 --attr system" -cli_args = ["*", "--attr", "system", "--files-only", "--limit", "10", "--columns", "all"] +id = "T37" +group = "attributes" +name = "T37 --attr system" +title = "T37 --attr system" +short_desc = "T37 --attr system" +cli_args = [ + "*", + "--attr", + "system", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "System" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T38" -group = "attributes" -name = "T38 --attr readonly" -title = "T38 --attr readonly" -short_desc = "T38 --attr readonly" -cli_args = ["*", "--attr", "readonly", "--files-only", "--limit", "10", "--columns", "all"] +id = "T38" +group = "attributes" +name = "T38 --attr readonly" +title = "T38 --attr readonly" +short_desc = "T38 --attr readonly" +cli_args = [ + "*", + "--attr", + "readonly", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Read-only" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T39" -group = "attributes" -name = "T39 --attr system,!hidden" -title = "T39 --attr system,!hidden" -short_desc = "T39 --attr system,!hidden" -cli_args = ["*", "--attr", "system,!hidden", "--files-only", "--limit", "10", "--columns", "all"] +id = "T39" +group = "attributes" +name = "T39 --attr system,!hidden" +title = "T39 --attr system,!hidden" +short_desc = "T39 --attr system,!hidden" +cli_args = [ + "*", + "--attr", + "system,!hidden", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "System" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test.column_checks]] column = "Hidden" -op = "ne" -value = "1" +op = "ne" +value = "1" [[test]] -id = "T68" -group = "attributes" -name = "T68 --attr archive" -title = "T68 --attr archive" -short_desc = "T68 --attr archive" -cli_args = ["*", "--attr", "archive", "--files-only", "--limit", "10", "--columns", "all"] +id = "T68" +group = "attributes" +name = "T68 --attr archive" +title = "T68 --attr archive" +short_desc = "T68 --attr archive" +cli_args = [ + "*", + "--attr", + "archive", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Archive" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T69" -group = "attributes" -name = "T69 --attr sparse" -title = "T69 --attr sparse" -short_desc = "T69 --attr sparse" -cli_args = ["*", "--attr", "sparse", "--files-only", "--limit", "10", "--columns", "all"] +id = "T69" +group = "attributes" +name = "T69 --attr sparse" +title = "T69 --attr sparse" +short_desc = "T69 --attr sparse" +cli_args = [ + "*", + "--attr", + "sparse", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Sparse" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T70" -group = "attributes" -name = "T70 --attr reparse" -title = "T70 --attr reparse" -short_desc = "T70 --attr reparse" -cli_args = ["*", "--attr", "reparse", "--limit", "10", "--columns", "all"] +id = "T70" +group = "attributes" +name = "T70 --attr reparse" +title = "T70 --attr reparse" +short_desc = "T70 --attr reparse" +cli_args = ["*", "--attr", "reparse", "--limit", "10", "--columns", "all"] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Reparse" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T71" -group = "attributes" -name = "T71 --attr offline" -title = "T71 --attr offline" -short_desc = "T71 --attr offline" -cli_args = ["*", "--attr", "offline", "--files-only", "--limit", "10", "--columns", "all"] +id = "T71" +group = "attributes" +name = "T71 --attr offline" +title = "T71 --attr offline" +short_desc = "T71 --attr offline" +cli_args = [ + "*", + "--attr", + "offline", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Offline" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T72" -group = "attributes" -name = "T72 --attr encrypted" -title = "T72 --attr encrypted" -short_desc = "T72 --attr encrypted" -cli_args = ["*", "--attr", "encrypted", "--files-only", "--limit", "10", "--columns", "all"] +id = "T72" +group = "attributes" +name = "T72 --attr encrypted" +title = "T72 --attr encrypted" +short_desc = "T72 --attr encrypted" +cli_args = [ + "*", + "--attr", + "encrypted", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] # Encrypted files are extremely rare — most datasets have zero. expect_min_rows = 0 expect_columns_all = true [[test.column_checks]] column = "Encrypted" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T73" -group = "attributes" -name = "T73 --attr !system" -title = "T73 --attr !system" -short_desc = "T73 --attr !system" -cli_args = ["*", "--attr", "!system", "--files-only", "--limit", "10", "--columns", "all"] +id = "T73" +group = "attributes" +name = "T73 --attr !system" +title = "T73 --attr !system" +short_desc = "T73 --attr !system" +cli_args = [ + "*", + "--attr", + "!system", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "System" -op = "ne" -value = "1" +op = "ne" +value = "1" [[test]] -id = "T74" -group = "attributes" -name = "T74 --attr hidden,system" -title = "T74 --attr hidden,system" -short_desc = "T74 --attr hidden,system" -cli_args = ["*", "--attr", "hidden,system", "--files-only", "--limit", "10", "--columns", "all"] +id = "T74" +group = "attributes" +name = "T74 --attr hidden,system" +title = "T74 --attr hidden,system" +short_desc = "T74 --attr hidden,system" +cli_args = [ + "*", + "--attr", + "hidden,system", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Hidden" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test.column_checks]] column = "System" -op = "eq" -value = "1" - +op = "eq" +value = "1" diff --git a/scripts/tests/definitions/05-time.toml b/scripts/tests/definitions/05-time.toml index c4c706479..8169b9a4c 100644 --- a/scripts/tests/definitions/05-time.toml +++ b/scripts/tests/definitions/05-time.toml @@ -2,51 +2,51 @@ # Auto-split from test-definitions.toml [[test]] -id = "T25" -group = "time" -name = "T25 --newer 7d" -title = "T25 --newer 7d" -short_desc = "Files modified in last 7 days — may return 0 on stale/offline index" -cli_args = ["*.log", "--newer", "7d", "--limit", "10"] +id = "T25" +group = "time" +name = "T25 --newer 7d" +title = "T25 --newer 7d" +short_desc = "Files modified in last 7 days — may return 0 on stale/offline index" +cli_args = ["*.log", "--newer", "7d", "--limit", "10"] expect_min_rows = 0 expect_max_rows = 10 [[test.column_checks]] column = "Name" -op = "ends_with" -value = ".log" -case = "lower" +op = "ends_with" +value = ".log" +case = "lower" [test.api_checks] result_has_key = ["rows", "records_scanned"] [[test]] -id = "T26" -group = "time" -name = "T26 --older 365d" -title = "T26 --older 365d" -short_desc = "Files modified more than 365d ago" -cli_args = ["*.doc", "--older", "365d", "--limit", "10"] +id = "T26" +group = "time" +name = "T26 --older 365d" +title = "T26 --older 365d" +short_desc = "Files modified more than 365d ago" +cli_args = ["*.doc", "--older", "365d", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 [[test.column_checks]] column = "Name" -op = "ends_with" -value = ".doc" -case = "lower" +op = "ends_with" +value = ".doc" +case = "lower" [test.api_checks] result_has_key = ["rows", "records_scanned"] total_count_min = 1 [[test]] -id = "T27" -group = "time" -name = "T27 --newer-created 30d" -title = "T27 --newer-created 30d" -short_desc = "Files created in last 30 days — active system should have many" -cli_args = ["*", "--newer-created", "30d", "--files-only", "--limit", "10"] +id = "T27" +group = "time" +name = "T27 --newer-created 30d" +title = "T27 --newer-created 30d" +short_desc = "Files created in last 30 days — active system should have many" +cli_args = ["*", "--newer-created", "30d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -54,12 +54,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T36" -group = "time" -name = "T36 --older-created 365d" -title = "T36 --older-created 365d" -short_desc = "Files created more than 1 year ago — most files on large system" -cli_args = ["*", "--older-created", "365d", "--files-only", "--limit", "10"] +id = "T36" +group = "time" +name = "T36 --older-created 365d" +title = "T36 --older-created 365d" +short_desc = "Files created more than 1 year ago — most files on large system" +cli_args = ["*", "--older-created", "365d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -67,12 +67,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T43" -group = "time" -name = "T43 --newer-accessed 7d" -title = "T43 --newer-accessed 7d" -short_desc = "Files accessed in last 7 days" -cli_args = ["*", "--newer-accessed", "7d", "--files-only", "--limit", "10"] +id = "T43" +group = "time" +name = "T43 --newer-accessed 7d" +title = "T43 --newer-accessed 7d" +short_desc = "Files accessed in last 7 days" +cli_args = ["*", "--newer-accessed", "7d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -80,12 +80,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T44" -group = "time" -name = "T44 --newer today" -title = "T44 --newer today" -short_desc = "Files modified today — narrow range, may return 0 on stale data" -cli_args = ["*", "--newer", "today", "--files-only", "--limit", "10"] +id = "T44" +group = "time" +name = "T44 --newer today" +title = "T44 --newer today" +short_desc = "Files modified today — narrow range, may return 0 on stale data" +cli_args = ["*", "--newer", "today", "--files-only", "--limit", "10"] expect_min_rows = 0 expect_max_rows = 10 @@ -93,12 +93,12 @@ expect_max_rows = 10 result_has_key = ["rows", "records_scanned", "total_count"] [[test]] -id = "T45" -group = "time" -name = "T45 --newer yesterday" -title = "T45 --newer yesterday" -short_desc = "Files modified since yesterday — slightly wider than today" -cli_args = ["*", "--newer", "yesterday", "--files-only", "--limit", "10"] +id = "T45" +group = "time" +name = "T45 --newer yesterday" +title = "T45 --newer yesterday" +short_desc = "Files modified since yesterday — slightly wider than today" +cli_args = ["*", "--newer", "yesterday", "--files-only", "--limit", "10"] expect_min_rows = 0 expect_max_rows = 10 @@ -106,12 +106,12 @@ expect_max_rows = 10 result_has_key = ["rows", "records_scanned", "total_count"] [[test]] -id = "T46" -group = "time" -name = "T46 --newer this_week" -title = "T46 --newer this_week" -short_desc = "Files modified this week — should have results on active system" -cli_args = ["*", "--newer", "this_week", "--files-only", "--limit", "10"] +id = "T46" +group = "time" +name = "T46 --newer this_week" +title = "T46 --newer this_week" +short_desc = "Files modified this week — should have results on active system" +cli_args = ["*", "--newer", "this_week", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -119,12 +119,12 @@ expect_max_rows = 10 total_count_min = 1 [[test]] -id = "T47" -group = "time" -name = "T47 --newer last_7d" -title = "T47 --newer last_7d" -short_desc = "Files modified in last 7 days — equivalent to this_week but rolling" -cli_args = ["*", "--newer", "last_7d", "--files-only", "--limit", "10"] +id = "T47" +group = "time" +name = "T47 --newer last_7d" +title = "T47 --newer last_7d" +short_desc = "Files modified in last 7 days — equivalent to this_week but rolling" +cli_args = ["*", "--newer", "last_7d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -132,12 +132,12 @@ expect_max_rows = 10 total_count_min = 1 [[test]] -id = "T48" -group = "time" -name = "T48 --newer last_30d" -title = "T48 --newer last_30d" -short_desc = "Files modified in last 30 days — must return results on active system" -cli_args = ["*", "--newer", "last_30d", "--files-only", "--limit", "10"] +id = "T48" +group = "time" +name = "T48 --newer last_30d" +title = "T48 --newer last_30d" +short_desc = "Files modified in last 30 days — must return results on active system" +cli_args = ["*", "--newer", "last_30d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -145,12 +145,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T49" -group = "time" -name = "T49 --newer this_month" -title = "T49 --newer this_month" -short_desc = "Files modified this month" -cli_args = ["*", "--newer", "this_month", "--files-only", "--limit", "10"] +id = "T49" +group = "time" +name = "T49 --newer this_month" +title = "T49 --newer this_month" +short_desc = "Files modified this month" +cli_args = ["*", "--newer", "this_month", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -158,12 +158,12 @@ expect_max_rows = 10 total_count_min = 1 [[test]] -id = "T50" -group = "time" -name = "T50 --newer this_year" -title = "T50 --newer this_year" -short_desc = "Files modified this year — wide range, many results expected" -cli_args = ["*", "--newer", "this_year", "--files-only", "--limit", "10"] +id = "T50" +group = "time" +name = "T50 --newer this_year" +title = "T50 --newer this_year" +short_desc = "Files modified this year — wide range, many results expected" +cli_args = ["*", "--newer", "this_year", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -171,12 +171,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T51" -group = "time" -name = "T51 --older last_year" -title = "T51 --older last_year" -short_desc = "Files older than last year — large dataset guarantees results" -cli_args = ["*", "--older", "last_year", "--files-only", "--limit", "10"] +id = "T51" +group = "time" +name = "T51 --older last_year" +title = "T51 --older last_year" +short_desc = "Files older than last year — large dataset guarantees results" +cli_args = ["*", "--older", "last_year", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -184,12 +184,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T52" -group = "time" -name = "T52 --newer last_90d" -title = "T52 --newer last_90d" -short_desc = "Files modified in last 90 days" -cli_args = ["*", "--newer", "last_90d", "--files-only", "--limit", "10"] +id = "T52" +group = "time" +name = "T52 --newer last_90d" +title = "T52 --newer last_90d" +short_desc = "Files modified in last 90 days" +cli_args = ["*", "--newer", "last_90d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -197,12 +197,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T53" -group = "time" -name = "T53 --newer last_365d" -title = "T53 --newer last_365d" -short_desc = "Files modified in last year — broadest recent range" -cli_args = ["*", "--newer", "last_365d", "--files-only", "--limit", "10"] +id = "T53" +group = "time" +name = "T53 --newer last_365d" +title = "T53 --newer last_365d" +short_desc = "Files modified in last year — broadest recent range" +cli_args = ["*", "--newer", "last_365d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -210,12 +210,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T54" -group = "time" -name = "T54 --newer-created today" -title = "T54 --newer-created today" -short_desc = "Files created today — may return 0 on stale index" -cli_args = ["*", "--newer-created", "today", "--files-only", "--limit", "10"] +id = "T54" +group = "time" +name = "T54 --newer-created today" +title = "T54 --newer-created today" +short_desc = "Files created today — may return 0 on stale index" +cli_args = ["*", "--newer-created", "today", "--files-only", "--limit", "10"] expect_min_rows = 0 expect_max_rows = 10 @@ -223,12 +223,19 @@ expect_max_rows = 10 result_has_key = ["rows", "records_scanned", "total_count"] [[test]] -id = "T55" -group = "time" -name = "T55 --newer-accessed this_week" -title = "T55 --newer-accessed this_week" -short_desc = "Files accessed this week" -cli_args = ["*", "--newer-accessed", "this_week", "--files-only", "--limit", "10"] +id = "T55" +group = "time" +name = "T55 --newer-accessed this_week" +title = "T55 --newer-accessed this_week" +short_desc = "Files accessed this week" +cli_args = [ + "*", + "--newer-accessed", + "this_week", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 expect_max_rows = 10 @@ -236,12 +243,21 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T56" -group = "time" -name = "T56 bounded time range (last_week)" -title = "T56 bounded time range (last_week)" -short_desc = "Bounded range: newer=last_week AND older=this_week — exactly last week's files" -cli_args = ["*", "--newer", "last_week", "--older", "this_week", "--files-only", "--limit", "10"] +id = "T56" +group = "time" +name = "T56 bounded time range (last_week)" +title = "T56 bounded time range (last_week)" +short_desc = "Bounded range: newer=last_week AND older=this_week — exactly last week's files" +cli_args = [ + "*", + "--newer", + "last_week", + "--older", + "this_week", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 expect_max_rows = 10 @@ -250,12 +266,12 @@ result_has_key = ["rows", "records_scanned"] total_count_min = 1 [[test]] -id = "T57" -group = "time" -name = "T57 --newer 2025-01-01" -title = "T57 --newer 2025-01-01" -short_desc = "Absolute date cutoff — all files modified since 2025-01-01" -cli_args = ["*", "--newer", "2025-01-01", "--files-only", "--limit", "10"] +id = "T57" +group = "time" +name = "T57 --newer 2025-01-01" +title = "T57 --newer 2025-01-01" +short_desc = "Absolute date cutoff — all files modified since 2025-01-01" +cli_args = ["*", "--newer", "2025-01-01", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -263,27 +279,45 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T77" -group = "time" -name = "T77 hidden + --newer last_30d" -title = "T77 hidden + --newer last_30d" -short_desc = "T77 hidden + --newer last_30d" -cli_args = ["*", "--attr", "hidden", "--newer", "last_30d", "--files-only", "--limit", "10", "--columns", "all"] +id = "T77" +group = "time" +name = "T77 hidden + --newer last_30d" +title = "T77 hidden + --newer last_30d" +short_desc = "T77 hidden + --newer last_30d" +cli_args = [ + "*", + "--attr", + "hidden", + "--newer", + "last_30d", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Hidden" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T81" -group = "time" -name = "T81 --newer-created this_year" -title = "T81 --newer-created this_year" -short_desc = "Files created this year — wide range on active system" -cli_args = ["*", "--newer-created", "this_year", "--files-only", "--limit", "10"] +id = "T81" +group = "time" +name = "T81 --newer-created this_year" +title = "T81 --newer-created this_year" +short_desc = "Files created this year — wide range on active system" +cli_args = [ + "*", + "--newer-created", + "this_year", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 expect_max_rows = 10 @@ -291,12 +325,19 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T82" -group = "time" -name = "T82 --newer-accessed last_week" -title = "T82 --newer-accessed last_week" -short_desc = "Files accessed in last week" -cli_args = ["*", "--newer-accessed", "last_week", "--files-only", "--limit", "10"] +id = "T82" +group = "time" +name = "T82 --newer-accessed last_week" +title = "T82 --newer-accessed last_week" +short_desc = "Files accessed in last week" +cli_args = [ + "*", + "--newer-accessed", + "last_week", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 expect_max_rows = 10 @@ -304,12 +345,12 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T86" -group = "time" -name = "T86 --older-accessed 365d" -title = "T86 --older-accessed 365d" -short_desc = "Files last accessed over 1 year ago — many on large system" -cli_args = ["*", "--older-accessed", "365d", "--files-only", "--limit", "10"] +id = "T86" +group = "time" +name = "T86 --older-accessed 365d" +title = "T86 --older-accessed 365d" +short_desc = "Files last accessed over 1 year ago — many on large system" +cli_args = ["*", "--older-accessed", "365d", "--files-only", "--limit", "10"] expect_min_rows = 1 expect_max_rows = 10 @@ -317,25 +358,42 @@ expect_max_rows = 10 total_count_min = 10 [[test]] -id = "T88" -group = "time" -name = "T88 name-only + hide-system + newer" -title = "T88 name-only + hide-system + newer" -short_desc = "T88 name-only + hide-system + newer" -cli_args = ["config", "--name-only", "--hide-system", "--newer", "last_90d", "--files-only", "--limit", "10"] +id = "T88" +group = "time" +name = "T88 name-only + hide-system + newer" +title = "T88 name-only + hide-system + newer" +short_desc = "T88 name-only + hide-system + newer" +cli_args = [ + "config", + "--name-only", + "--hide-system", + "--newer", + "last_90d", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 -validator = "T88" +validator = "T88" [[test]] -id = "T113" -group = "time" -name = "T113 --month jan + --newer last_365d" -title = "T113 --month jan + --newer last_365d" -short_desc = "Combined: January files modified in last year — intersection filter" -cli_args = ["*", "--month", "jan", "--newer", "last_365d", "--files-only", "--limit", "10"] +id = "T113" +group = "time" +name = "T113 --month jan + --newer last_365d" +title = "T113 --month jan + --newer last_365d" +short_desc = "Combined: January files modified in last year — intersection filter" +cli_args = [ + "*", + "--month", + "jan", + "--newer", + "last_365d", + "--files-only", + "--limit", + "10", +] expect_min_rows = 1 expect_max_rows = 10 [test.api_checks] total_count_min = 10 - diff --git a/scripts/tests/definitions/06-output.toml b/scripts/tests/definitions/06-output.toml index 3835d3f50..5bfbfb878 100644 --- a/scripts/tests/definitions/06-output.toml +++ b/scripts/tests/definitions/06-output.toml @@ -2,27 +2,27 @@ # Auto-split from test-definitions.toml [[test]] -id = "T20" -group = "output" -name = "T20 --format json" -title = "T20 --format json" -short_desc = "T20 --format json" -cli_args = ["*.rs", "--format", "json", "--limit", "5"] -validator = "T20" -targets = ["cli"] +id = "T20" +group = "output" +name = "T20 --format json" +title = "T20 --format json" +short_desc = "T20 --format json" +cli_args = ["*.rs", "--format", "json", "--limit", "5"] +validator = "T20" +targets = ["cli"] [test.api_checks] result_has_key = ["results"] total_count_min = 1 [[test]] -id = "T21" -group = "output" -name = "T21 --format json" -title = "T21 --format json" -short_desc = "T21 --format json (was table; piped stdout truncates column names)" -cli_args = ["*.rs", "--format", "json", "--limit", "5"] -targets = ["cli"] +id = "T21" +group = "output" +name = "T21 --format json" +title = "T21 --format json" +short_desc = "T21 --format json (was table; piped stdout truncates column names)" +cli_args = ["*.rs", "--format", "json", "--limit", "5"] +targets = ["cli"] stdout_contains = ["name", "size"] [test.api_checks] @@ -30,64 +30,82 @@ result_has_key = ["results"] total_count_min = 1 [[test]] -id = "T22" -group = "output" -name = "T22 --columns Name,Size,Path Only" -title = "T22 --columns Name,Size,Path Only" -short_desc = "T22 --columns Name,Size,Path Only" -cli_args = ["*.txt", "--columns", "Name,Size,Path Only", "--limit", "10"] -targets = ["cli"] +id = "T22" +group = "output" +name = "T22 --columns Name,Size,Path Only" +title = "T22 --columns Name,Size,Path Only" +short_desc = "T22 --columns Name,Size,Path Only" +cli_args = ["*.txt", "--columns", "Name,Size,Path Only", "--limit", "10"] +targets = ["cli"] stdout_contains = ["\"Name\",\"Size\",\"Path Only\""] expect_min_rows = 1 [[test]] -id = "T31" -group = "output" -name = "T31 --out file" -title = "Redirect output to file with --out" -short_desc = "Search results written to CSV file, verified by reading it back" -cli_args = ["*.rs", "--limit", "100", "--out", "test_cli_validation_out.csv"] -validator = "T31" -targets = ["cli"] +id = "T31" +group = "output" +name = "T31 --out file" +title = "Redirect output to file with --out" +short_desc = "Search results written to CSV file, verified by reading it back" +cli_args = ["*.rs", "--limit", "100", "--out", "test_cli_validation_out.csv"] +validator = "T31" +targets = ["cli"] [test.api_checks] result_has_key = ["results"] total_count_min = 1 [[test]] -id = "T79" -group = "output" -name = "T79 projection + json format" -title = "T79 projection + json format" -short_desc = "T79 projection + json format" -cli_args = ["*.rs", "--columns", "Name,Size,Modified", "--format", "json", "--limit", "5"] -validator = "T79" -targets = ["cli"] +id = "T79" +group = "output" +name = "T79 projection + json format" +title = "T79 projection + json format" +short_desc = "T79 projection + json format" +cli_args = [ + "*.rs", + "--columns", + "Name,Size,Modified", + "--format", + "json", + "--limit", + "5", +] +validator = "T79" +targets = ["cli"] [test.api_checks] result_has_key = ["results"] total_count_min = 1 [[test]] -id = "T80" -group = "output" -name = "T80 --columns all (wide)" -title = "T80 --columns all (wide)" -short_desc = "T80 --columns all (wide)" -cli_args = ["*.exe", "--columns", "all", "--limit", "5", "--drive", "C"] -targets = ["cli"] +id = "T80" +group = "output" +name = "T80 --columns all (wide)" +title = "T80 --columns all (wide)" +short_desc = "T80 --columns all (wide)" +cli_args = ["*.exe", "--columns", "all", "--limit", "5", "--drive", "C"] +targets = ["cli"] expect_columns_all = true expect_min_rows = 1 stdout_contains = ["\"Path\"", "\"Name\"", "\"Size\"", "\"Attributes\""] [[test]] -id = "T85" -group = "output" -name = "T85 json format + projection" -title = "T85 json format + projection" -short_desc = "T85 json format + projection (was table; piped stdout truncates column names)" -cli_args = ["*.dll", "--format", "json", "--columns", "Name,Size", "--limit", "5", "--drive", "C"] -targets = ["cli"] +id = "T85" +group = "output" +name = "T85 json format + projection" +title = "T85 json format + projection" +short_desc = "T85 json format + projection (was table; piped stdout truncates column names)" +cli_args = [ + "*.dll", + "--format", + "json", + "--columns", + "Name,Size", + "--limit", + "5", + "--drive", + "C", +] +targets = ["cli"] stdout_contains = ["name", "size"] [test.api_checks] @@ -95,28 +113,43 @@ result_has_key = ["results"] total_count_min = 1 [[test]] -id = "T159" -group = "output" -name = "T159 CSV with samples column" -title = "T159 CSV with samples column" -short_desc = "T159 CSV with samples column" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=3,sample=1", "--format", "csv"] +id = "T159" +group = "output" +name = "T159 CSV with samples column" +title = "T159 CSV with samples column" +short_desc = "T159 CSV with samples column" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=3,sample=1", + "--format", + "csv", +] stdout_contains = ["# buckets", "key,count,total_bytes", "samples", "drilldown"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.bucket_min_count = 1 [[test]] -id = "T164" -group = "output" -name = "T164 --histogram size:1048576 shorthand" -title = "T164 --histogram size:1048576 shorthand" -short_desc = "T164 --histogram size:1048576 shorthand" -cli_args = ["*", "--limit", "0", "--histogram", "size:1048576", "--format", "json"] -validator = "T164" +id = "T164" +group = "output" +name = "T164 --histogram size:1048576 shorthand" +title = "T164 --histogram size:1048576 shorthand" +short_desc = "T164 --histogram size:1048576 shorthand" +cli_args = [ + "*", + "--limit", + "0", + "--histogram", + "size:1048576", + "--format", + "json", +] +validator = "T164" [test.api_checks] expect_agg_results = 1 agg_kind_contains = ["buckets"] bucket_min_count = 1 - diff --git a/scripts/tests/definitions/07-search-mode.toml b/scripts/tests/definitions/07-search-mode.toml index 48fa9b5f1..efa9fd4b4 100644 --- a/scripts/tests/definitions/07-search-mode.toml +++ b/scripts/tests/definitions/07-search-mode.toml @@ -2,248 +2,356 @@ # Auto-split from test-definitions.toml [[test]] -id = "T88a" -group = "search_mode" -name = "T88a path: prefix" -title = "T88a path: prefix" -short_desc = "T88a path: prefix" -cli_args = ["path:*windows*", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88a" +group = "search_mode" +name = "T88a path: prefix" +title = "T88a path: prefix" +short_desc = "T88a path: prefix" +cli_args = [ + "path:*windows*", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T88a" +validator = "T88a" [[test]] -id = "T88b" -group = "search_mode" -name = "T88b dir: prefix" -title = "T88b dir: prefix" -short_desc = "T88b dir: prefix" -cli_args = ["dir:*system*", "--limit", "10", "--columns", "all"] +id = "T88b" +group = "search_mode" +name = "T88b dir: prefix" +title = "T88b dir: prefix" +short_desc = "T88b dir: prefix" +cli_args = ["dir:*system*", "--limit", "10", "--columns", "all"] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Directory Flag" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test]] -id = "T88c" -group = "search_mode" -name = "T88c file: prefix" -title = "T88c file: prefix" -short_desc = "T88c file: prefix" -cli_args = ["file:*.dll", "--limit", "10", "--columns", "all"] +id = "T88c" +group = "search_mode" +name = "T88c file: prefix" +title = "T88c file: prefix" +short_desc = "T88c file: prefix" +cli_args = ["file:*.dll", "--limit", "10", "--columns", "all"] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Directory Flag" -op = "ne" -value = "1" +op = "ne" +value = "1" [[test.column_checks]] column = "Name" -op = "ends_with" -value = ".dll" -case = "lower" +op = "ends_with" +value = ".dll" +case = "lower" [[test]] -id = "T88d" -group = "search_mode" -name = "T88d --begins-with" -title = "T88d --begins-with" -short_desc = "T88d --begins-with" -cli_args = ["--begins-with", "note", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88d" +group = "search_mode" +name = "T88d --begins-with" +title = "T88d --begins-with" +short_desc = "T88d --begins-with" +cli_args = [ + "--begins-with", + "note", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Name" -op = "starts_with" -value = "note" -case = "lower" +op = "starts_with" +value = "note" +case = "lower" [[test]] -id = "T88e" -group = "search_mode" -name = "T88e --ends-with" -title = "T88e --ends-with" -short_desc = "T88e --ends-with" -cli_args = ["--ends-with", ".log", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88e" +group = "search_mode" +name = "T88e --ends-with" +title = "T88e --ends-with" +short_desc = "T88e --ends-with" +cli_args = [ + "--ends-with", + ".log", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Name" -op = "ends_with" -value = ".log" -case = "lower" +op = "ends_with" +value = ".log" +case = "lower" [[test]] -id = "T88f" -group = "search_mode" -name = "T88f --contains" -title = "T88f --contains" -short_desc = "T88f --contains" -cli_args = ["--contains", "setup", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88f" +group = "search_mode" +name = "T88f --contains" +title = "T88f --contains" +short_desc = "T88f --contains" +cli_args = [ + "--contains", + "setup", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Name" -op = "contains" -value = "setup" -case = "lower" +op = "contains" +value = "setup" +case = "lower" [[test]] -id = "T88h" -group = "search_mode" -name = "T88h --in-path + pattern" -title = "T88h --in-path + pattern" -short_desc = "T88h --in-path + pattern" -cli_args = ["*.exe", "--in-path", "*windows*", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88h" +group = "search_mode" +name = "T88h --in-path + pattern" +title = "T88h --in-path + pattern" +short_desc = "T88h --in-path + pattern" +cli_args = [ + "*.exe", + "--in-path", + "*windows*", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Path Only" -op = "contains" -value = "windows" -case = "lower" +op = "contains" +value = "windows" +case = "lower" [[test.column_checks]] column = "Name" -op = "ends_with" -value = ".exe" -case = "lower" +op = "ends_with" +value = ".exe" +case = "lower" [[test]] -id = "T88i" -group = "search_mode" -name = "T88i path: vs --in-path" -title = "T88i path: vs --in-path" -short_desc = "T88i path: vs --in-path" -cli_args = ["path:*notepad*", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88i" +group = "search_mode" +name = "T88i path: vs --in-path" +title = "T88i path: vs --in-path" +short_desc = "T88i path: vs --in-path" +cli_args = [ + "path:*notepad*", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T88i" +validator = "T88i" [[test]] -id = "T88j" -group = "search_mode" -name = "T88j --contains + --not-contains" -title = "T88j --contains + --not-contains" -short_desc = "T88j --contains + --not-contains" -cli_args = ["--contains", "update", "--not-contains", "old", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88j" +group = "search_mode" +name = "T88j --contains + --not-contains" +title = "T88j --contains + --not-contains" +short_desc = "T88j --contains + --not-contains" +cli_args = [ + "--contains", + "update", + "--not-contains", + "old", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T88j" +validator = "T88j" [[test]] -id = "T88k" -group = "search_mode" -name = "T88k dir: prefix + sort treesize desc" -title = "T88k dir: prefix + sort treesize desc" -short_desc = "T88k dir: prefix + sort treesize desc" -cli_args = ["dir:*program*", "--sort", "treesize", "--sort-desc", "--limit", "10", "--columns", "all"] +id = "T88k" +group = "search_mode" +name = "T88k dir: prefix + sort treesize desc" +title = "T88k dir: prefix + sort treesize desc" +short_desc = "T88k dir: prefix + sort treesize desc" +cli_args = [ + "dir:*program*", + "--sort", + "treesize", + "--sort-desc", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T88k" +validator = "T88k" [[test.column_checks]] column = "Directory Flag" -op = "eq" -value = "1" +op = "eq" +value = "1" [[test.sort_checks]] column = "Tree Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T88l" -group = "search_mode" -name = "T88l --begins-with + --ext" -title = "T88l --begins-with + --ext" -short_desc = "T88l --begins-with + --ext" -cli_args = ["--begins-with", "win", "--ext", "exe,dll", "--files-only", "--limit", "10", "--columns", "all"] +id = "T88l" +group = "search_mode" +name = "T88l --begins-with + --ext" +title = "T88l --begins-with + --ext" +short_desc = "T88l --begins-with + --ext" +cli_args = [ + "--begins-with", + "win", + "--ext", + "exe,dll", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "T88l" +validator = "T88l" [[test]] -id = "T95" -group = "search_mode" -name = "T95 --in-path *windows*" -title = "T95 --in-path *windows*" -short_desc = "T95 --in-path *windows*" -cli_args = ["*.dll", "--in-path", "*windows*", "--files-only", "--limit", "10", "--columns", "all"] +id = "T95" +group = "search_mode" +name = "T95 --in-path *windows*" +title = "T95 --in-path *windows*" +short_desc = "T95 --in-path *windows*" +cli_args = [ + "*.dll", + "--in-path", + "*windows*", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Path Only" -op = "contains" -value = "windows" -case = "lower" +op = "contains" +value = "windows" +case = "lower" [[test]] -id = "T96" -group = "search_mode" -name = "T96 --in-path *system32*" -title = "T96 --in-path *system32*" -short_desc = "T96 --in-path *system32*" -cli_args = ["*.dll", "--in-path", "*system32*", "--files-only", "--limit", "10", "--columns", "all"] +id = "T96" +group = "search_mode" +name = "T96 --in-path *system32*" +title = "T96 --in-path *system32*" +short_desc = "T96 --in-path *system32*" +cli_args = [ + "*.dll", + "--in-path", + "*system32*", + "--files-only", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true [[test.column_checks]] column = "Path Only" -op = "contains" -value = "system32" -case = "lower" +op = "contains" +value = "system32" +case = "lower" [[test]] -id = "T117" -group = "search_mode" -name = "T117 type + in-path + size" -title = "T117 type + in-path + size" -short_desc = "System type, path contains 'windows', size > 1MB, sorted by size desc" -cli_args = ["*", "--type", "system", "--in-path", "*windows*", "--min-size", "1048576", "--files-only", "--sort", "-size", "--limit", "10", "--columns", "all"] +id = "T117" +group = "search_mode" +name = "T117 type + in-path + size" +title = "T117 type + in-path + size" +short_desc = "System type, path contains 'windows', size > 1MB, sorted by size desc" +cli_args = [ + "*", + "--type", + "system", + "--in-path", + "*windows*", + "--min-size", + "1048576", + "--files-only", + "--sort", + "-size", + "--limit", + "10", + "--columns", + "all", +] expect_min_rows = 1 expect_columns_all = true -validator = "type_system" +validator = "type_system" [[test.column_checks]] column = "Size" -op = "gte" -value = "1048576" +op = "gte" +value = "1048576" [[test.column_checks]] column = "Path Only" -op = "contains" -value = "windows" -case = "lower" +op = "contains" +value = "windows" +case = "lower" [[test.sort_checks]] column = "Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [test.api_checks] total_count_min = 1 [[test]] -id = "T129" -group = "search_mode" -name = "T129 regex case insensitive" -title = "Regex pattern matching (case insensitive)" -short_desc = "Regex search for system*.dll files" -cli_args = [">system.*\\.dll", "--files-only", "--limit", "10"] +id = "T129" +group = "search_mode" +name = "T129 regex case insensitive" +title = "Regex pattern matching (case insensitive)" +short_desc = "Regex search for system*.dll files" +cli_args = [">system.*\\.dll", "--files-only", "--limit", "10"] expect_min_rows = 1 stdout_contains = [".dll"] [test.api_checks] total_count_min = 1 - diff --git a/scripts/tests/definitions/08-combined.toml b/scripts/tests/definitions/08-combined.toml index 4c38a915a..0585eb6c2 100644 --- a/scripts/tests/definitions/08-combined.toml +++ b/scripts/tests/definitions/08-combined.toml @@ -2,97 +2,163 @@ # Auto-split from test-definitions.toml [[test]] -id = "T34" -group = "combined" -name = "T34 combined stress" -title = "T34 combined stress" -short_desc = "T34 combined stress" +id = "T34" +group = "combined" +name = "T34 combined stress" +title = "T34 combined stress" +short_desc = "T34 combined stress" # Removed --newer 365d: time-dependent filter fails on older MFT snapshots. -cli_args = ["*.pdf", "--files-only", "--min-size", "1048576", "--sort", "size", "--sort-desc", "--attr", "!hidden", "--limit", "10", "--format", "csv", "--columns", "Name,Size,Path Only"] +cli_args = [ + "*.pdf", + "--files-only", + "--min-size", + "1048576", + "--sort", + "size", + "--sort-desc", + "--attr", + "!hidden", + "--limit", + "10", + "--format", + "csv", + "--columns", + "Name,Size,Path Only", +] expect_min_rows = 1 -validator = "T34" +validator = "T34" [[test.sort_checks]] column = "Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [[test]] -id = "T75" -group = "combined" -name = "T75 size+time+ext combined" -title = "T75 size+time+ext combined" -short_desc = "T75 size+time+ext combined" -cli_args = ["*", "--ext", "exe,dll", "--min-size", "1048576", "--newer", "last_365d", "--files-only", "--sort", "size", "--sort-desc", "--limit", "10"] +id = "T75" +group = "combined" +name = "T75 size+time+ext combined" +title = "T75 size+time+ext combined" +short_desc = "T75 size+time+ext combined" +cli_args = [ + "*", + "--ext", + "exe,dll", + "--min-size", + "1048576", + "--newer", + "last_365d", + "--files-only", + "--sort", + "size", + "--sort-desc", + "--limit", + "10", +] expect_min_rows = 1 -validator = "T75" +validator = "T75" [[test]] -id = "T84" -group = "combined" -name = "T84 mega combined" -title = "T84 mega combined" -short_desc = "exe files, 10KB–1GB, not hidden/system, last year, sorted by size desc, drive C" -cli_args = ["*.exe", "--files-only", "--min-size", "10240", "--max-size", "1073741824", "--attr", "!hidden,!system", "--newer", "last_365d", "--sort", "-size", "--drive", "C", "--limit", "10", "--columns", "Name,Size,Modified,Path Only,Hidden,System"] +id = "T84" +group = "combined" +name = "T84 mega combined" +title = "T84 mega combined" +short_desc = "exe files, 10KB–1GB, not hidden/system, last year, sorted by size desc, drive C" +cli_args = [ + "*.exe", + "--files-only", + "--min-size", + "10240", + "--max-size", + "1073741824", + "--attr", + "!hidden,!system", + "--newer", + "last_365d", + "--sort", + "-size", + "--drive", + "C", + "--limit", + "10", + "--columns", + "Name,Size,Modified,Path Only,Hidden,System", +] expect_min_rows = 1 [[test.column_checks]] column = "Size" -op = "gte" -value = "10240" +op = "gte" +value = "10240" [[test.column_checks]] column = "Size" -op = "lte" -value = "1073741824" +op = "lte" +value = "1073741824" [[test.column_checks]] column = "Hidden" -op = "eq" -value = "0" +op = "eq" +value = "0" [[test.column_checks]] column = "System" -op = "eq" -value = "0" +op = "eq" +value = "0" [[test.sort_checks]] column = "Size" -order = "desc" -type = "u64" +order = "desc" +type = "u64" [test.api_checks] total_count_min = 1 [[test]] -id = "T100" -group = "combined" -name = "T100 bulkiness + size combined" -title = "T100 bulkiness + size combined" -short_desc = "T100 bulkiness + size combined" -cli_args = ["*", "--min-bulkiness", "500", "--min-size", "1048576", "--limit", "10"] +id = "T100" +group = "combined" +name = "T100 bulkiness + size combined" +title = "T100 bulkiness + size combined" +short_desc = "T100 bulkiness + size combined" +cli_args = [ + "*", + "--min-bulkiness", + "500", + "--min-size", + "1048576", + "--limit", + "10", +] expect_min_rows = 1 [[test.column_checks]] column = "Bulkiness" -op = "gte" -value = "500" +op = "gte" +value = "500" [[test.column_checks]] column = "Size" -op = "gte" -value = "1048576" +op = "gte" +value = "1048576" [[test]] -id = "T156" -group = "combined" -name = "T156 multiple --agg flags combined" -title = "T156 multiple --agg flags combined" -short_desc = "T156 multiple --agg flags combined" -cli_args = ["*", "--limit", "0", "--agg", "count", "--agg", "terms:extension,top=3", "--format", "json"] -validator = "T156" +id = "T156" +group = "combined" +name = "T156 multiple --agg flags combined" +title = "T156 multiple --agg flags combined" +short_desc = "T156 multiple --agg flags combined" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "count", + "--agg", + "terms:extension,top=3", + "--format", + "json", +] +validator = "T156" [test.api_checks] expect_agg_results = 2 agg_kind_contains = ["count", "buckets"] - diff --git a/scripts/tests/definitions/09-aggregation.toml b/scripts/tests/definitions/09-aggregation.toml index 9b8930034..fea975c1f 100644 --- a/scripts/tests/definitions/09-aggregation.toml +++ b/scripts/tests/definitions/09-aggregation.toml @@ -2,229 +2,242 @@ # Auto-split from test-definitions.toml [[test]] -id = "T119" -group = "aggregation" -name = "T119 agg count" -title = "T119 agg count" -short_desc = "T119 agg count" -cli_args = ["agg", "count"] +id = "T119" +group = "aggregation" +name = "T119 agg count" +title = "T119 agg count" +short_desc = "T119 agg count" +cli_args = ["agg", "count"] stdout_contains = ["=== count ===", "Total:"] -api_checks.result_has_key = ["records_scanned"] - -[[test]] -id = "T120" -group = "aggregation" -name = "T120 agg overview" -title = "T120 agg overview" -short_desc = "T120 agg overview" -cli_args = ["agg", "overview"] -stdout_contains = ["=== total_count ===", "=== files_vs_dirs ===", "=== by_type ===", "Total:"] +api_checks.result_has_key = ["records_scanned"] + +[[test]] +id = "T120" +group = "aggregation" +name = "T120 agg overview" +title = "T120 agg overview" +short_desc = "T120 agg overview" +cli_args = ["agg", "overview"] +stdout_contains = [ + "=== total_count ===", + "=== files_vs_dirs ===", + "=== by_type ===", + "Total:", +] api_checks.expect_agg_results = 5 api_checks.agg_label_contains = ["total_count", "files_vs_dirs", "by_type"] [[test]] -id = "T121" -group = "aggregation" -name = "T121 agg by_extension" -title = "T121 agg by_extension" -short_desc = "T121 agg by_extension" -cli_args = ["agg", "by_extension"] +id = "T121" +group = "aggregation" +name = "T121 agg by_extension" +title = "T121 agg by_extension" +short_desc = "T121 agg by_extension" +cli_args = ["agg", "by_extension"] stdout_contains = ["=== by_extension ===", "Count", "Total Size"] api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["by_extension"] [[test]] -id = "T122" -group = "aggregation" -name = "T122 agg by_type" -title = "T122 agg by_type" -short_desc = "T122 agg by_type" -cli_args = ["agg", "by_type"] +id = "T122" +group = "aggregation" +name = "T122 agg by_type" +title = "T122 agg by_type" +short_desc = "T122 agg by_type" +cli_args = ["agg", "by_type"] stdout_contains = ["=== by_type ===", "Count", "Total Size"] api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["by_type"] [[test]] -id = "T123" -group = "aggregation" -name = "T123 agg by_drive" -title = "T123 agg by_drive" -short_desc = "T123 agg by_drive" -cli_args = ["agg", "by_drive"] +id = "T123" +group = "aggregation" +name = "T123 agg by_drive" +title = "T123 agg by_drive" +short_desc = "T123 agg by_drive" +cli_args = ["agg", "by_drive"] stdout_contains = ["=== by_drive ===", "Count", "Total Size"] api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["by_drive"] [[test]] -id = "T124" -group = "aggregation" -name = "T124 agg by_size" -title = "T124 agg by_size" -short_desc = "T124 agg by_size" -cli_args = ["agg", "by_size"] +id = "T124" +group = "aggregation" +name = "T124 agg by_size" +title = "T124 agg by_size" +short_desc = "T124 agg by_size" +cli_args = ["agg", "by_size"] stdout_contains = ["=== by_size ===", "Count", "Total Size"] api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["by_size"] [[test]] -id = "T125" -group = "aggregation" -name = "T125 agg by_age" -title = "T125 agg by_age" -short_desc = "T125 agg by_age" -cli_args = ["agg", "by_age"] +id = "T125" +group = "aggregation" +name = "T125 agg by_age" +title = "T125 agg by_age" +short_desc = "T125 agg by_age" +cli_args = ["agg", "by_age"] stdout_contains = ["=== by_age ===", "Count", "Total Size"] api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["by_age"] [[test]] -id = "T126" -group = "aggregation" -name = "T126 agg count --format json" -title = "T126 agg count --format json" -short_desc = "T126 agg count --format json" -cli_args = ["agg", "count", "--format", "json"] -validator = "T126" +id = "T126" +group = "aggregation" +name = "T126 agg count --format json" +title = "T126 agg count --format json" +short_desc = "T126 agg count --format json" +cli_args = ["agg", "count", "--format", "json"] +validator = "T126" [test.api_checks] agg_kind_contains = ["count"] [[test]] -id = "T127" -group = "aggregation" -name = "T127 agg overview --format json" -title = "T127 agg overview --format json" -short_desc = "T127 agg overview --format json" -cli_args = ["agg", "overview", "--format", "json"] -validator = "T127" +id = "T127" +group = "aggregation" +name = "T127 agg overview --format json" +title = "T127 agg overview --format json" +short_desc = "T127 agg overview --format json" +cli_args = ["agg", "overview", "--format", "json"] +validator = "T127" [test.api_checks] agg_label_contains = ["total_count", "by_type"] bucket_min_count = 1 [[test]] -id = "T128" -group = "aggregation" -name = "T128 agg by_extension --format csv" -title = "T128 agg by_extension --format csv" -short_desc = "T128 agg by_extension --format csv" -cli_args = ["agg", "by_extension", "--format", "csv"] +id = "T128" +group = "aggregation" +name = "T128 agg by_extension --format csv" +title = "T128 agg by_extension --format csv" +short_desc = "T128 agg by_extension --format csv" +cli_args = ["agg", "by_extension", "--format", "csv"] stdout_contains = ["# by_extension", "key,count,total_bytes"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["by_extension"] [[test]] -id = "T130" -group = "aggregation" -name = "T130 agg top_folders preset" -title = "T130 agg top_folders preset" -short_desc = "T130 agg top_folders preset" -cli_args = ["agg", "top_folders"] +id = "T130" +group = "aggregation" +name = "T130 agg top_folders preset" +title = "T130 agg top_folders preset" +short_desc = "T130 agg top_folders preset" +cli_args = ["agg", "top_folders"] stdout_contains = ["=== top_folders ===", "Count", "Total Size"] api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["top_folders"] [[test]] -id = "T131" -group = "aggregation" -name = "T131 agg cleanup preset" -title = "T131 agg cleanup preset" -short_desc = "T131 agg cleanup preset" -cli_args = ["agg", "cleanup"] +id = "T131" +group = "aggregation" +name = "T131 agg cleanup preset" +title = "T131 agg cleanup preset" +short_desc = "T131 agg cleanup preset" +cli_args = ["agg", "cleanup"] stdout_contains = ["=== no_extension ===", "missing:"] api_checks.expect_agg_results = 3 api_checks.agg_label_contains = ["no_extension", "zero_byte_files"] [[test]] -id = "T132" -group = "aggregation" -name = "T132 agg duplicates preset" -title = "T132 agg duplicates preset" -short_desc = "T132 agg duplicates preset" -cli_args = ["agg", "duplicates"] +id = "T132" +group = "aggregation" +name = "T132 agg duplicates preset" +title = "T132 agg duplicates preset" +short_desc = "T132 agg duplicates preset" +cli_args = ["agg", "duplicates"] stdout_contains = ["=== duplicate_candidates ==="] api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["duplicate_candidates"] [[test]] -id = "T133" -group = "aggregation" -name = "T133 agg storage preset" -title = "T133 agg storage preset" -short_desc = "T133 agg storage preset" -cli_args = ["agg", "storage"] +id = "T133" +group = "aggregation" +name = "T133 agg storage preset" +title = "T133 agg storage preset" +short_desc = "T133 agg storage preset" +cli_args = ["agg", "storage"] stdout_contains = ["=== logical_size ===", "Count:", "Sum:"] api_checks.expect_agg_results = 3 api_checks.agg_label_contains = ["logical_size", "allocated_size"] [[test]] -id = "T134" -group = "aggregation" -name = "T134 agg activity preset" -title = "T134 agg activity preset" -short_desc = "T134 agg activity preset" -cli_args = ["agg", "activity"] +id = "T134" +group = "aggregation" +name = "T134 agg activity preset" +title = "T134 agg activity preset" +short_desc = "T134 agg activity preset" +cli_args = ["agg", "activity"] stdout_contains = ["=== modified_monthly ===", "Count", "Total Size"] api_checks.expect_agg_results = 2 api_checks.agg_label_contains = ["modified_monthly", "created_monthly"] [[test]] -id = "T135" -group = "aggregation" -name = "T135 agg media preset" -title = "T135 agg media preset" -short_desc = "T135 agg media preset" -cli_args = ["agg", "media"] +id = "T135" +group = "aggregation" +name = "T135 agg media preset" +title = "T135 agg media preset" +short_desc = "T135 agg media preset" +cli_args = ["agg", "media"] stdout_contains = ["=== media_type_breakdown ===", "Count", "Total Size"] api_checks.expect_agg_results = 3 api_checks.agg_label_contains = ["media_type_breakdown", "media_size_stats"] [[test]] -id = "T136" -group = "aggregation" -name = "T136 agg top_folders --format json" -title = "T136 agg top_folders --format json" -short_desc = "T136 agg top_folders --format json" -cli_args = ["agg", "top_folders", "--format", "json"] -validator = "T136" +id = "T136" +group = "aggregation" +name = "T136 agg top_folders --format json" +title = "T136 agg top_folders --format json" +short_desc = "T136 agg top_folders --format json" +cli_args = ["agg", "top_folders", "--format", "json"] +validator = "T136" [test.api_checks] agg_label_contains = ["top_folders"] bucket_min_count = 1 [[test]] -id = "T137" -group = "aggregation" -name = "T137 agg cleanup --format json" -title = "T137 agg cleanup --format json" -short_desc = "T137 agg cleanup --format json" -cli_args = ["agg", "cleanup", "--format", "json"] -validator = "T137" +id = "T137" +group = "aggregation" +name = "T137 agg cleanup --format json" +title = "T137 agg cleanup --format json" +short_desc = "T137 agg cleanup --format json" +cli_args = ["agg", "cleanup", "--format", "json"] +validator = "T137" [test.api_checks] agg_label_contains = ["no_extension", "zero_byte_files"] [[test]] -id = "T138" -group = "aggregation" -name = "T138 agg cleanup --format csv" -title = "T138 agg cleanup --format csv" -short_desc = "T138 agg cleanup --format csv" -cli_args = ["agg", "cleanup", "--format", "csv"] +id = "T138" +group = "aggregation" +name = "T138 agg cleanup --format csv" +title = "T138 agg cleanup --format csv" +short_desc = "T138 agg cleanup --format csv" +cli_args = ["agg", "cleanup", "--format", "csv"] stdout_contains = ["# no_extension", "value"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["no_extension"] [[test]] -id = "T139" -group = "aggregation" -name = "T139 --agg terms:extension,sample=3" -title = "T139 --agg terms:extension,sample=3" -short_desc = "T139 --agg terms:extension,sample=3" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=10,sample=3", "--format", "json"] -validator = "T139" +id = "T139" +group = "aggregation" +name = "T139 --agg terms:extension,sample=3" +title = "T139 --agg terms:extension,sample=3" +short_desc = "T139 --agg terms:extension,sample=3" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=10,sample=3", + "--format", + "json", +] +validator = "T139" [test.api_checks] expect_agg_results = 1 @@ -233,13 +246,21 @@ bucket_min_count = 1 bucket_has_samples = true [[test]] -id = "T140" -group = "aggregation" -name = "T140 --agg duplicates:size+name,sample=2" -title = "T140 --agg duplicates:size+name,sample=2" -short_desc = "T140 --agg duplicates:size+name,sample=2" -cli_args = ["*", "--limit", "0", "--agg", "duplicates:size+name,sample=2,top=50", "--format", "json"] -validator = "T140" +id = "T140" +group = "aggregation" +name = "T140 --agg duplicates:size+name,sample=2" +title = "T140 --agg duplicates:size+name,sample=2" +short_desc = "T140 --agg duplicates:size+name,sample=2" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "duplicates:size+name,sample=2,top=50", + "--format", + "json", +] +validator = "T140" [test.api_checks] expect_agg_results = 1 @@ -248,24 +269,24 @@ bucket_min_count = 1 bucket_has_samples = true [[test]] -id = "T145" -group = "aggregation" -name = "T145 --agg count + --rows mixed" -title = "T145 --agg count + --rows mixed" -short_desc = "T145 --agg count + --rows mixed" -cli_args = ["*.dll", "--limit", "5", "--agg", "count", "--rows", "--drive", "C"] -targets = ["cli"] +id = "T145" +group = "aggregation" +name = "T145 --agg count + --rows mixed" +title = "T145 --agg count + --rows mixed" +short_desc = "T145 --agg count + --rows mixed" +cli_args = ["*.dll", "--limit", "5", "--agg", "count", "--rows", "--drive", "C"] +targets = ["cli"] stdout_contains = ["# count"] expect_min_rows = 1 [[test]] -id = "T146" -group = "aggregation" -name = "T146 agg by_extension --format json (buckets)" -title = "T146 agg by_extension --format json (buckets)" -short_desc = "T146 agg by_extension --format json (buckets)" -cli_args = ["agg", "by_extension", "--format", "json"] -validator = "T146" +id = "T146" +group = "aggregation" +name = "T146 agg by_extension --format json (buckets)" +title = "T146 agg by_extension --format json (buckets)" +short_desc = "T146 agg by_extension --format json (buckets)" +cli_args = ["agg", "by_extension", "--format", "json"] +validator = "T146" [test.api_checks] expect_agg_results = 1 @@ -274,38 +295,38 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "T147" -group = "aggregation" -name = "T147 agg duplicates --format json" -title = "T147 agg duplicates --format json" -short_desc = "T147 agg duplicates --format json" -cli_args = ["agg", "duplicates", "--format", "json"] -validator = "T147" +id = "T147" +group = "aggregation" +name = "T147 agg duplicates --format json" +title = "T147 agg duplicates --format json" +short_desc = "T147 agg duplicates --format json" +cli_args = ["agg", "duplicates", "--format", "json"] +validator = "T147" [test.api_checks] agg_kind_contains = ["duplicates"] bucket_min_count = 1 [[test]] -id = "T148" -group = "aggregation" -name = "T148 agg by_drive --format csv" -title = "T148 agg by_drive --format csv" -short_desc = "T148 agg by_drive --format csv" -cli_args = ["agg", "by_drive", "--format", "csv"] +id = "T148" +group = "aggregation" +name = "T148 agg by_drive --format csv" +title = "T148 agg by_drive --format csv" +short_desc = "T148 agg by_drive --format csv" +cli_args = ["agg", "by_drive", "--format", "csv"] stdout_contains = ["# by_drive", "key,count,total_bytes"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["by_drive"] [[test]] -id = "T149" -group = "aggregation" -name = "T149 agg by_size --format json" -title = "T149 agg by_size --format json" -short_desc = "T149 agg by_size --format json" -cli_args = ["agg", "by_size", "--format", "json"] -validator = "T149" +id = "T149" +group = "aggregation" +name = "T149 agg by_size --format json" +title = "T149 agg by_size --format json" +short_desc = "T149 agg by_size --format json" +cli_args = ["agg", "by_size", "--format", "json"] +validator = "T149" [test.api_checks] agg_label_contains = ["by_size"] @@ -313,13 +334,21 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "T150" -group = "aggregation" -name = "T150 --agg terms:extension,sample=2 JSON sample_rows" -title = "T150 --agg terms:extension,sample=2 JSON sample_rows" -short_desc = "T150 --agg terms:extension,sample=2 JSON sample_rows" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=5,sample=2", "--format", "json"] -validator = "T150" +id = "T150" +group = "aggregation" +name = "T150 --agg terms:extension,sample=2 JSON sample_rows" +title = "T150 --agg terms:extension,sample=2 JSON sample_rows" +short_desc = "T150 --agg terms:extension,sample=2 JSON sample_rows" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=5,sample=2", + "--format", + "json", +] +validator = "T150" [test.api_checks] expect_agg_results = 1 @@ -327,13 +356,21 @@ agg_kind_contains = ["buckets"] bucket_has_samples = true [[test]] -id = "T151" -group = "aggregation" -name = "T151 --agg terms:extension,sample=2 JSON drilldown" -title = "T151 --agg terms:extension,sample=2 JSON drilldown" -short_desc = "T151 --agg terms:extension,sample=2 JSON drilldown" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=5,sample=2", "--format", "json"] -validator = "T151" +id = "T151" +group = "aggregation" +name = "T151 --agg terms:extension,sample=2 JSON drilldown" +title = "T151 --agg terms:extension,sample=2 JSON drilldown" +short_desc = "T151 --agg terms:extension,sample=2 JSON drilldown" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=5,sample=2", + "--format", + "json", +] +validator = "T151" [test.api_checks] expect_agg_results = 1 @@ -341,25 +378,41 @@ agg_kind_contains = ["buckets"] bucket_has_samples = true [[test]] -id = "T152" -group = "aggregation" -name = "T152 agg by_extension table with samples" -title = "T152 agg by_extension table with samples" -short_desc = "T152 agg by_extension table with samples" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=3,sample=1", "--format", "table"] +id = "T152" +group = "aggregation" +name = "T152 agg by_extension table with samples" +title = "T152 agg by_extension table with samples" +short_desc = "T152 agg by_extension table with samples" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=3,sample=1", + "--format", + "table", +] stdout_contains = ["=== buckets ===", "Count", "Total Size"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.bucket_min_count = 1 [[test]] -id = "T153" -group = "aggregation" -name = "T153 --agg terms:extension no sample" -title = "T153 --agg terms:extension no sample" -short_desc = "T153 --agg terms:extension no sample" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=3", "--format", "json"] -validator = "T153" +id = "T153" +group = "aggregation" +name = "T153 --agg terms:extension no sample" +title = "T153 --agg terms:extension no sample" +short_desc = "T153 --agg terms:extension no sample" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=3", + "--format", + "json", +] +validator = "T153" [test.api_checks] expect_agg_results = 1 @@ -367,13 +420,21 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "T154" -group = "aggregation" -name = "T154 rollup:drive power syntax" -title = "T154 rollup:drive power syntax" -short_desc = "T154 rollup:drive power syntax" -cli_args = ["*", "--limit", "0", "--agg", "rollup:drive,top=5", "--format", "json"] -validator = "T154" +id = "T154" +group = "aggregation" +name = "T154 rollup:drive power syntax" +title = "T154 rollup:drive power syntax" +short_desc = "T154 rollup:drive power syntax" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:drive,top=5", + "--format", + "json", +] +validator = "T154" [test.api_checks] expect_agg_results = 1 @@ -381,13 +442,21 @@ agg_kind_contains = ["rollup"] bucket_min_count = 1 [[test]] -id = "T155" -group = "aggregation" -name = "T155 rollup:path,depth=2 power syntax" -title = "T155 rollup:path,depth=2 power syntax" -short_desc = "T155 rollup:path,depth=2 power syntax" -cli_args = ["*", "--limit", "0", "--agg", "rollup:path,depth=2,top=10", "--format", "json"] -validator = "T155" +id = "T155" +group = "aggregation" +name = "T155 rollup:path,depth=2 power syntax" +title = "T155 rollup:path,depth=2 power syntax" +short_desc = "T155 rollup:path,depth=2 power syntax" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:path,depth=2,top=10", + "--format", + "json", +] +validator = "T155" [test.api_checks] expect_agg_results = 1 @@ -395,13 +464,21 @@ agg_kind_contains = ["rollup"] bucket_min_count = 1 [[test]] -id = "T157" -group = "aggregation" -name = "T157 --agg + search filter" -title = "T157 --agg + search filter" -short_desc = "T157 --agg + search filter" -cli_args = ["*.exe", "--limit", "0", "--agg", "terms:extension,top=3,sample=1", "--format", "json"] -validator = "T157" +id = "T157" +group = "aggregation" +name = "T157 --agg + search filter" +title = "T157 --agg + search filter" +short_desc = "T157 --agg + search filter" +cli_args = [ + "*.exe", + "--limit", + "0", + "--agg", + "terms:extension,top=3,sample=1", + "--format", + "json", +] +validator = "T157" [test.api_checks] expect_agg_results = 1 @@ -410,13 +487,21 @@ bucket_key_contains = ["exe"] bucket_has_samples = true [[test]] -id = "T158" -group = "aggregation" -name = "T158 sample row fields content" -title = "T158 sample row fields content" -short_desc = "T158 sample row fields content" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=3,sample=2", "--format", "json"] -validator = "T158" +id = "T158" +group = "aggregation" +name = "T158 sample row fields content" +title = "T158 sample row fields content" +short_desc = "T158 sample row fields content" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=3,sample=2", + "--format", + "json", +] +validator = "T158" [test.api_checks] expect_agg_results = 1 @@ -424,13 +509,21 @@ agg_kind_contains = ["buckets"] bucket_has_samples = true [[test]] -id = "T160" -group = "aggregation" -name = "T160 hist:size power syntax" -title = "T160 hist:size power syntax" -short_desc = "T160 hist:size power syntax" -cli_args = ["*", "--limit", "0", "--agg", "hist:size,interval=1048576", "--format", "json"] -validator = "T160" +id = "T160" +group = "aggregation" +name = "T160 hist:size power syntax" +title = "T160 hist:size power syntax" +short_desc = "T160 hist:size power syntax" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "hist:size,interval=1048576", + "--format", + "json", +] +validator = "T160" [test.api_checks] expect_agg_results = 1 @@ -438,13 +531,13 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "T161" -group = "aggregation" -name = "T161 stats:size power syntax" -title = "T161 stats:size power syntax" -short_desc = "T161 stats:size power syntax" -cli_args = ["*", "--limit", "0", "--agg", "stats:size", "--format", "json"] -validator = "T161" +id = "T161" +group = "aggregation" +name = "T161 stats:size power syntax" +title = "T161 stats:size power syntax" +short_desc = "T161 stats:size power syntax" +cli_args = ["*", "--limit", "0", "--agg", "stats:size", "--format", "json"] +validator = "T161" [test.api_checks] expect_agg_results = 1 @@ -452,13 +545,21 @@ agg_kind_contains = ["stats"] agg_stats_has_keys = ["count", "sum", "min", "max", "avg"] [[test]] -id = "T162" -group = "aggregation" -name = "T162 datehist:modified,calendar=month" -title = "T162 datehist:modified,calendar=month" -short_desc = "T162 datehist:modified,calendar=month" -cli_args = ["*", "--limit", "0", "--agg", "datehist:modified,calendar=month", "--format", "json"] -validator = "T162" +id = "T162" +group = "aggregation" +name = "T162 datehist:modified,calendar=month" +title = "T162 datehist:modified,calendar=month" +short_desc = "T162 datehist:modified,calendar=month" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "datehist:modified,calendar=month", + "--format", + "json", +] +validator = "T162" [test.api_checks] expect_agg_results = 1 @@ -466,13 +567,21 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "T163" -group = "aggregation" -name = "T163 range:size power syntax" -title = "T163 range:size power syntax" -short_desc = "T163 range:size power syntax" -cli_args = ["*", "--limit", "0", "--agg", "range:size,boundaries=0+1024+1048576+1073741824", "--format", "json"] -validator = "T163" +id = "T163" +group = "aggregation" +name = "T163 range:size power syntax" +title = "T163 range:size power syntax" +short_desc = "T163 range:size power syntax" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "range:size,boundaries=0+1024+1048576+1073741824", + "--format", + "json", +] +validator = "T163" [test.api_checks] expect_agg_results = 1 @@ -480,39 +589,63 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "T165" -group = "aggregation" -name = "T165 missing:extension power syntax" -title = "T165 missing:extension power syntax" -short_desc = "T165 missing:extension power syntax" -cli_args = ["*", "--limit", "0", "--agg", "missing:extension", "--format", "json"] -validator = "T165" +id = "T165" +group = "aggregation" +name = "T165 missing:extension power syntax" +title = "T165 missing:extension power syntax" +short_desc = "T165 missing:extension power syntax" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "missing:extension", + "--format", + "json", +] +validator = "T165" [test.api_checks] expect_agg_results = 1 agg_kind_contains = ["missing"] [[test]] -id = "T166" -group = "aggregation" -name = "T166 distinct:extension power syntax" -title = "T166 distinct:extension power syntax" -short_desc = "T166 distinct:extension power syntax" -cli_args = ["*", "--limit", "0", "--agg", "distinct:extension", "--format", "json"] -validator = "T166" +id = "T166" +group = "aggregation" +name = "T166 distinct:extension power syntax" +title = "T166 distinct:extension power syntax" +short_desc = "T166 distinct:extension power syntax" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "distinct:extension", + "--format", + "json", +] +validator = "T166" [test.api_checks] expect_agg_results = 1 agg_kind_contains = ["distinct"] [[test]] -id = "T167" -group = "aggregation" -name = "T167 sample_sort terms:extension,sample=2,sort=size" -title = "T167 sample_sort terms:extension,sample=2,sort=size" -short_desc = "T167 sample_sort terms:extension,sample=2,sort=size" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=3,sample=2,sort=size", "--format", "json"] -validator = "T167" +id = "T167" +group = "aggregation" +name = "T167 sample_sort terms:extension,sample=2,sort=size" +title = "T167 sample_sort terms:extension,sample=2,sort=size" +short_desc = "T167 sample_sort terms:extension,sample=2,sort=size" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=3,sample=2,sort=size", + "--format", + "json", +] +validator = "T167" [test.api_checks] expect_agg_results = 1 @@ -520,112 +653,141 @@ agg_kind_contains = ["buckets"] bucket_has_samples = true [[test]] -id = "T170" -group = "aggregation" -name = "T170 rollup:drive table format" -title = "T170 rollup:drive table format" -short_desc = "T170 rollup:drive table format" -cli_args = ["*", "--limit", "0", "--agg", "rollup:drive,top=5", "--format", "table"] +id = "T170" +group = "aggregation" +name = "T170 rollup:drive table format" +title = "T170 rollup:drive table format" +short_desc = "T170 rollup:drive table format" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:drive,top=5", + "--format", + "table", +] stdout_contains = ["=== rollup ===", "Count", "Total Size"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["rollup"] [[test]] -id = "T171" -group = "aggregation" -name = "T171 rollup:path CSV format" -title = "T171 rollup:path CSV format" -short_desc = "T171 rollup:path CSV format" -cli_args = ["*", "--limit", "0", "--agg", "rollup:path,depth=1,top=5", "--format", "csv"] +id = "T171" +group = "aggregation" +name = "T171 rollup:path CSV format" +title = "T171 rollup:path CSV format" +short_desc = "T171 rollup:path CSV format" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:path,depth=1,top=5", + "--format", + "csv", +] stdout_contains = ["# rollup", "key,count,total_bytes"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["rollup"] [[test]] -id = "T172" -group = "aggregation" -name = "T172 agg overview --format csv" -title = "T172 agg overview --format csv" -short_desc = "T172 agg overview --format csv" -cli_args = ["agg", "overview", "--format", "csv"] +id = "T172" +group = "aggregation" +name = "T172 agg overview --format csv" +title = "T172 agg overview --format csv" +short_desc = "T172 agg overview --format csv" +cli_args = ["agg", "overview", "--format", "csv"] stdout_contains = ["# total_count", "# files_vs_dirs", "# by_type"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 3 [[test]] -id = "T173" -group = "aggregation" -name = "T173 agg by_age --format json" -title = "T173 agg by_age --format json" -short_desc = "T173 agg by_age --format json" -cli_args = ["agg", "by_age", "--format", "json"] -validator = "T173" +id = "T173" +group = "aggregation" +name = "T173 agg by_age --format json" +title = "T173 agg by_age --format json" +short_desc = "T173 agg by_age --format json" +cli_args = ["agg", "by_age", "--format", "json"] +validator = "T173" [test.api_checks] agg_label_contains = ["by_age"] bucket_min_count = 1 [[test]] -id = "T174" -group = "aggregation" -name = "T174 agg storage --format json" -title = "T174 agg storage --format json" -short_desc = "T174 agg storage --format json" -cli_args = ["agg", "storage", "--format", "json"] -validator = "T174" +id = "T174" +group = "aggregation" +name = "T174 agg storage --format json" +title = "T174 agg storage --format json" +short_desc = "T174 agg storage --format json" +cli_args = ["agg", "storage", "--format", "json"] +validator = "T174" [test.api_checks] agg_label_contains = ["logical_size", "waste_by_drive"] bucket_min_count = 1 [[test]] -id = "T175" -group = "aggregation" -name = "T175 agg activity --format json" -title = "T175 agg activity --format json" -short_desc = "T175 agg activity --format json" -cli_args = ["agg", "activity", "--format", "json"] -validator = "T175" +id = "T175" +group = "aggregation" +name = "T175 agg activity --format json" +title = "T175 agg activity --format json" +short_desc = "T175 agg activity --format json" +cli_args = ["agg", "activity", "--format", "json"] +validator = "T175" [test.api_checks] agg_label_contains = ["modified_monthly", "created_monthly"] bucket_min_count = 1 [[test]] -id = "T176" -group = "aggregation" -name = "T176 agg media --format json" -title = "T176 agg media --format json" -short_desc = "T176 agg media --format json" -cli_args = ["agg", "media", "--format", "json"] -validator = "T176" +id = "T176" +group = "aggregation" +name = "T176 agg media --format json" +title = "T176 agg media --format json" +short_desc = "T176 agg media --format json" +cli_args = ["agg", "media", "--format", "json"] +validator = "T176" [test.api_checks] agg_label_contains = ["media_type_breakdown", "media_size_stats"] bucket_min_count = 1 [[test]] -id = "T177" -group = "aggregation" -name = "T177 agg duplicates --format csv" -title = "T177 agg duplicates --format csv" -short_desc = "T177 agg duplicates --format csv" -cli_args = ["agg", "duplicates", "--format", "csv"] -stdout_contains = ["# duplicate_candidates", "key,copies,file_size,total_bytes,reclaimable,verified"] -cli_checks.expect_min_rows = 1 +id = "T177" +group = "aggregation" +name = "T177 agg duplicates --format csv" +title = "T177 agg duplicates --format csv" +short_desc = "T177 agg duplicates --format csv" +cli_args = ["agg", "duplicates", "--format", "csv"] +stdout_contains = [ + "# duplicate_candidates", + "key,copies,file_size,total_bytes,reclaimable,verified", +] +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["duplicate_candidates"] [[test]] -id = "S3A.1" -group = "aggregation" -name = "S3A.1 next_cursor in JSON output" -title = "S3A.1 next_cursor in JSON output" -short_desc = "S3A.1 next_cursor in JSON output" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=50", "--agg-page-size", "2", "--format", "json"] -validator = "S3A.1" +id = "S3A.1" +group = "aggregation" +name = "S3A.1 next_cursor in JSON output" +title = "S3A.1 next_cursor in JSON output" +short_desc = "S3A.1 next_cursor in JSON output" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=50", + "--agg-page-size", + "2", + "--format", + "json", +] +validator = "S3A.1" [test.api_checks] expect_agg_results = 1 @@ -633,25 +795,45 @@ agg_kind_contains = ["buckets"] agg_has_cursor = true [[test]] -id = "S3A.2" -group = "aggregation" -name = "S3A.2 table cursor hint" -title = "S3A.2 table cursor hint" -short_desc = "S3A.2 table cursor hint" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=50", "--agg-page-size", "2", "--format", "table"] +id = "S3A.2" +group = "aggregation" +name = "S3A.2 table cursor hint" +title = "S3A.2 table cursor hint" +short_desc = "S3A.2 table cursor hint" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=50", + "--agg-page-size", + "2", + "--format", + "table", +] stdout_contains = ["=== buckets ===", "Count", "Total Size"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.bucket_min_count = 1 [[test]] -id = "S3A.3" -group = "aggregation" -name = "S3A.3 --agg-page-size limits buckets" -title = "S3A.3 --agg-page-size limits buckets" -short_desc = "S3A.3 --agg-page-size limits buckets" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=50", "--agg-page-size", "3", "--format", "json"] -validator = "S3A.3" +id = "S3A.3" +group = "aggregation" +name = "S3A.3 --agg-page-size limits buckets" +title = "S3A.3 --agg-page-size limits buckets" +short_desc = "S3A.3 --agg-page-size limits buckets" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=50", + "--agg-page-size", + "3", + "--format", + "json", +] +validator = "S3A.3" [test.api_checks] expect_agg_results = 1 @@ -659,13 +841,20 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "S3A.4" -group = "aggregation" -name = "S3A.4 aggregate --agg-page-size" -title = "S3A.4 aggregate --agg-page-size" -short_desc = "S3A.4 aggregate --agg-page-size" -cli_args = ["aggregate", "by_extension", "--format", "json", "--agg-page-size", "5"] -validator = "S3A.4" +id = "S3A.4" +group = "aggregation" +name = "S3A.4 aggregate --agg-page-size" +title = "S3A.4 aggregate --agg-page-size" +short_desc = "S3A.4 aggregate --agg-page-size" +cli_args = [ + "aggregate", + "by_extension", + "--format", + "json", + "--agg-page-size", + "5", +] +validator = "S3A.4" [test.api_checks] expect_agg_results = 1 @@ -673,13 +862,21 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "S3C.1" -group = "aggregation" -name = "S3C.1 nested rollup JSON sub_buckets" -title = "S3C.1 nested rollup JSON sub_buckets" -short_desc = "S3C.1 nested rollup JSON sub_buckets" -cli_args = ["*", "--limit", "0", "--agg", "rollup:drive,sub=terms:type,top=3", "--format", "json"] -validator = "S3C.1" +id = "S3C.1" +group = "aggregation" +name = "S3C.1 nested rollup JSON sub_buckets" +title = "S3C.1 nested rollup JSON sub_buckets" +short_desc = "S3C.1 nested rollup JSON sub_buckets" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:drive,sub=terms:type,top=3", + "--format", + "json", +] +validator = "S3C.1" [test.api_checks] expect_agg_results = 1 @@ -690,72 +887,120 @@ bucket_min_count = 1 # RPC response (CLI formatter handles sub-rows internally). [[test]] -id = "S3C.2" -group = "aggregation" -name = "S3C.2 nested rollup table rendering" -title = "S3C.2 nested rollup table rendering" -short_desc = "S3C.2 nested rollup table rendering" -cli_args = ["*", "--limit", "0", "--agg", "rollup:drive,sub=terms:type,top=3", "--format", "table"] +id = "S3C.2" +group = "aggregation" +name = "S3C.2 nested rollup table rendering" +title = "S3C.2 nested rollup table rendering" +short_desc = "S3C.2 nested rollup table rendering" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:drive,sub=terms:type,top=3", + "--format", + "table", +] stdout_contains = ["=== rollup ===", "Count", "Total Size"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["rollup"] [[test]] -id = "S3C.3" -group = "aggregation" -name = "S3C.3 nested rollup CSV" -title = "S3C.3 nested rollup CSV" -short_desc = "S3C.3 nested rollup CSV" -cli_args = ["*", "--limit", "0", "--agg", "rollup:drive,sub=terms:extension,top=3", "--format", "csv"] +id = "S3C.3" +group = "aggregation" +name = "S3C.3 nested rollup CSV" +title = "S3C.3 nested rollup CSV" +short_desc = "S3C.3 nested rollup CSV" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:drive,sub=terms:extension,top=3", + "--format", + "csv", +] stdout_contains = ["# rollup", "key,count,total_bytes"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["rollup"] [[test]] -id = "S3D.1" -group = "aggregation" -name = "S3D.1 truncated hint in table" -title = "S3D.1 truncated hint in table" -short_desc = "S3D.1 truncated hint in table" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=1", "--format", "table"] +id = "S3D.1" +group = "aggregation" +name = "S3D.1 truncated hint in table" +title = "S3D.1 truncated hint in table" +short_desc = "S3D.1 truncated hint in table" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=1", + "--format", + "table", +] stdout_contains = ["=== buckets ===", "more groups"] api_checks.expect_agg_results = 1 api_checks.bucket_min_count = 1 [[test]] -id = "S3D.2" -group = "aggregation" -name = "S3D.2 other_count in table" -title = "S3D.2 other_count in table" -short_desc = "S3D.2 other_count in table" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=2", "--format", "table"] +id = "S3D.2" +group = "aggregation" +name = "S3D.2 other_count in table" +title = "S3D.2 other_count in table" +short_desc = "S3D.2 other_count in table" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=2", + "--format", + "table", +] stdout_contains = ["=== buckets ===", "more groups"] api_checks.expect_agg_results = 1 api_checks.bucket_min_count = 2 [[test]] -id = "S3D.3" -group = "aggregation" -name = "S3D.3 JSON truncation metadata" -title = "S3D.3 JSON truncation metadata" -short_desc = "S3D.3 JSON truncation metadata" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=1", "--format", "json"] -validator = "S3D.3" +id = "S3D.3" +group = "aggregation" +name = "S3D.3 JSON truncation metadata" +title = "S3D.3 JSON truncation metadata" +short_desc = "S3D.3 JSON truncation metadata" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=1", + "--format", + "json", +] +validator = "S3D.3" [test.api_checks] expect_agg_results = 1 agg_kind_contains = ["buckets"] [[test]] -id = "S3D.4" -group = "aggregation" -name = "S3D.4 JSON exact flag" -title = "S3D.4 JSON exact flag" -short_desc = "S3D.4 JSON exact flag" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=5", "--format", "json"] -validator = "S3D.4" +id = "S3D.4" +group = "aggregation" +name = "S3D.4 JSON exact flag" +title = "S3D.4 JSON exact flag" +short_desc = "S3D.4 JSON exact flag" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=5", + "--format", + "json", +] +validator = "S3D.4" [test.api_checks] expect_agg_results = 1 @@ -763,25 +1008,41 @@ agg_kind_contains = ["buckets"] agg_exact = true [[test]] -id = "S3E.1" -group = "aggregation" -name = "S3E.1 CSV metadata comments" -title = "S3E.1 CSV metadata comments" -short_desc = "S3E.1 CSV metadata comments" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=1", "--format", "csv"] +id = "S3E.1" +group = "aggregation" +name = "S3E.1 CSV metadata comments" +title = "S3E.1 CSV metadata comments" +short_desc = "S3E.1 CSV metadata comments" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=1", + "--format", + "csv", +] stdout_contains = ["# buckets", "key,count,total_bytes", "# other_count="] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.bucket_min_count = 1 [[test]] -id = "S3G.1" -group = "aggregation" -name = "S3G.1 sample_sort desc JSON" -title = "S3G.1 sample_sort desc JSON" -short_desc = "S3G.1 sample_sort desc JSON" -cli_args = ["*", "--limit", "0", "--agg", "terms:drive,top=3,sample=3,sort=-size", "--format", "json"] -validator = "S3G.1" +id = "S3G.1" +group = "aggregation" +name = "S3G.1 sample_sort desc JSON" +title = "S3G.1 sample_sort desc JSON" +short_desc = "S3G.1 sample_sort desc JSON" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:drive,top=3,sample=3,sort=-size", + "--format", + "json", +] +validator = "S3G.1" [test.api_checks] expect_agg_results = 1 @@ -789,36 +1050,56 @@ agg_kind_contains = ["buckets"] bucket_has_samples = true [[test]] -id = "S3G.3" -group = "aggregation" -name = "S3G.3 --rows with --agg" -title = "S3G.3 --rows with --agg" -short_desc = "S3G.3 --rows with --agg" -cli_args = ["*.exe", "--limit", "5", "--agg", "count", "--rows"] -targets = ["cli"] +id = "S3G.3" +group = "aggregation" +name = "S3G.3 --rows with --agg" +title = "S3G.3 --rows with --agg" +short_desc = "S3G.3 --rows with --agg" +cli_args = ["*.exe", "--limit", "5", "--agg", "count", "--rows"] +targets = ["cli"] stdout_contains = ["# count"] expect_min_rows = 1 [[test]] -id = "S3G.7" -group = "aggregation" -name = "S3G.7 rollup path depth=3" -title = "S3G.7 rollup path depth=3" -short_desc = "S3G.7 rollup path depth=3" -cli_args = ["*", "--limit", "0", "--agg", "rollup:path,depth=3,top=5", "--format", "table"] +id = "S3G.7" +group = "aggregation" +name = "S3G.7 rollup path depth=3" +title = "S3G.7 rollup path depth=3" +short_desc = "S3G.7 rollup path depth=3" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "rollup:path,depth=3,top=5", + "--format", + "table", +] stdout_contains = ["=== rollup ===", "Count", "Total Size"] -cli_checks.expect_min_rows = 1 +cli_checks.expect_min_rows = 1 api_checks.expect_agg_results = 1 api_checks.agg_label_contains = ["rollup"] [[test]] -id = "S3G.8" -group = "aggregation" -name = "S3G.8 multiple --agg mixed" -title = "S3G.8 multiple --agg mixed" -short_desc = "S3G.8 multiple --agg mixed" -cli_args = ["*", "--limit", "0", "--agg", "count", "--agg", "terms:extension,top=3", "--agg", "stats:size", "--format", "json"] -validator = "S3G.8" +id = "S3G.8" +group = "aggregation" +name = "S3G.8 multiple --agg mixed" +title = "S3G.8 multiple --agg mixed" +short_desc = "S3G.8 multiple --agg mixed" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "count", + "--agg", + "terms:extension,top=3", + "--agg", + "stats:size", + "--format", + "json", +] +validator = "S3G.8" [test.api_checks] expect_agg_results = 3 @@ -827,14 +1108,22 @@ agg_kind_contains = ["count", "buckets", "stats"] # ── S3B: Facet Values (API-only, method not yet implemented) ───────────── [[test]] -id = "S3B.1" -group = "aggregation" -name = "S3B.1 facet_values paged (top 3)" -title = "Facet values via terms aggregation with top=3" -short_desc = "terms:extension top=3 — simulates facet_values pagination" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=3", "--format", "json"] -validator = "S3B.1" -targets = ["api"] +id = "S3B.1" +group = "aggregation" +name = "S3B.1 facet_values paged (top 3)" +title = "Facet values via terms aggregation with top=3" +short_desc = "terms:extension top=3 — simulates facet_values pagination" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=3", + "--format", + "json", +] +validator = "S3B.1" +targets = ["api"] [test.api_checks] expect_agg_results = 1 @@ -842,14 +1131,22 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "S3B.2" -group = "aggregation" -name = "S3B.2 facet_values all ext (top 10000)" -title = "Facet values via terms aggregation with top=10000" -short_desc = "terms:extension top=10000 — returns all extension facet values" -cli_args = ["*", "--limit", "0", "--agg", "terms:extension,top=10000", "--format", "json"] -validator = "S3B.2" -targets = ["api"] +id = "S3B.2" +group = "aggregation" +name = "S3B.2 facet_values all ext (top 10000)" +title = "Facet values via terms aggregation with top=10000" +short_desc = "terms:extension top=10000 — returns all extension facet values" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "terms:extension,top=10000", + "--format", + "json", +] +validator = "S3B.2" +targets = ["api"] [test.api_checks] expect_agg_results = 1 @@ -859,13 +1156,20 @@ bucket_min_count = 1 # ── S3misc: Miscellaneous aggregation tests ────────────────────────────── [[test]] -id = "S3misc.1" -group = "aggregation" -name = "S3misc.1 raw power syntax" -title = "Raw aggregation power syntax" -short_desc = "terms:extension,top=5,sample=2,sort=size via raw kind" -cli_args = ["*.exe", "--files-only", "--agg", "terms:extension,top=5,sample=2,sort=size", "--format", "json"] -validator = "S3misc.1" +id = "S3misc.1" +group = "aggregation" +name = "S3misc.1 raw power syntax" +title = "Raw aggregation power syntax" +short_desc = "terms:extension,top=5,sample=2,sort=size via raw kind" +cli_args = [ + "*.exe", + "--files-only", + "--agg", + "terms:extension,top=5,sample=2,sort=size", + "--format", + "json", +] +validator = "S3misc.1" [test.api_checks] expect_agg_results = 1 @@ -873,26 +1177,33 @@ agg_kind_contains = ["buckets"] bucket_has_samples = true [[test]] -id = "S3misc.2" -group = "aggregation" -name = "S3misc.2 count agg" -title = "Count aggregation" -short_desc = "Simple count aggregation" -cli_args = ["*.exe", "--files-only", "--agg", "count", "--format", "json"] -validator = "S3misc.2" +id = "S3misc.2" +group = "aggregation" +name = "S3misc.2 count agg" +title = "Count aggregation" +short_desc = "Simple count aggregation" +cli_args = ["*.exe", "--files-only", "--agg", "count", "--format", "json"] +validator = "S3misc.2" [test.api_checks] expect_agg_results = 1 agg_kind_contains = ["count"] [[test]] -id = "S3misc.3" -group = "aggregation" -name = "S3misc.3 include_rows=false" -title = "Aggregation-only query (no rows)" -short_desc = "include_rows=false returns only aggs, no rows" -cli_args = ["*", "--files-only", "--agg", "terms:extension,top=5", "--format", "json"] -validator = "S3misc.3" +id = "S3misc.3" +group = "aggregation" +name = "S3misc.3 include_rows=false" +title = "Aggregation-only query (no rows)" +short_desc = "include_rows=false returns only aggs, no rows" +cli_args = [ + "*", + "--files-only", + "--agg", + "terms:extension,top=5", + "--format", + "json", +] +validator = "S3misc.3" [test.api_checks] expect_agg_results = 1 @@ -900,13 +1211,22 @@ agg_kind_contains = ["buckets"] bucket_min_count = 1 [[test]] -id = "S3misc.4" -group = "aggregation" -name = "S3misc.4 include_rows=true" -title = "Aggregation with rows" -short_desc = "include_rows=true returns both rows and aggs" -cli_args = ["*.exe", "--files-only", "--limit", "5", "--agg", "terms:extension,top=3", "--format", "json"] -validator = "S3misc.4" +id = "S3misc.4" +group = "aggregation" +name = "S3misc.4 include_rows=true" +title = "Aggregation with rows" +short_desc = "include_rows=true returns both rows and aggs" +cli_args = [ + "*.exe", + "--files-only", + "--limit", + "5", + "--agg", + "terms:extension,top=3", + "--format", + "json", +] +validator = "S3misc.4" [test.api_checks] expect_agg_results = 1 @@ -915,13 +1235,21 @@ agg_kind_contains = ["buckets"] # ─── Stage 4C: Duplicate verification ──────────────────────────────────── [[test]] -id = "S4C.1" -group = "aggregation" -name = "S4C.1 duplicates verify=first_bytes JSON" -title = "Duplicates with first_bytes verification" -short_desc = "duplicates:size+name,verify=first_bytes produces JSON with verified field" -cli_args = ["*", "--limit", "0", "--agg", "duplicates:size+name,verify=first_bytes", "--format", "json"] -validator = "S4C.1" +id = "S4C.1" +group = "aggregation" +name = "S4C.1 duplicates verify=first_bytes JSON" +title = "Duplicates with first_bytes verification" +short_desc = "duplicates:size+name,verify=first_bytes produces JSON with verified field" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "duplicates:size+name,verify=first_bytes", + "--format", + "json", +] +validator = "S4C.1" [test.api_checks] expect_agg_results = 1 @@ -929,13 +1257,21 @@ agg_kind_contains = ["duplicates"] bucket_min_count = 1 [[test]] -id = "S4C.2" -group = "aggregation" -name = "S4C.2 duplicates verify=sha256 JSON" -title = "Duplicates with sha256 verification" -short_desc = "duplicates:size+name,verify=sha256 produces JSON with verified field" -cli_args = ["*", "--limit", "0", "--agg", "duplicates:size+name,verify=sha256", "--format", "json"] -validator = "S4C.2" +id = "S4C.2" +group = "aggregation" +name = "S4C.2 duplicates verify=sha256 JSON" +title = "Duplicates with sha256 verification" +short_desc = "duplicates:size+name,verify=sha256 produces JSON with verified field" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "duplicates:size+name,verify=sha256", + "--format", + "json", +] +validator = "S4C.2" [test.api_checks] expect_agg_results = 1 @@ -943,13 +1279,21 @@ agg_kind_contains = ["duplicates"] bucket_min_count = 1 [[test]] -id = "S4C.3" -group = "aggregation" -name = "S4C.3 duplicates verify=first_bytes,verify_bytes=8192" -title = "Duplicates with custom verify_bytes" -short_desc = "verify_bytes=8192 is accepted without error" -cli_args = ["*", "--limit", "0", "--agg", "duplicates:size+name,verify=first_bytes,verify_bytes=8192", "--format", "json"] -validator = "S4C.3" +id = "S4C.3" +group = "aggregation" +name = "S4C.3 duplicates verify=first_bytes,verify_bytes=8192" +title = "Duplicates with custom verify_bytes" +short_desc = "verify_bytes=8192 is accepted without error" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "duplicates:size+name,verify=first_bytes,verify_bytes=8192", + "--format", + "json", +] +validator = "S4C.3" [test.api_checks] expect_agg_results = 1 @@ -957,13 +1301,21 @@ agg_kind_contains = ["duplicates"] bucket_min_count = 1 [[test]] -id = "S4C.4" -group = "aggregation" -name = "S4C.4 duplicates no verify (default)" -title = "Duplicates without verification" -short_desc = "Default duplicates mode produces no verified field" -cli_args = ["*", "--limit", "0", "--agg", "duplicates:size+name", "--format", "json"] -validator = "S4C.4" +id = "S4C.4" +group = "aggregation" +name = "S4C.4 duplicates no verify (default)" +title = "Duplicates without verification" +short_desc = "Default duplicates mode produces no verified field" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "duplicates:size+name", + "--format", + "json", +] +validator = "S4C.4" [test.api_checks] expect_agg_results = 1 @@ -971,13 +1323,21 @@ agg_kind_contains = ["duplicates"] bucket_min_count = 1 [[test]] -id = "S4C.5" -group = "aggregation" -name = "S4C.5 duplicates verify=hash alias" -title = "verify=hash alias for sha256" -short_desc = "verify=hash is accepted as alias for sha256" -cli_args = ["*", "--limit", "0", "--agg", "duplicates:size+name,verify=hash", "--format", "json"] -validator = "S4C.5" +id = "S4C.5" +group = "aggregation" +name = "S4C.5 duplicates verify=hash alias" +title = "verify=hash alias for sha256" +short_desc = "verify=hash is accepted as alias for sha256" +cli_args = [ + "*", + "--limit", + "0", + "--agg", + "duplicates:size+name,verify=hash", + "--format", + "json", +] +validator = "S4C.5" [test.api_checks] expect_agg_results = 1 agg_kind_contains = ["duplicates"] @@ -986,4 +1346,3 @@ bucket_min_count = 1 # ═══════════════════════════════════════════════════════════════ # Coverage-gap tests: untested CLI flags # ═══════════════════════════════════════════════════════════════ - diff --git a/scripts/tests/definitions/10-error.toml b/scripts/tests/definitions/10-error.toml index bfcfffa6c..a5847398e 100644 --- a/scripts/tests/definitions/10-error.toml +++ b/scripts/tests/definitions/10-error.toml @@ -2,10 +2,10 @@ # Auto-split from test-definitions.toml [[test]] -id = "T40" -group = "error" -name = "T40 no results (graceful)" -title = "T40 no results (graceful)" +id = "T40" +group = "error" +name = "T40 no results (graceful)" +title = "T40 no results (graceful)" # A pattern that matches zero records must exit 0 with **empty # stdout** — no header, no CSV, nothing. This keeps pipelines # clean: `uffs | wc -l` prints `0` on no-match rather than @@ -13,8 +13,8 @@ title = "T40 no results (graceful)" # still checks `rows == []` and `total_count == 0`, pinning the # "no error" invariant the test was originally written to # enforce. -short_desc = "Nonsense pattern returns 0 rows, empty stdout, exit 0" -cli_args = ["xyzzy_nonexistent_file_pattern_12345", "--limit", "10"] +short_desc = "Nonsense pattern returns 0 rows, empty stdout, exit 0" +cli_args = ["xyzzy_nonexistent_file_pattern_12345", "--limit", "10"] expect_min_rows = 0 expect_max_rows = 0 expect_exit_code = 0 @@ -25,4 +25,3 @@ stdout_not_contains = ["\"Path\"", "Name,"] [test.api_checks] result_has_key = ["rows", "records_scanned"] total_count_min = 0 - diff --git a/scripts/tests/definitions/12-mcp.toml b/scripts/tests/definitions/12-mcp.toml index 901b936df..f09f64c4c 100644 --- a/scripts/tests/definitions/12-mcp.toml +++ b/scripts/tests/definitions/12-mcp.toml @@ -6,241 +6,241 @@ # ── Session bootstrap ──────────────────────────────────────────────────── [[test]] -id = "M100" -group = "mcp-session" -name = "M100 protocol version" -title = "Agent session: protocol = 2024-11-05" -short_desc = "MCP initialize returns protocolVersion 2024-11-05" -targets = ["mcp"] -mcp_method = "initialize" -mcp_checks = { path_equals = { "/protocolVersion" = "2024-11-05" } } - -[[test]] -id = "M101" -group = "mcp-session" -name = "M101 server name" -title = "Agent session: server identifies as uffs" -short_desc = "MCP initialize returns serverInfo.name = uffs" -targets = ["mcp"] -mcp_method = "initialize" -mcp_checks = { path_equals = { "/serverInfo/name" = "uffs" } } - -[[test]] -id = "M102" -group = "mcp-session" -name = "M102 tools capability" -title = "Agent session: server declares tools capability" -short_desc = "Initialize result has capabilities.tools" -targets = ["mcp"] -mcp_method = "initialize" -mcp_checks = { path_exists = ["/capabilities/tools"] } - -[[test]] -id = "M103" -group = "mcp-session" -name = "M103 resources capability" -title = "Agent session: server declares resources capability" -short_desc = "Initialize result has capabilities.resources" -targets = ["mcp"] -mcp_method = "initialize" -mcp_checks = { path_exists = ["/capabilities/resources"] } - -[[test]] -id = "M104" -group = "mcp-session" -name = "M104 prompts capability" -title = "Agent session: server declares prompts capability" -short_desc = "Initialize result has capabilities.prompts" -targets = ["mcp"] -mcp_method = "initialize" -mcp_checks = { path_exists = ["/capabilities/prompts"] } +id = "M100" +group = "mcp-session" +name = "M100 protocol version" +title = "Agent session: protocol = 2024-11-05" +short_desc = "MCP initialize returns protocolVersion 2024-11-05" +targets = ["mcp"] +mcp_method = "initialize" +mcp_checks = { path_equals = { "/protocolVersion" = "2024-11-05" } } + +[[test]] +id = "M101" +group = "mcp-session" +name = "M101 server name" +title = "Agent session: server identifies as uffs" +short_desc = "MCP initialize returns serverInfo.name = uffs" +targets = ["mcp"] +mcp_method = "initialize" +mcp_checks = { path_equals = { "/serverInfo/name" = "uffs" } } + +[[test]] +id = "M102" +group = "mcp-session" +name = "M102 tools capability" +title = "Agent session: server declares tools capability" +short_desc = "Initialize result has capabilities.tools" +targets = ["mcp"] +mcp_method = "initialize" +mcp_checks = { path_exists = ["/capabilities/tools"] } + +[[test]] +id = "M103" +group = "mcp-session" +name = "M103 resources capability" +title = "Agent session: server declares resources capability" +short_desc = "Initialize result has capabilities.resources" +targets = ["mcp"] +mcp_method = "initialize" +mcp_checks = { path_exists = ["/capabilities/resources"] } + +[[test]] +id = "M104" +group = "mcp-session" +name = "M104 prompts capability" +title = "Agent session: server declares prompts capability" +short_desc = "Initialize result has capabilities.prompts" +targets = ["mcp"] +mcp_method = "initialize" +mcp_checks = { path_exists = ["/capabilities/prompts"] } # ── Tool discovery ─────────────────────────────────────────────────────── [[test]] -id = "M110" -group = "mcp-discovery" -name = "M110 tools/list" -title = "Agent discovers tools" -short_desc = "tools/list returns a non-empty tool array" -targets = ["mcp"] -mcp_method = "tools/list" -mcp_checks = { path_exists = ["/tools"] } +id = "M110" +group = "mcp-discovery" +name = "M110 tools/list" +title = "Agent discovers tools" +short_desc = "tools/list returns a non-empty tool array" +targets = ["mcp"] +mcp_method = "tools/list" +mcp_checks = { path_exists = ["/tools"] } [[test]] -id = "M111" -group = "mcp-discovery" -name = "M111 tool inputSchema" -title = "Every tool has inputSchema for the agent" -short_desc = "First tool in tools/list has an inputSchema" -targets = ["mcp"] -mcp_method = "tools/list" -mcp_checks = { path_exists = ["/tools/0/inputSchema"] } +id = "M111" +group = "mcp-discovery" +name = "M111 tool inputSchema" +title = "Every tool has inputSchema for the agent" +short_desc = "First tool in tools/list has an inputSchema" +targets = ["mcp"] +mcp_method = "tools/list" +mcp_checks = { path_exists = ["/tools/0/inputSchema"] } [[test]] -id = "M112" -group = "mcp-discovery" -name = "M112 tool annotations" -title = "Every tool declares readOnlyHint annotation" -short_desc = "First tool has annotations object" -targets = ["mcp"] -mcp_method = "tools/list" -mcp_checks = { path_exists = ["/tools/0/annotations"] } +id = "M112" +group = "mcp-discovery" +name = "M112 tool annotations" +title = "Every tool declares readOnlyHint annotation" +short_desc = "First tool has annotations object" +targets = ["mcp"] +mcp_method = "tools/list" +mcp_checks = { path_exists = ["/tools/0/annotations"] } # ── Resource discovery & reads ─────────────────────────────────────────── [[test]] -id = "M200" -group = "mcp-resources" -name = "M200 resources/list" -title = "Agent discovers resources" -short_desc = "resources/list returns non-empty resource array" -targets = ["mcp"] -mcp_method = "resources/list" -mcp_checks = { path_exists = ["/resources"] } - -[[test]] -id = "M201" -group = "mcp-resources" -name = "M201 schema/fields" -title = "Agent reads field catalog" -short_desc = "resources/read uffs://schema/fields returns contents" -targets = ["mcp"] -mcp_method = "resources/read" -mcp_params = { uri = "uffs://schema/fields" } -mcp_checks = { path_exists = ["/contents"] } - -[[test]] -id = "M202" -group = "mcp-resources" -name = "M202 schema/search" -title = "Agent reads search schema" -short_desc = "resources/read uffs://schema/search returns contents" -targets = ["mcp"] -mcp_method = "resources/read" -mcp_params = { uri = "uffs://schema/search" } -mcp_checks = { path_exists = ["/contents"] } - - -[[test]] -id = "M204" -group = "mcp-resources" -name = "M204 presets/aggregate" -title = "Agent reads aggregate presets" -short_desc = "resources/read uffs://presets/aggregate returns contents" -targets = ["mcp"] -mcp_method = "resources/read" -mcp_params = { uri = "uffs://presets/aggregate" } -mcp_checks = { path_exists = ["/contents"] } - -[[test]] -id = "M205" -group = "mcp-resources" -name = "M205 drives resource" -title = "Agent reads drives resource" -short_desc = "resources/read uffs://drives returns contents" -targets = ["mcp"] -mcp_method = "resources/read" -mcp_params = { uri = "uffs://drives" } -mcp_checks = { path_exists = ["/contents"] } - -[[test]] -id = "M206" -group = "mcp-resources" -name = "M206 status resource" -title = "Agent reads status resource" -short_desc = "resources/read uffs://status returns contents" -targets = ["mcp"] -mcp_method = "resources/read" -mcp_params = { uri = "uffs://status" } -mcp_checks = { path_exists = ["/contents"] } +id = "M200" +group = "mcp-resources" +name = "M200 resources/list" +title = "Agent discovers resources" +short_desc = "resources/list returns non-empty resource array" +targets = ["mcp"] +mcp_method = "resources/list" +mcp_checks = { path_exists = ["/resources"] } + +[[test]] +id = "M201" +group = "mcp-resources" +name = "M201 schema/fields" +title = "Agent reads field catalog" +short_desc = "resources/read uffs://schema/fields returns contents" +targets = ["mcp"] +mcp_method = "resources/read" +mcp_params = { uri = "uffs://schema/fields" } +mcp_checks = { path_exists = ["/contents"] } + +[[test]] +id = "M202" +group = "mcp-resources" +name = "M202 schema/search" +title = "Agent reads search schema" +short_desc = "resources/read uffs://schema/search returns contents" +targets = ["mcp"] +mcp_method = "resources/read" +mcp_params = { uri = "uffs://schema/search" } +mcp_checks = { path_exists = ["/contents"] } + + +[[test]] +id = "M204" +group = "mcp-resources" +name = "M204 presets/aggregate" +title = "Agent reads aggregate presets" +short_desc = "resources/read uffs://presets/aggregate returns contents" +targets = ["mcp"] +mcp_method = "resources/read" +mcp_params = { uri = "uffs://presets/aggregate" } +mcp_checks = { path_exists = ["/contents"] } + +[[test]] +id = "M205" +group = "mcp-resources" +name = "M205 drives resource" +title = "Agent reads drives resource" +short_desc = "resources/read uffs://drives returns contents" +targets = ["mcp"] +mcp_method = "resources/read" +mcp_params = { uri = "uffs://drives" } +mcp_checks = { path_exists = ["/contents"] } + +[[test]] +id = "M206" +group = "mcp-resources" +name = "M206 status resource" +title = "Agent reads status resource" +short_desc = "resources/read uffs://status returns contents" +targets = ["mcp"] +mcp_method = "resources/read" +mcp_params = { uri = "uffs://status" } +mcp_checks = { path_exists = ["/contents"] } # ── Prompt discovery & usage ───────────────────────────────────────────── [[test]] -id = "M500" -group = "mcp-prompts" -name = "M500 prompts/list" -title = "Agent discovers prompts" -short_desc = "prompts/list returns non-empty prompt array" -targets = ["mcp"] -mcp_method = "prompts/list" -mcp_checks = { path_exists = ["/prompts"] } - -[[test]] -id = "M501" -group = "mcp-prompts" -name = "M501 find_large_files" -title = "Agent uses prompt: find_large_files" -short_desc = "prompts/get find_large_files returns messages" -targets = ["mcp"] -mcp_method = "prompts/get" -mcp_params = { name = "find_large_files" } -mcp_checks = { path_exists = ["/messages"] } - -[[test]] -id = "M502" -group = "mcp-prompts" -name = "M502 find_by_extension" -title = "Agent uses prompt: find_by_extension(pdf)" -short_desc = "prompts/get with extension argument returns messages" -targets = ["mcp"] -mcp_method = "prompts/get" -mcp_params = { name = "find_by_extension", arguments = { extension = "pdf" } } -mcp_checks = { path_exists = ["/messages"] } - -[[test]] -id = "M503" -group = "mcp-prompts" -name = "M503 disk_usage_report" -title = "Agent uses prompt: disk_usage_report" -short_desc = "prompts/get disk_usage_report returns messages" -targets = ["mcp"] -mcp_method = "prompts/get" -mcp_params = { name = "disk_usage_report" } -mcp_checks = { path_exists = ["/messages"] } - -[[test]] -id = "M504" -group = "mcp-prompts" -name = "M504 cleanup_report" -title = "Agent uses prompt: cleanup_report" -short_desc = "prompts/get cleanup_report returns messages" -targets = ["mcp"] -mcp_method = "prompts/get" -mcp_params = { name = "cleanup_report" } -mcp_checks = { path_exists = ["/messages"] } - -[[test]] -id = "M505" -group = "mcp-prompts" -name = "M505 recent_changes" -title = "Agent uses prompt: recent_changes" -short_desc = "prompts/get recent_changes returns messages" -targets = ["mcp"] -mcp_method = "prompts/get" -mcp_params = { name = "recent_changes" } -mcp_checks = { path_exists = ["/messages"] } - -[[test]] -id = "M506" -group = "mcp-prompts" -name = "M506 duplicate_investigation" -title = "Agent uses prompt: duplicate_investigation" -short_desc = "prompts/get duplicate_investigation returns messages" -targets = ["mcp"] -mcp_method = "prompts/get" -mcp_params = { name = "duplicate_investigation" } -mcp_checks = { path_exists = ["/messages"] } +id = "M500" +group = "mcp-prompts" +name = "M500 prompts/list" +title = "Agent discovers prompts" +short_desc = "prompts/list returns non-empty prompt array" +targets = ["mcp"] +mcp_method = "prompts/list" +mcp_checks = { path_exists = ["/prompts"] } + +[[test]] +id = "M501" +group = "mcp-prompts" +name = "M501 find_large_files" +title = "Agent uses prompt: find_large_files" +short_desc = "prompts/get find_large_files returns messages" +targets = ["mcp"] +mcp_method = "prompts/get" +mcp_params = { name = "find_large_files" } +mcp_checks = { path_exists = ["/messages"] } + +[[test]] +id = "M502" +group = "mcp-prompts" +name = "M502 find_by_extension" +title = "Agent uses prompt: find_by_extension(pdf)" +short_desc = "prompts/get with extension argument returns messages" +targets = ["mcp"] +mcp_method = "prompts/get" +mcp_params = { name = "find_by_extension", arguments = { extension = "pdf" } } +mcp_checks = { path_exists = ["/messages"] } + +[[test]] +id = "M503" +group = "mcp-prompts" +name = "M503 disk_usage_report" +title = "Agent uses prompt: disk_usage_report" +short_desc = "prompts/get disk_usage_report returns messages" +targets = ["mcp"] +mcp_method = "prompts/get" +mcp_params = { name = "disk_usage_report" } +mcp_checks = { path_exists = ["/messages"] } + +[[test]] +id = "M504" +group = "mcp-prompts" +name = "M504 cleanup_report" +title = "Agent uses prompt: cleanup_report" +short_desc = "prompts/get cleanup_report returns messages" +targets = ["mcp"] +mcp_method = "prompts/get" +mcp_params = { name = "cleanup_report" } +mcp_checks = { path_exists = ["/messages"] } + +[[test]] +id = "M505" +group = "mcp-prompts" +name = "M505 recent_changes" +title = "Agent uses prompt: recent_changes" +short_desc = "prompts/get recent_changes returns messages" +targets = ["mcp"] +mcp_method = "prompts/get" +mcp_params = { name = "recent_changes" } +mcp_checks = { path_exists = ["/messages"] } + +[[test]] +id = "M506" +group = "mcp-prompts" +name = "M506 duplicate_investigation" +title = "Agent uses prompt: duplicate_investigation" +short_desc = "prompts/get duplicate_investigation returns messages" +targets = ["mcp"] +mcp_method = "prompts/get" +mcp_params = { name = "duplicate_investigation" } +mcp_checks = { path_exists = ["/messages"] } # ── Protocol hygiene ───────────────────────────────────────────────────── [[test]] -id = "M700" -group = "mcp-hygiene" -name = "M700 unknown method" -title = "Unknown method returns error (not crash)" -short_desc = "Calling a nonexistent method returns JSON-RPC error" -targets = ["mcp"] -mcp_method = "this_method_does_not_exist" -mcp_checks = { expect_rpc_error = true } +id = "M700" +group = "mcp-hygiene" +name = "M700 unknown method" +title = "Unknown method returns error (not crash)" +short_desc = "Calling a nonexistent method returns JSON-RPC error" +targets = ["mcp"] +mcp_method = "this_method_does_not_exist" +mcp_checks = { expect_rpc_error = true } diff --git a/scripts/tests/definitions/14-shmem.toml b/scripts/tests/definitions/14-shmem.toml index 2c9ce4b33..4ad45bdb9 100644 --- a/scripts/tests/definitions/14-shmem.toml +++ b/scripts/tests/definitions/14-shmem.toml @@ -15,112 +15,112 @@ # ── S1: Basic shmem trigger (*.dll, all drives, no limit) ────────────── [[test]] -id = "S5A" -group = "shmem" -name = "S5A shmem *.dll no limit" -title = "shmem transport triggers for >100K results" -short_desc = "Search *.dll with no limit — should produce >100K results and use shmem" -long_desc = """ +id = "S5A" +group = "shmem" +name = "S5A shmem *.dll no limit" +title = "shmem transport triggers for >100K results" +short_desc = "Search *.dll with no limit — should produce >100K results and use shmem" +long_desc = """ Sends a search for *.dll with no explicit limit. The daemon's search engine returns all matching rows (typically 400K+). Since the count exceeds SHMEM_THRESHOLD (100K), the daemon writes to a memory-mapped file and the response contains shmem_path + shmem_count instead of inline rows.""" -targets = ["api"] -rpc_method = "search" -rpc_params = '{"pattern":"*.dll"}' +targets = ["api"] +rpc_method = "search" +rpc_params = '{"pattern":"*.dll"}' expect_min_rows = 0 -api_checks.result_has_key = ["shmem_path", "shmem_count"] +api_checks.result_has_key = ["shmem_path", "shmem_count"] api_checks.shmem_count_min = 100000 # ── S2: Shmem with drive filter (still >100K) ───────────────────────── [[test]] -id = "S5B" -group = "shmem" -name = "S5B shmem *.dll drive C" -title = "shmem transport with drive filter" -short_desc = "Search *.dll on drive C only — should still exceed shmem threshold" -long_desc = """ +id = "S5B" +group = "shmem" +name = "S5B shmem *.dll drive C" +title = "shmem transport with drive filter" +short_desc = "Search *.dll on drive C only — should still exceed shmem threshold" +long_desc = """ Filters to drive C only. With ~3.4M records on C:, *.dll should still return well over 100K results and trigger shmem.""" -targets = ["api"] -rpc_method = "search" -rpc_params = '{"pattern":"*.dll","drives":["C"]}' +targets = ["api"] +rpc_method = "search" +rpc_params = '{"pattern":"*.dll","drives":["C"]}' expect_min_rows = 0 -api_checks.result_has_key = ["shmem_path", "shmem_count"] +api_checks.result_has_key = ["shmem_path", "shmem_count"] api_checks.shmem_count_min = 10000 # ── S3: Below threshold — shmem NOT used ─────────────────────────────── [[test]] -id = "S5C" -group = "shmem" -name = "S5C no-shmem small result" -title = "small result set uses inline rows (no shmem)" -short_desc = "Search with limit=50 — should return inline rows, no shmem_path" -long_desc = """ +id = "S5C" +group = "shmem" +name = "S5C no-shmem small result" +title = "small result set uses inline rows (no shmem)" +short_desc = "Search with limit=50 — should return inline rows, no shmem_path" +long_desc = """ Sends a search with explicit limit=50. The result set is well under SHMEM_THRESHOLD so the daemon returns inline rows. Verifies that shmem_path is NOT present in the response.""" -targets = ["api"] -rpc_method = "search" -rpc_params = '{"pattern":"*.dll","limit":50}' +targets = ["api"] +rpc_method = "search" +rpc_params = '{"pattern":"*.dll","limit":50}' expect_min_rows = 50 expect_max_rows = 50 -validator = "no_shmem" +validator = "no_shmem" # ── S4: Shmem with broad pattern (all files) ────────────────────────── [[test]] -id = "S5D" -group = "shmem" -name = "S5D shmem match-all drive G" -title = "shmem with match-all on small drive" -short_desc = "Search * on drive G (~15K records) — no shmem expected" -long_desc = """ +id = "S5D" +group = "shmem" +name = "S5D shmem match-all drive G" +title = "shmem with match-all on small drive" +short_desc = "Search * on drive G (~15K records) — no shmem expected" +long_desc = """ Drive G has only ~15K records, so a match-all search returns <100K rows and should NOT trigger shmem. Verifies the threshold boundary.""" -targets = ["api"] -rpc_method = "search" -rpc_params = '{"pattern":"*","drives":["G"],"limit":20000}' +targets = ["api"] +rpc_method = "search" +rpc_params = '{"pattern":"*","drives":["G"],"limit":20000}' expect_min_rows = 100 -validator = "no_shmem" +validator = "no_shmem" # ── S5: CLI large search (shmem transparent) ────────────────────────── [[test]] -id = "S5E" -group = "shmem" -name = "S5E cli large search" -title = "CLI large search — shmem transparent" -short_desc = "CLI search *.xml with high limit works correctly via shmem" -long_desc = """ +id = "S5E" +group = "shmem" +name = "S5E cli large search" +title = "CLI large search — shmem transparent" +short_desc = "CLI search *.xml with high limit works correctly via shmem" +long_desc = """ The CLI uses UffsClient which reads shmem transparently. This test sends a CLI search that should exceed the shmem threshold and verifies the output is correct. The CLI should report the full result count.""" -targets = ["cli"] -cli_args = ["*.xml", "--format", "csv"] -cli_format = "csv" +targets = ["cli"] +cli_args = ["*.xml", "--format", "csv"] +cli_format = "csv" expect_min_rows = 1000 stdout_contains = ["Name"] # ── S6: Shmem file cleanup verification ─────────────────────────────── [[test]] -id = "S5F" -group = "shmem" -name = "S5F shmem cleanup" -title = "shmem file is deleted after client reads it" -short_desc = "After reading shmem results, the file should be cleaned up" -long_desc = """ +id = "S5F" +group = "shmem" +name = "S5F shmem cleanup" +title = "shmem file is deleted after client reads it" +short_desc = "After reading shmem results, the file should be cleaned up" +long_desc = """ This is validated automatically by the S5A test — the api_checks validator reads the shmem file header and then deletes it. If the file was already deleted (by a concurrent reader), the test reports which layer consumed it.""" -targets = ["api"] -rpc_method = "search" -rpc_params = '{"pattern":"*.json"}' +targets = ["api"] +rpc_method = "search" +rpc_params = '{"pattern":"*.json"}' expect_min_rows = 0 -api_checks.result_has_key = ["shmem_path", "shmem_count"] +api_checks.result_has_key = ["shmem_path", "shmem_count"] api_checks.shmem_count_min = 100000