From 85bfece875ed5ada1d2158cd0e7606d6060d066e Mon Sep 17 00:00:00 2001 From: timoheimonen Date: Fri, 25 Sep 2026 20:49:41 +0300 Subject: [PATCH] Prune redundant tests and remove unused helpers --- documents/LATENCY_WHITEPAPER.md | 18 +- documents/LLM_MEMORY_PROFILE_WHITEPAPER.md | 3 +- documents/PROJECT_STRUCTURE.md | 11 +- documents/TECHNICAL_SPECIFICATION.md | 2 +- src/benchmark/benchmark_executor.cpp | 15 +- src/benchmark/benchmark_executor.h | 1 - src/core/config/constants.h | 2 - src/core/memory/memory_utils.cpp | 50 +- src/core/memory/memory_utils.h | 17 +- src/llm_memory/llm_kv_layout.cpp | 216 -- src/llm_memory/llm_kv_layout.h | 87 - src/llm_memory/llm_metal_backend.h | 11 - src/llm_memory/llm_metal_backend.mm | 67 - src/llm_memory/llm_runner.cpp | 21 - src/llm_memory/llm_runner.h | 2 - src/llm_memory/llm_work_plan.cpp | 38 - src/llm_memory/llm_work_plan.h | 24 - src/output/console/messages/messages_api.h | 2 - .../console/messages/results_messages.cpp | 5 - src/utils/json_utils.cpp | 78 +- src/utils/json_utils.h | 30 +- tests/test_analysis.cpp | 63 +- tests/test_benchmark_executor.cpp | 39 - tests/test_benchmark_runner.cpp | 210 +- tests/test_benchmark_work_plan.cpp | 38 +- tests/test_buffer_manager.cpp | 85 +- tests/test_config.cpp | 1103 +++------ tests/test_core_to_core_cli.cpp | 143 +- tests/test_core_to_core_runner.cpp | 67 - tests/test_descriptive_statistics.cpp | 17 - tests/test_executable_cli.cpp | 1007 ++------- tests/test_gpu_bandwidth.cpp | 408 +--- tests/test_gpu_metal_backend.cpp | 3 +- tests/test_gpu_work_plan.cpp | 114 +- tests/test_hash_utils.cpp | 82 +- tests/test_json_output_session.cpp | 236 +- tests/test_json_schema.cpp | 298 +-- tests/test_json_utils.cpp | 73 +- tests/test_llm_memory_config.cpp | 538 +---- tests/test_llm_memory_contract.cpp | 1699 -------------- tests/test_llm_memory_executor.cpp | 423 +--- tests/test_llm_memory_json.cpp | 932 +++----- tests/test_llm_memory_kernels.cpp | 613 ++--- tests/test_llm_memory_output.cpp | 303 +-- tests/test_llm_memory_runner.cpp | 2013 +++++------------ tests/test_llm_memory_work_plan.cpp | 1941 ++++------------ tests/test_llm_metal_backend.cpp | 1664 +++----------- tests/test_memory_manager.cpp | 15 - tests/test_memory_utils.cpp | 148 +- tests/test_messages.cpp | 1280 +++-------- tests/test_mode_selector.cpp | 87 +- tests/test_numeric_utils.cpp | 13 - tests/test_pattern_benchmark.cpp | 406 +--- tests/test_pattern_validation.cpp | 29 +- tests/test_pattern_work_plan.cpp | 110 +- tests/test_script_examples.py | 45 +- tests/test_standard_kernels.cpp | 156 +- tests/test_statistics.cpp | 57 +- tests/test_statistics_renderer.cpp | 16 +- tests/test_sweep_runner.cpp | 563 +---- tests/test_system_info.cpp | 62 +- tests/test_timer.cpp | 11 - tests/test_tlb_chain.cpp | 141 +- tests/test_tlb_measurement_scheduler.cpp | 95 +- tests/test_tlb_sweep_planner.cpp | 6 - tests/test_utils.cpp | 24 - 66 files changed, 3751 insertions(+), 14325 deletions(-) delete mode 100644 tests/test_llm_memory_contract.cpp diff --git a/documents/LATENCY_WHITEPAPER.md b/documents/LATENCY_WHITEPAPER.md index 797d3ca..ceb5371 100644 --- a/documents/LATENCY_WHITEPAPER.md +++ b/documents/LATENCY_WHITEPAPER.md @@ -134,20 +134,12 @@ so stride must be a multiple of 8. The following behaviors are explicitly covered by unit tests: -- **Null buffers, zero/misaligned strides, and one-node buffers are rejected with exact reasons** - (`SetupLatencyChainRejectsInvalidInputsWithExactReasons`) -- **The minimum two-node chain is accepted at pointer-sized and default strides** - (`SetupLatencyChainAcceptsMinimumTwoNodeChainsAtSupportedStrides`) -- **A constructed chain visits every aligned in-buffer node exactly once and returns to its head** - (`SetupLatencyChainCreatesValidChain`) -- **Pointer count, touched-page count, page size, and stride diagnostics are exact** - (`SetupLatencyChainCollectsDiagnostics`) -- **A locality window that cannot contain two stride-spaced nodes is rejected** - (`SetupLatencyChainWithTooSmallTlbLocalityFails`) -- **A locality-using box mode with a zero locality window is rejected** - (`SetupLatencyChainWithBoxModeAndZeroLocalityFails`) +- **Null buffers, zero/misaligned strides, one-node buffers, unusable locality windows, and box modes without + locality are rejected with exact reasons** (`SetupLatencyChainRejectsInvalidInputsWithExactReasons`). +- **Two-node and four-node chains at pointer-sized and default strides visit every aligned in-buffer node exactly + once and return to the head** (`SetupLatencyChainCreatesValidChain`). - **Equal explicit seeds reproduce the chain, different seeds change it, and same-random box mode reuses one - permutation across boxes** (`ExplicitSeedsAreReproducibleAndSameRandomModeReusesBoxPermutation`) + permutation across boxes** (`ExplicitSeedsAreReproducibleAndSameRandomModeReusesBoxPermutation`). These tests validate setup correctness and failure handling before timing is run. diff --git a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md index 93588b0..e411b83 100644 --- a/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md +++ b/documents/LLM_MEMORY_PROFILE_WHITEPAPER.md @@ -793,8 +793,7 @@ The maintained examples are in [CPU kernel tests](../tests/test_llm_memory_kerne [Metal helper tests](../tests/test_llm_metal_checksum.mm) for `SharedAffineLanesExposeBoundedContentCollisions`. The [Metal backend tests](../tests/test_llm_metal_backend.cpp) contain -`DecodePagedPermutationAndPaddingHooksAreDetectedIntegration` and -`PrefillPagedPermutationAndPaddingHooksAreDetectedIntegration`. Their real table/blit mutations differ from +`PagedPermutationAndPaddingHooksAreDetectedIntegration`, which covers decode and prefill. Its real table/blit mutations differ from `force_timed_checksum_mismatch` (host readback altered after execution) and `force_kv_write_validation_mismatch` (forced validation boolean), which test result handling. Pure independent checksum-oracle goldens test arithmetic; they are not GPU execution evidence. Algorithms and their existing identities remain unchanged by this fault model. diff --git a/documents/PROJECT_STRUCTURE.md b/documents/PROJECT_STRUCTURE.md index ae7bbe4..0988842 100644 --- a/documents/PROJECT_STRUCTURE.md +++ b/documents/PROJECT_STRUCTURE.md @@ -195,7 +195,7 @@ Core infrastructure for configuration, memory management, macOS system introspec | `buffer_manager.h` | Defines the phase-local `BenchmarkBuffers` and command-local `PatternBuffers` RAII owners | | `buffer_allocator.h` / `.cpp` | Atomically allocates the pattern source/destination pair and performs overflow-safe peak-concurrent memory accounting | | `buffer_initializer.h` / `.cpp` | Initializes deterministic pattern source data and the zeroed pattern destination before execution | -| `memory_utils.h` / `.cpp` | Cache-line alignment helpers, deterministic/unseeded latency-chain construction and diagnostics, and basic buffer initialization | +| `memory_utils.h` / `.cpp` | Cache-line alignment helpers, deterministic/unseeded latency-chain construction, and basic buffer initialization | #### src/core/system/ @@ -375,12 +375,11 @@ installed. All test `.cpp` and `.mm` files are picked up automatically by the Ma |---|---|---| | `test_config.cpp` | `ConfigTest` | Strict whole-token CLI/sweep parsing, validation, defaults, and derived buffer/access math | | `test_signal_handler.cpp` | `BenchmarkSignalMaskGuardTest` | Exact thread-mask restoration, caller-preserved blocked signals, and nested scope ownership | -| `test_messages.cpp` | `Messages*Test` | Console message string functions across all categories | +| `test_messages.cpp` | `Messages*Test` | Exact help and formatted console message contracts | | `test_memory_utils.cpp` | `MemoryUtilsTest` | Memory helpers: pointer-chase chain construction and verification | | `test_memory_manager.cpp` | `MemoryManagerTest` | Injected mmap/madvise policy, failures, and exact RAII unmapping | | `test_numeric_utils.cpp` | `NumericUtilsTest` | Overflow-safe arithmetic, duration calibration, pilot counts, and quantization boundaries | -| `test_llm_memory_contract.cpp` | `LlmMemoryContractTest` | Independent executable specification for all eight decode/prefill backend/layout methodology identities, payloads, paged geometry/lookup goldens, append and full-prompt writes, checksum/event-trace identities, descriptor ABI layouts, and schema acceptance; it does not exercise production LLM code | -| `test_llm_memory_config.cpp` | `LlmMemoryConfigTest` | Config/status defaults, exact standalone decode/prefill parsing including paged prefill, phase/layout rules and stable reasons, strict decimal errors, raw output values, help isolation, worker/seed resolution, geometry/work-limit preflight, and incompatible options | +| `test_llm_memory_config.cpp` | `LlmMemoryConfigTest` | Exact standalone decode/prefill parsing including paged prefill, phase/layout rules and stable reasons, strict decimal errors, raw output values, help isolation, worker/seed resolution, geometry/work-limit preflight, and incompatible options | | `test_llm_memory_work_plan.cpp` | `LlmMemoryWorkPlanTest` | Production checked decode/prefill geometry and payloads, physical/padding/table/lookup/accounted math including paged-prefill `N+2*M`, deterministic permutation/hash, CPU token/block ownership, Metal cyclic block-owner scheduling and component identities, memory budget, descriptor/range/layout invariants, worker reduction, scenario caps/calibration, and cyclic order | | `test_llm_memory_executor.cpp` | `LlmMemoryExecutorTest` | CPU phase/layout dispatch, atomic mapping/table-preparation failure seams, contiguous and paged decode/prefill descriptor materialization, independent checksums, padding canaries, synchronized timing, QoS, cancellation, and fake-kernel validation | | `test_llm_memory_runner.cpp` | `LlmMemoryRunnerTest` | Fake-backend lifecycle and generic decode/prefill task seams, lifecycle unsupported/failure handling, common identity/timing/completion/validation acceptance, exact calibration/single-unit/freeze/frozen-warmup order, cyclic measurements, status/counter/aggregate semantics, interruption, release/checkpoint precedence, auxiliary budgeting, and runner exception boundaries | @@ -388,7 +387,7 @@ installed. All test `.cpp` and `.mm` files are picked up automatically by the Ma | `test_llm_metal_checksum.mm` | `LlmMetalChecksumHelperIntegrationTest` | Real-device checks of shared MSL affine checksum collision boundaries and six-component modulo reduction across partial SIMD/threadgroups | | `test_llm_memory_output.cpp` | `LlmMemoryOutputTest` | Exact decode/prefill, contiguous/paged, and Metal device/resource/table/task K/V-write/lookup/padding plus cyclic owner/per-threadgroup cost formatting, interpretation text, and deduplicated warnings | | `test_llm_memory_kernels.cpp` | `LlmMemoryKernelIntegrationTest` | Real contiguous/paged decode/prefill ARM64 descriptor/kernel scenarios, paged tails/lookups, full-prompt writes, exact partial tiled scans, checksum, padding, worker-count, and AAPCS64 coverage | -| `test_llm_metal_backend.cpp` | `LlmMetalBackendTest`, `LlmMetalBackendIntegrationTest`, `LlmMetalBackendFailureInjectionIntegrationTest` | Pure capability, contiguous/paged segmentation, argument-buffer, decode/prefill ABIs, source-hash and prefill loop-order audit, phase-aware checksum/reason mapping, paged-prefill `N+2*M`, cyclic no-duplication ownership and per-threadgroup accounted-byte evidence, plus real-device selected-profile compilation, Tier-2 slots, private table/K/V initialization, exact-tail decode, full-prompt/tiled-prefix prefill, final-ordinal write/padding detection, interruption/failure cleanup, and idempotent release | +| `test_llm_metal_backend.cpp` | `LlmMetalBackendTest`, `LlmMetalBackendIntegrationTest`, `LlmMetalBackendFailureInjectionIntegrationTest` | Pure capability, contiguous/paged segmentation, argument-buffer, decode/prefill ABIs, source provenance and prefill loop-order audit, phase-aware checksum/reason mapping, paged-prefill `N+2*M`, cyclic no-duplication ownership and per-threadgroup accounted-byte evidence, plus real-device selected-profile compilation, Tier-2 slots, private table/K/V initialization, exact-tail decode, full-prompt/tiled-prefix prefill, final-ordinal write/padding detection, interruption/failure cleanup, and idempotent release | | `test_buffer_manager.cpp` | `BufferManagerTest` | Pattern mapping policy, atomic allocation cleanup, initialized content, validation, and peak accounting | | `test_benchmark_executor.cpp` | `BenchmarkExecutorTest` | Injected phase/chain failures, continuous latency sampling, and hardware executor contracts | | `test_benchmark_runner.cpp` | `BenchmarkStatisticsCollectorTest`, `BenchmarkRunnerTest` | Status-bearing aggregation, schema-3 retained snapshots, checkpointing, interruption, and runner exception/failure seams | @@ -403,7 +402,7 @@ installed. All test `.cpp` and `.mm` files are picked up automatically by the Ma | `test_llm_result_verifier.py` | Independent LLM verifier | Eight current profiles, arithmetic goldens, status/population and semantic mutations, bounded input and build provenance | | `test_script_examples.py` | Python script-example entry test | Compatible standard schema-3/methodology inputs, provenance-version handling, completion and field-shape checks, and metric extraction for the bundled standard-memory examples, including the optional jq-backed shell path when jq is installed | | `test_json_output_session.cpp` | `JsonOutputTargetTest`, `JsonOutputSessionTest` | Exact sentinel/path classification, lazy checkpoint dispatch, atomic-file parity, stdout routing/restoration, and stream failure containment | -| `test_json_utils.cpp` | `JsonUtilsTest`, `JsonFileWriterTest` | JSON parse/statistics and atomic writer success/failure contracts | +| `test_json_utils.cpp` | `JsonUtilsTest`, `JsonFileWriterTest` | UTC timestamp/statistics and atomic writer success/failure contracts | | `test_output_printer.cpp` | `OutputPrinterTest`, `OutputPrinterCustomCacheUnitsTest` | Status-aware partial output, mode/cache composition, and custom-cache size-unit boundaries | | `test_sweep_runner.cpp` | `SweepRunnerTest` | Current-only nested standard schema-3 completion classification, complete/partial/interrupted/failed attempt accounting, schema-1 envelopes, and file/lazy-stdout checkpoint behavior | | `test_sweep_utils.cpp` | `SweepUtilsTest` | Shared sweep parsing, empty-dimension behavior, and overflow-safe Cartesian counts | diff --git a/documents/TECHNICAL_SPECIFICATION.md b/documents/TECHNICAL_SPECIFICATION.md index e30bf4c..8df40ea 100644 --- a/documents/TECHNICAL_SPECIFICATION.md +++ b/documents/TECHNICAL_SPECIFICATION.md @@ -1482,7 +1482,7 @@ Recommended validation commands: - GPU help smoke check: `./memory_benchmark --gpu-bandwidth --help` - LLM help smoke check: `./memory_benchmark --llm-memory --help` - Deterministic LLM tests: - `./test_runner '--gtest_filter=ModeSelectorTest.*:LlmMemoryContractTest.*:LlmMemoryConfigTest.*:LlmMemoryWorkPlanTest.*:LlmMemoryExecutorTest.*:LlmMemoryRunnerTest.*:LlmMemoryJsonTest.*:LlmMemoryOutputTest.*:MessagesTest.*'` + `./test_runner '--gtest_filter=ModeSelectorTest.*:LlmMemoryConfigTest.*:LlmMemoryWorkPlanTest.*:LlmMemoryExecutorTest.*:LlmMemoryRunnerTest.*:LlmMemoryJsonTest.*:LlmMemoryOutputTest.*:Messages*Test.*'` - Real LLM ARM64 and bounded executable transport contracts: `./test_runner '--gtest_filter=*LlmMemory*Integration*:*ExecutableCliIntegration*'` - LLM Metal decode and prefill for contiguous and paged KV: diff --git a/src/benchmark/benchmark_executor.cpp b/src/benchmark/benchmark_executor.cpp index b87fc1d..1ac9a6b 100644 --- a/src/benchmark/benchmark_executor.cpp +++ b/src/benchmark/benchmark_executor.cpp @@ -287,7 +287,6 @@ int prepare_cache_latency_buffers(BenchmarkConfig& config, BenchmarkBuffers& buf config.custom_buffer_size, config.latency_stride_bytes, config.latency_tlb_locality_bytes, - nullptr, config.latency_chain_mode, derive_benchmark_seed(config.benchmark_seed, kSeedDomainCustomLatency)); @@ -303,7 +302,6 @@ int prepare_cache_latency_buffers(BenchmarkConfig& config, BenchmarkBuffers& buf config.l1_buffer_size, config.latency_stride_bytes, config.latency_tlb_locality_bytes, - nullptr, config.latency_chain_mode, derive_benchmark_seed(config.benchmark_seed, kSeedDomainL1Latency)) != EXIT_SUCCESS) { @@ -321,7 +319,6 @@ int prepare_cache_latency_buffers(BenchmarkConfig& config, BenchmarkBuffers& buf config.l2_buffer_size, config.latency_stride_bytes, config.latency_tlb_locality_bytes, - nullptr, config.latency_chain_mode, derive_benchmark_seed(config.benchmark_seed, kSeedDomainL2Latency)) != EXIT_SUCCESS) { @@ -352,7 +349,6 @@ int prepare_main_memory_latency_buffer(BenchmarkConfig& config, BenchmarkBuffers config.buffer_size, config.latency_stride_bytes, config.latency_tlb_locality_bytes, - nullptr, config.latency_chain_mode, derive_benchmark_seed(config.benchmark_seed, kSeedDomainMainLatency)); @@ -443,8 +439,7 @@ void run_paired_locality_comparison(void* buffer, const LatencyChainMode mode = use_locality ? LatencyChainMode::RandomInBoxRandomBox : LatencyChainMode::GlobalRandom; - if (setup_latency_chain(buffer, buffer_size, stride_bytes, locality_bytes, - nullptr, mode, seed) != EXIT_SUCCESS) { + if (setup_latency_chain(buffer, buffer_size, stride_bytes, locality_bytes, mode, seed) != EXIT_SUCCESS) { return false; } show_progress(); @@ -1047,14 +1042,6 @@ BenchmarkResults run_single_benchmark_loop(BenchmarkConfig& config, throw std::runtime_error( Messages::benchmark_reason_prepare_failed(enabled_phase.name)); } - if (test_hooks != nullptr && test_hooks->fail_latency_chain_setup && - (enabled_phase.phase == Phase::CacheLatency || - enabled_phase.phase == Phase::MainLatency) && - test_hooks->fail_latency_chain_setup(enabled_phase.name)) { - throw std::runtime_error( - Messages::benchmark_reason_latency_chain_setup_failed( - enabled_phase.name)); - } switch (enabled_phase.phase) { case Phase::MainBandwidth: { BenchmarkBuffers phase_buffers; diff --git a/src/benchmark/benchmark_executor.h b/src/benchmark/benchmark_executor.h index ff41088..7c644c9 100644 --- a/src/benchmark/benchmark_executor.h +++ b/src/benchmark/benchmark_executor.h @@ -38,7 +38,6 @@ struct BenchmarkExecutionState; /** @brief Optional kernel-free fault seams for phase preparation tests. */ struct BenchmarkExecutorTestHooks { std::function fail_phase_preparation; - std::function fail_latency_chain_setup; }; /** diff --git a/src/core/config/constants.h b/src/core/config/constants.h index a311264..8d5f47c 100644 --- a/src/core/config/constants.h +++ b/src/core/config/constants.h @@ -259,8 +259,6 @@ namespace Constants { "llm-kv-layout-geometry-v1"; constexpr const char* LLM_KV_LAYOUT_PLAN_IDENTITY_VERSION = "llm-kv-layout-plan-v1"; - constexpr const char* LLM_PAGED_DECODE_WORKLOAD_IDENTITY_VERSION = - "llm-paged-decode-workload-v1"; constexpr const char* LLM_PAGED_CPU_EXECUTION_IDENTITY_VERSION = "llm-paged-cpu-execution-v1"; constexpr const char* LLM_PAGED_METAL_EXECUTION_IDENTITY_VERSION = diff --git a/src/core/memory/memory_utils.cpp b/src/core/memory/memory_utils.cpp index 632f0b8..2c8145f 100644 --- a/src/core/memory/memory_utils.cpp +++ b/src/core/memory/memory_utils.cpp @@ -24,7 +24,6 @@ */ #include "core/memory/memory_utils.h" -#include "core/system/page_size.h" #include "output/console/messages/messages_api.h" #include #include @@ -244,17 +243,9 @@ bool latency_chain_mode_uses_locality(LatencyChainMode mode) { */ static int setup_latency_chain_impl(void *buffer, size_t buffer_size, size_t stride, size_t tlb_locality_bytes, - LatencyChainDiagnostics* diagnostics, LatencyChainMode mode, std::optional deterministic_seed) { - if (diagnostics != nullptr) { - diagnostics->pointer_count = 0; - diagnostics->unique_pages_touched = 0; - diagnostics->page_size_bytes = 0; - diagnostics->stride_bytes = stride; - } - // Validate input parameters if (buffer == nullptr) { std::cerr << Messages::error_prefix() << Messages::error_buffer_pointer_null_latency_chain() << std::endl; @@ -282,20 +273,6 @@ static int setup_latency_chain_impl(void *buffer, size_t buffer_size, size_t str return EXIT_FAILURE; } - const size_t page_size = test_hooks_active - ? active_test_hooks.page_size_bytes - : get_system_page_size_bytes(); - const bool collect_page_diagnostics = (diagnostics != nullptr && page_size > 0); - std::vector page_seen; - size_t unique_pages_touched = 0; - size_t base_page_offset = 0; - if (collect_page_diagnostics) { - base_page_offset = reinterpret_cast(buffer) % page_size; - const size_t span_with_offset = base_page_offset + buffer_size; - const size_t page_count = (span_with_offset + page_size - 1) / page_size; - page_seen.assign(page_count, 0); - } - const LatencyChainMode effective_mode = resolve_latency_chain_mode(mode, tlb_locality_bytes); if (latency_chain_mode_uses_locality(effective_mode) && tlb_locality_bytes == 0) { @@ -358,18 +335,6 @@ static int setup_latency_chain_impl(void *buffer, size_t buffer_size, size_t str return EXIT_FAILURE; } - if (collect_page_diagnostics) { - const size_t page_index = (base_page_offset + current_offset) / page_size; - if (page_index >= page_seen.size()) { - std::cerr << Messages::error_prefix() << Messages::error_offset_exceeds_bounds(page_index, page_seen.size() - 1) << std::endl; - return EXIT_FAILURE; - } - if (page_seen[page_index] == 0) { - page_seen[page_index] = 1; - ++unique_pages_touched; - } - } - uintptr_t *current_loc = (uintptr_t *)(base_ptr + current_offset); // Calculate the memory address of the *next* element in the shuffled sequence. @@ -389,31 +354,20 @@ static int setup_latency_chain_impl(void *buffer, size_t buffer_size, size_t str *current_loc = next_addr; } - if (diagnostics != nullptr) { - diagnostics->pointer_count = num_pointers; - diagnostics->unique_pages_touched = unique_pages_touched; - diagnostics->page_size_bytes = page_size; - diagnostics->stride_bytes = stride; - } - return EXIT_SUCCESS; } int setup_latency_chain(void* buffer, size_t buffer_size, size_t stride, size_t tlb_locality_bytes, - LatencyChainDiagnostics* diagnostics, LatencyChainMode mode) { - return setup_latency_chain_impl(buffer, buffer_size, stride, tlb_locality_bytes, - diagnostics, mode, std::nullopt); + return setup_latency_chain_impl(buffer, buffer_size, stride, tlb_locality_bytes, mode, std::nullopt); } int setup_latency_chain(void* buffer, size_t buffer_size, size_t stride, size_t tlb_locality_bytes, - LatencyChainDiagnostics* diagnostics, LatencyChainMode mode, uint64_t deterministic_seed) { - return setup_latency_chain_impl(buffer, buffer_size, stride, tlb_locality_bytes, - diagnostics, mode, deterministic_seed); + return setup_latency_chain_impl(buffer, buffer_size, stride, tlb_locality_bytes, mode, deterministic_seed); } /** diff --git a/src/core/memory/memory_utils.h b/src/core/memory/memory_utils.h index 6069a0e..ec23234 100644 --- a/src/core/memory/memory_utils.h +++ b/src/core/memory/memory_utils.h @@ -73,20 +73,8 @@ LatencyChainMode resolve_latency_chain_mode(LatencyChainMode mode, size_t tlb_lo */ bool latency_chain_mode_uses_locality(LatencyChainMode mode); -/** - * @struct LatencyChainDiagnostics - * @brief Diagnostics collected during latency-chain setup - */ -struct LatencyChainDiagnostics { - size_t pointer_count = 0; ///< Number of nodes in the pointer chain - size_t unique_pages_touched = 0; ///< Unique virtual pages touched by chain nodes - size_t page_size_bytes = 0; ///< System page size used for page accounting - size_t stride_bytes = 0; ///< Stride used to build the chain -}; - -/** Deterministic platform/random inputs used only by memory utility unit tests. */ +/** Deterministic random input used only by memory utility unit tests. */ struct MemoryUtilsTestHooks { - size_t page_size_bytes = 0; uint64_t generated_seed = 0; }; @@ -137,7 +125,6 @@ inline size_t alignment_offset_to_cache_line(void* ptr) { * @param buffer_size Size of the buffer in bytes * @param stride Stride size in bytes between linked pointers * @param tlb_locality_bytes Optional TLB-locality window in bytes (0 = global random chain) - * @param diagnostics Optional diagnostics output for page-touch analysis * @param mode Pointer-chain construction policy * @return EXIT_SUCCESS on success, EXIT_FAILURE on error * @@ -146,7 +133,6 @@ inline size_t alignment_offset_to_cache_line(void* ptr) { */ int setup_latency_chain(void* buffer, size_t buffer_size, size_t stride, size_t tlb_locality_bytes = 0, - LatencyChainDiagnostics* diagnostics = nullptr, LatencyChainMode mode = LatencyChainMode::Auto); /** @@ -154,7 +140,6 @@ int setup_latency_chain(void* buffer, size_t buffer_size, size_t stride, */ int setup_latency_chain(void* buffer, size_t buffer_size, size_t stride, size_t tlb_locality_bytes, - LatencyChainDiagnostics* diagnostics, LatencyChainMode mode, uint64_t deterministic_seed); diff --git a/src/llm_memory/llm_kv_layout.cpp b/src/llm_memory/llm_kv_layout.cpp index a81ae19..9487cd7 100644 --- a/src/llm_memory/llm_kv_layout.cpp +++ b/src/llm_memory/llm_kv_layout.cpp @@ -58,14 +58,6 @@ bool checked_subtract(size_t lhs, size_t rhs, size_t& out) { return true; } -bool valid_scenario(LlmScenario scenario) { - return static_cast(scenario) < kLlmScenarioCount; -} - -bool scenario_uses_kv(LlmScenario scenario) { - return scenario == LlmScenario::KvOnly || scenario == LlmScenario::Mixed; -} - bool is_lowercase_sha256(std::string_view value) { return value.size() == 64 && std::all_of(value.begin(), value.end(), [](char character) { @@ -911,157 +903,6 @@ materialize_llm_kv_block_table_in_place( return result; } -LlmPagedDecodeWorkloadPlan build_llm_paged_decode_workload_plan( - const LlmKvLayoutPlan& layout, LlmScenario scenario, size_t work_units, - size_t effective_model_payload_bytes_per_work_unit, - const LlmKvPermutationIdentity& permutation) { - LlmPagedDecodeWorkloadPlan plan; - plan.scenario = scenario; - plan.work_units = work_units; - plan.effective_model_payload_bytes_per_work_unit = - effective_model_payload_bytes_per_work_unit; - plan.maximum_work_units_by_work_unit_cap = - Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT; - if (!layout.valid) { - plan.reason_code = layout.reason_code; - return plan; - } - plan.layout_geometry_identity = layout.geometry_identity; - plan.layout_identity = - serialize_llm_kv_layout_identity(layout, permutation); - if (plan.layout_identity.empty()) { - plan.reason_code = LlmKvLayoutReason::INVALID_LAYOUT_IDENTITY; - return plan; - } - if (!valid_scenario(scenario)) { - plan.reason_code = LlmKvLayoutReason::INVALID_SCENARIO; - return plan; - } - if (work_units == 0) { - plan.reason_code = LlmKvLayoutReason::WORK_UNIT_COUNT_ZERO; - return plan; - } - if (work_units > plan.maximum_work_units_by_work_unit_cap) { - plan.reason_code = LlmKvLayoutReason::WORK_UNIT_CAP_EXCEEDED; - return plan; - } - - if (scenario_uses_kv(scenario)) { - size_t doubled_blocks = 0; - if (!NumericUtils::checked_multiply(layout.blocks_per_sequence, 2, - doubled_blocks) || - !NumericUtils::checked_add( - doubled_blocks, 1, - plan.layout_metadata_lookup_count_per_layer_sequence)) { - plan.reason_code = LlmKvLayoutReason::LOOKUP_COUNT_OVERFLOW; - return plan; - } - size_t layer_sequences = 0; - if (!NumericUtils::checked_multiply(layout.layer_count, - layout.batch_size, - layer_sequences) || - !NumericUtils::checked_multiply( - layer_sequences, - plan.layout_metadata_lookup_count_per_layer_sequence, - plan.layout_metadata_lookup_count_per_work_unit)) { - plan.reason_code = LlmKvLayoutReason::LOOKUP_COUNT_OVERFLOW; - return plan; - } - if (!NumericUtils::checked_multiply( - plan.layout_metadata_lookup_count_per_work_unit, - Constants::LLM_KV_BLOCK_TABLE_ENTRY_BYTES, - plan.layout_metadata_read_bytes_per_work_unit)) { - plan.reason_code = LlmKvLayoutReason::LOOKUP_BYTES_OVERFLOW; - return plan; - } - } - - if (!NumericUtils::checked_add( - plan.effective_model_payload_bytes_per_work_unit, - plan.layout_metadata_read_bytes_per_work_unit, - plan.accounted_bytes_per_work_unit)) { - plan.reason_code = LlmKvLayoutReason::TASK_TOTAL_OVERFLOW; - return plan; - } - if (plan.accounted_bytes_per_work_unit == 0) { - plan.reason_code = LlmKvLayoutReason::ACCOUNTED_BYTES_ZERO; - return plan; - } - plan.maximum_work_units_by_guardrail = - Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK / - plan.accounted_bytes_per_work_unit; - plan.effective_maximum_work_units = - std::min(plan.maximum_work_units_by_work_unit_cap, - plan.maximum_work_units_by_guardrail); - if (plan.effective_maximum_work_units == 0) { - plan.reason_code = LlmKvLayoutReason::GUARDRAIL_BELOW_ONE_WORK_UNIT; - return plan; - } - if (!NumericUtils::checked_multiply( - plan.effective_model_payload_bytes_per_work_unit, work_units, - plan.effective_model_payload_bytes) || - !NumericUtils::checked_multiply( - plan.layout_metadata_lookup_count_per_work_unit, work_units, - plan.layout_metadata_lookup_count) || - !NumericUtils::checked_multiply( - plan.layout_metadata_read_bytes_per_work_unit, work_units, - plan.layout_metadata_read_bytes) || - !NumericUtils::checked_multiply(plan.accounted_bytes_per_work_unit, - work_units, - plan.task_accounted_bytes)) { - plan.reason_code = LlmKvLayoutReason::TASK_TOTAL_OVERFLOW; - return plan; - } - if (plan.task_accounted_bytes > - Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK) { - plan.reason_code = - LlmKvLayoutReason::TASK_ACCOUNTED_BYTES_CAP_EXCEEDED; - return plan; - } - - plan.identity = Constants::LLM_PAGED_DECODE_WORKLOAD_IDENTITY_VERSION; - append_identity_field(plan.identity, "layout_identity", - plan.layout_identity); - append_identity_field(plan.identity, "scenario", - llm_scenario_to_string(plan.scenario)); - append_identity_field(plan.identity, "work_units", plan.work_units); - append_identity_field( - plan.identity, "effective_model_payload_bytes_per_work_unit", - plan.effective_model_payload_bytes_per_work_unit); - append_identity_field( - plan.identity, "layout_metadata_lookup_count_per_layer_sequence", - plan.layout_metadata_lookup_count_per_layer_sequence); - append_identity_field( - plan.identity, "layout_metadata_lookup_count_per_work_unit", - plan.layout_metadata_lookup_count_per_work_unit); - append_identity_field( - plan.identity, "layout_metadata_read_bytes_per_work_unit", - plan.layout_metadata_read_bytes_per_work_unit); - append_identity_field(plan.identity, "accounted_bytes_per_work_unit", - plan.accounted_bytes_per_work_unit); - append_identity_field(plan.identity, - "effective_model_payload_bytes", - plan.effective_model_payload_bytes); - append_identity_field(plan.identity, - "layout_metadata_lookup_count", - plan.layout_metadata_lookup_count); - append_identity_field(plan.identity, - "layout_metadata_read_bytes", - plan.layout_metadata_read_bytes); - append_identity_field(plan.identity, "task_accounted_bytes", - plan.task_accounted_bytes); - append_identity_field(plan.identity, "maximum_work_units_by_guardrail", - plan.maximum_work_units_by_guardrail); - append_identity_field(plan.identity, - "maximum_work_units_by_work_unit_cap", - plan.maximum_work_units_by_work_unit_cap); - append_identity_field(plan.identity, "effective_maximum_work_units", - plan.effective_maximum_work_units); - plan.valid = true; - plan.reason_code = LlmKvLayoutReason::VALID; - return plan; -} - LlmKvCpuOwnershipPlan build_llm_paged_decode_kv_cpu_ownership_plan( const LlmKvLayoutPlan& layout, size_t worker_count, const LlmKvStopRequested& stop_requested) { @@ -1601,60 +1442,3 @@ bool validate_llm_kv_layout_identity( matcher.string_field("permutation_sha256", permutation.sha256) && matcher.complete(); } - -std::string serialize_llm_kv_cpu_execution_identity( - const LlmPagedDecodeWorkloadPlan& workload, - const LlmKvCpuOwnershipPlan& ownership) { - if (!workload.valid || !ownership.valid || - workload.layout_geometry_identity.empty() || - workload.layout_geometry_identity != - ownership.layout_geometry_identity || - !has_identity_version( - workload.layout_geometry_identity, - Constants::LLM_KV_LAYOUT_GEOMETRY_IDENTITY_VERSION) || - !has_identity_version( - workload.layout_identity, - Constants::LLM_KV_LAYOUT_PLAN_IDENTITY_VERSION) || - !has_identity_version( - workload.identity, - Constants::LLM_PAGED_DECODE_WORKLOAD_IDENTITY_VERSION) || - !has_identity_version( - ownership.identity, - Constants::LLM_PAGED_CPU_EXECUTION_IDENTITY_VERSION)) { - return {}; - } - std::string identity = - Constants::LLM_PAGED_CPU_EXECUTION_IDENTITY_VERSION; - append_identity_field(identity, "workload_identity", workload.identity); - append_identity_field(identity, "ownership_identity", - ownership.identity); - return identity; -} - -std::string serialize_llm_kv_metal_execution_identity( - const LlmPagedDecodeWorkloadPlan& workload, - const LlmKvMetalSegmentPlan& segments) { - if (!workload.valid || !segments.valid || - workload.layout_geometry_identity.empty() || - workload.layout_geometry_identity != - segments.layout_geometry_identity || - !has_identity_version( - workload.layout_geometry_identity, - Constants::LLM_KV_LAYOUT_GEOMETRY_IDENTITY_VERSION) || - !has_identity_version( - workload.layout_identity, - Constants::LLM_KV_LAYOUT_PLAN_IDENTITY_VERSION) || - !has_identity_version( - workload.identity, - Constants::LLM_PAGED_DECODE_WORKLOAD_IDENTITY_VERSION) || - !has_identity_version( - segments.identity, - Constants::LLM_PAGED_METAL_EXECUTION_IDENTITY_VERSION)) { - return {}; - } - std::string identity = - Constants::LLM_PAGED_METAL_EXECUTION_IDENTITY_VERSION; - append_identity_field(identity, "workload_identity", workload.identity); - append_identity_field(identity, "segment_identity", segments.identity); - return identity; -} diff --git a/src/llm_memory/llm_kv_layout.h b/src/llm_memory/llm_kv_layout.h index 7228c27..9040d85 100644 --- a/src/llm_memory/llm_kv_layout.h +++ b/src/llm_memory/llm_kv_layout.h @@ -75,22 +75,6 @@ inline constexpr const char* TRANSIENT_BYTES_OVERFLOW = "layout-transient-bytes-overflow"; inline constexpr const char* MEMORY_BUDGET_OVERFLOW = "layout-memory-budget-overflow"; -inline constexpr const char* LOOKUP_COUNT_OVERFLOW = - "decode-lookup-count-overflow"; -inline constexpr const char* LOOKUP_BYTES_OVERFLOW = - "decode-lookup-bytes-overflow"; -inline constexpr const char* WORK_UNIT_COUNT_ZERO = "work-unit-count-zero"; -inline constexpr const char* WORK_UNIT_CAP_EXCEEDED = - "work-unit-cap-exceeded"; -inline constexpr const char* TASK_TOTAL_OVERFLOW = - "decode-task-total-overflow"; -inline constexpr const char* ACCOUNTED_BYTES_ZERO = - "accounted-bytes-per-work-unit-zero"; -inline constexpr const char* GUARDRAIL_BELOW_ONE_WORK_UNIT = - "guardrail-below-one-work-unit"; -inline constexpr const char* TASK_ACCOUNTED_BYTES_CAP_EXCEEDED = - "task-accounted-bytes-cap-exceeded"; -inline constexpr const char* INVALID_SCENARIO = "invalid-scenario"; inline constexpr const char* WORKER_COUNT_ZERO = "worker-count-zero"; inline constexpr const char* OWNERSHIP_COUNT_OVERFLOW = "block-ownership-count-overflow"; @@ -351,51 +335,6 @@ materialize_llm_kv_block_table_in_place( size_t hash_chunk_entries = 1024, const LlmKvStopRequested& stop_requested = {}); -/** Checked paged metadata accounting for one complete decode task. */ -struct LlmPagedDecodeWorkloadPlan { - bool valid = false; - std::string reason_code = LlmKvLayoutReason::INVALID_SCENARIO; - LlmScenario scenario = LlmScenario::WeightsOnly; - std::string layout_geometry_identity; - std::string layout_identity; - size_t work_units = 0; - size_t effective_model_payload_bytes_per_work_unit = 0; - size_t layout_metadata_lookup_count_per_layer_sequence = 0; - size_t layout_metadata_lookup_count_per_work_unit = 0; - size_t layout_metadata_read_bytes_per_work_unit = 0; - size_t accounted_bytes_per_work_unit = 0; - size_t maximum_work_units_by_work_unit_cap = 0; - size_t maximum_work_units_by_guardrail = 0; - size_t effective_maximum_work_units = 0; - size_t effective_model_payload_bytes = 0; - size_t layout_metadata_lookup_count = 0; - size_t layout_metadata_read_bytes = 0; - size_t task_accounted_bytes = 0; - std::string identity; -}; - -/** - * Resolve checked decode lookup and model-plus-metadata task totals. - * - * The workload identity is always reconstructed from `layout` and a complete, - * self-consistent materialized permutation identity; callers cannot substitute - * an arbitrary layout string. Both the shared work-unit ceiling and the 64 GiB - * accounted-byte guardrail apply. - * - * @param layout Valid geometry used to materialize `permutation`. - * @param scenario Decode scenario whose KV lookups are being accounted. - * @param work_units Positive number of complete decode steps in the task. - * @param effective_model_payload_bytes_per_work_unit Scenario payload without - * layout-metadata bytes. - * @param permutation Frozen materialized table identity matching `layout`. - * @return A complete canonical task plan, or a stable invalid reason. - * @throws std::bad_alloc If canonical identity storage cannot be allocated. - */ -LlmPagedDecodeWorkloadPlan build_llm_paged_decode_workload_plan( - const LlmKvLayoutPlan& layout, LlmScenario scenario, size_t work_units, - size_t effective_model_payload_bytes_per_work_unit, - const LlmKvPermutationIdentity& permutation); - /** One contiguous logical-block range owned by exactly one CPU worker. */ struct LlmKvCpuBlockAssignment { size_t layer_index = 0; @@ -549,30 +488,4 @@ bool validate_llm_kv_layout_identity( const LlmKvPermutationIdentity& permutation, std::string_view identity) noexcept; -/** - * Canonically bind a frozen workload to CPU ownership semantics. - * - * Both typed inputs must be valid, carry canonical component identities, and - * reference the same geometry identity. - * - * @return Length-prefixed identity, or empty for invalid/mismatched inputs. - * @throws std::bad_alloc If identity storage cannot be allocated. - */ -std::string serialize_llm_kv_cpu_execution_identity( - const LlmPagedDecodeWorkloadPlan& workload, - const LlmKvCpuOwnershipPlan& ownership); - -/** - * Canonically bind a frozen workload to Metal segment semantics. - * - * Both typed inputs must be valid, carry canonical component identities, and - * reference the same geometry identity. - * - * @return Length-prefixed identity, or empty for invalid/mismatched inputs. - * @throws std::bad_alloc If identity storage cannot be allocated. - */ -std::string serialize_llm_kv_metal_execution_identity( - const LlmPagedDecodeWorkloadPlan& workload, - const LlmKvMetalSegmentPlan& segments); - #endif // LLM_KV_LAYOUT_H diff --git a/src/llm_memory/llm_metal_backend.h b/src/llm_memory/llm_metal_backend.h index bc62caf..7849339 100644 --- a/src/llm_memory/llm_metal_backend.h +++ b/src/llm_memory/llm_metal_backend.h @@ -521,17 +521,6 @@ uint32_t llm_metal_paged_pattern_word(uint64_t seed, uint64_t layer_index, uint64_t physical_block, uint64_t block_word_index) noexcept; -/** Return one deterministic decode append word in the pool-address domain. */ -uint32_t llm_metal_decode_append_word(uint64_t scenario_seed, uint64_t work_unit, uint64_t layer_index, - uint64_t batch_index, uint64_t absolute_word_index, - LlmMetalResourcePool pool) noexcept; - -/** Return one deterministic prefill full-prompt write word. */ -uint32_t llm_metal_prefill_write_word( - uint64_t scenario_seed, uint64_t work_unit, uint64_t layer_index, - uint64_t batch_index, uint64_t absolute_word_index, - LlmMetalResourcePool pool) noexcept; - /** Calculate expected timed W/K/V accumulators without reading resource bytes. */ LlmMetalChecksumOracle calculate_llm_metal_decode_contiguous_checksum( const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan) noexcept; diff --git a/src/llm_memory/llm_metal_backend.mm b/src/llm_memory/llm_metal_backend.mm index 54fc3f0..295f2be 100644 --- a/src/llm_memory/llm_metal_backend.mm +++ b/src/llm_memory/llm_metal_backend.mm @@ -495,40 +495,6 @@ uint32_t llm_metal_paged_pattern_word(uint64_t seed, uint64_t layer_index, static_cast(block_word_index + 1U); } -uint32_t llm_metal_decode_append_word(uint64_t scenario_seed, uint64_t work_unit, uint64_t layer_index, - uint64_t batch_index, uint64_t absolute_word_index, - LlmMetalResourcePool pool) noexcept { - const uint32_t pool_domain = pool == LlmMetalResourcePool::K - ? LlmMetalKernelContract::kAppendKeyDomain - : LlmMetalKernelContract::kAppendValueDomain; - return static_cast(scenario_seed) + - LlmMetalKernelContract::kAppendWorkUnitMultiplier * static_cast(work_unit + 1U) + - LlmMetalKernelContract::kAppendLayerMultiplier * static_cast(layer_index + 1U) + - LlmMetalKernelContract::kAppendBatchMultiplier * static_cast(batch_index + 1U) + - LlmMetalKernelContract::kAppendWordMultiplier * static_cast(absolute_word_index + 1U) + - pool_domain; -} - -uint32_t llm_metal_prefill_write_word( - uint64_t scenario_seed, uint64_t work_unit, uint64_t layer_index, - uint64_t batch_index, uint64_t absolute_word_index, - LlmMetalResourcePool pool) noexcept { - const uint32_t pool_domain = - pool == LlmMetalResourcePool::K - ? LlmMetalKernelContract::kPrefillWriteKeyDomain - : LlmMetalKernelContract::kPrefillWriteValueDomain; - return static_cast(scenario_seed) + - LlmMetalKernelContract::kAppendWorkUnitMultiplier * - static_cast(work_unit + 1U) + - LlmMetalKernelContract::kAppendLayerMultiplier * - static_cast(layer_index + 1U) + - LlmMetalKernelContract::kAppendBatchMultiplier * - static_cast(batch_index + 1U) + - LlmMetalKernelContract::kAppendWordMultiplier * - static_cast(absolute_word_index + 1U) + - pool_domain; -} - bool equal_llm_metal_checksum(const LlmMetalDualMod32Checksum& left, const LlmMetalDualMod32Checksum& right) noexcept { return left.weight.a == right.weight.a && left.weight.b == right.weight.b && left.k.a == right.k.a && @@ -2565,39 +2531,6 @@ LlmMetalGridPlan build_llm_metal_grid_plan(const LlmMetalGridRequest& request) { plan.reason_code = LlmMetalPlanReason::VECTOR_ITERATION_CAP_EXCEEDED; return plan; } - if (!request.owner_accounted_bytes.empty()) { - if (request.owner_accounted_bytes.size() != request.owner_count) { - plan.reason_code = LlmMetalPlanReason::OWNER_COST_COUNT_MISMATCH; - return plan; - } - try { - plan.threadgroup_accounted_bytes.assign(plan.actual_threadgroups, 0); - } catch (const std::bad_alloc&) { - plan.reason_code = LlmMetalPlanReason::PLANNER_ALLOCATION_FAILED; - return plan; - } catch (const std::length_error&) { - plan.reason_code = LlmMetalPlanReason::PLANNER_ALLOCATION_FAILED; - return plan; - } - for (size_t owner = 0; owner < request.owner_count; ++owner) { - const size_t threadgroup = owner % plan.actual_threadgroups; - if (!NumericUtils::checked_add(plan.threadgroup_accounted_bytes[threadgroup], - request.owner_accounted_bytes[owner], - plan.threadgroup_accounted_bytes[threadgroup])) { - plan.reason_code = LlmMetalPlanReason::OWNER_COUNT_OVERFLOW; - return plan; - } - } - const auto [minimum, maximum] = - std::minmax_element(plan.threadgroup_accounted_bytes.begin(), plan.threadgroup_accounted_bytes.end()); - plan.minimum_threadgroup_accounted_bytes = *minimum; - plan.maximum_threadgroup_accounted_bytes = *maximum; - if (!checked_subtract(plan.maximum_threadgroup_accounted_bytes, plan.minimum_threadgroup_accounted_bytes, - plan.threadgroup_accounted_imbalance_bytes)) { - plan.reason_code = LlmMetalPlanReason::OWNER_COUNT_OVERFLOW; - return plan; - } - } if (!populate_llm_metal_grid_identity(request, plan)) { plan.reason_code = LlmMetalPlanReason::PLANNER_ALLOCATION_FAILED; return plan; diff --git a/src/llm_memory/llm_runner.cpp b/src/llm_memory/llm_runner.cpp index feaf268..1bed1db 100644 --- a/src/llm_memory/llm_runner.cpp +++ b/src/llm_memory/llm_runner.cpp @@ -2358,27 +2358,6 @@ int run_llm_memory_suite(const LlmMemoryConfig& config, } } -const char* llm_runner_task_kind_to_string(LlmRunnerTaskKind kind) noexcept { - switch (kind) { - case LlmRunnerTaskKind::Warmup: - return "warmup"; - case LlmRunnerTaskKind::Calibration: - return "calibration"; - case LlmRunnerTaskKind::Measurement: - return "measurement"; - } - return "unknown"; -} - -const char* llm_checkpoint_kind_to_string(LlmCheckpointKind kind) noexcept { - switch (kind) { - case LlmCheckpointKind::MeasurementTerminal: - return "measurement-terminal"; - case LlmCheckpointKind::CommandTerminal: - return "command-terminal"; - } - return "unknown"; -} void prepare_llm_result_snapshot(LlmMemoryResult& result) { ++result.snapshot_preparation_attempts; diff --git a/src/llm_memory/llm_runner.h b/src/llm_memory/llm_runner.h index dc6f48a..699b548 100644 --- a/src/llm_memory/llm_runner.h +++ b/src/llm_memory/llm_runner.h @@ -376,7 +376,5 @@ int run_llm_memory_suite(const LlmMemoryConfig& config, std::string_view canonicalize_llm_result_reason_code(std::string_view reason_code) noexcept; /** Stable token helpers for runner task and checkpoint identities. */ -const char* llm_runner_task_kind_to_string(LlmRunnerTaskKind kind) noexcept; -const char* llm_checkpoint_kind_to_string(LlmCheckpointKind kind) noexcept; #endif // LLM_RUNNER_H diff --git a/src/llm_memory/llm_work_plan.cpp b/src/llm_memory/llm_work_plan.cpp index bda7605..cf4b7e3 100644 --- a/src/llm_memory/llm_work_plan.cpp +++ b/src/llm_memory/llm_work_plan.cpp @@ -4937,44 +4937,6 @@ LlmMemoryWorkPlanDraft prepare_llm_memory_work_plan( return prepare_llm_memory_work_plan(request, stop_requested); } -LlmMemoryWorkPlan build_llm_memory_work_plan( - const LlmMemoryConfig& config, size_t available_workers, - size_t available_memory_bytes, size_t mapping_granularity_bytes, - size_t checksum_auxiliary_bytes, - size_t orchestration_auxiliary_bytes, - const LlmKvStopRequested& stop_requested) { - const LlmMemoryConfigValidation validation = - validate_llm_memory_config(config); - if (!validation.valid) { - return invalid_config_plan(validation.reason_code); - } - - LlmMemoryWorkPlanRequest request; - request.geometry = {validation.active_weight_bytes, - config.layer_count, - config.query_head_count, - config.kv_head_count, - config.head_dimension, - config.kv_element_bytes, - config.visible_context_tokens, - config.batch_size, - config.kv_block_tokens, - config.phase, - config.kv_layout}; - request.geometry.prompt_tokens = config.prompt_tokens; - request.geometry.attention_query_tile_tokens = - config.attention_query_tile_tokens; - request.backend = config.backend; - request.requested_workers = config.requested_workers; - request.available_workers = available_workers; - request.available_memory_bytes = available_memory_bytes; - request.mapping_granularity_bytes = mapping_granularity_bytes; - request.checksum_auxiliary_bytes = checksum_auxiliary_bytes; - request.orchestration_auxiliary_bytes = orchestration_auxiliary_bytes; - request.base_seed = config.seed; - return build_llm_memory_work_plan(request, stop_requested); -} - bool readmit_llm_memory_work_plan( LlmMemoryWorkPlan& plan, size_t checksum_auxiliary_bytes, size_t orchestration_auxiliary_bytes) noexcept { diff --git a/src/llm_memory/llm_work_plan.h b/src/llm_memory/llm_work_plan.h index 538e0e6..0f16699 100644 --- a/src/llm_memory/llm_work_plan.h +++ b/src/llm_memory/llm_work_plan.h @@ -776,7 +776,6 @@ struct LlmMetalGridRequest { size_t work_units = 0; size_t paged_semantic_lookups = 0; size_t serial_range_visits_per_lane = 0; - std::vector owner_accounted_bytes; LlmMetalPipelineCapabilities pipeline; LlmMetalPlanningLimits limits; }; @@ -1199,29 +1198,6 @@ LlmMemoryWorkPlan build_llm_memory_work_plan( const LlmMemoryWorkPlanRequest& request, const LlmKvStopRequested& stop_requested = {}); -/** - * Validate a resolved config and build its pointer-free work plan. - * - * @param config Resolved command configuration; the function does not retain a - * reference to it. - * @param available_workers Detected CPU worker capacity, or zero for Metal. - * @param available_memory_bytes Current logical memory-admission sample. - * @param mapping_granularity_bytes Host mapping/page granularity used for - * checked resource rounding. - * @param checksum_auxiliary_bytes Retained checksum-support storage charged to - * the plan's memory budget. - * @param orchestration_auxiliary_bytes Retained runner/output storage charged - * to the plan's memory budget. - * @param stop_requested Optional synchronous predicate polled while a paged - * block table is initialized, validated, and hashed. - * @return A reason-bearing plan. Invalid plans contain no executable templates. - */ -LlmMemoryWorkPlan build_llm_memory_work_plan( - const LlmMemoryConfig& config, size_t available_workers, - size_t available_memory_bytes, size_t mapping_granularity_bytes, - size_t checksum_auxiliary_bytes, size_t orchestration_auxiliary_bytes, - const LlmKvStopRequested& stop_requested = {}); - /** Re-evaluate one already-materialized plan with finalized auxiliary bytes. */ bool readmit_llm_memory_work_plan( LlmMemoryWorkPlan& plan, size_t checksum_auxiliary_bytes, diff --git a/src/output/console/messages/messages_api.h b/src/output/console/messages/messages_api.h index 825a6f9..c121b7c 100644 --- a/src/output/console/messages/messages_api.h +++ b/src/output/console/messages/messages_api.h @@ -548,8 +548,6 @@ const std::string& benchmark_reason_unknown_loop_exception(); std::string benchmark_reason_coordinator_exception(const std::string& error); const std::string& benchmark_reason_unknown_coordinator_exception(); std::string benchmark_reason_prepare_failed(const std::string& phase_name); -std::string benchmark_reason_latency_chain_setup_failed( - const std::string& phase_name); const std::string& benchmark_reason_invalid_bandwidth_plan(); const std::string& benchmark_reason_no_worker_partition(); const std::string& benchmark_reason_copy_payload_overflow(); diff --git a/src/output/console/messages/results_messages.cpp b/src/output/console/messages/results_messages.cpp index 0c3b72e..2f7017b 100644 --- a/src/output/console/messages/results_messages.cpp +++ b/src/output/console/messages/results_messages.cpp @@ -383,9 +383,4 @@ std::string benchmark_reason_coordinator_exception(const std::string& error) { return "standard benchmark coordinator exception: " + error; } -std::string benchmark_reason_latency_chain_setup_failed( - const std::string& phase_name) { - return "failed to construct " + phase_name + " latency chain"; -} - } // namespace Messages diff --git a/src/utils/json_utils.cpp b/src/utils/json_utils.cpp index a4e7570..40fc108 100644 --- a/src/utils/json_utils.cpp +++ b/src/utils/json_utils.cpp @@ -23,16 +23,13 @@ * @brief JSON utility function implementations * * Provides implementations for JSON-related utility functions including - * statistical calculations, JSON parsing from strings and files with - * comprehensive error handling and validation. + * statistical calculations and UTC timestamp formatting. */ #include "json_utils.h" #include #include -#include -#include #include #include @@ -77,76 +74,3 @@ nlohmann::json calculate_json_statistics(const std::vector& values) { return json_statistics; } - -// Parse JSON from a string with validation -// Returns true on success, false on error -// On error, error_message is populated with a descriptive error message -bool parse_json_from_string(const std::string& json_string, nlohmann::json& result, std::string& error_message) { - error_message.clear(); - if (json_string.empty()) { - error_message = "Empty JSON string"; - return false; - } - - try { - result = nlohmann::json::parse(json_string); - return true; - } catch (const nlohmann::json::parse_error& e) { - std::ostringstream oss; - oss << "JSON parse error at position " << e.byte << ": " << e.what(); - error_message = oss.str(); - return false; - } catch (const nlohmann::json::exception& e) { - std::ostringstream oss; - oss << "JSON exception: " << e.what(); - error_message = oss.str(); - return false; - } catch (const std::exception& e) { - std::ostringstream oss; - oss << "Unexpected error during JSON parsing: " << e.what(); - error_message = oss.str(); - return false; - } -} - -// Parse JSON from a file with validation -// Returns true on success, false on error -// On error, error_message is populated with a descriptive error message -bool parse_json_from_file(const std::string& file_path, nlohmann::json& result, std::string& error_message) { - error_message.clear(); - std::filesystem::path path(file_path); - - // Check if file exists - if (!std::filesystem::exists(path)) { - error_message = "File does not exist: " + file_path; - return false; - } - - // Check if it's a regular file - if (!std::filesystem::is_regular_file(path)) { - error_message = "Path is not a regular file: " + file_path; - return false; - } - - // Try to open the file - std::ifstream file(path); - if (!file.is_open()) { - std::ostringstream oss; - oss << "Failed to open file: " << file_path; - error_message = oss.str(); - return false; - } - - // Read file content - std::ostringstream content; - content << file.rdbuf(); - file.close(); - - if (content.str().empty()) { - error_message = "File is empty: " + file_path; - return false; - } - - // Parse JSON from file content - return parse_json_from_string(content.str(), result, error_message); -} diff --git a/src/utils/json_utils.h b/src/utils/json_utils.h index ddc95a0..fa6ebe3 100644 --- a/src/utils/json_utils.h +++ b/src/utils/json_utils.h @@ -20,15 +20,13 @@ /** * @file json_utils.h - * @brief JSON utility functions for parsing, generation, and statistical calculations + * @brief JSON timestamp and statistical calculation utilities * * This file provides utility functions for JSON operations used throughout the benchmark application: * - Statistical calculations (average, median, percentiles, standard deviation) with JSON output - * - Validated JSON parsing from strings and files with comprehensive error handling * - JSON generation helpers for structured benchmark result output * - * These utilities wrap the nlohmann/json library to provide consistent error handling - * and statistical analysis for benchmark results exported to JSON format. + * Statistical output uses nlohmann/json to preserve unavailable values as JSON null. * * @note Uses nlohmann/json library (MIT License): https://github.com/nlohmann/json */ @@ -68,28 +66,4 @@ std::string build_utc_timestamp( */ nlohmann::json calculate_json_statistics(const std::vector& values); -/** - * @brief Parse JSON from a string with validation - * - * @param json_string Input string containing JSON data - * @param result Output parameter to store parsed JSON object - * @param error_message Output parameter for error description on failure - * @return true on successful parsing, false on error - * - * @note On error, error_message contains detailed parsing error information - */ -bool parse_json_from_string(const std::string& json_string, nlohmann::json& result, std::string& error_message); - -/** - * @brief Parse JSON from a file with validation - * - * @param file_path Path to the JSON file to parse - * @param result Output parameter to store parsed JSON object - * @param error_message Output parameter for error description on failure - * @return true on successful parsing, false on error - * - * @note On error, error_message contains detailed file I/O or parsing error information - */ -bool parse_json_from_file(const std::string& file_path, nlohmann::json& result, std::string& error_message); - #endif // JSON_UTILS_H diff --git a/tests/test_analysis.cpp b/tests/test_analysis.cpp index 9c4e8da..ef1928d 100644 --- a/tests/test_analysis.cpp +++ b/tests/test_analysis.cpp @@ -160,22 +160,6 @@ TEST(AnalysisTest, CoordinatorStopsBeforeFirstTaskWithExactCounters) { EXPECT_FALSE(result_json["tlb_analysis"].contains("status_reason")); } -TEST(AnalysisTest, CoordinatorRejectsMissingPassExecutor) { - BenchmarkConfig config; - config.tlb_sweep_density = TlbSweepDensity::Low; - TlbAnalysisExecutionSeam seam = make_tlb_execution_seam(); - bool observed = false; - seam.observe_summary = [&](const TlbAnalysisCoordinatorSummary&) { observed = true; }; - - nlohmann::ordered_json result_json = {{"stale", true}}; - - EXPECT_EQ(run_tlb_analysis_collect_silently( - config, []() { return false; }, seam, result_json), - EXIT_FAILURE); - EXPECT_FALSE(observed); - EXPECT_TRUE(result_json.empty()); -} - TEST(AnalysisTest, CoordinatorCollectRetainsPartialInterruptedAndErrorEvidence) { struct TestCase { TlbScheduleExecutionStatus pass_status; @@ -372,6 +356,15 @@ TEST(AnalysisTest, RobustBoundaryAcceptsPersistentStepWithIndependentValidation) EXPECT_TRUE(boundary.validation.passed); EXPECT_EQ(boundary.discovery.persistence_points_passed, 2u); EXPECT_GT(boundary.discovery.effect_ci.lower_ns, boundary.discovery.noise_floor_ns); + const TlbBoundaryDetection second = detect_tlb_boundary_robust(localities, discovery, &validation, 0, 0, 123456); + + ASSERT_TRUE(boundary.detected); + ASSERT_TRUE(second.detected); + EXPECT_EQ(boundary.confidence, second.confidence); + EXPECT_DOUBLE_EQ(boundary.discovery.effect_ci.lower_ns, second.discovery.effect_ci.lower_ns); + EXPECT_DOUBLE_EQ(boundary.discovery.effect_ci.upper_ns, second.discovery.effect_ci.upper_ns); + EXPECT_DOUBLE_EQ(boundary.validation.effect_ci.lower_ns, second.validation.effect_ci.lower_ns); + EXPECT_DOUBLE_EQ(boundary.validation.effect_ci.upper_ns, second.validation.effect_ci.upper_ns); } TEST(AnalysisTest, RobustBoundaryRejectsSingleSpikeThatReturnsToBaseline) { @@ -408,26 +401,6 @@ TEST(AnalysisTest, RobustBoundaryRequiresIndependentValidationEvidence) { EXPECT_EQ(boundary.candidates.front().validation.rejection_reason, "effect-below-minimum"); } -TEST(AnalysisTest, RobustBoundaryBootstrapIsDeterministicForSameSeed) { - const size_t page_size = 16 * Constants::BYTES_PER_KB; - const std::vector localities = { - page_size, 2 * page_size, 4 * page_size, 8 * page_size, 16 * page_size, 32 * page_size, - }; - const TlbRoundPointMatrix discovery = make_round_point_matrix({0.10, 0.12, 2.10, 2.20, 2.25, 2.30}); - const TlbRoundPointMatrix validation = make_round_point_matrix({0.08, 0.10, 2.00, 2.15, 2.20, 2.25}); - - const TlbBoundaryDetection first = detect_tlb_boundary_robust(localities, discovery, &validation, 0, 0, 4242); - const TlbBoundaryDetection second = detect_tlb_boundary_robust(localities, discovery, &validation, 0, 0, 4242); - - ASSERT_TRUE(first.detected); - ASSERT_TRUE(second.detected); - EXPECT_EQ(first.confidence, second.confidence); - EXPECT_DOUBLE_EQ(first.discovery.effect_ci.lower_ns, second.discovery.effect_ci.lower_ns); - EXPECT_DOUBLE_EQ(first.discovery.effect_ci.upper_ns, second.discovery.effect_ci.upper_ns); - EXPECT_DOUBLE_EQ(first.validation.effect_ci.lower_ns, second.validation.effect_ci.lower_ns); - EXPECT_DOUBLE_EQ(first.validation.effect_ci.upper_ns, second.validation.effect_ci.upper_ns); -} - TEST(AnalysisTest, TranslationDeltaMatrixPreservesRoundAndPointCoordinates) { const std::vector localities = {16384, 32768}; TlbMeasurementRecord base; @@ -534,7 +507,7 @@ TEST(AnalysisTest, DetectBoundaryMultiPointPersistenceSurvivesNoiseDip) { 64 * Constants::BYTES_PER_MB, }; const std::vector latencies_ns = { - 10.0, 10.1, 20.0, 12.5, 23.0, 24.0, 25.0, + 10.0, 10.1, 20.0, 12.0, 23.0, 24.0, 25.0, }; const TlbBoundaryDetection boundary = detect_tlb_boundary(localities, latencies_ns, 0); @@ -594,19 +567,3 @@ TEST(AnalysisTest, DetectBoundaryIqrDistinguishesNoisyAndClearSteps) { EXPECT_EQ(clear_boundary.boundary_index, 7u); EXPECT_TRUE(clear_boundary.persistent_jump); } - -TEST(AnalysisTest, DetectBoundaryLastPointStrongStepGetsMediumConfidence) { - // Last sweep point with a strong step (>8 ns) should get Medium confidence - // (not Low) even though there are no future points for persistence. - const std::vector localities = { - 64 * Constants::BYTES_PER_KB, 128 * Constants::BYTES_PER_KB, 256 * Constants::BYTES_PER_KB, - 512 * Constants::BYTES_PER_KB, 1 * Constants::BYTES_PER_MB, - }; - const std::vector latencies_ns = {10.0, 10.1, 10.2, 10.3, 20.0}; - - const TlbBoundaryDetection boundary = detect_tlb_boundary(localities, latencies_ns, 0); - ASSERT_TRUE(boundary.detected); - EXPECT_EQ(boundary.boundary_index, 4u); - EXPECT_TRUE(boundary.persistent_jump); - EXPECT_NE(boundary.confidence, "Low"); -} diff --git a/tests/test_benchmark_executor.cpp b/tests/test_benchmark_executor.cpp index f4341b6..8881029 100644 --- a/tests/test_benchmark_executor.cpp +++ b/tests/test_benchmark_executor.cpp @@ -133,19 +133,6 @@ TEST(BenchmarkExecutorTest, ParallelExecutorConsumesAlignedUnevenChunksExactly) EXPECT_EQ(covered, buffer.size()); } -TEST(BenchmarkExecutorTest, WriteTestRejectsZeroThreadCountWithoutModifyingBuffer) { - std::array buffer{}; - buffer.fill(0xCD); - - auto timer_opt = HighResTimer::create(); - ASSERT_TRUE(timer_opt.has_value()); - - EXPECT_EQ(run_write_test(buffer.data(), buffer.size(), 1, 0, *timer_opt), 0.0); - - const bool unchanged = std::all_of(buffer.begin(), buffer.end(), [](unsigned char value) { return value == 0xCD; }); - EXPECT_TRUE(unchanged); -} - TEST(BenchmarkExecutorTest, ParallelTimingStopsBeforeWorkerTeardown) { const ScopedDeterministicTimerSystemCalls timer_system_calls; std::array buffer{}; @@ -286,29 +273,3 @@ TEST(BenchmarkExecutorTest, InjectedPreparationFailureCoversEveryPhaseBoundary) EXPECT_NE(error_output.find(expected_reason), std::string::npos) << failing_phase; } } - -TEST(BenchmarkExecutorTest, InjectedLatencyChainFailureCoversCacheAndMainPhases) { - const ScopedDeterministicTimerSystemCalls timer_system_calls; - BenchmarkConfig config = build_injected_failure_config(); - config.only_bandwidth = false; - config.only_latency = false; - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - - for (const auto& phase : std::array, 2>{std::pair{"cache-latency", 2}, - std::pair{"main-latency", 3}}) { - BenchmarkExecutorTestHooks hooks; - hooks.fail_latency_chain_setup = [&](const std::string& phase_name) { return phase_name == phase.first; }; - const std::string expected_reason = Messages::benchmark_reason_latency_chain_setup_failed(phase.first); - std::string caught_reason; - testing::internal::CaptureStderr(); - try { - static_cast(run_single_benchmark_loop(config, phase.second, *timer, nullptr, &hooks)); - } catch (const std::runtime_error& error) { - caught_reason = error.what(); - } - const std::string error_output = testing::internal::GetCapturedStderr(); - EXPECT_EQ(caught_reason, expected_reason) << phase.first; - EXPECT_NE(error_output.find(expected_reason), std::string::npos) << phase.first; - } -} diff --git a/tests/test_benchmark_runner.cpp b/tests/test_benchmark_runner.cpp index a733c39..cb65c30 100644 --- a/tests/test_benchmark_runner.cpp +++ b/tests/test_benchmark_runner.cpp @@ -271,57 +271,43 @@ TEST(BenchmarkRunnerTest, InjectedTimerCreationFailureIsReportedAndCheckpointed) EXPECT_NE(error_output.find(stats.status_reason), std::string::npos); } -TEST(BenchmarkRunnerTest, InjectedLoopExceptionIsFailedAndCheckpointedWithExactReason) { +TEST(BenchmarkRunnerTest, TypedAndUnknownLoopExceptionsAreContainedAndCheckpointedWithExactReason) { const ScopedDeterministicTimerSystemCalls timer_system_calls; - BenchmarkConfig config; - config.loop_count = 1; - config.output_file = "/tmp/benchmark-runner-hook-unused.json"; - BenchmarkStatistics stats; - size_t checkpoints = 0; - BenchmarkRunnerTestHooks hooks; - inject_deterministic_elapsed(hooks); - hooks.execute_loop = [](BenchmarkConfig&, int, HighResTimer&, BenchmarkExecutionState*) -> BenchmarkResults { - throw std::runtime_error("injected loop failure"); - }; - hooks.checkpoint = [&](const BenchmarkConfig&, const BenchmarkStatistics& snapshot, double, bool) { - ++checkpoints; - EXPECT_EQ(snapshot.status, BenchmarkRunStatus::Failed); - EXPECT_EQ(snapshot.status_reason, "injected loop failure"); - return EXIT_SUCCESS; - }; - - testing::internal::CaptureStderr(); - const int result = run_all_benchmarks(config, stats, &hooks); - const std::string error = testing::internal::GetCapturedStderr(); - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_EQ(stats.status, BenchmarkRunStatus::Failed); - EXPECT_EQ(stats.status_reason, "injected loop failure"); - EXPECT_EQ(stats.completed_loops, 0u); - EXPECT_TRUE(stats.loop_results.empty()); - EXPECT_EQ(checkpoints, 1u); - EXPECT_EQ(error, Messages::error_benchmark_loop(0, "injected loop failure") + "\n"); -} - -TEST(BenchmarkRunnerTest, UnknownLoopExceptionIsContainedWithCentralizedReason) { - const ScopedDeterministicTimerSystemCalls timer_system_calls; - BenchmarkConfig config; - config.loop_count = 1; - BenchmarkStatistics stats; - BenchmarkRunnerTestHooks hooks; - hooks.execute_loop = [](BenchmarkConfig&, int, HighResTimer&, BenchmarkExecutionState*) -> BenchmarkResults { - throw 7; - }; - - testing::internal::CaptureStderr(); - int result = EXIT_SUCCESS; - EXPECT_NO_THROW(result = run_all_benchmarks(config, stats, &hooks)); - const std::string error = testing::internal::GetCapturedStderr(); - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_EQ(stats.status, BenchmarkRunStatus::Failed); - EXPECT_EQ(stats.status_reason, Messages::benchmark_reason_unknown_loop_exception()); - EXPECT_EQ(error, Messages::error_benchmark_loop(0, stats.status_reason) + "\n"); + for (bool typed_exception : {true, false}) { + SCOPED_TRACE(typed_exception); + const std::string reason = + typed_exception ? "injected loop failure" : Messages::benchmark_reason_unknown_loop_exception(); + BenchmarkConfig config; + config.loop_count = 1; + config.output_file = "/tmp/benchmark-runner-hook-unused.json"; + BenchmarkStatistics stats; + size_t checkpoints = 0; + BenchmarkRunnerTestHooks hooks; + inject_deterministic_elapsed(hooks); + hooks.execute_loop = [typed_exception](BenchmarkConfig&, int, HighResTimer&, + BenchmarkExecutionState*) -> BenchmarkResults { + if (typed_exception) throw std::runtime_error("injected loop failure"); + throw 7; + }; + hooks.checkpoint = [&](const BenchmarkConfig&, const BenchmarkStatistics& snapshot, double, bool) { + ++checkpoints; + EXPECT_EQ(snapshot.status, BenchmarkRunStatus::Failed); + EXPECT_EQ(snapshot.status_reason, reason); + return EXIT_SUCCESS; + }; + + testing::internal::CaptureStderr(); + const int result = run_all_benchmarks(config, stats, &hooks); + const std::string error = testing::internal::GetCapturedStderr(); + + EXPECT_EQ(result, EXIT_FAILURE); + EXPECT_EQ(stats.status, BenchmarkRunStatus::Failed); + EXPECT_EQ(stats.status_reason, reason); + EXPECT_EQ(stats.completed_loops, 0u); + EXPECT_TRUE(stats.loop_results.empty()); + EXPECT_EQ(checkpoints, 1u); + EXPECT_EQ(error, Messages::error_benchmark_loop(0, reason) + "\n"); + } } TEST(BenchmarkRunnerTest, StopHookExceptionIsContainedAtCoordinatorBoundary) { @@ -402,50 +388,6 @@ TEST(BenchmarkRunnerTest, InjectedCheckpointFailurePreservesCompletedLoopButFail EXPECT_EQ(stats.completed_loops, 1u); } -TEST(BenchmarkRunnerTest, StopBeforeFirstLoopCheckpointFailureTakesPrecedence) { - const ScopedDeterministicTimerSystemCalls timer_system_calls; - BenchmarkConfig config; - config.loop_count = 2; - config.output_file = "/tmp/benchmark-runner-hook-unused.json"; - config.only_bandwidth = true; - BenchmarkStatistics stats; - size_t executed_loops = 0; - size_t checkpoints = 0; - BenchmarkRunnerTestHooks hooks; - inject_deterministic_elapsed(hooks); - hooks.stop_requested = [] { return true; }; - hooks.execute_loop = [&](BenchmarkConfig&, int, HighResTimer&, - BenchmarkExecutionState*) { - ++executed_loops; - return BenchmarkResults{}; - }; - hooks.checkpoint = [&](const BenchmarkConfig&, - const BenchmarkStatistics& snapshot, double, bool) { - ++checkpoints; - EXPECT_EQ(snapshot.status, BenchmarkRunStatus::Interrupted); - EXPECT_EQ(snapshot.status_reason, Messages::msg_interrupted_by_user()); - EXPECT_EQ(snapshot.completed_loops, 0u); - EXPECT_TRUE(snapshot.loop_results.empty()); - return EXIT_FAILURE; - }; - - testing::internal::CaptureStdout(); - const int result = run_all_benchmarks(config, stats, &hooks); - const std::string human_output = testing::internal::GetCapturedStdout(); - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_EQ(stats.status, BenchmarkRunStatus::Failed); - EXPECT_EQ(stats.status_reason, - Messages::benchmark_reason_checkpoint_failed()); - EXPECT_EQ(stats.planned_loops, 2u); - EXPECT_EQ(stats.completed_loops, 0u); - EXPECT_TRUE(stats.loop_results.empty()); - EXPECT_EQ(executed_loops, 0u); - EXPECT_EQ(checkpoints, 1u); - EXPECT_TRUE(human_output.empty()); - EXPECT_FALSE(should_write_standard_final_json(JsonOutputKind::File, result)); -} - TEST(BenchmarkRunnerTest, CommandBoundaryFinalizationNeverRetriesFailedFileCheckpoint) { EXPECT_FALSE(should_write_standard_final_json(JsonOutputKind::Disabled, @@ -522,40 +464,6 @@ TEST(BenchmarkRunnerTest, std::string::npos); } -TEST(BenchmarkRunnerTest, InjectedStopBetweenLoopsPreservesCompletedLoop) { - const ScopedDeterministicTimerSystemCalls timer_system_calls; - BenchmarkConfig config; - config.loop_count = 3; - config.output_file = "/tmp/benchmark-runner-hook-unused.json"; - config.only_bandwidth = true; - BenchmarkStatistics stats; - size_t stop_checks = 0; - size_t checkpoints = 0; - BenchmarkRunnerTestHooks hooks; - inject_deterministic_elapsed(hooks); - hooks.stop_requested = [&] { return stop_checks++ >= 1; }; - hooks.execute_loop = [](BenchmarkConfig&, int loop, HighResTimer&, BenchmarkExecutionState*) { - BenchmarkResults results; - results.status = BenchmarkRunStatus::Complete; - results.loop_index = static_cast(loop); - return results; - }; - hooks.checkpoint = [&](const BenchmarkConfig&, const BenchmarkStatistics&, double, bool) { - ++checkpoints; - return EXIT_SUCCESS; - }; - - testing::internal::CaptureStdout(); - const int result = run_all_benchmarks(config, stats, &hooks); - static_cast(testing::internal::GetCapturedStdout()); - - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(stats.status, BenchmarkRunStatus::Interrupted); - EXPECT_EQ(stats.completed_loops, 1u); - EXPECT_EQ(stats.loop_results.size(), 1u); - EXPECT_EQ(checkpoints, 2u); -} - TEST(BenchmarkRunnerTest, StdoutInterruptionRetainsOneLoopAndEmitsOneSchema3TerminalDocument) { const ScopedDeterministicTimerSystemCalls timer_system_calls; @@ -621,11 +529,6 @@ TEST(BenchmarkRunnerTest, nlohmann::ordered_json parsed_payload; ASSERT_NO_THROW(parsed_payload = nlohmann::ordered_json::parse(json_output)); - EXPECT_EQ(parsed_payload, terminal_payload); - ASSERT_TRUE(parsed_payload.contains("configuration")); - EXPECT_EQ(parsed_payload["configuration"]["benchmark_schema_version"], - Constants::BENCHMARK_JSON_SCHEMA_VERSION); - EXPECT_EQ(parsed_payload["configuration"]["output_file"], "-"); EXPECT_EQ(parsed_payload["status"], "interrupted"); EXPECT_FALSE(parsed_payload["results_complete"].get()); EXPECT_FALSE(parsed_payload["conclusions_valid"].get()); @@ -639,47 +542,6 @@ TEST(BenchmarkRunnerTest, EXPECT_NE(human_output.find("Interrupted by user"), std::string::npos); } -TEST(BenchmarkRunnerTest, - StdoutSessionSuppressesFailureCheckpointAndEmitsOneTerminalPayload) { - BenchmarkConfig config; - config.loop_count = 1; - config.output_file = "-"; - config.only_bandwidth = true; - BenchmarkStatistics stats; - BenchmarkRunnerTestHooks hooks; - hooks.force_timer_creation_failure = true; - inject_deterministic_elapsed(hooks); - - int run_result = EXIT_SUCCESS; - int output_result = EXIT_FAILURE; - nlohmann::ordered_json terminal_payload; - testing::internal::CaptureStdout(); - testing::internal::CaptureStderr(); - { - JsonOutputSession session(make_json_output_target(config.output_file)); - run_result = run_all_benchmarks(config, stats, &hooks, &session); - terminal_payload = build_results_json(config, stats, 1.0); - output_result = session.write_final(terminal_payload); - } - const std::string error_output = testing::internal::GetCapturedStderr(); - const std::string json_output = testing::internal::GetCapturedStdout(); - - EXPECT_EQ(run_result, EXIT_FAILURE); - EXPECT_EQ(output_result, EXIT_SUCCESS); - EXPECT_EQ(stats.status, BenchmarkRunStatus::Failed); - EXPECT_EQ(stats.status_reason, Messages::error_timer_creation_failed()); - EXPECT_EQ(json_output, terminal_payload.dump(2) + "\n"); - EXPECT_EQ(nlohmann::ordered_json::parse(json_output), terminal_payload); - EXPECT_EQ(terminal_payload["configuration"]["benchmark_schema_version"], - Constants::BENCHMARK_JSON_SCHEMA_VERSION); - EXPECT_EQ(terminal_payload["configuration"]["output_file"], "-"); - EXPECT_EQ(terminal_payload["status"], "failed"); - EXPECT_FALSE(terminal_payload["results_complete"].get()); - EXPECT_FALSE(terminal_payload["conclusions_valid"].get()); - EXPECT_NE(error_output.find(Messages::error_timer_creation_failed()), - std::string::npos); -} - TEST(BenchmarkRunnerTest, FileSessionPersistsTheExistingCompletedLoopCheckpoint) { const ScopedDeterministicTimerSystemCalls timer_system_calls; diff --git a/tests/test_benchmark_work_plan.cpp b/tests/test_benchmark_work_plan.cpp index 4977663..ad066ad 100644 --- a/tests/test_benchmark_work_plan.cpp +++ b/tests/test_benchmark_work_plan.cpp @@ -50,19 +50,25 @@ TEST(BenchmarkWorkPlanTest, StateIndexesCoverEveryTargetOperationExactlyOnce) { EXPECT_TRUE(std::all_of(latency_indexes.begin(), latency_indexes.end(), [](bool covered) { return covered; })); } -TEST(BenchmarkWorkPlanTest, RejectsInvalidBandwidthParameters) { - EXPECT_EQ(build_benchmark_bandwidth_work_plan(0, 1, 1, BenchmarkTarget::MainMemory, BenchmarkOperation::Read).status, - BenchmarkMeasurementStatus::Invalid); - EXPECT_EQ( - build_benchmark_bandwidth_work_plan(4096, 0, 1, BenchmarkTarget::MainMemory, BenchmarkOperation::Read).status, - BenchmarkMeasurementStatus::Invalid); - EXPECT_EQ( - build_benchmark_bandwidth_work_plan(4096, 1, 0, BenchmarkTarget::MainMemory, BenchmarkOperation::Read).status, - BenchmarkMeasurementStatus::Invalid); - const BenchmarkWorkPlan latency_operation = - build_benchmark_bandwidth_work_plan(4096, 1, 1, BenchmarkTarget::MainMemory, BenchmarkOperation::Latency); - EXPECT_EQ(latency_operation.status, BenchmarkMeasurementStatus::Invalid); - EXPECT_EQ(latency_operation.status_reason, Messages::benchmark_reason_invalid_bandwidth_plan()); +TEST(BenchmarkWorkPlanTest, RejectsInvalidBandwidthParametersAndPayloadOverflow) { + struct InvalidCase { + size_t bytes; + int threads; + int passes; + BenchmarkOperation operation; + }; + for (const InvalidCase& entry : + {InvalidCase{0, 1, 1, BenchmarkOperation::Read}, InvalidCase{4096, 0, 1, BenchmarkOperation::Read}, + InvalidCase{4096, 1, 0, BenchmarkOperation::Read}, InvalidCase{4096, 1, 1, BenchmarkOperation::Latency}, + InvalidCase{std::numeric_limits::max(), 1, 1, BenchmarkOperation::Copy}}) { + SCOPED_TRACE(::testing::Message() << entry.bytes << "/" << entry.threads << "/" << entry.passes); + const BenchmarkWorkPlan plan = build_benchmark_bandwidth_work_plan(entry.bytes, entry.threads, entry.passes, + BenchmarkTarget::MainMemory, entry.operation); + EXPECT_EQ(plan.status, BenchmarkMeasurementStatus::Invalid); + if (entry.operation == BenchmarkOperation::Latency) { + EXPECT_EQ(plan.status_reason, Messages::benchmark_reason_invalid_bandwidth_plan()); + } + } } TEST(BenchmarkWorkPlanTest, CoversUnevenBufferExactlyWithAlignedBoundaries) { @@ -95,12 +101,6 @@ TEST(BenchmarkWorkPlanTest, ReducesWorkersForTinyBuffers) { EXPECT_EQ(plan.workers[0].span_bytes, 64u); } -TEST(BenchmarkWorkPlanTest, RejectsPayloadOverflow) { - BenchmarkWorkPlan plan = build_benchmark_bandwidth_work_plan(std::numeric_limits::max(), 1, 1, - BenchmarkTarget::MainMemory, BenchmarkOperation::Copy); - EXPECT_EQ(plan.status, BenchmarkMeasurementStatus::Invalid); -} - TEST(BenchmarkWorkPlanTest, PassUpdatesRejectLimitsAndOverflowWithoutMutation) { BenchmarkWorkPlan ordinary = build_benchmark_bandwidth_work_plan(4096, 1, 3, BenchmarkTarget::MainMemory, BenchmarkOperation::Read); diff --git a/tests/test_buffer_manager.cpp b/tests/test_buffer_manager.cpp index c28e215..56edc32 100644 --- a/tests/test_buffer_manager.cpp +++ b/tests/test_buffer_manager.cpp @@ -148,69 +148,30 @@ TEST_F(BufferManagerTest, InvalidPatternAllocationRequestsFailBeforeSystemCalls) } } -TEST_F(BufferManagerTest, PatternInitializationRejectsInvalidInputs) { - struct InvalidInputCase { +TEST_F(BufferManagerTest, PeakAccountingUsesModeSpecificLiveBuffers) { + struct AllocationCase { const char* name; - bool allocate_mappings; - size_t buffer_size; - std::string expected_reason; + size_t main_mb; + bool only_latency; + bool patterns; + bool custom_cache; + size_t expected_mb; }; - const InvalidInputCase cases[] = { - {"missing mappings", false, 512, Messages::error_main_buffers_not_allocated()}, - {"zero size", true, 0, Messages::error_buffer_size_zero_generic()}, - }; - - for (const InvalidInputCase& test_case : cases) { - SCOPED_TRACE(test_case.name); - PatternBuffers buffers; - if (test_case.allocate_mappings) { - const BenchmarkConfig config = make_pattern_config(); - ASSERT_EQ(allocate_pattern_buffers(config, buffers), EXIT_SUCCESS); - } - - testing::internal::CaptureStderr(); - const int result = initialize_pattern_buffers(buffers, test_case.buffer_size); - const std::string error = testing::internal::GetCapturedStderr(); - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_EQ(error, Messages::error_prefix() + test_case.expected_reason + "\n"); + for (const AllocationCase& entry : {AllocationCase{"cache peak", 1, false, false, false, 6}, + AllocationCase{"latency custom cache", 0, true, false, true, 43}, + AllocationCase{"patterns", 1, false, true, true, 2}}) { + SCOPED_TRACE(entry.name); + BenchmarkConfig config; + config.buffer_size = entry.main_mb * Constants::BYTES_PER_MB; + config.l1_buffer_size = Constants::BYTES_PER_MB; + config.l2_buffer_size = 2 * Constants::BYTES_PER_MB; + config.only_latency = entry.only_latency; + config.run_patterns = entry.patterns; + config.use_custom_cache_size = entry.custom_cache; + config.custom_buffer_size = 43 * Constants::BYTES_PER_MB; + config.max_total_allowed_mb = 0; + size_t total_memory_bytes = 0; + ASSERT_EQ(calculate_total_allocation_bytes(config, total_memory_bytes), EXIT_SUCCESS); + EXPECT_EQ(total_memory_bytes, entry.expected_mb * Constants::BYTES_PER_MB); } } - -TEST_F(BufferManagerTest, PeakAccountingUsesLargerCachePhaseInsteadOfMainPhase) { - BenchmarkConfig config; - config.buffer_size = Constants::BYTES_PER_MB; - config.l1_buffer_size = Constants::BYTES_PER_MB; - config.l2_buffer_size = 2 * Constants::BYTES_PER_MB; - config.max_total_allowed_mb = 0; - - size_t total_memory_bytes = 0; - ASSERT_EQ(calculate_total_allocation_bytes(config, total_memory_bytes), EXIT_SUCCESS); - EXPECT_EQ(total_memory_bytes, 6 * Constants::BYTES_PER_MB); -} - -TEST_F(BufferManagerTest, PeakAccountingHandlesLatencyOnlyCustomCacheWithMainDisabled) { - BenchmarkConfig config; - config.buffer_size = 0; - config.only_latency = true; - config.use_custom_cache_size = true; - config.custom_buffer_size = 43 * Constants::BYTES_PER_MB; - config.max_total_allowed_mb = 0; - - size_t total_memory_bytes = 0; - ASSERT_EQ(calculate_total_allocation_bytes(config, total_memory_bytes), EXIT_SUCCESS); - EXPECT_EQ(total_memory_bytes, 43 * Constants::BYTES_PER_MB); -} - -TEST_F(BufferManagerTest, PeakAccountingPatternsSkipLatencyAndCacheBuffers) { - BenchmarkConfig config; - config.buffer_size = Constants::BYTES_PER_MB; - config.run_patterns = true; - config.use_custom_cache_size = true; - config.custom_buffer_size = 43 * Constants::BYTES_PER_MB; - config.max_total_allowed_mb = 0; - - size_t total_memory_bytes = 0; - ASSERT_EQ(calculate_total_allocation_bytes(config, total_memory_bytes), EXIT_SUCCESS); - EXPECT_EQ(total_memory_bytes, 2 * Constants::BYTES_PER_MB); -} diff --git a/tests/test_config.cpp b/tests/test_config.cpp index aa5462b..8f63ef5 100644 --- a/tests/test_config.cpp +++ b/tests/test_config.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -79,24 +80,6 @@ std::string expected_invalid_value(const std::string& option, const std::string& } // namespace -// Test default configuration values -TEST(ConfigTest, DefaultValues) { - BenchmarkConfig config; - EXPECT_EQ(config.buffer_size_mb, Constants::DEFAULT_BUFFER_SIZE_MB); - EXPECT_EQ(config.iterations, Constants::DEFAULT_ITERATIONS); - EXPECT_EQ(config.loop_count, Constants::DEFAULT_LOOP_COUNT); - EXPECT_EQ(config.latency_stride_bytes, static_cast(Constants::LATENCY_STRIDE_BYTES)); - EXPECT_EQ(config.latency_tlb_locality_bytes, Constants::DEFAULT_LATENCY_TLB_LOCALITY_KB * Constants::BYTES_PER_KB); - EXPECT_EQ(config.tlb_seed, 0u); - EXPECT_FALSE(config.user_specified_tlb_seed); - EXPECT_EQ(config.pattern_seed, 0u); - EXPECT_FALSE(config.user_specified_pattern_seed); - EXPECT_EQ(config.benchmark_seed, 0u); - EXPECT_FALSE(config.user_specified_benchmark_seed); - EXPECT_EQ(config.custom_cache_size_kb_ll, -1); - EXPECT_FALSE(config.use_custom_cache_size); -} - TEST(ConfigTest, ParsesEquivalentLongAndShortOptions) { struct ValidAliasCase { const char* label; @@ -150,32 +133,6 @@ TEST(ConfigTest, ParseSweepValid) { EXPECT_EQ(config.sweep_max_runs, 4u); } -TEST(ConfigTest, ParseOutputPreservesStdoutSentinelAndLiteralDashPath) { - struct OutputTargetCase { - const char* mode; - const char* output; - bool patterns; - }; - - const OutputTargetCase cases[] = { - {"--benchmark", "-", false}, - {"--patterns", "./-", true}, - }; - - for (const OutputTargetCase& test_case : cases) { - SCOPED_TRACE(test_case.output); - BenchmarkConfig config; - const char* argv[] = {"program", "--output", test_case.output, - test_case.mode}; - - ASSERT_EQ(parse_arguments(4, const_cast(argv), config), - EXIT_SUCCESS); - EXPECT_EQ(config.output_file, test_case.output); - EXPECT_EQ(config.run_patterns, test_case.patterns); - EXPECT_EQ(config.run_benchmark, !test_case.patterns); - } -} - TEST(ConfigTest, ParseFlagShapedOutputTargetsRemainOpaqueAcrossGeneralModes) { struct ModeCase { const char* option; @@ -183,7 +140,7 @@ TEST(ConfigTest, ParseFlagShapedOutputTargetsRemainOpaqueAcrossGeneralModes) { }; const char* output_options[] = {"-o", "--output"}; - const char* output_targets[] = {"-T", "--analyze-tlb", "-k", "--cache-size"}; + const char* output_targets[] = {"-", "./-", "-T", "--analyze-tlb", "-k", "--cache-size"}; const ModeCase modes[] = {{"--benchmark", false}, {"--patterns", true}}; size_t case_index = 0; @@ -260,37 +217,15 @@ TEST(ConfigTest, ParseOutputPrescanSkipsExactlyOneFlagShapedValue) { } } -TEST(ConfigTest, ParseFlagShapedOutputTargetDoesNotInventMode) { - BenchmarkConfig config; - const CapturedParseResult parsed = parse_capturing_stderr({"program", "--output", "-T"}, config); - - ASSERT_EQ(parsed.result, EXIT_SUCCESS) << parsed.stderr_output; - EXPECT_EQ(config.output_file, "-T"); - EXPECT_FALSE(config.run_benchmark); - EXPECT_FALSE(config.run_patterns); - EXPECT_FALSE(config.analyze_tlb); - EXPECT_TRUE(parsed.stderr_output.empty()) << parsed.stderr_output; -} - -TEST(ConfigTest, ParseMissingOutputValueUsesNormalParserDiagnostic) { - BenchmarkConfig config; - const CapturedParseResult parsed = parse_capturing_stderr({"program", "--benchmark", "--output"}, config); - const std::string expected_diagnostic = - Messages::error_prefix() + Messages::error_missing_value("--output") + "\n"; - - EXPECT_EQ(parsed.result, EXIT_FAILURE); - EXPECT_EQ(parsed.stderr_output.find(expected_diagnostic), 0u) << parsed.stderr_output; -} - -TEST(ConfigTest, ParseDuplicateOutputUsesNormalParserDiagnostic) { - BenchmarkConfig config; - const CapturedParseResult parsed = - parse_capturing_stderr({"program", "--benchmark", "--output", "-T", "--output", "second.json"}, config); - const std::string expected_diagnostic = - Messages::error_prefix() + Messages::error_duplicate_option("--output") + "\n"; - - EXPECT_EQ(parsed.result, EXIT_FAILURE); - EXPECT_EQ(parsed.stderr_output.find(expected_diagnostic), 0u) << parsed.stderr_output; +TEST(ConfigTest, ParseMissingValuesUseNormalParserDiagnostics) { + for (const char* option : {"--output", "--cache-size", "--iterations"}) { + SCOPED_TRACE(option); + BenchmarkConfig config; + const CapturedParseResult parsed = parse_capturing_stderr({"program", "--benchmark", option}, config); + EXPECT_EQ(parsed.result, EXIT_FAILURE); + EXPECT_EQ(parsed.stderr_output.find(Messages::error_prefix() + Messages::error_missing_value(option) + "\n"), 0u) + << parsed.stderr_output; + } } TEST(ConfigTest, RejectsMalformedNumericTokensWithCentralizedErrors) { @@ -355,15 +290,13 @@ TEST(ConfigTest, RejectsMalformedSweepListsAndNumericValues) { constexpr const char* kInvalidSignedReason = "must be an integer without whitespace, a plus sign, or trailing characters"; const InvalidSweepCase cases[] = { + {"latency-samples=100,200", "unsupported sweep parameter: latency-samples", false}, {"buffer-size=1,,2", "sweep value list cannot contain empty values", false}, {"buffer-size=1x", kInvalidSignedReason, false}, {"cache-size=16x", kInvalidSignedReason, false}, {"threads=1x", kInvalidSignedReason, false}, {"latency-tlb-locality-kb=16x", kInvalidSignedReason, false}, {"latency-stride-bytes=64x", kInvalidSignedReason, false}, - {"buffer-size= 1", kInvalidSignedReason, false}, - {"buffer-size=+1", kInvalidSignedReason, false}, - {"buffer-size=9223372036854775808", "out of range", false}, {"latency-stride-bytes=64x", kInvalidSignedReason, true}, }; @@ -418,258 +351,170 @@ TEST(ConfigTest, RejectsSweepValuesOutsideSemanticDomains) { } } -TEST(ConfigTest, ParseSweepRejectsUnsupportedParameter) { - BenchmarkConfig config; - const char* argv[] = {"program", "--benchmark", "--sweep", "latency-samples=100,200"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateSweepRejectsTooManyRuns) { - BenchmarkConfig config; - config.run_benchmark = true; - config.run_sweep = true; - config.output_file = "sweep.json"; - config.sweep_max_runs = 3; - config.sweep_specs = { - {SweepParameter::BufferSizeMb, "buffer-size", {{"128"}, {"256"}}}, - {SweepParameter::Threads, "threads", {{"1"}, {"2"}}}, +TEST(ConfigTest, ValidateSweepRejectsInvalidConfigurationsWithExactDiagnostics) { + struct SweepCase { + const char* name; + std::function configure; + std::string diagnostic; }; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateSweepRejectsPatternCacheSize) { - BenchmarkConfig config; - config.run_patterns = true; - config.run_sweep = true; - config.output_file = "sweep.json"; - config.sweep_specs = { - {SweepParameter::CacheSizeKb, "cache-size", {{"1024"}}}, + const SweepCase cases[] = { + {"run limit", + [](BenchmarkConfig& config) { + config.run_benchmark = true; + config.sweep_max_runs = 3; + config.sweep_specs = {{SweepParameter::BufferSizeMb, "buffer-size", {{"128"}, {"256"}}}, + {SweepParameter::Threads, "threads", {{"1"}, {"2"}}}}; + }, + Messages::error_sweep_too_many_runs(4, 3)}, + {"pattern cache", + [](BenchmarkConfig& config) { + config.run_patterns = true; + config.sweep_specs = {{SweepParameter::CacheSizeKb, "cache-size", {{"1024"}}}}; + }, + Messages::error_sweep_parameter_not_allowed("cache-size", "--patterns")}, + {"missing output", + [](BenchmarkConfig& config) { + config.run_benchmark = true; + config.output_file.clear(); + config.sweep_specs = {{SweepParameter::BufferSizeMb, "buffer-size", {{"128"}}}}; + }, + Messages::error_sweep_requires_output()}, + {"TLB global random", + [](BenchmarkConfig& config) { + config.analyze_tlb = true; + SweepValue value; + value.raw_value = "global-random"; + value.latency_chain_mode = LatencyChainMode::GlobalRandom; + config.sweep_specs = {{SweepParameter::LatencyChainMode, "latency-chain-mode", {value}}}; + }, + Messages::error_analyze_tlb_global_random_unsupported()}, + {"duplicate stride", + [](BenchmarkConfig& config) { + config.analyze_tlb = true; + SweepValue first, second; + first.integer_value = 64; + second.integer_value = 128; + config.sweep_specs = {{SweepParameter::LatencyStrideBytes, "latency-stride-bytes", {first}}, + {SweepParameter::LatencyStrideBytes, "latency-stride-bytes", {second}}}; + }, + Messages::error_duplicate_sweep_parameter("latency-stride-bytes")}, }; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateSweepRequiresOutput) { - BenchmarkConfig config; - config.run_benchmark = true; - config.run_sweep = true; - config.sweep_specs = { - {SweepParameter::BufferSizeMb, "buffer-size", {{"128"}}}, - }; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateAnalyzeTlbSweepRejectsGlobalRandom) { - BenchmarkConfig config; - config.analyze_tlb = true; - config.run_sweep = true; - config.output_file = "sweep.json"; - SweepValue value; - value.raw_value = "global-random"; - value.latency_chain_mode = LatencyChainMode::GlobalRandom; - config.sweep_specs = { - {SweepParameter::LatencyChainMode, "latency-chain-mode", {value}}, - }; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateAnalyzeTlbRejectsStrideLargerThanPage) { - BenchmarkConfig config; - config.analyze_tlb = true; - config.latency_stride_bytes = scoped_config_test_hooks.page_size_bytes() * 2; - - EXPECT_EQ(validate_config(config), EXIT_FAILURE); + for (const SweepCase& test_case : cases) { + SCOPED_TRACE(test_case.name); + BenchmarkConfig config; + config.run_sweep = true; + config.output_file = "sweep.json"; + test_case.configure(config); + testing::internal::CaptureStderr(); + const int result = validate_config(config); + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(result, EXIT_FAILURE); + EXPECT_EQ(diagnostic, Messages::error_prefix() + test_case.diagnostic + "\n"); + } } -TEST(ConfigTest, ValidateAnalyzeTlbAllowsAlignedStrideThatDoesNotDividePage) { - BenchmarkConfig config; - config.analyze_tlb = true; - config.latency_stride_bytes = 136; - +TEST(ConfigTest, ValidateAnalyzeTlbStrideBoundsForDirectAndSweepModes) { + const size_t page_size = scoped_config_test_hooks.page_size_bytes(); ASSERT_EQ(136u % sizeof(uintptr_t), 0u); - ASSERT_NE(scoped_config_test_hooks.page_size_bytes() % config.latency_stride_bytes, 0u); - EXPECT_EQ(validate_config(config), EXIT_SUCCESS); + ASSERT_NE(page_size % 136u, 0u); + for (const size_t stride : {size_t{136}, page_size * 2}) { + for (bool sweep : {false, true}) { + SCOPED_TRACE(stride); + SCOPED_TRACE(sweep); + BenchmarkConfig config; + config.analyze_tlb = true; + config.run_sweep = sweep; + if (sweep) { + config.output_file = "sweep.json"; + SweepValue value; + value.integer_value = static_cast(stride); + config.sweep_specs = {{SweepParameter::LatencyStrideBytes, "latency-stride-bytes", {value}}}; + } else { + config.latency_stride_bytes = stride; + } + testing::internal::CaptureStderr(); + const int result = validate_config(config); + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(result, stride <= page_size ? EXIT_SUCCESS : EXIT_FAILURE); + EXPECT_EQ(diagnostic, stride <= page_size + ? "" + : Messages::error_prefix() + + Messages::error_analyze_tlb_stride_exceeds_page(stride, page_size) + "\n"); + } + } } -TEST(ConfigTest, ValidateAnalyzeTlbSweepAllowsAlignedNonDivisorStride) { - BenchmarkConfig config; - config.analyze_tlb = true; - config.run_sweep = true; - config.output_file = "sweep.json"; - SweepValue value; - value.raw_value = "136"; - value.integer_value = 136; - config.sweep_specs = { - {SweepParameter::LatencyStrideBytes, "latency-stride-bytes", {value}}, +TEST(ConfigTest, ParseSemanticErrorsHaveExactDiagnostics) { + struct InvalidCase { + const char* option; + const char* value; + std::string reason; }; - - EXPECT_EQ(validate_config(config), EXIT_SUCCESS); -} - -TEST(ConfigTest, ValidateSweepRejectsDuplicateParameter) { - BenchmarkConfig config; - config.analyze_tlb = true; - config.run_sweep = true; - config.output_file = "sweep.json"; - SweepValue first; - first.raw_value = "64"; - first.integer_value = 64; - SweepValue second; - second.raw_value = "128"; - second.integer_value = 128; - config.sweep_specs = { - {SweepParameter::LatencyStrideBytes, "latency-stride-bytes", {first}}, - {SweepParameter::LatencyStrideBytes, "latency-stride-bytes", {second}}, + const InvalidCase cases[] = { + {"--cache-size", "8", + Messages::error_cache_size_invalid(Constants::MIN_CACHE_SIZE_KB, Constants::MAX_CACHE_SIZE_KB, + Constants::MAX_CACHE_SIZE_KB / 1024)}, + {"--cache-size", "1100000", + Messages::error_cache_size_invalid(Constants::MIN_CACHE_SIZE_KB, Constants::MAX_CACHE_SIZE_KB, + Constants::MAX_CACHE_SIZE_KB / 1024)}, + {"--latency-stride-bytes", "0", + Messages::error_latency_stride_invalid(0, 1, std::numeric_limits::max())}, + {"--latency-chain-mode", "unknown-mode", Messages::error_latency_chain_mode_invalid()}, + {"--latency-tlb-locality-kb", "-1", + Messages::error_latency_tlb_locality_invalid( + -1, static_cast(std::numeric_limits::max() / Constants::BYTES_PER_KB))}, + {"--threads", "0", Messages::error_threads_invalid(0, 1, std::numeric_limits::max())}, + {"--latency-samples", "0", Messages::error_latency_samples_invalid(0, 1, std::numeric_limits::max())}, + {"-unknown", "value", ""}, }; - - EXPECT_EQ(validate_config(config), EXIT_FAILURE); -} - -// Test parsing custom cache size -// The code parses --cache-size in a first pass, then skips it in the second pass -// (since it was already parsed). This allows --cache-size to work correctly. -TEST(ConfigTest, ParseCustomCacheSize) { - for (const char* option : {"-k", "--cache-size"}) { - SCOPED_TRACE(option); + for (const InvalidCase& test_case : cases) { + SCOPED_TRACE(test_case.option); + SCOPED_TRACE(test_case.value); BenchmarkConfig config; - const CapturedParseResult parsed = parse_capturing_stderr({"program", option, "256"}, config); - - // The code parses the option in the first pass, then skips it in the - // second pass because it was already parsed and validated. - EXPECT_EQ(parsed.result, EXIT_SUCCESS); - EXPECT_EQ(config.custom_cache_size_kb_ll, 256); - EXPECT_TRUE(config.use_custom_cache_size); - EXPECT_TRUE(parsed.stderr_output.empty()) << parsed.stderr_output; + const CapturedParseResult parsed = parse_capturing_stderr({"program", test_case.option, test_case.value}, config); + EXPECT_EQ(parsed.result, EXIT_FAILURE); + const std::string diagnostic = test_case.reason.empty() + ? Messages::error_prefix() + Messages::error_unknown_option(test_case.option) + : expected_invalid_value(test_case.option, test_case.value, test_case.reason); + EXPECT_EQ(parsed.stderr_output.find(diagnostic + "\n"), 0u) << parsed.stderr_output; } } -TEST(ConfigTest, ParseMissingCacheSizeValueFailsInFirstPass) { - BenchmarkConfig config; - const CapturedParseResult parsed = parse_capturing_stderr({"program", "--cache-size"}, config); - - EXPECT_EQ(parsed.result, EXIT_FAILURE); - EXPECT_NE(parsed.stderr_output.find(Messages::error_prefix() + Messages::error_missing_value("--cache-size")), - std::string::npos); -} - -// Test parsing invalid cache size (too small) -TEST(ConfigTest, ParseInvalidCacheSizeTooSmall) { - BenchmarkConfig config; - const char* argv[] = {"program", "--cache-size", "8"}; // Below minimum of 16 KB - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -// Test parsing invalid cache size (too large) -TEST(ConfigTest, ParseInvalidCacheSizeTooLarge) { - BenchmarkConfig config; - const char* argv[] = {"program", "--cache-size", "1100000"}; // Above maximum of 1048576 KB (1 GB) - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - // Test parsing cache size zero (validated later; allowed only with --only-latency) -TEST(ConfigTest, ParseCacheSizeZero) { - BenchmarkConfig config; - const char* argv[] = {"program", "--cache-size", "0"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(config.custom_cache_size_kb_ll, 0); - EXPECT_TRUE(config.use_custom_cache_size); -} - -// Test parsing buffer size zero (validated later; allowed only with --only-latency) -TEST(ConfigTest, ParseBufferSizeZero) { - BenchmarkConfig config; - const char* argv[] = {"program", "--buffer-size", "0"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(config.buffer_size_mb, 0u); -} - -TEST(ConfigTest, ParseLatencyTlbLocalityValid) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-tlb-locality-kb", "16"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(config.latency_tlb_locality_bytes, static_cast(16) * Constants::BYTES_PER_KB); -} - -TEST(ConfigTest, ParseLatencyStrideValid) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-stride-bytes", "64"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(config.latency_stride_bytes, 64u); -} - -TEST(ConfigTest, ParseLatencyStrideInvalidZero) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-stride-bytes", "0"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseLatencyChainModeValid) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-chain-mode", "same-random-in-box"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(config.latency_chain_mode, LatencyChainMode::SameRandomInBoxIncreasingBox); -} - -TEST(ConfigTest, ParseLatencyChainModeInvalid) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-chain-mode", "unknown-mode"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseLatencyTlbLocalityZeroDisables) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-tlb-locality-kb", "0"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(config.latency_tlb_locality_bytes, 0u); -} - -TEST(ConfigTest, ParseLatencyTlbLocalityInvalidNegative) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-tlb-locality-kb", "-1"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); +TEST(ConfigTest, ParseExplicitMemoryAndLatencyOptions) { + struct ValidCase { + const char* option; + const char* value; + std::function verify; + }; + const ValidCase cases[] = { + {"--cache-size", "0", + [](const BenchmarkConfig& config) { + EXPECT_EQ(config.custom_cache_size_kb_ll, 0); + EXPECT_TRUE(config.use_custom_cache_size); + }}, + {"--buffer-size", "0", [](const BenchmarkConfig& config) { EXPECT_EQ(config.buffer_size_mb, 0u); }}, + {"--latency-tlb-locality-kb", "16", + [](const BenchmarkConfig& config) { + EXPECT_EQ(config.latency_tlb_locality_bytes, 16u * Constants::BYTES_PER_KB); + }}, + {"--latency-tlb-locality-kb", "0", + [](const BenchmarkConfig& config) { EXPECT_EQ(config.latency_tlb_locality_bytes, 0u); }}, + {"--latency-stride-bytes", "64", + [](const BenchmarkConfig& config) { EXPECT_EQ(config.latency_stride_bytes, 64u); }}, + {"--latency-chain-mode", "same-random-in-box", + [](const BenchmarkConfig& config) { + EXPECT_EQ(config.latency_chain_mode, LatencyChainMode::SameRandomInBoxIncreasingBox); + }}, + }; + for (const ValidCase& test_case : cases) { + SCOPED_TRACE(std::string(test_case.option) + " " + test_case.value); + BenchmarkConfig config; + const CapturedParseResult parsed = parse_capturing_stderr({"program", test_case.option, test_case.value}, config); + ASSERT_EQ(parsed.result, EXIT_SUCCESS) << parsed.stderr_output; + EXPECT_TRUE(parsed.stderr_output.empty()); + test_case.verify(config); + } } TEST(ConfigTest, ParseAnalyzeTlbStandaloneAppliesSafeDefaultsAndGeneratedSeed) { @@ -688,231 +533,97 @@ TEST(ConfigTest, ParseAnalyzeTlbStandaloneAppliesSafeDefaultsAndGeneratedSeed) { } } -TEST(ConfigTest, ParseAnalyzeTlbExplicitSweepLimitOverridesSafeDefault) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--sweep-max-runs", "24"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(config.sweep_max_runs, 24u); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithOtherArgumentsFails) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--buffer-size", "512"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithExplicitSeedSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--seed", "18446744073709551615"}; - - EXPECT_EQ(parse_arguments(4, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_EQ(config.tlb_seed, std::numeric_limits::max()); - EXPECT_TRUE(config.user_specified_tlb_seed); -} - -TEST(ConfigTest, ParseAnalyzeTlbRejectsDuplicateSeed) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--seed", "42", "--seed", "43"}; - - EXPECT_EQ(parse_arguments(6, const_cast(argv), config), EXIT_FAILURE); -} - -TEST(ConfigTest, ParsePatternsWithExplicitSeedSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--patterns", "--seed", "18446744073709551615"}; - - EXPECT_EQ(parse_arguments(4, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_EQ(config.pattern_seed, std::numeric_limits::max()); - EXPECT_TRUE(config.user_specified_pattern_seed); -} - -TEST(ConfigTest, ParsePatternsUsesGeneratedSeedAndDetectedCoreCountByDefault) { - BenchmarkConfig config; - const char* argv[] = {"program", "--patterns"}; - - ASSERT_EQ(parse_arguments(2, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_EQ(config.pattern_seed, scoped_config_test_hooks.generated_seed()); - EXPECT_FALSE(config.user_specified_pattern_seed); - EXPECT_EQ(config.num_threads, scoped_config_test_hooks.total_logical_cores()); - EXPECT_FALSE(config.user_specified_threads); -} - -TEST(ConfigTest, ParseBenchmarkWithExplicitSeedSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--benchmark", "--seed", "18446744073709551615"}; - - EXPECT_EQ(parse_arguments(4, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_EQ(config.benchmark_seed, std::numeric_limits::max()); - EXPECT_TRUE(config.user_specified_benchmark_seed); - EXPECT_FALSE(config.user_specified_pattern_seed); -} - -TEST(ConfigTest, ParseBenchmarkGeneratesSeedWhenOmitted) { - BenchmarkConfig config; - const char* argv[] = {"program", "--benchmark"}; - - EXPECT_EQ(parse_arguments(2, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_EQ(config.benchmark_seed, scoped_config_test_hooks.generated_seed()); - EXPECT_FALSE(config.user_specified_benchmark_seed); -} - -TEST(ConfigTest, ParsePatternsHonorsExplicitThreadCount) { - BenchmarkConfig config; - const char* argv[] = {"program", "--patterns", "--threads", "1"}; - - ASSERT_EQ(parse_arguments(4, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_EQ(config.num_threads, 1); - EXPECT_TRUE(config.user_specified_threads); -} - -TEST(ConfigTest, ParsePatternsRejectsDuplicateSeed) { - BenchmarkConfig config; - const char* argv[] = {"program", "--patterns", "--seed", "42", "--seed", "43"}; - - EXPECT_EQ(parse_arguments(6, const_cast(argv), config), EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateConfigRejectsPatternSeedWithoutPatterns) { - BenchmarkConfig config; - config.pattern_seed = 42; - config.user_specified_pattern_seed = true; - - EXPECT_EQ(validate_config(config), EXIT_FAILURE); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithOutputFirstSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--output", "tlb.json", "--analyze-tlb"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_TRUE(config.analyze_tlb); - EXPECT_EQ(config.output_file, "tlb.json"); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithLatencyStrideSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--latency-stride-bytes", "128"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_TRUE(config.analyze_tlb); - EXPECT_EQ(config.latency_stride_bytes, 128u); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithLatencyChainModeSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--latency-chain-mode", "random-box"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_TRUE(config.analyze_tlb); - EXPECT_EQ(config.latency_chain_mode, LatencyChainMode::RandomInBoxRandomBox); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithGlobalRandomLatencyChainModeFails) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--latency-chain-mode", "global-random"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithTlbDensityLowSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--tlb-density", "low"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_TRUE(config.analyze_tlb); - EXPECT_EQ(config.tlb_sweep_density, TlbSweepDensity::Low); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithTlbDensityMediumSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--tlb-density", "medium"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_TRUE(config.analyze_tlb); - EXPECT_EQ(config.tlb_sweep_density, TlbSweepDensity::Medium); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithTlbDensityHighSucceeds) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--tlb-density", "high"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_TRUE(config.analyze_tlb); - EXPECT_EQ(config.tlb_sweep_density, TlbSweepDensity::High); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithInvalidTlbDensityFails) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--tlb-density", "ultra"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithInvalidLatencyStrideFails) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--latency-stride-bytes", "0"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithUnalignedLatencyStrideFails) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--latency-stride-bytes", "65"}; - int argc = 4; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseAnalyzeTlbWithMissingOutputValueFails) { - BenchmarkConfig config; - const char* argv[] = {"program", "--analyze-tlb", "--output"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); +TEST(ConfigTest, ParseAnalyzeTlbExplicitOptions) { + struct ValidCase { + const char* option; + const char* value; + std::function verify; + }; + const ValidCase cases[] = { + {"--sweep-max-runs", "24", [](const BenchmarkConfig& config) { EXPECT_EQ(config.sweep_max_runs, 24u); }}, + {"--seed", "18446744073709551615", + [](const BenchmarkConfig& config) { + EXPECT_EQ(config.tlb_seed, std::numeric_limits::max()); + EXPECT_TRUE(config.user_specified_tlb_seed); + }}, + {"--latency-stride-bytes", "128", + [](const BenchmarkConfig& config) { EXPECT_EQ(config.latency_stride_bytes, 128u); }}, + {"--latency-chain-mode", "random-box", + [](const BenchmarkConfig& config) { + EXPECT_EQ(config.latency_chain_mode, LatencyChainMode::RandomInBoxRandomBox); + }}, + {"--tlb-density", "low", + [](const BenchmarkConfig& config) { EXPECT_EQ(config.tlb_sweep_density, TlbSweepDensity::Low); }}, + {"--tlb-density", "medium", + [](const BenchmarkConfig& config) { EXPECT_EQ(config.tlb_sweep_density, TlbSweepDensity::Medium); }}, + {"--tlb-density", "high", + [](const BenchmarkConfig& config) { EXPECT_EQ(config.tlb_sweep_density, TlbSweepDensity::High); }}, + }; + for (const ValidCase& test_case : cases) { + SCOPED_TRACE(std::string(test_case.option) + " " + test_case.value); + BenchmarkConfig config; + const CapturedParseResult parsed = + parse_capturing_stderr({"program", "--analyze-tlb", test_case.option, test_case.value}, config); + ASSERT_EQ(parsed.result, EXIT_SUCCESS) << parsed.stderr_output; + EXPECT_TRUE(parsed.stderr_output.empty()); + EXPECT_TRUE(config.analyze_tlb); + test_case.verify(config); + } } -TEST(ConfigTest, ParseThreadsInvalidZero) { - BenchmarkConfig config; - const char* argv[] = {"program", "--threads", "0"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); +TEST(ConfigTest, ParseAnalyzeTlbRejectsInvalidOptionsWithExactDiagnostics) { + struct InvalidCase { + std::vector options; + std::string diagnostic; + }; + const InvalidCase cases[] = { + {{"--buffer-size", "512"}, Messages::error_analyze_tlb_must_be_used_alone()}, + {{"--seed", "42", "--seed", "43"}, Messages::error_duplicate_option("--seed")}, + {{"--latency-chain-mode", "global-random"}, Messages::error_analyze_tlb_global_random_unsupported()}, + {{"--tlb-density", "ultra"}, + Messages::error_invalid_value("--tlb-density", "ultra", "must be one of: low, medium, high")}, + {{"--latency-stride-bytes", "0"}, + Messages::error_invalid_value( + "--latency-stride-bytes", "0", + Messages::error_latency_stride_invalid(0, 1, std::numeric_limits::max()))}, + {{"--latency-stride-bytes", "65"}, Messages::error_latency_stride_alignment(65, sizeof(void*))}, + {{"--output"}, Messages::error_missing_value("--output")}, + }; + for (const InvalidCase& test_case : cases) { + SCOPED_TRACE(testing::PrintToString(test_case.options)); + std::vector arguments = {"program", "--analyze-tlb"}; + arguments.insert(arguments.end(), test_case.options.begin(), test_case.options.end()); + BenchmarkConfig config; + const CapturedParseResult parsed = parse_capturing_stderr(arguments, config); + EXPECT_EQ(parsed.result, EXIT_FAILURE); + EXPECT_EQ(parsed.stderr_output.find(Messages::error_prefix() + test_case.diagnostic + "\n"), 0u) + << parsed.stderr_output; + } } -TEST(ConfigTest, ParseLatencySamplesInvalidZero) { - BenchmarkConfig config; - const char* argv[] = {"program", "--latency-samples", "0"}; - int argc = 3; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); +TEST(ConfigTest, ParseStandardAndPatternSeedsAndThreadCounts) { + for (bool patterns : {false, true}) { + for (bool explicit_seed : {false, true}) { + for (bool explicit_threads : {false, true}) { + SCOPED_TRACE(patterns); + SCOPED_TRACE(explicit_seed); + SCOPED_TRACE(explicit_threads); + std::vector arguments = {"program", patterns ? "--patterns" : "--benchmark"}; + if (explicit_seed) arguments.insert(arguments.end(), {"--seed", "18446744073709551615"}); + if (explicit_threads) arguments.insert(arguments.end(), {"--threads", "1"}); + BenchmarkConfig config; + const CapturedParseResult parsed = parse_capturing_stderr(arguments, config); + ASSERT_EQ(parsed.result, EXIT_SUCCESS) << parsed.stderr_output; + EXPECT_TRUE(parsed.stderr_output.empty()); + EXPECT_EQ(patterns ? config.pattern_seed : config.benchmark_seed, + explicit_seed ? std::numeric_limits::max() : scoped_config_test_hooks.generated_seed()); + EXPECT_EQ(config.user_specified_pattern_seed, patterns && explicit_seed); + EXPECT_EQ(config.user_specified_benchmark_seed, !patterns && explicit_seed); + EXPECT_EQ(config.num_threads, explicit_threads ? 1 : scoped_config_test_hooks.total_logical_cores()); + EXPECT_EQ(config.user_specified_threads, explicit_threads); + } + } + } } TEST(ConfigTest, ParseDuplicateValueOptionsRejected) { @@ -931,37 +642,27 @@ TEST(ConfigTest, ParseDuplicateValueOptionsRejected) { {"--latency-chain-mode", "auto", "global-random"}, {"--latency-tlb-locality-kb", "16", "32"}, {"--threads", "1", "2"}, - {"--output", "first.json", "second.json"}, + {"--output", "-T", "second.json"}, {"--cache-size", "256", "512"}, + {"--seed", "42", "43"}, }; for (const DuplicateOptionCase& test_case : cases) { - SCOPED_TRACE(test_case.option); - BenchmarkConfig config; - const char* argv[] = {"program", test_case.option, test_case.first_value, test_case.option, test_case.second_value}; - int argc = 5; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); + for (const char* mode : {"--benchmark", "--patterns"}) { + SCOPED_TRACE(test_case.option); + SCOPED_TRACE(mode); + BenchmarkConfig config; + const CapturedParseResult parsed = parse_capturing_stderr( + {"program", mode, test_case.option, test_case.first_value, test_case.option, test_case.second_value}, config); + EXPECT_EQ(parsed.result, EXIT_FAILURE); + EXPECT_EQ(parsed.stderr_output.find(Messages::error_prefix() + + Messages::error_duplicate_option(test_case.option) + "\n"), + 0u) + << parsed.stderr_output; + } } } -// Test parsing missing value for option -TEST(ConfigTest, ParseMissingValue) { - BenchmarkConfig config; - const char* argv[] = {"program", "--iterations"}; - int argc = 2; - - int result = parse_arguments(argc, const_cast(argv), config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, RejectsUnknownSingleDashLongOption) { - BenchmarkConfig config; - EXPECT_EQ(parse_capturing_stderr({"program", "-unknown", "value"}, config).result, - EXIT_FAILURE); -} - // Test parsing help flag TEST(ConfigTest, ParseHelpFlag) { BenchmarkConfig config; @@ -973,55 +674,47 @@ TEST(ConfigTest, ParseHelpFlag) { EXPECT_TRUE(config.help_printed); } -TEST(ConfigTest, ValidateParsedOnlyBandwidthRejectsLatencySamples) { - BenchmarkConfig config; - const char* argv[] = {"program", "--benchmark", "--only-bandwidth", "--latency-samples", "10"}; - int argc = 5; - - ASSERT_EQ(parse_arguments(argc, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_TRUE(config.only_bandwidth); - EXPECT_TRUE(config.user_specified_latency_samples); - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateParsedOnlyBandwidthRejectsCacheSize) { - BenchmarkConfig config; - const char* argv[] = {"program", "--benchmark", "--only-bandwidth", "--cache-size", "256"}; - int argc = 5; - - ASSERT_EQ(parse_arguments(argc, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_TRUE(config.only_bandwidth); - EXPECT_TRUE(config.use_custom_cache_size); - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateParsedOnlyLatencyRejectsIterations) { - BenchmarkConfig config; - const char* argv[] = {"program", "--benchmark", "--only-latency", "--iterations", "10"}; - int argc = 5; - - ASSERT_EQ(parse_arguments(argc, const_cast(argv), config), EXIT_SUCCESS); - EXPECT_TRUE(config.only_latency); - EXPECT_TRUE(config.user_specified_iterations); - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ParseBenchmarkAndPatternsAreMutuallyExclusiveInEitherOrder) { - const std::vector> cases = { - {"program", "--benchmark", "--patterns"}, - {"program", "--patterns", "--benchmark"}, +TEST(ConfigTest, ValidateParsedIncompatibleFlagsWithExactDiagnostics) { + struct InvalidCase { + std::vector arguments; + bool BenchmarkConfig::* first_flag; + bool BenchmarkConfig::* second_flag; + std::string diagnostic; }; - - for (const std::vector& arguments : cases) { - SCOPED_TRACE(arguments[1]); + const InvalidCase cases[] = { + {{"program", "--benchmark", "--only-bandwidth", "--latency-samples", "10"}, + &BenchmarkConfig::only_bandwidth, + &BenchmarkConfig::user_specified_latency_samples, + Messages::error_only_bandwidth_with_latency_samples()}, + {{"program", "--benchmark", "--only-bandwidth", "--cache-size", "256"}, + &BenchmarkConfig::only_bandwidth, + &BenchmarkConfig::use_custom_cache_size, + Messages::error_only_bandwidth_with_cache_size()}, + {{"program", "--benchmark", "--only-latency", "--iterations", "10"}, + &BenchmarkConfig::only_latency, + &BenchmarkConfig::user_specified_iterations, + Messages::error_only_latency_with_iterations()}, + {{"program", "--benchmark", "--only-bandwidth", "--only-latency"}, + &BenchmarkConfig::only_bandwidth, + &BenchmarkConfig::only_latency, + Messages::error_incompatible_flags()}, + {{"program", "--patterns", "--only-bandwidth"}, + &BenchmarkConfig::run_patterns, + &BenchmarkConfig::only_bandwidth, + Messages::error_only_flags_with_patterns()}, + }; + for (const InvalidCase& test_case : cases) { + SCOPED_TRACE(testing::PrintToString(test_case.arguments)); BenchmarkConfig config; - EXPECT_EQ(parse_capturing_stderr(arguments, config).result, EXIT_FAILURE); + const CapturedParseResult parsed = parse_capturing_stderr(test_case.arguments, config); + ASSERT_EQ(parsed.result, EXIT_SUCCESS) << parsed.stderr_output; + EXPECT_TRUE(config.*test_case.first_flag); + EXPECT_TRUE(config.*test_case.second_flag); + testing::internal::CaptureStderr(); + const int result = validate_config(config); + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(result, EXIT_FAILURE); + EXPECT_EQ(diagnostic, Messages::error_prefix() + test_case.diagnostic + "\n"); } } @@ -1084,24 +777,6 @@ TEST(ConfigTest, CalculateAccessCountsUsesExactLinearScaling) { } // Test validate_config rejects mutually exclusive flags -TEST(ConfigTest, ValidateConfigRejectsOnlyBandwidthAndOnlyLatency) { - BenchmarkConfig config; - config.only_bandwidth = true; - config.only_latency = true; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -// Test validate_config rejects only-flags with pattern mode -TEST(ConfigTest, ValidateConfigRejectsOnlyFlagsWithPatterns) { - BenchmarkConfig config; - config.run_patterns = true; - config.only_bandwidth = true; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} // Test mode-aware per-buffer capping based on required main buffer count TEST(ConfigTest, ValidateConfigModeAwareBufferCap) { @@ -1134,116 +809,74 @@ TEST(ConfigTest, ValidateConfigModeAwareBufferCap) { EXPECT_EQ(lat_only.buffer_size_mb, expected_cap(lat_only, 1)); } -TEST(ConfigTest, ValidateConfigRejectsBufferSizeZeroWithoutOnlyLatency) { - BenchmarkConfig config; - config.buffer_size_mb = 0; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateConfigRejectsCacheSizeZeroWithoutOnlyLatency) { - BenchmarkConfig config; - config.custom_cache_size_kb_ll = 0; - config.use_custom_cache_size = true; - config.custom_cache_size_bytes = 0; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateConfigAllowsCacheOnlyLatencyMode) { - BenchmarkConfig config; - config.run_benchmark = true; - config.only_latency = true; - config.buffer_size_mb = 0; - config.custom_cache_size_kb_ll = 8096; - config.use_custom_cache_size = true; - config.custom_cache_size_bytes = static_cast(8096) * Constants::BYTES_PER_KB; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_SUCCESS); -} - -TEST(ConfigTest, ValidateConfigAllowsMainOnlyLatencyMode) { - BenchmarkConfig config; - config.run_benchmark = true; - config.only_latency = true; - config.buffer_size_mb = 16; - config.custom_cache_size_kb_ll = 0; - config.use_custom_cache_size = true; - config.custom_cache_size_bytes = 0; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_SUCCESS); -} - -TEST(ConfigTest, ValidateConfigRejectsOnlyLatencyWithNoTargets) { - BenchmarkConfig config; - config.run_benchmark = true; - config.only_latency = true; - config.buffer_size_mb = 0; - config.custom_cache_size_kb_ll = 0; - config.use_custom_cache_size = true; - config.custom_cache_size_bytes = 0; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateConfigRejectsLatencyTlbLocalityNotPageMultiple) { - BenchmarkConfig config; - const size_t page_size = scoped_config_test_hooks.page_size_bytes(); - config.latency_tlb_locality_bytes = page_size + Constants::BYTES_PER_KB; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateConfigRejectsLatencyStrideNotPointerAligned) { - BenchmarkConfig config; - config.latency_stride_bytes = sizeof(uintptr_t) + 1; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateConfigRejectsLatencyTlbLocalityTooSmallForStride) { - BenchmarkConfig config; - const size_t page_size = scoped_config_test_hooks.page_size_bytes(); - config.latency_stride_bytes = page_size; - config.latency_tlb_locality_bytes = page_size; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); -} - -TEST(ConfigTest, ValidateConfigAllowsLatencyTlbLocalityForStride) { - BenchmarkConfig config; - const size_t page_size = scoped_config_test_hooks.page_size_bytes(); - config.latency_stride_bytes = page_size; - config.latency_tlb_locality_bytes = page_size * 2; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_SUCCESS); -} - -TEST(ConfigTest, ValidateConfigRejectsLatencyChainModeWithoutLocality) { - BenchmarkConfig config; - config.latency_chain_mode = LatencyChainMode::SameRandomInBoxIncreasingBox; - config.latency_tlb_locality_bytes = 0; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_FAILURE); +TEST(ConfigTest, ValidateConfigLatencyTargetCombinations) { + struct TargetCase { + bool only_latency; + unsigned long buffer_mb; + long long cache_kb; + std::string diagnostic; + }; + const TargetCase cases[] = { + {false, 0, -1, Messages::error_buffersize_zero_requires_only_latency()}, + {false, 512, 0, Messages::error_cache_size_zero_requires_only_latency()}, + {true, 0, 8096, ""}, + {true, 16, 0, ""}, + {true, 0, 0, Messages::error_only_latency_requires_latency_target()}, + }; + for (const TargetCase& test_case : cases) { + SCOPED_TRACE(test_case.only_latency); + SCOPED_TRACE(test_case.buffer_mb); + SCOPED_TRACE(test_case.cache_kb); + BenchmarkConfig config; + config.run_benchmark = true; + config.only_latency = test_case.only_latency; + config.buffer_size_mb = test_case.buffer_mb; + config.custom_cache_size_kb_ll = test_case.cache_kb; + config.use_custom_cache_size = test_case.cache_kb >= 0; + config.custom_cache_size_bytes = + test_case.cache_kb < 0 ? 0 : static_cast(test_case.cache_kb) * Constants::BYTES_PER_KB; + testing::internal::CaptureStderr(); + const int result = validate_config(config); + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(result, test_case.diagnostic.empty() ? EXIT_SUCCESS : EXIT_FAILURE); + EXPECT_EQ(diagnostic, test_case.diagnostic.empty() ? "" : Messages::error_prefix() + test_case.diagnostic + "\n"); + } } -TEST(ConfigTest, ValidateConfigAllowsGlobalLatencyChainModeWithoutLocality) { - BenchmarkConfig config; - config.latency_chain_mode = LatencyChainMode::GlobalRandom; - config.latency_tlb_locality_bytes = 0; - - int result = validate_config(config); - EXPECT_EQ(result, EXIT_SUCCESS); +TEST(ConfigTest, ValidateConfigLatencyLocalityAndStrideConstraints) { + struct LocalityCase { + size_t stride; + size_t locality; + LatencyChainMode mode; + std::string diagnostic; + }; + const size_t page = scoped_config_test_hooks.page_size_bytes(); + const LocalityCase cases[] = { + {Constants::LATENCY_STRIDE_BYTES, page + Constants::BYTES_PER_KB, LatencyChainMode::Auto, + Messages::error_latency_tlb_locality_page_multiple(page / Constants::BYTES_PER_KB + 1, + page / Constants::BYTES_PER_KB)}, + {sizeof(uintptr_t) + 1, page, LatencyChainMode::Auto, + Messages::error_latency_stride_alignment(sizeof(uintptr_t) + 1, sizeof(uintptr_t))}, + {page, page, LatencyChainMode::Auto, Messages::error_latency_tlb_locality_too_small_for_stride(page, page)}, + {page, page * 2, LatencyChainMode::Auto, ""}, + {Constants::LATENCY_STRIDE_BYTES, 0, LatencyChainMode::SameRandomInBoxIncreasingBox, + Messages::error_latency_chain_mode_requires_locality("same-random-in-box")}, + {Constants::LATENCY_STRIDE_BYTES, 0, LatencyChainMode::GlobalRandom, ""}, + }; + for (const LocalityCase& test_case : cases) { + SCOPED_TRACE(test_case.stride); + SCOPED_TRACE(test_case.locality); + SCOPED_TRACE(static_cast(test_case.mode)); + BenchmarkConfig config; + config.latency_stride_bytes = test_case.stride; + config.latency_tlb_locality_bytes = test_case.locality; + config.latency_chain_mode = test_case.mode; + testing::internal::CaptureStderr(); + const int result = validate_config(config); + const std::string diagnostic = testing::internal::GetCapturedStderr(); + EXPECT_EQ(result, test_case.diagnostic.empty() ? EXIT_SUCCESS : EXIT_FAILURE); + EXPECT_EQ(diagnostic, test_case.diagnostic.empty() ? "" : Messages::error_prefix() + test_case.diagnostic + "\n"); + } } TEST(ConfigTest, ValidateConfigAnalyzeTlbSkipsUnrelatedStandardModeRules) { diff --git a/tests/test_core_to_core_cli.cpp b/tests/test_core_to_core_cli.cpp index b19ac9e..681e939 100644 --- a/tests/test_core_to_core_cli.cpp +++ b/tests/test_core_to_core_cli.cpp @@ -84,49 +84,22 @@ TEST(CoreToCoreCliTest, ParsesDefaultStandaloneModeValues) { } TEST(CoreToCoreCliTest, ParsesOptionalModeArguments) { - // Optional standalone flags should override defaults without enabling help mode. - CoreToCoreLatencyConfig config; - const int parse_result = - parse_with_args({"memory_benchmark", - "--analyze-core2core", - "--count", - "5", - "--latency-samples", - "128", - "--output", - "core2core.json"}, - config); - - EXPECT_EQ(parse_result, EXIT_SUCCESS); - EXPECT_FALSE(config.help_requested); - EXPECT_EQ(config.loop_count, 5); - EXPECT_EQ(config.latency_sample_count, 128); - EXPECT_EQ(config.output_file, "core2core.json"); -} - -TEST(CoreToCoreCliTest, ParsesShortModeArguments) { - CoreToCoreLatencyConfig config; - const int parse_result = - parse_with_args({"memory_benchmark", "-C", "-r", "5", "-n", "128", "-o", "core2core.json"}, config); - - EXPECT_EQ(parse_result, EXIT_SUCCESS); - EXPECT_FALSE(config.help_requested); - EXPECT_EQ(config.loop_count, 5); - EXPECT_EQ(config.latency_sample_count, 128); - EXPECT_EQ(config.output_file, "core2core.json"); -} - -TEST(CoreToCoreCliTest, PreservesRawStdoutSentinelAndExplicitDotDashFile) { - for (const std::string& raw_output : {std::string("-"), std::string("./-")}) { - SCOPED_TRACE(raw_output); - CoreToCoreLatencyConfig config; - const int parse_result = parse_with_args( - {"memory_benchmark", "--analyze-core2core", "--output", - raw_output}, - config); - - EXPECT_EQ(parse_result, EXIT_SUCCESS); - EXPECT_EQ(config.output_file, raw_output); + for (bool short_options : {false, true}) { + for (const char* output : {"core2core.json", "-", "./-"}) { + SCOPED_TRACE(short_options); + SCOPED_TRACE(output); + CoreToCoreLatencyConfig config; + const CapturedCoreCliParse parsed = parse_capturing_stderr( + {"memory_benchmark", short_options ? "-C" : "--analyze-core2core", short_options ? "-r" : "--count", "5", + short_options ? "-n" : "--latency-samples", "128", short_options ? "-o" : "--output", output}, + config); + ASSERT_EQ(parsed.result, EXIT_SUCCESS) << parsed.stderr_output; + EXPECT_TRUE(parsed.stderr_output.empty()); + EXPECT_FALSE(config.help_requested); + EXPECT_EQ(config.loop_count, 5); + EXPECT_EQ(config.latency_sample_count, 128); + EXPECT_EQ(config.output_file, output); + } } } @@ -191,60 +164,27 @@ TEST(CoreToCoreCliTest, ParsesSweepArguments) { EXPECT_EQ(config.sweep_specs[1].values[1].integer_value, 8); } -TEST(CoreToCoreCliTest, RejectsUnsupportedSweepParameter) { - CoreToCoreLatencyConfig config; - const int parse_result = - parse_with_args({"memory_benchmark", - "--analyze-core2core", - "--output", - "core2core_sweep.json", - "--sweep", - "threads=1,2"}, - config); - - EXPECT_EQ(parse_result, EXIT_FAILURE); -} - -TEST(CoreToCoreCliTest, RejectsSweepWithoutOutput) { - CoreToCoreLatencyConfig config; - const int parse_result = - parse_with_args({"memory_benchmark", "--analyze-core2core", "--sweep", "count=1,2"}, config); - - EXPECT_EQ(parse_result, EXIT_FAILURE); -} - -TEST(CoreToCoreCliTest, RejectsSweepExceedingMaxRuns) { - CoreToCoreLatencyConfig config; - const int parse_result = - parse_with_args({"memory_benchmark", - "--analyze-core2core", - "--output", - "core2core_sweep.json", - "--sweep", - "count=1,2", - "--sweep", - "latency-samples=4,8", - "--sweep-max-runs", - "3"}, - config); - - EXPECT_EQ(parse_result, EXIT_FAILURE); -} - -TEST(CoreToCoreCliTest, RejectsDuplicateSweepParameters) { - CoreToCoreLatencyConfig config; - const int parse_result = parse_with_args( - {"memory_benchmark", - "--analyze-core2core", - "--output", - "core2core_sweep.json", - "--sweep", - "count=1,2", - "--sweep", - "count=3,4"}, - config); - - EXPECT_EQ(parse_result, EXIT_FAILURE); +TEST(CoreToCoreCliTest, RejectsInvalidSweepConfigurationsWithCentralizedDiagnostics) { + struct SweepCase { + std::vector options; + std::string diagnostic; + }; + const SweepCase cases[] = { + {{"--sweep", "count=1,2"}, Messages::error_sweep_requires_output()}, + {{"--output", "sweep.json", "--sweep", "count=1,2", "--sweep", "latency-samples=4,8", "--sweep-max-runs", "3"}, + Messages::error_sweep_too_many_runs(4, 3)}, + {{"--output", "sweep.json", "--sweep", "count=1,2", "--sweep", "count=3,4"}, + Messages::error_duplicate_option("--sweep count")}, + }; + for (const SweepCase& test_case : cases) { + SCOPED_TRACE(testing::PrintToString(test_case.options)); + std::vector arguments = {"memory_benchmark", "--analyze-core2core"}; + arguments.insert(arguments.end(), test_case.options.begin(), test_case.options.end()); + CoreToCoreLatencyConfig config; + const CapturedCoreCliParse parsed = parse_capturing_stderr(arguments, config); + EXPECT_EQ(parsed.result, EXIT_FAILURE); + EXPECT_EQ(first_output_line(parsed.stderr_output), Messages::error_prefix() + test_case.diagnostic); + } } TEST(CoreToCoreCliTest, RejectsUnknownOptionsInStandaloneMode) { @@ -289,11 +229,6 @@ TEST(CoreToCoreCliTest, RejectsMalformedNumericTokensWithCentralizedErrors) { {"--count", "5junk", kInvalidReason}, {"--latency-samples", "8junk", kInvalidReason}, {"--sweep-max-runs", "4junk", kInvalidReason}, - {"--count", " 5", kInvalidReason}, - {"--count", "5 ", kInvalidReason}, - {"--count", "+5", kInvalidReason}, - {"--count", "abc", kInvalidReason}, - {"--count", "9223372036854775808", "out of range"}, {"--count", "0", positive_range_reason.c_str()}, {"--count", "-1", positive_range_reason.c_str()}, }; @@ -324,15 +259,13 @@ TEST(CoreToCoreCliTest, RejectsMalformedSweepListsAndValues) { constexpr const char* kInvalidReason = "must be an integer without whitespace, a plus sign, or trailing characters"; const InvalidSweepCase cases[] = { + {"threads=1,2", "--sweep", "threads=1,2", "unsupported core-to-core sweep parameter: threads"}, {"count", "--sweep", "count", "sweep must use key=value1,value2 syntax"}, {"count=,1", "--sweep", "count=,1", "sweep value list cannot contain empty values"}, {"count=1,", "--sweep", "count=1,", "sweep value list cannot contain empty values"}, {"count=1,,2", "--sweep", "count=1,,2", "sweep value list cannot contain empty values"}, {"count=1x", "--count", "1x", kInvalidReason}, {"latency-samples=4x", "--latency-samples", "4x", kInvalidReason}, - {"count= 1", "--count", " 1", kInvalidReason}, - {"count=+1", "--count", "+1", kInvalidReason}, - {"count=9223372036854775808", "--count", "9223372036854775808", "out of range"}, }; for (const InvalidSweepCase& test_case : cases) { diff --git a/tests/test_core_to_core_runner.cpp b/tests/test_core_to_core_runner.cpp index dfa2802..42add23 100644 --- a/tests/test_core_to_core_runner.cpp +++ b/tests/test_core_to_core_runner.cpp @@ -139,30 +139,6 @@ TEST(CoreToCoreRunnerTest, LoopRecordSampleRangesCountOnlySamplesAppendedToPool) EXPECT_EQ(scenario_result.completed_loops, 1u); } -TEST(CoreToCoreRunnerTest, DeterministicFailureSeamsReturnExplicitStatus) { - const ScopedDeterministicTimerSystemCalls timer_system_calls; - const ScenarioDescriptor scenario = { - Constants::CORE_TO_CORE_SCENARIO_NO_AFFINITY, - Constants::CORE_TO_CORE_AFFINITY_HINT_DISABLED, - Constants::CORE_TO_CORE_AFFINITY_TAG_NONE, - Constants::CORE_TO_CORE_AFFINITY_TAG_NONE, - }; - const CoreToCoreWorkPlan plan = make_small_work_plan(); - - ScenarioMeasurement timer_failure; - CoreToCoreFailureInjection fail_timer; - fail_timer.fail_timer_creation = true; - EXPECT_FALSE(execute_single_scenario(scenario, plan, 0, timer_failure, &fail_timer)); - EXPECT_EQ(timer_failure.status, CoreToCoreMeasurementStatus::Failed); - EXPECT_EQ(timer_failure.status_reason, "timer-creation-failed"); - - ScenarioMeasurement responder_failure; - CoreToCoreFailureInjection fail_responder; - fail_responder.fail_responder_startup = true; - EXPECT_FALSE(execute_single_scenario(scenario, plan, 0, responder_failure, &fail_responder)); - EXPECT_EQ(responder_failure.status_reason, "responder-thread-startup-failed"); -} - TEST(CoreToCoreRunnerTest, InitiatorStartupFailureCleansUpResponderIntegration) { const ScenarioDescriptor scenario = { Constants::CORE_TO_CORE_SCENARIO_NO_AFFINITY, @@ -177,46 +153,3 @@ TEST(CoreToCoreRunnerTest, InitiatorStartupFailureCleansUpResponderIntegration) EXPECT_FALSE(execute_single_scenario(scenario, plan, 0, initiator_failure, &fail_initiator)); EXPECT_EQ(initiator_failure.status_reason, "initiator-thread-startup-failed"); } - -TEST(CoreToCoreRunnerTest, ExecuteSingleScenarioProducesHeadlineAndSamplesIntegration) { - const ScenarioDescriptor scenario = { - Constants::CORE_TO_CORE_SCENARIO_NO_AFFINITY, - Constants::CORE_TO_CORE_AFFINITY_HINT_DISABLED, - Constants::CORE_TO_CORE_AFFINITY_TAG_NONE, - Constants::CORE_TO_CORE_AFFINITY_TAG_NONE, - }; - ScenarioMeasurement measurement; - const CoreToCoreWorkPlan plan = make_small_work_plan(); - - const bool ok = execute_single_scenario(scenario, plan, 1, measurement); - - ASSERT_TRUE(ok); - EXPECT_EQ(measurement.status, CoreToCoreMeasurementStatus::Measured); - EXPECT_GT(measurement.round_trip_ns, 0.0); - ASSERT_EQ(measurement.samples_ns.size(), 1u); - EXPECT_GT(measurement.samples_ns[0], 0.0); - EXPECT_FALSE(std::isnan(measurement.round_trip_ns)); - EXPECT_FALSE(std::isinf(measurement.round_trip_ns)); -} - -TEST(CoreToCoreRunnerTest, ExecuteSingleScenarioSupportsZeroSamplesIntegration) { - const ScenarioDescriptor scenario = { - Constants::CORE_TO_CORE_SCENARIO_DIFFERENT_AFFINITY, - Constants::CORE_TO_CORE_AFFINITY_HINT_ENABLED, - Constants::CORE_TO_CORE_AFFINITY_TAG_PRIMARY, - Constants::CORE_TO_CORE_AFFINITY_TAG_SECONDARY, - }; - ScenarioMeasurement measurement; - const CoreToCoreWorkPlan plan = make_small_work_plan(); - - const bool ok = execute_single_scenario(scenario, plan, 0, measurement); - - ASSERT_TRUE(ok); - EXPECT_EQ(measurement.status, CoreToCoreMeasurementStatus::Measured); - EXPECT_GT(measurement.round_trip_ns, 0.0); - EXPECT_TRUE(measurement.samples_ns.empty()); - EXPECT_EQ(measurement.initiator_hint.affinity_requested, scenario.apply_affinity); - EXPECT_EQ(measurement.responder_hint.affinity_requested, scenario.apply_affinity); - EXPECT_EQ(measurement.initiator_hint.affinity_tag, scenario.initiator_affinity_tag); - EXPECT_EQ(measurement.responder_hint.affinity_tag, scenario.responder_affinity_tag); -} diff --git a/tests/test_descriptive_statistics.cpp b/tests/test_descriptive_statistics.cpp index eb962c1..a4bd301 100644 --- a/tests/test_descriptive_statistics.cpp +++ b/tests/test_descriptive_statistics.cpp @@ -21,23 +21,6 @@ #include "utils/descriptive_statistics.h" -TEST(DescriptiveStatisticsTest, EmptyPopulationReturnsUndefinedDefaults) { - const DescriptiveStatistics statistics = calculate_descriptive_statistics({}); - - EXPECT_EQ(statistics.sample_count, 0U); - EXPECT_DOUBLE_EQ(statistics.average, 0.0); - EXPECT_DOUBLE_EQ(statistics.min, 0.0); - EXPECT_DOUBLE_EQ(statistics.max, 0.0); - EXPECT_DOUBLE_EQ(statistics.median, 0.0); - EXPECT_DOUBLE_EQ(statistics.p90, 0.0); - EXPECT_DOUBLE_EQ(statistics.p95, 0.0); - EXPECT_DOUBLE_EQ(statistics.p99, 0.0); - EXPECT_DOUBLE_EQ(statistics.stddev, 0.0); - EXPECT_DOUBLE_EQ(statistics.coefficient_of_variation_pct, 0.0); - EXPECT_FALSE(statistics.coefficient_of_variation_defined); - EXPECT_DOUBLE_EQ(statistics.median_absolute_deviation, 0.0); -} - TEST(DescriptiveStatisticsTest, SingleNonzeroSampleHasDefinedZeroVariation) { const DescriptiveStatistics statistics = calculate_descriptive_statistics({42.0}); diff --git a/tests/test_executable_cli.cpp b/tests/test_executable_cli.cpp index 25b3399..41b9d04 100644 --- a/tests/test_executable_cli.cpp +++ b/tests/test_executable_cli.cpp @@ -729,240 +729,88 @@ std::vector bounded_llm_metal_prefill_arguments( output_target}; } -std::vector bounded_llm_metal_paged_prefill_arguments( - const std::string& output_target, size_t loop_count = 3) { - std::vector arguments = - bounded_llm_metal_prefill_arguments(output_target, loop_count); - arguments.insert(arguments.end() - 2, - {"--kv-layout", "paged", "--kv-block-tokens", "4"}); - return arguments; -} } // namespace -TEST(ExecutableCliIntegrationTest, NoArgumentsShowsHelpAndReturnsSuccessIntegration) { - const CliResult result = run_memory_benchmark({}); - - EXPECT_EQ(result.exit_code, EXIT_SUCCESS); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("Usage:"), std::string::npos); - EXPECT_NE(result.output.find("--benchmark"), std::string::npos); -} - -TEST(ExecutableCliIntegrationTest, HelpFlagShowsHelpAndReturnsSuccessIntegration) { - const CliResult result = run_memory_benchmark({"-h"}); - - EXPECT_EQ(result.exit_code, EXIT_SUCCESS); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("Usage:"), std::string::npos); - EXPECT_NE(result.output.find("--patterns"), std::string::npos); - EXPECT_NE(result.output.find("--gpu-bandwidth"), std::string::npos); -} - -TEST(ExecutableCliIntegrationTest, - HelpWithStdoutTargetRemainsHumanInEitherOrderIntegration) { - for (const std::vector& arguments : { - std::vector{"--benchmark", "--output", "-", - "--help"}, - std::vector{"--benchmark", "--help", "--output", - "-"}, - }) { - SCOPED_TRACE(testing::PrintToString(arguments)); - const CliResult result = run_memory_benchmark(arguments); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_SUCCESS); - expect_no_runtime_banner(result); - EXPECT_NE(result.stdout_output.find("Usage:"), std::string::npos); - EXPECT_NE(result.stdout_output.find("--benchmark"), std::string::npos); - EXPECT_TRUE(result.stderr_output.empty()) << result.stderr_output; - EXPECT_FALSE(nlohmann::json::accept(result.stdout_output)); - expect_no_dash_transport_artifacts(result); - } -} - -TEST(ExecutableCliIntegrationTest, GpuHelpUsesDedicatedStandaloneParserIntegration) { - const CliResult result = - run_memory_benchmark({"--gpu-bandwidth", "--help"}); - - EXPECT_EQ(result.exit_code, EXIT_SUCCESS); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find( - "Usage: ./memory_benchmark --gpu-bandwidth [options]"), - std::string::npos); - EXPECT_NE(result.output.find("minimum: 64 MB"), std::string::npos); - EXPECT_NE(result.output.find("default: 3"), std::string::npos); -} - -TEST(ExecutableCliIntegrationTest, - GpuHelpWithStdoutTargetRemainsHumanInEitherOrderIntegration) { - for (const std::vector& arguments : { - std::vector{"--gpu-bandwidth", "--output", "-", - "--help"}, - std::vector{"--gpu-bandwidth", "--help", - "--output", "-"}, - }) { - SCOPED_TRACE(testing::PrintToString(arguments)); - const CliResult result = run_memory_benchmark(arguments); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_SUCCESS); - expect_no_runtime_banner(result); - EXPECT_NE(result.stdout_output.find( - "Usage: ./memory_benchmark --gpu-bandwidth [options]"), - std::string::npos); - EXPECT_NE(result.stdout_output.find("minimum: 64 MB"), - std::string::npos); - EXPECT_TRUE(result.stderr_output.empty()) << result.stderr_output; - EXPECT_FALSE(nlohmann::json::accept(result.stdout_output)); - expect_no_dash_transport_artifacts(result); +TEST(ExecutableCliIntegrationTest, HelpRemainsHumanAcrossModesAndOutputOrderingIntegration) { + struct HelpCase { + std::vector arguments; + const char* heading; + const char* detail; + }; + std::vector cases = { + {{}, "Usage:", "--benchmark"}, + {{"--threads", "1"}, "Usage:", "--threads "}, + }; + const HelpCase modes[] = { + {{"--benchmark"}, "Usage:", "--benchmark"}, + {{"--gpu-bandwidth"}, "Usage: ./memory_benchmark --gpu-bandwidth [options]", "minimum: 64 MB"}, + {{"--llm-memory"}, "Usage: ./memory_benchmark --llm-memory [options]", "schema 2"}, + {{"--llm-memory", "--llm-memory-backend", "metal"}, + "Usage: ./memory_benchmark --llm-memory [options]", + "schema 2"}, + {{"--analyze-core2core"}, "Usage:", "--analyze-core2core"}, + }; + for (const HelpCase& mode : modes) { + for (bool help_first : {false, true}) { + HelpCase test_case = mode; + if (help_first) { + test_case.arguments.insert(test_case.arguments.begin() + 1, "--help"); + test_case.arguments.insert(test_case.arguments.end(), {"--output", "-"}); + } else { + test_case.arguments.insert(test_case.arguments.end(), {"--output", "-", "--help"}); + } + cases.push_back(std::move(test_case)); + } } -} - -TEST(ExecutableCliIntegrationTest, - LlmHelpWithStdoutTargetRemainsHumanInEitherOrderIntegration) { - for (const std::vector& arguments : { - std::vector{"--llm-memory", "--output", "-", - "--help"}, - std::vector{"--llm-memory", "--help", "--output", - "-"}, - std::vector{"--llm-memory", - "--llm-memory-backend", "metal", - "--output", "-", "--help"}, - std::vector{"--llm-memory", "--help", - "--llm-memory-backend", "metal", - "--output", "-"}, - }) { - SCOPED_TRACE(testing::PrintToString(arguments)); - const CliResult result = run_memory_benchmark(arguments); - + for (const HelpCase& test_case : cases) { + SCOPED_TRACE(testing::PrintToString(test_case.arguments)); + const CliResult result = run_memory_benchmark(test_case.arguments); expect_process_completed(result); EXPECT_EQ(result.exit_code, EXIT_SUCCESS); expect_no_runtime_banner(result); - EXPECT_NE(result.stdout_output.find( - "Usage: ./memory_benchmark --llm-memory [options]"), - std::string::npos); - EXPECT_NE(result.stdout_output.find("schema 2"), std::string::npos); + EXPECT_NE(result.stdout_output.find(test_case.heading), std::string::npos); + EXPECT_NE(result.stdout_output.find(test_case.detail), std::string::npos); EXPECT_TRUE(result.stderr_output.empty()) << result.stderr_output; EXPECT_FALSE(nlohmann::json::accept(result.stdout_output)); expect_no_dash_transport_artifacts(result); } } -TEST(ExecutableCliIntegrationTest, - InvalidLlmStdoutTargetLeavesStdoutEmptyIntegration) { - const CliResult result = run_memory_benchmark( - {"--llm-memory", "--weight-size-mb", "1", "--output", "-"}); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find( - Messages::error_llm_memory_missing_required_option( - "--layers")), - std::string::npos) - << result.stderr_output; - expect_no_dash_transport_artifacts(result); -} - -TEST(ExecutableCliIntegrationTest, - LlmMetalPreflightRejectsThreadsBeforeOutputSessionIntegration) { - struct InvalidCase { - std::string name; - std::vector arguments; - const char* reason_code; - }; - - std::vector backend_before_threads = - bounded_llm_metal_arguments("-"); - backend_before_threads.insert(backend_before_threads.end() - 2, - {"--threads", "1"}); - - std::vector threads_before_backend = bounded_llm_arguments("-"); - threads_before_backend.insert(threads_before_backend.end() - 2, - {"--llm-memory-backend", "metal"}); - - const std::vector cases = { - {"backend-before-threads", std::move(backend_before_threads), - LlmMemoryConfigReason::THREADS_NOT_APPLICABLE}, - {"threads-before-backend", std::move(threads_before_backend), - LlmMemoryConfigReason::THREADS_NOT_APPLICABLE}, - }; - - for (const InvalidCase& test_case : cases) { - SCOPED_TRACE(test_case.name); - const CliResult result = run_memory_benchmark(test_case.arguments); - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE) << result.output; - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find( - Messages::error_llm_memory_config_invalid( - test_case.reason_code)), - std::string::npos) - << result.stderr_output; - expect_no_dash_transport_artifacts(result); +TEST(ExecutableCliIntegrationTest, LlmMetalStdoutIsCompleteOrOneTerminalUnsupportedDocumentIntegration) { + for (bool prefill : {false, true}) { + for (bool paged : {false, true}) { + SCOPED_TRACE(prefill); + SCOPED_TRACE(paged); + std::vector arguments = + prefill ? bounded_llm_metal_prefill_arguments("-") : bounded_llm_metal_arguments("-"); + if (paged) { + arguments.insert(arguments.end() - 2, {"--kv-layout", "paged", "--kv-block-tokens", prefill ? "4" : "2"}); + } + const CliResult result = run_memory_benchmark(arguments); + expect_process_completed(result); + const nlohmann::json json = parse_single_stdout_json(result); + if (prefill) { + expect_bounded_metal_prefill_result( + result, json, paged ? "paged" : "contiguous", + paged ? "llm-memory-v2-metal-prefill-paged" : "llm-memory-v2-metal-prefill-contiguous", + paged ? "llm-metal-prefill-paged-full-prompt-affine32-v1" + : "llm-metal-prefill-contiguous-full-prompt-affine32-v1", + paged ? "llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1" : "llm-metal-dual-mod32-v1"); + } else { + expect_complete_or_unsupported_metal_result( + result, json, paged ? "paged" : "contiguous", + paged ? "llm-memory-v2-metal-decode-paged" : "llm-memory-v2-metal-decode-contiguous"); + EXPECT_EQ(json["phase"], "decode"); + EXPECT_EQ(json["kv_layout"], paged ? "paged" : "contiguous"); + if (paged) EXPECT_EQ(json["configuration"]["kv_block_tokens"], 2U); + } + expect_single_runtime_banner(result); + expect_no_dash_transport_artifacts(result); + } } } -TEST(ExecutableCliIntegrationTest, - LlmMetalStdoutIsCompleteOrOneTerminalUnsupportedDocumentIntegration) { - const CliResult result = - run_memory_benchmark(bounded_llm_metal_arguments("-")); - - expect_process_completed(result); - const nlohmann::json json = parse_single_stdout_json(result); - expect_complete_or_unsupported_metal_result(result, json); - expect_single_runtime_banner(result); - expect_no_dash_transport_artifacts(result); -} - -TEST(ExecutableCliIntegrationTest, - LlmMetalPagedStdoutIsCompleteOrOneTerminalUnsupportedDocumentIntegration) { - std::vector arguments = bounded_llm_metal_arguments("-"); - arguments.insert(arguments.end() - 2, - {"--kv-layout", "paged", "--kv-block-tokens", "2"}); - const CliResult result = run_memory_benchmark(arguments); - - expect_process_completed(result); - const nlohmann::json json = parse_single_stdout_json(result); - expect_complete_or_unsupported_metal_result( - result, json, "paged", "llm-memory-v2-metal-decode-paged"); - EXPECT_EQ(json["phase"], "decode"); - EXPECT_EQ(json["kv_layout"], "paged"); - EXPECT_EQ(json["configuration"]["kv_block_tokens"], 2U); - expect_single_runtime_banner(result); - expect_no_dash_transport_artifacts(result); -} - -TEST(ExecutableCliIntegrationTest, - LlmMetalPrefillStdoutIsCompleteOrCapabilityUnsupportedIntegration) { - const CliResult result = - run_memory_benchmark(bounded_llm_metal_prefill_arguments("-")); - - expect_process_completed(result); - const nlohmann::json json = parse_single_stdout_json(result); - expect_bounded_metal_prefill_result(result, json); - expect_single_runtime_banner(result); - expect_no_dash_transport_artifacts(result); -} - -TEST(ExecutableCliIntegrationTest, - LlmMetalPagedPrefillStdoutIsCompleteOrCapabilityUnsupportedIntegration) { - const CliResult result = run_memory_benchmark( - bounded_llm_metal_paged_prefill_arguments("-")); - - expect_process_completed(result); - const nlohmann::json json = parse_single_stdout_json(result); - expect_bounded_metal_prefill_result( - result, json, "paged", "llm-memory-v2-metal-prefill-paged", - "llm-metal-prefill-paged-full-prompt-affine32-v1", - "llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1"); - expect_single_runtime_banner(result); - expect_no_dash_transport_artifacts(result); -} - TEST(ExecutableCliIntegrationTest, LlmMetalFileIsAtomicCompleteOrTerminalUnsupportedDocumentIntegration) { const TemporaryJsonFile output("llm_metal_schema_v1"); @@ -984,30 +832,6 @@ TEST(ExecutableCliIntegrationTest, << result.output; } -TEST(ExecutableCliIntegrationTest, - LlmMetalPagedPrefillFileIsAtomicCompleteOrCapabilityUnsupportedIntegration) { - const TemporaryJsonFile output("llm_metal_paged_prefill_schema_v1"); - const CliResult result = run_memory_benchmark( - bounded_llm_metal_paged_prefill_arguments(output.path())); - - expect_process_completed(result); - ASSERT_EQ(access(output.path().c_str(), F_OK), 0) << result.output; - EXPECT_EQ(access((output.path() + ".tmp").c_str(), F_OK), -1); - const nlohmann::json json = - nlohmann::json::parse(read_file(output.path())); - expect_bounded_metal_prefill_result( - result, json, "paged", "llm-memory-v2-metal-prefill-paged", - "llm-metal-prefill-paged-full-prompt-affine32-v1", - "llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1"); - EXPECT_EQ(json["configuration"]["output_file"], output.path()); - expect_single_runtime_banner(result); - EXPECT_EQ(count_occurrences( - result.stdout_output, - Messages::msg_results_saved_to(output.path())), - 1u) - << result.output; -} - TEST(ExecutableCliIntegrationTest, LlmPrefillPagedWritesCompleteSchemaV2Integration) { std::vector arguments = bounded_llm_prefill_arguments("-"); arguments.insert(arguments.end() - 2, {"--kv-layout", "paged", "--kv-block-tokens", "4"}); @@ -1047,45 +871,33 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillPagedWritesCompleteSchemaV2Integrat expect_no_dash_transport_artifacts(result); } -TEST(ExecutableCliIntegrationTest, - LlmJsonIntegerAndPeakPreflightsRejectBeforeAllocationIntegration) { - { - std::vector arguments = bounded_llm_arguments("-", 1); - const auto query_heads = std::find(arguments.begin(), arguments.end(), - "--query-heads"); - ASSERT_NE(query_heads, arguments.end()); - ASSERT_NE(std::next(query_heads), arguments.end()); - *std::next(query_heads) = - std::to_string(Constants::LLM_JSON_MAX_SAFE_INTEGER + 1); - - const CliResult result = run_memory_benchmark(arguments); - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find("json-integer-out-of-range"), - std::string::npos) - << result.stderr_output; - expect_no_dash_transport_artifacts(result); - } - - { +TEST(ExecutableCliIntegrationTest, LlmJsonIntegerAndPeakPreflightsRejectBeforeAllocationIntegration) { + struct PreflightCase { + const char* option; + uint64_t value; + const char* reason; + bool runtime_banner; + }; + const PreflightCase cases[] = { + {"--query-heads", Constants::LLM_JSON_MAX_SAFE_INTEGER + 1, "json-integer-out-of-range", false}, + {"--count", Constants::LLM_JSON_MAX_SAFE_INTEGER / 3, "json-output-peak-bytes-overflow", true}, + }; + for (const PreflightCase& test_case : cases) { + SCOPED_TRACE(test_case.option); std::vector arguments = bounded_llm_arguments("-", 1); - const auto count_value = std::find(arguments.begin(), arguments.end(), - "--count"); - ASSERT_NE(count_value, arguments.end()); - ASSERT_NE(std::next(count_value), arguments.end()); - *std::next(count_value) = std::to_string( - Constants::LLM_JSON_MAX_SAFE_INTEGER / 3); - + const auto option = std::find(arguments.begin(), arguments.end(), test_case.option); + ASSERT_NE(option, arguments.end()); + ASSERT_NE(std::next(option), arguments.end()); + *std::next(option) = std::to_string(test_case.value); const CliResult result = run_memory_benchmark(arguments); expect_process_completed(result); EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_single_runtime_banner(result); + if (test_case.runtime_banner) + expect_single_runtime_banner(result); + else + expect_no_runtime_banner(result); EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find("json-output-peak-bytes-overflow"), - std::string::npos) - << result.stderr_output; + EXPECT_NE(result.stderr_output.find(test_case.reason), std::string::npos) << result.stderr_output; expect_no_dash_transport_artifacts(result); } } @@ -1106,72 +918,84 @@ TEST(ExecutableCliIntegrationTest, } } -TEST(ExecutableCliIntegrationTest, - LlmWritesSingleCompleteSchemaV2DocumentToStdoutIntegration) { - const CliResult result = run_memory_benchmark(bounded_llm_arguments("-")); +TEST(ExecutableCliIntegrationTest, LlmStdoutIsCompleteAndSeededMetadataIsReproducibleIntegration) { + nlohmann::json first; + for (int repetition = 0; repetition < 2; ++repetition) { + SCOPED_TRACE(repetition); + const CliResult result = run_memory_benchmark(bounded_llm_arguments("-")); - expect_process_completed(result); - ASSERT_EQ(result.exit_code, EXIT_SUCCESS) << result.stderr_output; - const nlohmann::json json = parse_single_stdout_json(result); - ASSERT_TRUE(json.is_object()) << result.stdout_output; - EXPECT_EQ(json["software"]["version"], SOFTVERSION); - EXPECT_TRUE(json["software"]["timestamp"].is_string()); - EXPECT_EQ(json["schema_version"], 2); - EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); - EXPECT_EQ(json["backend"], "cpu"); - EXPECT_EQ(json["phase"], "decode"); - EXPECT_EQ(json["kv_layout"], "contiguous"); - EXPECT_EQ(json["methodology_version"], - Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION); - EXPECT_EQ(json["status"], "complete"); - EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["run_accepted"].get()); - EXPECT_TRUE(json["scenario_order_balance_complete"].get()); - EXPECT_EQ(json["configuration"]["output_file"], "-"); - EXPECT_EQ(json["configuration"]["base_seed_uint64_decimal"], "42"); - EXPECT_EQ(json["configuration"]["iterations"], 1u); - EXPECT_EQ(json["configuration"]["loop_count"], 3u); - EXPECT_EQ(json["configuration"]["resolved_sources"]["backend"], - "default"); - EXPECT_EQ(json["configuration"]["resolved_sources"]["phase"], - "default"); - EXPECT_EQ(json["configuration"]["resolved_sources"]["kv_layout"], - "default"); - EXPECT_EQ(json["resolved_plan"]["backend"], "cpu"); - EXPECT_EQ(json["resolved_plan"]["phase"], "decode"); - EXPECT_EQ(json["resolved_plan"]["kv_layout"], "contiguous"); - EXPECT_EQ(json["resolved_plan"]["work_unit_kind"], "decode_step"); - EXPECT_TRUE(json["backend_evidence"]["cpu"].is_object()); - EXPECT_TRUE(json["backend_evidence"]["metal"].is_null()); - EXPECT_EQ(json["counters"]["planned_measurements"], 9u); - EXPECT_EQ(json["counters"]["measured_measurements"], 9u); - ASSERT_EQ(json["measurements"].size(), 9u); - for (const nlohmann::json& measurement : json["measurements"]) { - EXPECT_EQ(measurement["status"], "measured"); - EXPECT_EQ(canonical_llm_plan(json, measurement)["work_unit_kind"], "decode_step"); - EXPECT_EQ(canonical_llm_plan(json, measurement)["work_units"], 1u); - EXPECT_EQ(measurement["completed_work_units"], 1u); - expect_complete_llm_named_checks(json, measurement); - EXPECT_TRUE(measurement["checksum"]["checksum_valid"].get()); - EXPECT_TRUE( - measurement["synthetic_work_unit_latency_seconds"].is_number()); - EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"] - .is_number()); - EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_number()); - } - expect_complete_llm_checkpoint_lifecycle(json); + expect_process_completed(result); + ASSERT_EQ(result.exit_code, EXIT_SUCCESS) << result.stderr_output; + const nlohmann::json json = parse_single_stdout_json(result); + ASSERT_TRUE(json.is_object()) << result.stdout_output; + EXPECT_EQ(json["software"]["version"], SOFTVERSION); + EXPECT_TRUE(json["software"]["timestamp"].is_string()); + EXPECT_EQ(json["schema_version"], 2); + EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); + EXPECT_EQ(json["backend"], "cpu"); + EXPECT_EQ(json["phase"], "decode"); + EXPECT_EQ(json["kv_layout"], "contiguous"); + EXPECT_EQ(json["methodology_version"], Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION); + EXPECT_EQ(json["status"], "complete"); + EXPECT_TRUE(json["results_complete"].get()); + EXPECT_TRUE(json["run_accepted"].get()); + EXPECT_TRUE(json["scenario_order_balance_complete"].get()); + EXPECT_EQ(json["configuration"]["output_file"], "-"); + EXPECT_EQ(json["configuration"]["base_seed_uint64_decimal"], "42"); + EXPECT_EQ(json["configuration"]["iterations"], 1u); + EXPECT_EQ(json["configuration"]["loop_count"], 3u); + EXPECT_EQ(json["configuration"]["resolved_sources"]["backend"], "default"); + EXPECT_EQ(json["configuration"]["resolved_sources"]["phase"], "default"); + EXPECT_EQ(json["configuration"]["resolved_sources"]["kv_layout"], "default"); + EXPECT_EQ(json["resolved_plan"]["backend"], "cpu"); + EXPECT_EQ(json["resolved_plan"]["phase"], "decode"); + EXPECT_EQ(json["resolved_plan"]["kv_layout"], "contiguous"); + EXPECT_EQ(json["resolved_plan"]["work_unit_kind"], "decode_step"); + EXPECT_TRUE(json["backend_evidence"]["cpu"].is_object()); + EXPECT_TRUE(json["backend_evidence"]["metal"].is_null()); + EXPECT_EQ(json["counters"]["planned_measurements"], 9u); + EXPECT_EQ(json["counters"]["measured_measurements"], 9u); + ASSERT_EQ(json["measurements"].size(), 9u); + for (const nlohmann::json& measurement : json["measurements"]) { + EXPECT_EQ(measurement["status"], "measured"); + EXPECT_EQ(canonical_llm_plan(json, measurement)["work_unit_kind"], "decode_step"); + EXPECT_EQ(canonical_llm_plan(json, measurement)["work_units"], 1u); + EXPECT_EQ(measurement["completed_work_units"], 1u); + expect_complete_llm_named_checks(json, measurement); + EXPECT_TRUE(measurement["checksum"]["checksum_valid"].get()); + EXPECT_TRUE(measurement["synthetic_work_unit_latency_seconds"].is_number()); + EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_number()); + EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_number()); + } + expect_complete_llm_checkpoint_lifecycle(json); - expect_single_runtime_banner(result); - EXPECT_EQ(result.stdout_output.find(Messages::config_header(SOFTVERSION)), - std::string::npos); - EXPECT_NE(result.stderr_output.find(Messages::report_llm_memory_header( - "cpu", "decode", "decode_step", "contiguous")), - std::string::npos) - << result.stderr_output; - EXPECT_EQ(count_occurrences(result.output, - Messages::msg_results_saved_to("")), - 0u); - expect_no_dash_transport_artifacts(result); + expect_single_runtime_banner(result); + EXPECT_EQ(result.stdout_output.find(Messages::config_header(SOFTVERSION)), std::string::npos); + EXPECT_NE( + result.stderr_output.find(Messages::report_llm_memory_header("cpu", "decode", "decode_step", "contiguous")), + std::string::npos) + << result.stderr_output; + EXPECT_EQ(count_occurrences(result.output, Messages::msg_results_saved_to("")), 0u); + expect_no_dash_transport_artifacts(result); + if (repetition == 0) { + first = json; + continue; + } + for (const char* field : {"configuration", "seeds"}) { + SCOPED_TRACE(field); + EXPECT_EQ(first[field], json[field]); + } + for (const char* field : {"methodology", "geometry", "model_work_plan", "scenario_plans"}) { + SCOPED_TRACE(field); + EXPECT_EQ(first["resolved_plan"][field], json["resolved_plan"][field]); + } + ASSERT_EQ(first["measurements"].size(), json["measurements"].size()); + for (size_t index = 0; index < first["measurements"].size(); ++index) { + SCOPED_TRACE(index); + EXPECT_EQ(first["measurements"][index]["plan_ref"], json["measurements"][index]["plan_ref"]); + EXPECT_EQ(first["measurements"][index]["checksum"], json["measurements"][index]["checksum"]); + } + } } TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchemaV2Integration) { @@ -1209,39 +1033,6 @@ TEST(ExecutableCliIntegrationTest, LlmPrefillWritesExactCompleteContiguousSchema const nlohmann::json& resolved_plan = json["resolved_plan"]; EXPECT_EQ(resolved_plan["phase"], "prefill"); EXPECT_EQ(resolved_plan["work_unit_kind"], "prefill_operation"); - EXPECT_EQ(resolved_plan["methodology"]["context_policy"], "full-prompt-population-with-tiled-causal-prefix-scans"); - const nlohmann::json& geometry = resolved_plan["geometry"]; - EXPECT_TRUE(geometry["decode"].is_null()); - ASSERT_TRUE(geometry["prefill"].is_object()); - EXPECT_EQ(geometry["prefill"]["prompt_tokens"], 5u); - EXPECT_EQ(geometry["prefill"]["attention_query_tile_tokens"], 2u); - EXPECT_EQ(geometry["prefill"]["tile_count"], "3"); - EXPECT_EQ(geometry["prefill"]["attention_prefix_token_visits_per_sequence"], "11"); - EXPECT_EQ(resolved_plan["model_context"]["prefill"]["causal_token_pairs_per_sequence"], "15"); - EXPECT_EQ(resolved_plan["model_context"]["prefill"]["logical_attention_pairs"], "15"); - EXPECT_EQ(resolved_plan["model_context"]["prefill"]["logical_attention_fma_terms"], "120"); - EXPECT_EQ(geometry["k_or_v_sequence_visible_bytes"], "40"); - EXPECT_EQ(geometry["k_mapping_bytes"], "40"); - EXPECT_EQ(geometry["v_mapping_bytes"], "40"); - EXPECT_EQ(geometry["kv_capacity_bytes"], "80"); - EXPECT_EQ(geometry["weight_read_bytes_per_work_unit"], "1048576"); - EXPECT_EQ(geometry["kv_read_bytes_per_work_unit"], "176"); - EXPECT_EQ(geometry["kv_write_bytes_per_work_unit"], "80"); - EXPECT_EQ(geometry["kv_only_effective_model_payload_bytes_per_work_unit"], "256"); - EXPECT_EQ(geometry["mixed_effective_model_payload_bytes_per_work_unit"], "1048832"); - EXPECT_EQ(geometry["total_data_mapping_bytes"], "1048656"); - EXPECT_TRUE(geometry["traffic_crossover_numerator"].is_null()); - EXPECT_TRUE(geometry["traffic_crossover_denominator"].is_null()); - EXPECT_TRUE(geometry["traffic_crossover_context_tokens"].is_null()); - - const nlohmann::json& traffic = json["aggregates"]["traffic_diagnostics"]; - EXPECT_TRUE(traffic["traffic_crossover_numerator"].is_null()); - EXPECT_TRUE(traffic["traffic_crossover_denominator"].is_null()); - EXPECT_TRUE(traffic["traffic_crossover_context_tokens"].is_null()); - EXPECT_TRUE(traffic["current_visible_context_tokens"].is_null()); - EXPECT_TRUE(traffic["current_weight_to_kv_read_payload_ratio"].is_null()); - EXPECT_TRUE(traffic["current_context_classification"].is_null()); - const nlohmann::json& prefill_evidence = json["backend_evidence"]["cpu"]["prefill"]; EXPECT_EQ(prefill_evidence["cost_unit"], "worker-cost"); EXPECT_EQ(prefill_evidence["sequence_descriptors_per_scenario_per_worker"], 1u); @@ -1399,151 +1190,6 @@ TEST(ExecutableCliIntegrationTest, << result.output; } -TEST(ExecutableCliIntegrationTest, - LlmExactGeometryAndSeededWorkloadMetadataAreReproducibleIntegration) { - const CliResult first_result = - run_memory_benchmark(bounded_llm_arguments("-", 1)); - const CliResult second_result = - run_memory_benchmark(bounded_llm_arguments("-", 1)); - - expect_process_completed(first_result); - expect_process_completed(second_result); - ASSERT_EQ(first_result.exit_code, EXIT_SUCCESS) - << first_result.stderr_output; - ASSERT_EQ(second_result.exit_code, EXIT_SUCCESS) - << second_result.stderr_output; - const nlohmann::json first = parse_single_stdout_json(first_result); - const nlohmann::json second = parse_single_stdout_json(second_result); - ASSERT_TRUE(first.is_object()); - ASSERT_TRUE(second.is_object()); - - const nlohmann::json& resolved_plan = first["resolved_plan"]; - const nlohmann::json& geometry = resolved_plan["geometry"]; - EXPECT_EQ(geometry["phase"], "decode"); - EXPECT_EQ(geometry["work_unit_kind"], "decode_step"); - EXPECT_EQ(geometry["decode"]["visible_context_tokens"], 2u); - EXPECT_TRUE(geometry["prefill"].is_null()); - EXPECT_EQ(geometry["active_weight_bytes_per_work_unit"], "1048576"); - EXPECT_EQ(geometry["layer_count"], 1u); - EXPECT_EQ(geometry["query_head_count"], 1u); - EXPECT_EQ(geometry["kv_head_count"], 1u); - EXPECT_EQ(geometry["query_heads_per_kv_head"], 1u); - EXPECT_EQ(geometry["head_dimension"], 8u); - EXPECT_EQ(geometry["kv_element_bytes"], "1"); - EXPECT_EQ(geometry["batch_size"], 1u); - EXPECT_EQ(geometry["kv_vector_bytes"], "8"); - EXPECT_EQ(geometry["k_or_v_record_bytes_per_layer"], "8"); - EXPECT_EQ(geometry["kv_record_bytes_per_layer"], "16"); - EXPECT_EQ(geometry["kv_bytes_per_visible_token"], "16"); - EXPECT_EQ(geometry["k_or_v_sequence_visible_bytes"], "16"); - EXPECT_EQ(geometry["k_mapping_bytes"], "16"); - EXPECT_EQ(geometry["v_mapping_bytes"], "16"); - EXPECT_EQ(geometry["kv_capacity_bytes"], "32"); - EXPECT_EQ(geometry["weight_read_bytes_per_work_unit"], "1048576"); - EXPECT_EQ(geometry["kv_read_bytes_per_work_unit"], "32"); - EXPECT_EQ(geometry["kv_write_bytes_per_work_unit"], "16"); - EXPECT_EQ(geometry["kv_only_effective_model_payload_bytes_per_work_unit"], - "48"); - EXPECT_EQ(geometry["mixed_effective_model_payload_bytes_per_work_unit"], - "1048624"); - EXPECT_EQ(geometry["total_data_mapping_bytes"], "1048608"); - EXPECT_EQ(geometry["traffic_crossover_numerator"], "1048576"); - EXPECT_EQ(geometry["traffic_crossover_denominator"], "16"); - EXPECT_DOUBLE_EQ( - geometry["traffic_crossover_context_tokens"].get(), 65536.0); - - const nlohmann::json& layout = resolved_plan["layout"]; - EXPECT_EQ(layout["kv_layout"], "contiguous"); - EXPECT_TRUE(layout["kv_block_tokens"].is_null()); - EXPECT_TRUE(layout["block_table_bytes"].is_null()); - EXPECT_TRUE(layout["permutation_algorithm_version"].is_null()); - - const nlohmann::json& resources = resolved_plan["resources"]; - EXPECT_EQ(resources["weight_logical_bytes"], "1048576"); - EXPECT_EQ(resources["k_logical_bytes"], "16"); - EXPECT_EQ(resources["v_logical_bytes"], "16"); - EXPECT_EQ(resources["k_physical_length_bytes"], "16"); - EXPECT_EQ(resources["v_physical_length_bytes"], "16"); - EXPECT_EQ(resources["k_layout_padding_bytes"], "0"); - EXPECT_EQ(resources["v_layout_padding_bytes"], "0"); - EXPECT_TRUE(resources["block_table_bytes"].is_null()); - - const nlohmann::json& components = - resolved_plan["component_identities"]; - EXPECT_EQ(components["logical_profile_version"], - Constants::LLM_LOGICAL_PROFILE_VERSION); - EXPECT_EQ(components["kv_layout_version"], - Constants::LLM_CONTIGUOUS_KV_LAYOUT_VERSION); - EXPECT_TRUE(components["permutation_version"].is_null()); - EXPECT_EQ(components["backend_executor_version"], - Constants::LLM_CPU_EXECUTOR_VERSION); - EXPECT_EQ(components["resource_abi_version"], - Constants::LLM_DESCRIPTOR_ABI_VERSION); - EXPECT_EQ(components["schedule_version"], - Constants::LLM_CPU_SCHEDULE_VERSION); - EXPECT_EQ(components["timer_policy_version"], - Constants::LLM_CPU_TIMER_POLICY_VERSION); - EXPECT_TRUE(components["msl_revision"].is_null()); - EXPECT_TRUE(components["msl_source_sha256"].is_null()); - - const nlohmann::json& scenarios = - resolved_plan["scenario_plans"]; - ASSERT_EQ(scenarios.size(), 3u); - EXPECT_EQ(scenarios[0]["scenario"], "weights_only"); - EXPECT_EQ(scenarios[0]["work_unit_kind"], "decode_step"); - EXPECT_EQ(scenarios[0]["kv_write_kind"], "none"); - EXPECT_EQ(scenarios[0]["work_units"], 1u); - EXPECT_EQ(scenarios[0]["weight_read_bytes_per_work_unit"], "1048576"); - EXPECT_EQ(scenarios[0]["kv_read_bytes_per_work_unit"], "0"); - EXPECT_EQ(scenarios[0]["kv_write_bytes_per_work_unit"], "0"); - EXPECT_EQ(scenarios[0]["effective_model_payload_bytes"], "1048576"); - EXPECT_EQ(scenarios[0]["task_accounted_bytes"], "1048576"); - EXPECT_EQ(scenarios[1]["scenario"], "kv_only"); - EXPECT_EQ(scenarios[1]["kv_write_kind"], "current_token_append"); - EXPECT_EQ(scenarios[1]["weight_read_bytes_per_work_unit"], "0"); - EXPECT_EQ(scenarios[1]["kv_read_bytes_per_work_unit"], "32"); - EXPECT_EQ(scenarios[1]["kv_write_bytes_per_work_unit"], "16"); - EXPECT_EQ(scenarios[1]["effective_model_payload_bytes"], "48"); - EXPECT_EQ(scenarios[1]["task_accounted_bytes"], "48"); - EXPECT_EQ(scenarios[2]["scenario"], "mixed"); - EXPECT_EQ(scenarios[2]["kv_write_kind"], "current_token_append"); - EXPECT_EQ(scenarios[2]["weight_read_bytes_per_work_unit"], "1048576"); - EXPECT_EQ(scenarios[2]["kv_read_bytes_per_work_unit"], "32"); - EXPECT_EQ(scenarios[2]["kv_write_bytes_per_work_unit"], "16"); - EXPECT_EQ(scenarios[2]["effective_model_payload_bytes"], "1048624"); - EXPECT_EQ(scenarios[2]["task_accounted_bytes"], "1048624"); - for (const nlohmann::json& scenario : scenarios) { - EXPECT_EQ(scenario["layout_metadata_lookup_count_per_work_unit"], "0"); - EXPECT_EQ(scenario["layout_metadata_read_bytes_per_work_unit"], "0"); - EXPECT_EQ(scenario["layout_metadata_lookup_count"], "0"); - EXPECT_EQ(scenario["layout_metadata_read_bytes"], "0"); - } - EXPECT_EQ(first["counters"]["planned_work_units"], "3"); - EXPECT_EQ(first["counters"]["planned_effective_model_payload_bytes"], - "2097248"); - EXPECT_EQ(first["counters"]["planned_task_accounted_bytes"], "2097248"); - EXPECT_EQ(first["counters"]["planned_layout_metadata_lookup_count"], "0"); - EXPECT_EQ(first["counters"]["planned_layout_metadata_read_bytes"], "0"); - - EXPECT_EQ(first["configuration"], second["configuration"]); - EXPECT_EQ(first["resolved_plan"]["methodology"], - second["resolved_plan"]["methodology"]); - EXPECT_EQ(first["resolved_plan"]["geometry"], - second["resolved_plan"]["geometry"]); - EXPECT_EQ(first["seeds"], second["seeds"]); - EXPECT_EQ(first["resolved_plan"]["model_work_plan"], - second["resolved_plan"]["model_work_plan"]); - EXPECT_EQ(first["resolved_plan"]["scenario_plans"], - second["resolved_plan"]["scenario_plans"]); - ASSERT_EQ(first["measurements"].size(), second["measurements"].size()); - for (size_t index = 0; index < first["measurements"].size(); ++index) { - EXPECT_EQ(first["measurements"][index]["plan_ref"], - second["measurements"][index]["plan_ref"]); - EXPECT_EQ(first["measurements"][index]["checksum"], - second["measurements"][index]["checksum"]); - } -} - TEST(ExecutableCliIntegrationTest, LlmPagedTasksResetAppendSlotsBetweenScenariosIntegration) { std::vector arguments = bounded_llm_arguments("-", 3); @@ -1564,75 +1210,6 @@ TEST(ExecutableCliIntegrationTest, } } -TEST(ExecutableCliIntegrationTest, - LlmDotDashAndFlagShapedOutputsRemainOrdinaryFilesIntegration) { - for (const std::string& target : {"./-", "-G"}) { - SCOPED_TRACE(target); - const CliResult result = - run_memory_benchmark(bounded_llm_arguments(target, 1)); - - expect_process_completed(result); - ASSERT_EQ(result.exit_code, EXIT_SUCCESS) << result.output; - ASSERT_NE(result.directory, nullptr); - const std::filesystem::path output_path = - result.directory->path() / (target == "./-" ? "-" : target); - ASSERT_TRUE(std::filesystem::is_regular_file(output_path)); - EXPECT_FALSE(std::filesystem::exists(output_path.string() + ".tmp")); - const nlohmann::json json = - nlohmann::json::parse(read_file(output_path.string())); - EXPECT_EQ(json["configuration"]["output_file"], target); - EXPECT_EQ(json["mode"], Constants::LLM_JSON_MODE_NAME); - EXPECT_EQ(json["status"], "complete"); - EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["run_accepted"].get()); - EXPECT_FALSE(json["scenario_order_balance_complete"].get()); - } -} - -TEST(ExecutableCliIntegrationTest, GpuModeConflictIsOrderIndependentIntegration) { - for (const std::vector& arguments : { - std::vector{"--gpu-bandwidth", - "--analyze-core2core"}, - std::vector{"--analyze-core2core", - "--gpu-bandwidth"}}) { - const CliResult result = run_memory_benchmark(arguments); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("mutually exclusive"), std::string::npos); - EXPECT_EQ(result.output.find("Running GPU memory bandwidth"), - std::string::npos); - } -} - -TEST(ExecutableCliIntegrationTest, GpuMinimumFailsBeforeOutputIntegration) { - const TemporaryJsonFile output("gpu_below_minimum"); - const CliResult result = run_memory_benchmark( - {"--gpu-bandwidth", "--buffer-size", "63", "--output", - output.path()}); - - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("at least 64 MB"), std::string::npos); - EXPECT_EQ(access(output.path().c_str(), F_OK), -1); -} - -TEST(ExecutableCliIntegrationTest, - InvalidGpuStdoutTargetLeavesStdoutEmptyIntegration) { - const CliResult result = run_memory_benchmark( - {"--gpu-bandwidth", "--buffer-size", "63", "--output", "-"}); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_EQ(result.stderr_output, - Messages::error_prefix() + - Messages::error_gpu_buffer_size_below_minimum( - 63, Constants::GPU_MIN_BUFFER_SIZE_MB) + - "\n"); - expect_no_dash_transport_artifacts(result); -} - TEST(ExecutableCliIntegrationTest, GpuStdoutEmitsOneSchemaV1ResultOnSupportedOrUnsupportedMetalIntegration) { const CliResult result = run_memory_benchmark( @@ -1787,102 +1364,45 @@ TEST(ExecutableCliIntegrationTest, GpuAutomaticCalibrationFreezesPlansIntegratio } } -TEST(ExecutableCliIntegrationTest, OptionsWithoutModeShowHelpAndReturnSuccessIntegration) { - const CliResult result = run_memory_benchmark({"--threads", "1"}); - - EXPECT_EQ(result.exit_code, EXIT_SUCCESS); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("Usage:"), std::string::npos); - EXPECT_NE(result.output.find("--threads "), std::string::npos); -} - -TEST(ExecutableCliIntegrationTest, InvalidStandardModeConfigReturnsFailureIntegration) { - const CliResult result = run_memory_benchmark( - {"--benchmark", "--only-bandwidth", "--latency-samples", "1"}); - - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("--only-bandwidth cannot be used with --latency-samples"), std::string::npos); -} - -TEST(ExecutableCliIntegrationTest, - InvalidStdoutTargetCommandLeavesStdoutEmptyIntegration) { - const CliResult result = run_memory_benchmark( - {"--benchmark", "--only-bandwidth", "--latency-samples", "1", - "--output", "-"}); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find( - Messages::error_only_bandwidth_with_latency_samples()), - std::string::npos) - << result.stderr_output; - expect_no_dash_transport_artifacts(result); -} - -TEST(ExecutableCliIntegrationTest, - ParseErrorWithStdoutTargetLeavesStdoutEmptyIntegration) { - const std::string invalid_option = "--not-a-real-option"; - const CliResult result = run_memory_benchmark( - {"--benchmark", "--output", "-", invalid_option}); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find( - Messages::error_unknown_option(invalid_option)), - std::string::npos) - << result.stderr_output; - expect_no_dash_transport_artifacts(result); -} - -TEST(ExecutableCliIntegrationTest, CoreToCoreArgumentsAreRoutedBeforeNormalParserIntegration) { - const CliResult result = run_memory_benchmark( - {"--analyze-core2core", "--buffer-size", "256"}); - - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("--analyze-core2core allows only optional"), std::string::npos); -} - -TEST(ExecutableCliIntegrationTest, - InvalidCoreToCoreStdoutTargetLeavesStdoutEmptyIntegration) { - const CliResult result = run_memory_benchmark( - {"--analyze-core2core", "--output", "-", "--buffer-size", "256"}); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find( - Messages::error_analyze_core_to_core_must_be_used_alone()), - std::string::npos) - << result.stderr_output; - expect_no_dash_transport_artifacts(result); -} - -TEST(ExecutableCliIntegrationTest, - CoreToCoreHelpWithStdoutTargetRemainsHumanInEitherOrderIntegration) { - for (const std::vector& arguments : { - std::vector{"--analyze-core2core", "--output", "-", - "--help"}, - std::vector{"--analyze-core2core", "--help", - "--output", "-"}, - }) { - SCOPED_TRACE(testing::PrintToString(arguments)); - const CliResult result = run_memory_benchmark(arguments); - +TEST(ExecutableCliIntegrationTest, InvalidCommandsFailBeforeOutputAcrossModesIntegration) { + const TemporaryJsonFile output("tlb_invalid_stride_sweep"); + const long page_size = ::sysconf(_SC_PAGESIZE); + ASSERT_GT(page_size, 0); + struct InvalidCase { + std::vector arguments; + std::string diagnostic; + bool diagnostic_is_complete = false; + }; + const InvalidCase cases[] = { + {{"--benchmark", "--only-bandwidth", "--latency-samples", "1", "--output", "-"}, + Messages::error_only_bandwidth_with_latency_samples()}, + {{"--llm-memory", "--weight-size-mb", "1", "--output", "-"}, + Messages::error_llm_memory_missing_required_option("--layers")}, + {{"--gpu-bandwidth", "--buffer-size", "63", "--output", "-"}, + Messages::error_gpu_buffer_size_below_minimum(63, Constants::GPU_MIN_BUFFER_SIZE_MB), + true}, + {{"--benchmark", "--output", "-", "--not-a-real-option"}, Messages::error_unknown_option("--not-a-real-option")}, + {{"--analyze-core2core", "--output", "-", "--buffer-size", "256"}, + Messages::error_analyze_core_to_core_must_be_used_alone()}, + {{"--analyze-tlb", "--sweep", "latency-stride-bytes=64,130", "--output", output.path()}, + Messages::error_latency_stride_alignment(130, sizeof(uintptr_t))}, + {{"--analyze-tlb", "--latency-stride-bytes", "32768", "--output", "-"}, + Messages::error_analyze_tlb_stride_exceeds_page(32768, static_cast(page_size))}, + {{"--analyze-tlb", "--sweep", "latency-stride-bytes=64,130", "--output", "-"}, + Messages::error_latency_stride_alignment(130, sizeof(uintptr_t))}, + }; + for (const InvalidCase& test_case : cases) { + SCOPED_TRACE(testing::PrintToString(test_case.arguments)); + const CliResult result = run_memory_benchmark(test_case.arguments); expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_SUCCESS); + EXPECT_EQ(result.exit_code, EXIT_FAILURE); expect_no_runtime_banner(result); - EXPECT_NE(result.stdout_output.find("Usage:"), std::string::npos); - EXPECT_NE(result.stdout_output.find("--analyze-core2core"), - std::string::npos); - EXPECT_TRUE(result.stderr_output.empty()) << result.stderr_output; - EXPECT_FALSE(nlohmann::json::accept(result.stdout_output)); + EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; + const std::string diagnostic = Messages::error_prefix() + test_case.diagnostic + "\n"; + EXPECT_EQ(result.stderr_output.find(diagnostic), 0u) << result.stderr_output; + if (test_case.diagnostic_is_complete) EXPECT_EQ(result.stderr_output, diagnostic); + EXPECT_EQ(result.stderr_output.find(Messages::msg_running_sweep(2)), std::string::npos); + EXPECT_EQ(access(output.path().c_str(), F_OK), -1); expect_no_dash_transport_artifacts(result); } } @@ -2016,40 +1536,6 @@ TEST(ExecutableCliIntegrationTest, expect_no_dash_transport_artifacts(result); } -TEST(ExecutableCliIntegrationTest, AnalyzeTlbInvalidStrideSweepFailsBeforeExecutionIntegration) { - const TemporaryJsonFile output("tlb_invalid_stride_sweep"); - - const CliResult result = run_memory_benchmark({ - "--analyze-tlb", "--sweep", "latency-stride-bytes=64,130", - "--output", output.path()}); - - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_NE(result.output.find("must be a multiple of 8 bytes"), std::string::npos); - EXPECT_EQ(result.output.find("Running sweep"), std::string::npos); - EXPECT_EQ(access(output.path().c_str(), F_OK), -1); -} - -TEST(ExecutableCliIntegrationTest, - AnalyzeTlbInvalidPreflightWithStdoutTargetLeavesStdoutEmptyIntegration) { - const CliResult result = run_memory_benchmark({ - "--analyze-tlb", "--latency-stride-bytes", "32768", "--output", - "-"}); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - const long page_size = ::sysconf(_SC_PAGESIZE); - ASSERT_GT(page_size, 0); - EXPECT_NE(result.stderr_output.find( - Messages::error_analyze_tlb_stride_exceeds_page( - 32768, static_cast(page_size))), - std::string::npos) - << result.stderr_output; - expect_no_dash_transport_artifacts(result); -} - TEST(ExecutableCliIntegrationTest, AnalyzeTlbWritesSingleJsonDocumentToStdoutIntegration) { const CliResult result = run_memory_benchmark( @@ -2148,37 +1634,6 @@ TEST(ExecutableCliIntegrationTest, expect_no_dash_transport_artifacts(result); } -TEST(ExecutableCliIntegrationTest, - ExplicitDotDashOutputRemainsOrdinaryFileTargetIntegration) { - const CliResult result = run_memory_benchmark({ - "--benchmark", "--only-bandwidth", "--buffer-size", "1", - "--iterations", "1", "--count", "1", "--threads", "1", - "--output", "./-"}); - - expect_process_completed(result); - ASSERT_EQ(result.exit_code, EXIT_SUCCESS) << result.output; - ASSERT_NE(result.directory, nullptr); - const std::filesystem::path output_path = result.directory->path() / "-"; - ASSERT_TRUE(std::filesystem::exists(output_path)); - EXPECT_FALSE(std::filesystem::exists(result.directory->path() / "-.tmp")); - const nlohmann::json json = - nlohmann::json::parse(read_file(output_path.string())); - EXPECT_EQ(json["configuration"]["output_file"], "./-"); - EXPECT_EQ(json["configuration"]["mode"], "benchmark"); - EXPECT_EQ(json["configuration"]["benchmark_schema_version"], - Constants::BENCHMARK_JSON_SCHEMA_VERSION); - EXPECT_EQ(json["status"], "complete"); - EXPECT_TRUE(json["results_complete"].get()); - EXPECT_TRUE(json["conclusions_valid"].get()); - EXPECT_NE(result.stdout_output.find(Messages::msg_running_benchmarks()), - std::string::npos) - << result.stdout_output; - EXPECT_NE(result.stdout_output.find("Results saved to:"), - std::string::npos) - << result.stdout_output; - EXPECT_FALSE(nlohmann::json::accept(result.stdout_output)); -} - TEST(ExecutableCliIntegrationTest, FlagShapedOutputValuesRemainStandardFileTargetsIntegration) { const auto verify_target = [](const std::string& target) { @@ -2220,14 +1675,15 @@ TEST(ExecutableCliIntegrationTest, << result.output; EXPECT_EQ(result.output.find("mutually exclusive"), std::string::npos) << result.output; + EXPECT_NE(result.stdout_output.find(Messages::msg_running_benchmarks()), std::string::npos); + EXPECT_FALSE(nlohmann::json::accept(result.stdout_output)); EXPECT_EQ(count_occurrences(result.output, Messages::msg_results_saved_to("")), 1u) << result.output; }; - for (const char* target : - {"-G", "-T", "--analyze-tlb", "-k", "--cache-size"}) { + for (const char* target : {"./-", "-T", "--cache-size"}) { SCOPED_TRACE(target); verify_target(target); } @@ -2323,26 +1779,6 @@ TEST(ExecutableCliIntegrationTest, expect_no_dash_transport_artifacts(result); } -TEST(ExecutableCliIntegrationTest, - InvalidTlbSweepWithStdoutTargetLeavesStdoutEmptyIntegration) { - const CliResult result = run_memory_benchmark({ - "--analyze-tlb", "--sweep", "latency-stride-bytes=64,130", - "--output", "-"}); - - expect_process_completed(result); - EXPECT_EQ(result.exit_code, EXIT_FAILURE); - expect_no_runtime_banner(result); - EXPECT_TRUE(result.stdout_output.empty()) << result.stdout_output; - EXPECT_NE(result.stderr_output.find( - Messages::error_latency_stride_alignment( - 130, sizeof(uintptr_t))), - std::string::npos) - << result.stderr_output; - EXPECT_EQ(result.stderr_output.find(Messages::msg_running_sweep(2)), - std::string::npos); - expect_no_dash_transport_artifacts(result); -} - TEST(ExecutableCliIntegrationTest, PatternSweepPrintsOneBannerAcrossNestedLoopsIntegration) { const TemporaryJsonFile output("pattern_sweep_banner"); @@ -2372,41 +1808,6 @@ TEST(ExecutableCliIntegrationTest, PatternSweepPrintsOneBannerAcrossNestedLoopsI EXPECT_EQ(access((output.path() + ".tmp").c_str(), F_OK), -1); } -TEST(ExecutableCliIntegrationTest, - PatternSweepWritesSingleTerminalJsonToStdoutIntegration) { - const CliResult result = run_memory_benchmark({ - "--patterns", "--iterations", "1", "--count", "1", "--threads", - "1", "--seed", "42", "--sweep", "buffer-size=1,2", - "--sweep-max-runs", "2", "--output", "-"}); - - expect_process_completed(result); - ASSERT_EQ(result.exit_code, EXIT_SUCCESS) << result.stderr_output; - const nlohmann::json json = parse_single_stdout_json(result); - EXPECT_EQ(json["configuration"]["mode"], - Constants::SWEEP_JSON_MODE_NAME); - EXPECT_EQ(json["configuration"]["base_mode"], - Constants::PATTERNS_JSON_MODE_NAME); - EXPECT_EQ(json["configuration"]["sweep_schema_version"], 1); - EXPECT_EQ(json["status"], "complete"); - EXPECT_EQ(json["planned_runs"], 2u); - EXPECT_EQ(json["attempted_runs"], 2u); - EXPECT_EQ(json["completed_runs"], 2u); - EXPECT_TRUE(json["conclusions_valid"].get()); - ASSERT_EQ(json["runs"].size(), 2u); - for (const nlohmann::json& run : json["runs"]) { - EXPECT_EQ(run["status"], "complete"); - EXPECT_EQ(run["result"]["configuration"]["pattern_schema_version"], - 3); - EXPECT_EQ(run["result"]["status"], "complete"); - EXPECT_TRUE(run["result"]["results_complete"].get()); - } - - EXPECT_EQ(count_occurrences(result.output, - Messages::msg_results_saved_to("")), - 0u); - expect_no_dash_transport_artifacts(result); -} - TEST(ExecutableCliIntegrationTest, PatternModeRunsPatternOrchestrationIntegration) { const CliResult result = run_memory_benchmark( {"--patterns", "--buffer-size", "1", "--iterations", "1", "--count", "1"}); diff --git a/tests/test_gpu_bandwidth.cpp b/tests/test_gpu_bandwidth.cpp index c617dd6..5119918 100644 --- a/tests/test_gpu_bandwidth.cpp +++ b/tests/test_gpu_bandwidth.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -74,29 +73,6 @@ class TemporaryGpuJsonFile { std::filesystem::path path_; }; -class WriteFailingGpuStreamBuffer : public std::streambuf { - protected: - std::streamsize xsputn(const char*, std::streamsize) override { - return 0; - } - - int_type overflow(int_type) override { return traits_type::eof(); } -}; - -class ScopedGpuStdoutBuffer { - public: - explicit ScopedGpuStdoutBuffer(std::streambuf* replacement) - : original_(std::cout.rdbuf(replacement)) {} - - ~ScopedGpuStdoutBuffer() { std::cout.rdbuf(original_); } - - ScopedGpuStdoutBuffer(const ScopedGpuStdoutBuffer&) = delete; - ScopedGpuStdoutBuffer& operator=(const ScopedGpuStdoutBuffer&) = delete; - - private: - std::streambuf* original_; -}; - size_t operation_index(GpuOperation operation) { return static_cast(operation); } struct PhaseCall { @@ -524,20 +500,6 @@ TEST_F(GpuBandwidthParserTest, UserValuesAndMaximumSeedAreExact) { EXPECT_TRUE(config.user_specified_seed); } -TEST_F(GpuBandwidthParserTest, OutputTargetSpellingIsRetainedExactly) { - for (const std::string& output_target : {"-", "./-"}) { - SCOPED_TRACE(output_target); - const std::vector arguments = { - "memory_benchmark", "--gpu-bandwidth", "--buffer-size", "64", - "--seed", "1", "--output", output_target}; - GpuBandwidthConfig config; - - ASSERT_EQ(parse_gpu_arguments(arguments, config), EXIT_SUCCESS); - EXPECT_EQ(config.output_file, output_target); - EXPECT_EQ(config.argv, arguments); - } -} - TEST_F(GpuBandwidthParserTest, RejectsMissingAndDuplicateOutputOptionsWithCentralizedDiagnostics) { struct OutputErrorCase { @@ -578,24 +540,8 @@ TEST_F(GpuBandwidthParserTest, StrictDuplicateAndIncompatibleOptionsAreRejected) {"memory_benchmark", "-G", "--count", "2147483648"}, {"memory_benchmark", "-G", "-b", "64", "--buffer-size", "64"}, {"memory_benchmark", "-G", "--seed", "1", "--seed", "2"}, - {"memory_benchmark", "-G", "--analyze-core2core"}, {"memory_benchmark", "-G", "--unknown"}, {"memory_benchmark", "-G", "--iterations"}, - {"memory_benchmark", "-G", "--threads", "1"}, - {"memory_benchmark", "-G", "--cache-size", "1"}, - {"memory_benchmark", "-G", "--latency-samples", "1"}, - {"memory_benchmark", "-G", "--latency-stride-bytes", "256"}, - {"memory_benchmark", "-G", "--latency-chain-mode", "auto"}, - {"memory_benchmark", "-G", "--latency-tlb-locality-kb", "1024"}, - {"memory_benchmark", "-G", "--tlb-density", "high"}, - {"memory_benchmark", "-G", "--only-bandwidth"}, - {"memory_benchmark", "-G", "--only-latency"}, - {"memory_benchmark", "-G", "--non-cacheable"}, - {"memory_benchmark", "-G", "--benchmark"}, - {"memory_benchmark", "-G", "--patterns"}, - {"memory_benchmark", "-G", "--analyze-tlb"}, - {"memory_benchmark", "-G", "--sweep", "buffer-size=64"}, - {"memory_benchmark", "-G", "--sweep-max-runs", "1"}, {"memory_benchmark", "-G", "-G"}, {"memory_benchmark", "-G", "--help", "--help"}, }; @@ -1017,69 +963,77 @@ TEST(GpuRunnerTest, CalibrationFailureFinalizesTailByPendingStopState) { } } -TEST(GpuRunnerTest, ProductionFinalSaveFailureWinsCalibrationFailure) { - FakeGpuBackend backend; - backend.timed_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuTimedResult& timed) { - timed.status = GpuBackendStatus::Failed; - timed.command_status = GpuCommandStatus::Error; - timed.reason_code = "calibration-timed-command-failed"; - }; - GpuBandwidthConfig config = automatic_config(); - config.output_file = "/tmp/" + std::string(5000, 'x') + "/gpu.json"; - GpuRunResult result; - - testing::internal::CaptureStderr(); - const int status = run_gpu_bandwidth_suite(config, backend, result); - const std::string error = testing::internal::GetCapturedStderr(); - EXPECT_EQ(status, EXIT_FAILURE); - EXPECT_NE(error.find("Failed to write file"), std::string::npos); - EXPECT_EQ(result.status, GpuRunStatus::Failed); - EXPECT_EQ(result.reason_code, "checkpoint-write-failed"); - EXPECT_FALSE(result.results_complete); - EXPECT_FALSE(result.conclusions_valid); -} - -TEST(GpuRunnerTest, ProductionFinalSaveFailureWinsRunnerException) { - FakeGpuBackend backend; - GpuBandwidthConfig config = explicit_config(1); - config.output_file = "/tmp/" + std::string(5000, 'x') + "/gpu.json"; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = []() -> bool { throw std::runtime_error("injected-stop-failure"); }; - - testing::internal::CaptureStderr(); - const int status = run_gpu_bandwidth_suite(config, backend, result, hooks); - const std::string error = testing::internal::GetCapturedStderr(); - EXPECT_EQ(status, EXIT_FAILURE); - EXPECT_NE(error.find("Failed to write file"), std::string::npos); - EXPECT_EQ(result.status, GpuRunStatus::Failed); - EXPECT_EQ(result.reason_code, "checkpoint-write-failed"); - EXPECT_FALSE(result.results_complete); - EXPECT_FALSE(result.conclusions_valid); +TEST(GpuRunnerTest, ProductionFinalSaveFailureWinsCalibrationFailureOrException) { + for (bool throw_from_hook : {false, true}) { + SCOPED_TRACE(throw_from_hook); + FakeGpuBackend backend; + GpuBandwidthConfig config = throw_from_hook ? explicit_config(1) : automatic_config(); + config.output_file = "/tmp/" + std::string(5000, 'x') + "/gpu.json"; + GpuRunnerTestHooks hooks; + if (throw_from_hook) { + hooks.stop_requested = []() -> bool { throw std::runtime_error("injected-stop-failure"); }; + } else { + backend.timed_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuTimedResult& timed) { + timed.status = GpuBackendStatus::Failed; + timed.command_status = GpuCommandStatus::Error; + timed.reason_code = "calibration-timed-command-failed"; + }; + } + GpuRunResult result; + testing::internal::CaptureStderr(); + const int status = run_gpu_bandwidth_suite(config, backend, result, hooks); + const std::string error = testing::internal::GetCapturedStderr(); + EXPECT_EQ(status, EXIT_FAILURE); + EXPECT_NE(error.find("Failed to write file"), std::string::npos); + EXPECT_EQ(result.status, GpuRunStatus::Failed); + EXPECT_EQ(result.reason_code, "checkpoint-write-failed"); + EXPECT_FALSE(result.results_complete); + EXPECT_FALSE(result.conclusions_valid); + } } TEST(GpuRunnerTest, InvalidTimerStopsBeforeValidationAndProducesNullValue) { - FakeGpuBackend backend; - backend.timed_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuTimedResult& timed) { - timed.gpu_elapsed_seconds = 0.0; - timed.gpu_end_seconds = timed.gpu_start_seconds; - }; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = []() { return false; }; + for (double elapsed : {0.0, std::numeric_limits::quiet_NaN()}) { + SCOPED_TRACE(elapsed); + FakeGpuBackend backend; + backend.timed_mutator = [elapsed](const GpuBackendAttemptRequest&, size_t, GpuTimedResult& timed) { + timed.gpu_elapsed_seconds = elapsed; + timed.gpu_end_seconds = timed.gpu_start_seconds; + }; + GpuRunResult result; + GpuRunnerTestHooks hooks; + hooks.stop_requested = []() { return false; }; - EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(1), backend, result, hooks), EXIT_FAILURE); - ASSERT_EQ(result.status, GpuRunStatus::Failed); - ASSERT_EQ(result.measurements[0].status, GpuMeasurementStatus::Invalid); - EXPECT_EQ(result.measurements[0].reason_code, "invalid-gpu-timestamp"); - EXPECT_FALSE(result.measurements[0].value_gb_s.has_value()); - EXPECT_EQ(result.measurements[0].validation.validation_status, GpuValidationStatus::NotRunTimerInvalid); - EXPECT_EQ(backend.validation_calls, 0U); - EXPECT_EQ(result.counters.completed_measurements, 1U); - EXPECT_EQ(result.counters.validated_measurements, 0U); - EXPECT_EQ(result.counters.attempted_measurements, 1U); - EXPECT_EQ(result.counters.terminal_measurements, result.counters.planned_measurements); - expect_unstarted_failed_tail(result, 1); + EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(1), backend, result, hooks), EXIT_FAILURE); + ASSERT_EQ(result.status, GpuRunStatus::Failed); + ASSERT_EQ(result.measurements[0].status, GpuMeasurementStatus::Invalid); + EXPECT_EQ(result.measurements[0].reason_code, "invalid-gpu-timestamp"); + EXPECT_FALSE(result.measurements[0].value_gb_s.has_value()); + EXPECT_EQ(result.measurements[0].validation.validation_status, GpuValidationStatus::NotRunTimerInvalid); + EXPECT_EQ(backend.validation_calls, 0U); + EXPECT_EQ(result.counters.completed_measurements, 1U); + EXPECT_EQ(result.counters.validated_measurements, 0U); + EXPECT_EQ(result.counters.attempted_measurements, 1U); + EXPECT_EQ(result.counters.terminal_measurements, result.counters.planned_measurements); + expect_unstarted_failed_tail(result, 1); + const Json output = build_gpu_bandwidth_json(explicit_config(1), result); + EXPECT_EQ(output["measurements"][0]["status"], "invalid"); + EXPECT_TRUE(output["measurements"][0]["value_gb_s"].is_null()); + if (std::isfinite(elapsed)) { + EXPECT_DOUBLE_EQ(output["measurements"][0]["timed"]["gpu_elapsed_seconds"].get(), elapsed); + } else { + EXPECT_TRUE(output["measurements"][0]["timed"]["gpu_elapsed_seconds"].is_null()); + } + for (size_t index = 1; index < output["measurements"].size(); ++index) { + EXPECT_EQ(output["measurements"][index]["status"], "failed"); + EXPECT_EQ(output["measurements"][index]["reason_code"], "not-run-after-runtime-failure"); + EXPECT_TRUE(output["measurements"][index]["value_gb_s"].is_null()); + EXPECT_EQ(output["measurements"][index]["warmup"]["status"], "not-run"); + EXPECT_EQ(output["measurements"][index]["warmup"]["command_buffer_status"], "not-run"); + } + EXPECT_TRUE(output["aggregates"]["read"]["headline_gb_s"].is_null()); + EXPECT_TRUE(output["aggregates"]["read"]["statistics"].is_null()); + } } TEST(GpuRunnerTest, TimedAccumulatorMismatchIsInvalidAfterValidation) { @@ -1098,24 +1052,6 @@ TEST(GpuRunnerTest, TimedAccumulatorMismatchIsInvalidAfterValidation) { EXPECT_FALSE(result.measurements[0].value_gb_s.has_value()); } -TEST(GpuRunnerTest, ValidationCommandErrorPreservesRawDiagnostic) { - FakeGpuBackend backend; - backend.validation_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuValidationResult& validation) { - validation.status = GpuBackendStatus::Failed; - validation.command_status = GpuCommandStatus::Error; - validation.validation_status = GpuValidationStatus::Error; - validation.reason_code = "validation-command-failed"; - validation.error = {"MTLCommandBufferErrorDomain", 4, "localized GPU fault"}; - }; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = []() { return false; }; - EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(1), backend, result, hooks), EXIT_FAILURE); - EXPECT_EQ(result.measurements[0].status, GpuMeasurementStatus::Failed); - EXPECT_EQ(result.measurements[0].reason_code, "validation-command-failed"); - EXPECT_EQ(result.measurements[0].validation.error.description, "localized GPU fault"); -} - TEST(GpuRunnerTest, StopBeforeTaskStartsNoBackendPhaseAndInterruptsAllSlots) { FakeGpuBackend backend; GpuRunResult result; @@ -1170,61 +1106,47 @@ TEST(GpuRunnerTest, StopDuringWarmupCompletesStartedTaskBeforeTail) { EXPECT_EQ(checkpoints[0].measurements[0].status, GpuMeasurementStatus::Measured); } -TEST(GpuRunnerTest, FailureWinsOverStopAndPreventsValidationAfterTimedError) { - FakeGpuBackend backend; - bool stop = false; - backend.timed_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuTimedResult& timed) { - timed.status = GpuBackendStatus::Failed; - timed.command_status = GpuCommandStatus::Error; - timed.reason_code = "timed-command-failed"; - }; - backend.phase_hook = [&](const std::string& phase, size_t occurrence, const GpuBackendAttemptRequest&) { - if (phase == "timed" && occurrence == 1) { - stop = true; - } - }; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = [&]() { return stop; }; - - EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(2), backend, result, hooks), EXIT_FAILURE); - EXPECT_EQ(result.status, GpuRunStatus::Failed); - EXPECT_EQ(result.reason_code, "timed-command-failed"); - EXPECT_TRUE(result.interruption_requested); - EXPECT_EQ(result.measurements[0].status, GpuMeasurementStatus::Failed); - EXPECT_EQ(backend.validation_calls, 0U); - expect_unstarted_interrupted_tail(result, 1); -} - -TEST(GpuRunnerTest, ValidationMismatchWinsOverSimultaneousStop) { - FakeGpuBackend backend; - bool stop = false; - backend.validation_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuValidationResult& validation) { - validation.validation_status = GpuValidationStatus::Mismatch; - validation.reason_code = "final-checksum-mismatch"; - ++validation.actual_final_checksum.first; - }; - backend.phase_hook = [&](const std::string& phase, size_t occurrence, const GpuBackendAttemptRequest&) { - if (phase == "validation" && occurrence == 1) { - stop = true; +TEST(GpuRunnerTest, TimedFailureAndValidationMismatchWinOverSimultaneousStop) { + for (bool validation_failure : {false, true}) { + SCOPED_TRACE(validation_failure); + FakeGpuBackend backend; + bool stop = false; + const char* phase = validation_failure ? "validation" : "timed"; + const char* reason = validation_failure ? "final-checksum-mismatch" : "timed-command-failed"; + if (validation_failure) { + backend.validation_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuValidationResult& validation) { + validation.validation_status = GpuValidationStatus::Mismatch; + validation.reason_code = "final-checksum-mismatch"; + ++validation.actual_final_checksum.first; + }; + } else { + backend.timed_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuTimedResult& timed) { + timed.status = GpuBackendStatus::Failed; + timed.command_status = GpuCommandStatus::Error; + timed.reason_code = "timed-command-failed"; + }; } - }; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = [&]() { return stop; }; - - EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(2), backend, result, hooks), EXIT_FAILURE); - EXPECT_EQ(result.status, GpuRunStatus::Failed); - EXPECT_EQ(result.reason_code, "final-checksum-mismatch"); - EXPECT_TRUE(result.interruption_requested); - ASSERT_EQ(result.measurements[0].status, GpuMeasurementStatus::Invalid); - EXPECT_EQ(result.measurements[0].reason_code, "final-checksum-mismatch"); - EXPECT_FALSE(result.measurements[0].value_gb_s.has_value()); - EXPECT_EQ(backend.timed_calls, 1U); - EXPECT_EQ(backend.validation_calls, 1U); - EXPECT_EQ(result.counters.attempted_measurements, 1U); - EXPECT_EQ(result.counters.completed_measurements, 1U); - expect_unstarted_interrupted_tail(result, 1); + backend.phase_hook = [&](const std::string& current, size_t occurrence, const GpuBackendAttemptRequest&) { + if (current == phase && occurrence == 1) stop = true; + }; + GpuRunResult result; + GpuRunnerTestHooks hooks; + hooks.stop_requested = [&]() { return stop; }; + EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(2), backend, result, hooks), EXIT_FAILURE); + EXPECT_EQ(result.status, GpuRunStatus::Failed); + EXPECT_EQ(result.reason_code, reason); + EXPECT_TRUE(result.interruption_requested); + ASSERT_FALSE(result.measurements.empty()); + EXPECT_EQ(result.measurements[0].status, + validation_failure ? GpuMeasurementStatus::Invalid : GpuMeasurementStatus::Failed); + EXPECT_EQ(result.measurements[0].reason_code, reason); + EXPECT_FALSE(result.measurements[0].value_gb_s.has_value()); + EXPECT_EQ(backend.timed_calls, 1U); + EXPECT_EQ(backend.validation_calls, validation_failure ? 1U : 0U); + EXPECT_EQ(result.counters.attempted_measurements, 1U); + EXPECT_EQ(result.counters.completed_measurements, validation_failure ? 1U : 0U); + expect_unstarted_interrupted_tail(result, 1); + } } TEST(GpuRunnerTest, StopInLastTaskKeepsAllMeasurementsButInvalidatesRun) { @@ -1349,94 +1271,6 @@ TEST(GpuRunnerTest, std::filesystem::exists(file_target.path().string() + ".tmp")); } -TEST(GpuRunnerTest, - StdoutSessionSerializesUnsupportedResultOnceWithoutIntermediateOutput) { - FakeGpuBackend backend; - backend.initialization.status = GpuBackendStatus::Unsupported; - backend.initialization.reason_code = - "required-gpu-family-unsupported"; - GpuBandwidthConfig config = explicit_config(); - config.output_file = "-"; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = []() { return false; }; - - int status = EXIT_SUCCESS; - int final_write_status = EXIT_FAILURE; - Json payload; - testing::internal::CaptureStdout(); - { - JsonOutputSession session(make_json_output_target( - config.output_file, JsonFilePathPolicy::PreserveRaw)); - status = run_gpu_bandwidth_suite(config, backend, result, session, - hooks); - payload = build_gpu_bandwidth_json(config, result); - final_write_status = session.write_final(payload); - } - const std::string stdout_text = testing::internal::GetCapturedStdout(); - - EXPECT_EQ(status, EXIT_FAILURE); - EXPECT_EQ(final_write_status, EXIT_SUCCESS); - EXPECT_EQ(stdout_text, payload.dump(2) + "\n"); - ASSERT_TRUE(Json::accept(stdout_text)); - const Json parsed = Json::parse(stdout_text); - EXPECT_EQ(result.status, GpuRunStatus::Unsupported); - EXPECT_EQ(result.reason_code, "required-gpu-family-unsupported"); - EXPECT_FALSE(result.results_complete); - EXPECT_FALSE(result.conclusions_valid); - EXPECT_EQ(parsed["schema_version"], - Constants::GPU_JSON_SCHEMA_VERSION); - EXPECT_EQ(parsed["status"], "unsupported"); - EXPECT_FALSE(parsed["results_complete"].get()); - EXPECT_FALSE(parsed["conclusions_valid"].get()); - EXPECT_EQ(parsed["configuration"]["output_file"], "-"); - EXPECT_EQ(backend.lifecycle_log, - (std::vector{"initialize"})); -} - -TEST(GpuRunnerTest, - StdoutFinalWriteFailureWinsWithoutMutatingTerminalResult) { - FakeGpuBackend backend; - GpuBandwidthConfig config = explicit_config(); - config.output_file = "-"; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = []() { return false; }; - WriteFailingGpuStreamBuffer failing_stdout; - int run_status = EXIT_FAILURE; - int final_write_status = EXIT_SUCCESS; - Json payload_before; - Json payload_after; - - testing::internal::CaptureStderr(); - { - ScopedGpuStdoutBuffer stdout_buffer(&failing_stdout); - JsonOutputSession session(make_json_output_target( - config.output_file, JsonFilePathPolicy::PreserveRaw)); - run_status = run_gpu_bandwidth_suite(config, backend, result, session, - hooks); - payload_before = build_gpu_bandwidth_json(config, result); - final_write_status = session.write_final(payload_before); - payload_after = build_gpu_bandwidth_json(config, result); - } - const std::string errors = testing::internal::GetCapturedStderr(); - const int command_status = final_write_status == EXIT_SUCCESS - ? run_status - : EXIT_FAILURE; - - EXPECT_EQ(run_status, EXIT_SUCCESS); - EXPECT_EQ(final_write_status, EXIT_FAILURE); - EXPECT_EQ(command_status, EXIT_FAILURE); - EXPECT_EQ(payload_after, payload_before); - EXPECT_EQ(result.status, GpuRunStatus::Complete); - EXPECT_TRUE(result.results_complete); - EXPECT_TRUE(result.conclusions_valid); - EXPECT_NE(errors.find(Messages::error_json_stdout_write_failed( - Messages::json_stdout_reason_write_failed())), - std::string::npos) - << errors; -} - TEST(GpuRunnerTest, FileSessionCheckpointFailureKeepsPrecedenceAndRetryCadence) { TemporaryGpuJsonFile blocking_parent("checkpoint_failure_parent"); @@ -1586,30 +1420,6 @@ TEST(GpuJsonTest, SchemaV1UsesExactStringsAndMeasuredOnlyValues) { EXPECT_EQ(output["dram_residency"], "unverified"); } -TEST(GpuJsonTest, MissingAndNonFiniteValuesAreNull) { - FakeGpuBackend backend; - backend.timed_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuTimedResult& timed) { - timed.gpu_elapsed_seconds = std::numeric_limits::quiet_NaN(); - }; - GpuRunResult result; - GpuRunnerTestHooks hooks; - hooks.stop_requested = []() { return false; }; - EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(1), backend, result, hooks), EXIT_FAILURE); - const Json output = build_gpu_bandwidth_json(explicit_config(1), result); - EXPECT_EQ(output["measurements"][0]["status"], "invalid"); - EXPECT_TRUE(output["measurements"][0]["value_gb_s"].is_null()); - EXPECT_TRUE(output["measurements"][0]["timed"]["gpu_elapsed_seconds"].is_null()); - for (size_t index = 1; index < output["measurements"].size(); ++index) { - EXPECT_EQ(output["measurements"][index]["status"], "failed"); - EXPECT_EQ(output["measurements"][index]["reason_code"], "not-run-after-runtime-failure"); - EXPECT_TRUE(output["measurements"][index]["value_gb_s"].is_null()); - EXPECT_EQ(output["measurements"][index]["warmup"]["status"], "not-run"); - EXPECT_EQ(output["measurements"][index]["warmup"]["command_buffer_status"], "not-run"); - } - EXPECT_TRUE(output["aggregates"]["read"]["headline_gb_s"].is_null()); - EXPECT_TRUE(output["aggregates"]["read"]["statistics"].is_null()); -} - TEST(GpuJsonTest, StableReasonCodeAndRawNSErrorStaySeparate) { FakeGpuBackend backend; backend.validation_mutator = [](const GpuBackendAttemptRequest&, size_t, GpuValidationResult& validation) { @@ -1623,6 +1433,10 @@ TEST(GpuJsonTest, StableReasonCodeAndRawNSErrorStaySeparate) { GpuRunnerTestHooks hooks; hooks.stop_requested = []() { return false; }; EXPECT_EQ(run_gpu_bandwidth_suite(explicit_config(1), backend, result, hooks), EXIT_FAILURE); + ASSERT_FALSE(result.measurements.empty()); + EXPECT_EQ(result.measurements[0].status, GpuMeasurementStatus::Failed); + EXPECT_EQ(result.measurements[0].reason_code, "validation-command-failed"); + EXPECT_EQ(result.measurements[0].validation.error.description, "localized and unstable description"); const Json output = build_gpu_bandwidth_json(explicit_config(1), result); const Json& measurement = output["measurements"][0]; EXPECT_EQ(output["reason_code"], "validation-command-failed"); diff --git a/tests/test_gpu_metal_backend.cpp b/tests/test_gpu_metal_backend.cpp index 929989f..2537575 100644 --- a/tests/test_gpu_metal_backend.cpp +++ b/tests/test_gpu_metal_backend.cpp @@ -246,8 +246,7 @@ TEST_F(GpuMetalBackendIntegrationTest, InitializationAndRuntimeCompilationIntegr EXPECT_EQ(initialization_.compilation.floating_point_math, "not_applicable_integer_only"); EXPECT_EQ(initialization_.compilation.preprocessor_macros, "none"); EXPECT_EQ(initialization_.compilation.kernel_revision, "gpu-linear-word-mod32-tg-reduce-v2"); - EXPECT_EQ(initialization_.compilation.kernel_source_sha256, - "21def2d75d3545dba31aa4897ea57ec2fd0e4481cd86ce21725338ab0f322ac5"); + EXPECT_EQ(initialization_.compilation.kernel_source_sha256, canonical_gpu_kernel_source_sha256()); if (device.device_name.find("M4") != std::string::npos) { EXPECT_TRUE(initialization_.compilation.compiler_diagnostics.empty()); } diff --git a/tests/test_gpu_work_plan.cpp b/tests/test_gpu_work_plan.cpp index 7087252..ddee3f0 100644 --- a/tests/test_gpu_work_plan.cpp +++ b/tests/test_gpu_work_plan.cpp @@ -47,37 +47,6 @@ GpuWorkPlanRequest make_request(GpuOperation operation, size_t buffer_bytes, } // namespace -TEST(GpuWorkPlanTest, ConstantsMatchLockedGpuMethodology) { - EXPECT_EQ(Constants::GPU_DEFAULT_BUFFER_SIZE_MB, 512u); - EXPECT_EQ(Constants::GPU_MIN_BUFFER_SIZE_MB, 64u); - EXPECT_EQ(Constants::GPU_DEFAULT_LOOP_COUNT, 3u); - EXPECT_DOUBLE_EQ(Constants::GPU_CALIBRATION_TARGET_SECONDS, - Constants::BANDWIDTH_CALIBRATION_TARGET_SECONDS); - EXPECT_DOUBLE_EQ(Constants::GPU_CALIBRATION_MIN_SECONDS, 0.100); - EXPECT_DOUBLE_EQ(Constants::GPU_CALIBRATION_MAX_SECONDS, 0.250); - EXPECT_EQ(Constants::GPU_CALIBRATION_MAX_CORRECTIONS, 2u); - EXPECT_EQ(Constants::GPU_CALIBRATION_MIN_PILOT_BYTES, - 8 * Constants::BYTES_PER_MB); - EXPECT_EQ(Constants::GPU_MAX_DISPATCHES_PER_MEASUREMENT, 16384u); - EXPECT_EQ(Constants::GPU_MAX_EXACT_PAYLOAD_BYTES, - 64ULL * 1024ULL * Constants::BYTES_PER_MB); - EXPECT_EQ(Constants::GPU_VECTOR_WIDTH_BYTES, 16u); - EXPECT_EQ(Constants::GPU_THREADS_PER_THREADGROUP_CAP, 256u); - EXPECT_EQ(Constants::GPU_MAX_THREADGROUPS_PER_GRID, 8192u); - EXPECT_DOUBLE_EQ(Constants::GPU_STREAMING_CV_WARNING_PCT, 5.0); -} - -TEST(GpuWorkPlanTest, SharedCyclicOrderPreservesBenchmarkCompatibility) { - EXPECT_EQ(build_cyclic_order(4, 0), - (std::vector{0, 1, 2, 3})); - EXPECT_EQ(build_cyclic_order(4, 1), - (std::vector{1, 2, 3, 0})); - EXPECT_EQ(build_cyclic_order(4, std::numeric_limits::max()), - build_benchmark_cyclic_order( - 4, std::numeric_limits::max())); - EXPECT_TRUE(build_cyclic_order(0, 7).empty()); -} - TEST(GpuWorkPlanTest, OperationOrderRotatesReadWriteCopyAcrossLoops) { EXPECT_EQ(build_gpu_operation_order(0), (std::array{GpuOperation::Read, @@ -92,13 +61,15 @@ TEST(GpuWorkPlanTest, OperationOrderRotatesReadWriteCopyAcrossLoops) { GpuOperation::Read, GpuOperation::Write})); EXPECT_EQ(build_gpu_operation_order(3), build_gpu_operation_order(0)); + EXPECT_EQ(build_cyclic_order(4, 0), (std::vector{0, 1, 2, 3})); + EXPECT_EQ(build_cyclic_order(4, 1), (std::vector{1, 2, 3, 0})); + EXPECT_EQ(build_cyclic_order(4, std::numeric_limits::max()), (std::vector{3, 0, 1, 2})); + EXPECT_TRUE(build_cyclic_order(0, 7).empty()); } TEST(GpuWorkPlanTest, OperationSeedsHaveStableSplitMixDomains) { EXPECT_EQ(gpu_operation_seed_domain(GpuOperation::Read), 0x4750555f52454144ULL); - EXPECT_EQ(gpu_operation_seed_domain(GpuOperation::Write), - 0x4750555752495445ULL); EXPECT_EQ(gpu_operation_seed_domain(GpuOperation::Copy), 0x4750555f434f5059ULL); EXPECT_EQ(derive_gpu_operation_seed(42, GpuOperation::Read), @@ -139,8 +110,7 @@ TEST(GpuWorkPlanTest, PassLimitsUseExactPayloadsAndEffectiveCaps) { const GpuPassLimits tiny = calculate_gpu_pass_limits(1, GpuOperation::Read); ASSERT_TRUE(tiny.valid); - EXPECT_EQ(tiny.effective_maximum_passes, - Constants::GPU_MAX_DISPATCHES_PER_MEASUREMENT); + EXPECT_EQ(tiny.effective_maximum_passes, 16384u); EXPECT_TRUE(tiny.dispatch_cap_is_limiting); EXPECT_FALSE(tiny.payload_cap_is_limiting); } @@ -284,45 +254,36 @@ TEST(GpuWorkPlanTest, PlannerCalculatesExactPayloadAndRejectsExplicitCaps) { EXPECT_EQ(copy.bytes_per_pass, 8192u); EXPECT_EQ(copy.exact_payload_bytes, 40960u); - GpuWorkPlanRequest dispatch_exceeded = - make_request(GpuOperation::Read, 1, - Constants::GPU_MAX_DISPATCHES_PER_MEASUREMENT + 1); - dispatch_exceeded.explicit_iterations = true; - EXPECT_EQ(build_gpu_work_plan(dispatch_exceeded).reason_code, - GpuWorkPlanReason::EXPLICIT_DISPATCH_CAP_EXCEEDED); - - GpuWorkPlanRequest payload_exceeded = make_request( - GpuOperation::Copy, 64 * Constants::BYTES_PER_MB, 513); - payload_exceeded.explicit_iterations = true; - EXPECT_EQ(build_gpu_work_plan(payload_exceeded).reason_code, - GpuWorkPlanReason::EXPLICIT_PAYLOAD_CAP_EXCEEDED); - - payload_exceeded.passes = 512; - const GpuWorkPlan at_cap = build_gpu_work_plan(payload_exceeded); + struct InvalidCase { + GpuOperation operation; + size_t bytes; + size_t passes; + const char* automatic_reason; + const char* explicit_reason; + }; + for (const InvalidCase& entry : + {InvalidCase{GpuOperation::Read, 4096, 0, GpuWorkPlanReason::PASS_COUNT_ZERO, + GpuWorkPlanReason::PASS_COUNT_ZERO}, + InvalidCase{GpuOperation::Read, 1, Constants::GPU_MAX_DISPATCHES_PER_MEASUREMENT + 1, + GpuWorkPlanReason::DISPATCH_CAP_EXCEEDED, GpuWorkPlanReason::EXPLICIT_DISPATCH_CAP_EXCEEDED}, + InvalidCase{GpuOperation::Copy, 64 * Constants::BYTES_PER_MB, 513, GpuWorkPlanReason::PAYLOAD_CAP_EXCEEDED, + GpuWorkPlanReason::EXPLICIT_PAYLOAD_CAP_EXCEEDED}}) { + for (bool explicit_iterations : {false, true}) { + SCOPED_TRACE(::testing::Message() << entry.bytes << "/" << entry.passes << "/" << explicit_iterations); + GpuWorkPlanRequest request = make_request(entry.operation, entry.bytes, entry.passes); + request.explicit_iterations = explicit_iterations; + const GpuWorkPlan invalid = build_gpu_work_plan(request); + EXPECT_FALSE(invalid.valid); + EXPECT_EQ(invalid.reason_code, explicit_iterations ? entry.explicit_reason : entry.automatic_reason); + EXPECT_TRUE(invalid.plan_identity.empty()); + EXPECT_EQ(invalid.exact_payload_bytes, 0u); + } + } + GpuWorkPlanRequest at_cap_request = make_request(GpuOperation::Copy, 64 * Constants::BYTES_PER_MB, 512); + at_cap_request.explicit_iterations = true; + const GpuWorkPlan at_cap = build_gpu_work_plan(at_cap_request); ASSERT_TRUE(at_cap.valid) << at_cap.reason_code; - EXPECT_EQ(at_cap.exact_payload_bytes, - Constants::GPU_MAX_EXACT_PAYLOAD_BYTES); -} - -TEST(GpuWorkPlanTest, InvalidPlansRetainStableReasonsWithoutIdentity) { - GpuWorkPlanRequest zero_passes = - make_request(GpuOperation::Read, 4096, 0); - const GpuWorkPlan zero = build_gpu_work_plan(zero_passes); - EXPECT_FALSE(zero.valid); - EXPECT_EQ(zero.reason_code, GpuWorkPlanReason::PASS_COUNT_ZERO); - EXPECT_TRUE(zero.plan_identity.empty()); - EXPECT_EQ(zero.exact_payload_bytes, 0u); - - GpuWorkPlanRequest automatic_dispatch_exceeded = make_request( - GpuOperation::Read, 1, - Constants::GPU_MAX_DISPATCHES_PER_MEASUREMENT + 1); - EXPECT_EQ(build_gpu_work_plan(automatic_dispatch_exceeded).reason_code, - GpuWorkPlanReason::DISPATCH_CAP_EXCEEDED); - - GpuWorkPlanRequest automatic_payload_exceeded = make_request( - GpuOperation::Copy, 64 * Constants::BYTES_PER_MB, 513); - EXPECT_EQ(build_gpu_work_plan(automatic_payload_exceeded).reason_code, - GpuWorkPlanReason::PAYLOAD_CAP_EXCEEDED); + EXPECT_EQ(at_cap.exact_payload_bytes, Constants::GPU_MAX_EXACT_PAYLOAD_BYTES); } TEST(GpuWorkPlanTest, CalibrationHelpersScaleClampAndClassifyGuardrails) { @@ -425,7 +386,6 @@ TEST(GpuTimedAccumulatorOracleTest, GpuDualChecksum{2454305124U, 2861025294U}, GpuDualChecksum{1933397850U, 2368268164U}, GpuDualChecksum{1074271602U, 2541544478U}}; - const GpuDualChecksum zero; for (size_t operation_index = 0; operation_index < kOperations.size(); ++operation_index) { @@ -453,16 +413,10 @@ TEST(GpuTimedAccumulatorOracleTest, SCOPED_TRACE(::testing::Message() << "operation=" << gpu_operation_to_string(operation)); - EXPECT_NE(pass_24, zero); - EXPECT_NE(pass_64, zero); EXPECT_EQ(pass_24, kExpectedPass24[operation_index]); EXPECT_EQ(pass_64, kExpectedPass64[operation_index]); EXPECT_NE(pass_23, pass_24); EXPECT_NE(pass_24, pass_64); EXPECT_NE(pass_63, pass_64); - EXPECT_NE(pass_24.first, 0U); - EXPECT_NE(pass_24.second, 0U); - EXPECT_NE(pass_64.first, 0U); - EXPECT_NE(pass_64.second, 0U); } } diff --git a/tests/test_hash_utils.cpp b/tests/test_hash_utils.cpp index 3e609c1..fcbd10a 100644 --- a/tests/test_hash_utils.cpp +++ b/tests/test_hash_utils.cpp @@ -32,72 +32,22 @@ #include #include -namespace { - -constexpr std::array kPermutationLittleEndianBytes = { - 2, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, - 4, 0, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, -}; - -constexpr std::string_view kPermutationSha256 = - "9d1cfab79005723a285fec9a5716b53baa7a6c0501e3d17434bfb31ea88935d1"; - -std::string_view permutation_bytes() { - return {reinterpret_cast(kPermutationLittleEndianBytes.data()), - kPermutationLittleEndianBytes.size()}; -} - -} // namespace - -TEST(HashUtilsTest, EmptyInputMatchesStandardSha256Vector) { - EXPECT_EQ(HashUtils::sha256_hex(""), - "e3b0c44298fc1c149afbf4c8996fb924" - "27ae41e4649b934ca495991b7852b855"); -} - -TEST(HashUtilsTest, AbcMatchesStandardSha256Vector) { - EXPECT_EQ(HashUtils::sha256_hex("abc"), - "ba7816bf8f01cfea414140de5dae2223" - "b00361a396177a9cb410ff61f20015ad"); -} - -TEST(HashUtilsTest, NoallocEmptyInputMatchesStandardSha256Vector) { - std::array digest{}; - ASSERT_TRUE(HashUtils::sha256_hex_noalloc("", digest)); - EXPECT_EQ(std::string_view(digest.data(), digest.size()), - "e3b0c44298fc1c149afbf4c8996fb924" - "27ae41e4649b934ca495991b7852b855"); -} - -TEST(HashUtilsTest, NoallocKnownTextMatchesStandardSha256Vector) { - std::array digest{}; - ASSERT_TRUE(HashUtils::sha256_hex_noalloc("abc", digest)); - EXPECT_EQ(std::string_view(digest.data(), digest.size()), - "ba7816bf8f01cfea414140de5dae2223" - "b00361a396177a9cb410ff61f20015ad"); -} - -TEST(HashUtilsTest, NoallocEmbeddedNullsMatchAllocatingHelper) { - const std::string payload{"prefix\0middle\xffsuffix", 20}; - std::array digest{}; - ASSERT_TRUE(HashUtils::sha256_hex_noalloc(payload, digest)); - EXPECT_EQ(std::string_view(digest.data(), digest.size()), - HashUtils::sha256_hex(payload)); -} - -TEST(HashUtilsTest, IncrementalUpdatesMatchPermutationGoldenAcrossChunkBoundaries) { - const std::string_view bytes = permutation_bytes(); - EXPECT_EQ(HashUtils::sha256_hex(bytes), kPermutationSha256); - - constexpr std::array chunk_sizes = {1, 2, 3, 4, 5, 7, 8, 15, 31, 32}; - for (size_t chunk_size : chunk_sizes) { - SCOPED_TRACE(chunk_size); - HashUtils::Sha256Hasher hasher; - for (size_t offset = 0; offset < bytes.size(); offset += chunk_size) { - const size_t length = std::min(chunk_size, bytes.size() - offset); - hasher.update(bytes.substr(offset, length)); - } - EXPECT_EQ(hasher.finalize_hex(), kPermutationSha256); +TEST(HashUtilsTest, AllocatingAndNoallocHelpersMatchKnownSha256Vectors) { + struct Vector { + std::string input; + const char* digest; + }; + const Vector vectors[] = { + {"", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {"abc", "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"}, + {std::string{"prefix\0middle\xffsuffix", 20}, "fa5e64c9d422596327e5747f3a0b502b4bcf9f8ae8ab984652d48e88f501190c"}, + }; + for (const Vector& entry : vectors) { + SCOPED_TRACE(::testing::PrintToString(entry.input)); + EXPECT_EQ(HashUtils::sha256_hex(entry.input), entry.digest); + std::array digest{}; + ASSERT_TRUE(HashUtils::sha256_hex_noalloc(entry.input, digest)); + EXPECT_EQ(std::string_view(digest.data(), digest.size()), entry.digest); } } diff --git a/tests/test_json_output_session.cpp b/tests/test_json_output_session.cpp index 3a6570a..265950c 100644 --- a/tests/test_json_output_session.cpp +++ b/tests/test_json_output_session.cpp @@ -91,16 +91,7 @@ class FlushFailingStreamBuffer : public std::stringbuf { int sync() override { return -1; } }; -class ThrowingStreamBuffer : public std::streambuf { - protected: - std::streamsize xsputn(const char*, std::streamsize) override { - throw std::runtime_error("injected stream write exception"); - } - int_type overflow(int_type) override { - throw std::runtime_error("injected stream write exception"); - } -}; std::string read_text_file(const std::filesystem::path& path) { std::ifstream input(path); @@ -116,41 +107,31 @@ static_assert(!std::is_move_assignable_v); } // namespace -TEST(JsonOutputTargetTest, EmptyAndExactDashAreClassifiedBeforePaths) { - const JsonOutputTarget disabled = make_json_output_target( - "", JsonFilePathPolicy::ResolveAgainstCurrentDirectory); - EXPECT_EQ(disabled.kind, JsonOutputKind::Disabled); - EXPECT_TRUE(disabled.raw_value.empty()); - EXPECT_TRUE(disabled.file_path.empty()); - - for (JsonFilePathPolicy policy : { - JsonFilePathPolicy::ResolveAgainstCurrentDirectory, - JsonFilePathPolicy::PreserveRaw, - }) { - const JsonOutputTarget stdout_target = make_json_output_target("-", policy); - EXPECT_EQ(stdout_target.kind, JsonOutputKind::Stdout); - EXPECT_EQ(stdout_target.raw_value, "-"); - EXPECT_TRUE(stdout_target.file_path.empty()); +TEST(JsonOutputTargetTest, ClassifiesDisabledStdoutAndLiteralFileTargetsBeforePathResolution) { + const auto directory = std::filesystem::current_path(); + struct TargetCase { + const char* raw; + JsonOutputKind kind; + }; + for (const TargetCase& entry : + {TargetCase{"", JsonOutputKind::Disabled}, TargetCase{"-", JsonOutputKind::Stdout}, + TargetCase{"./-", JsonOutputKind::File}, TargetCase{"nested/result-with-dashes.json", JsonOutputKind::File}}) { + for (JsonFilePathPolicy policy : + {JsonFilePathPolicy::ResolveAgainstCurrentDirectory, JsonFilePathPolicy::PreserveRaw}) { + SCOPED_TRACE(entry.raw); + SCOPED_TRACE(static_cast(policy)); + const JsonOutputTarget target = make_json_output_target(entry.raw, policy); + EXPECT_EQ(target.kind, entry.kind); + EXPECT_EQ(target.raw_value, entry.raw); + const std::filesystem::path expected = entry.kind != JsonOutputKind::File ? std::filesystem::path{} + : policy == JsonFilePathPolicy::PreserveRaw + ? std::filesystem::path(entry.raw) + : directory / entry.raw; + EXPECT_EQ(target.file_path, expected); + } } } -TEST(JsonOutputTargetTest, LiteralDashPathsRemainFileTargets) { - const std::filesystem::path current_directory = - std::filesystem::current_path(); - const JsonOutputTarget dot_dash = make_json_output_target( - "./-", JsonFilePathPolicy::ResolveAgainstCurrentDirectory); - EXPECT_EQ(dot_dash.kind, JsonOutputKind::File); - EXPECT_EQ(dot_dash.raw_value, "./-"); - EXPECT_EQ(dot_dash.file_path, current_directory / std::filesystem::path("./-")); - - const JsonOutputTarget dashed_name = make_json_output_target( - "nested/result-with-dashes.json", - JsonFilePathPolicy::ResolveAgainstCurrentDirectory); - EXPECT_EQ(dashed_name.kind, JsonOutputKind::File); - EXPECT_EQ(dashed_name.file_path, - current_directory / "nested/result-with-dashes.json"); -} - TEST(JsonOutputTargetTest, FilePathPoliciesPreserveTheirExistingSemantics) { const std::string relative = "nested/../result.json"; const JsonOutputTarget resolved = make_json_output_target( @@ -176,33 +157,40 @@ TEST(JsonOutputTargetTest, FilePathPoliciesPreserveTheirExistingSemantics) { } } -TEST(JsonOutputSessionTest, DisabledOutputIsLazyAndDoesNotRouteStreams) { - std::ostringstream stdout_capture; - std::ostringstream stderr_capture; - size_t builder_calls = 0; - int checkpoint_result = EXIT_FAILURE; - int final_result = EXIT_FAILURE; - bool buffer_unchanged = false; +TEST(JsonOutputSessionTest, DisabledAndStdoutCheckpointsAreLazyWithoutFilePersistence) { + for (const std::string& target : {std::string{}, std::string{"-"}}) { + SCOPED_TRACE(target); + std::ostringstream stdout_capture; + std::ostringstream stderr_capture; + size_t builder_calls = 0; + int checkpoint_result = EXIT_FAILURE; + int final_result = EXIT_FAILURE; + bool buffer_unchanged = false; - { - ScopedStreamBuffers capture(stdout_capture.rdbuf(), stderr_capture.rdbuf()); - std::streambuf* installed_stdout = std::cout.rdbuf(); - JsonOutputSession session(make_json_output_target("")); - buffer_unchanged = std::cout.rdbuf() == installed_stdout; - checkpoint_result = session.checkpoint([&]() { - ++builder_calls; - return nlohmann::ordered_json{{"unused", true}}; - }); - final_result = session.write_final({{"unused", true}}); - std::cout << "ordinary stdout\n"; - } + { + ScopedStreamBuffers capture(stdout_capture.rdbuf(), stderr_capture.rdbuf()); + std::streambuf* installed_stdout = std::cout.rdbuf(); + JsonOutputSession session(make_json_output_target(target)); + buffer_unchanged = std::cout.rdbuf() == installed_stdout; + checkpoint_result = session.checkpoint([&]() { + ++builder_calls; + return nlohmann::ordered_json{{"unused", true}}; + }); + EXPECT_EQ(session.file_writer_attempts(), 0u); + EXPECT_EQ(session.successful_file_writes(), 0u); + if (target.empty()) { + final_result = session.write_final({{"unused", true}}); + std::cout << "ordinary stdout\n"; + } + } - EXPECT_TRUE(buffer_unchanged); - EXPECT_EQ(checkpoint_result, EXIT_SUCCESS); - EXPECT_EQ(final_result, EXIT_SUCCESS); - EXPECT_EQ(builder_calls, 0u); - EXPECT_EQ(stdout_capture.str(), "ordinary stdout\n"); - EXPECT_TRUE(stderr_capture.str().empty()); + EXPECT_EQ(buffer_unchanged, target.empty()); + EXPECT_EQ(checkpoint_result, EXIT_SUCCESS); + if (target.empty()) EXPECT_EQ(final_result, EXIT_SUCCESS); + EXPECT_EQ(builder_calls, 0u); + EXPECT_EQ(stdout_capture.str(), target.empty() ? "ordinary stdout\n" : ""); + EXPECT_TRUE(stderr_capture.str().empty()); + } } TEST(JsonOutputSessionTest, @@ -319,93 +307,28 @@ TEST(JsonOutputSessionTest, FileCheckpointsUseAtomicWriterAndRetainCadence) { EXPECT_TRUE(stderr_capture.str().empty()); } -TEST(JsonOutputSessionTest, SamePrebuiltPayloadIsEquivalentForFileAndStdout) { - TemporaryDirectory temporary("transport_equivalence"); - const std::filesystem::path target_path = temporary.path() / "result.json"; - const nlohmann::ordered_json payload = { - {"configuration", {{"schema_version", 7}}}, - {"status", "partial"}, - {"optional", nullptr}, +TEST(JsonOutputSessionTest, FailedStdoutWriteOrFlushReturnsExactFailureWithoutThrowing) { + WriteFailingStreamBuffer write_failure; + FlushFailingStreamBuffer flush_failure; + struct FailureCase { + const char* name; + std::streambuf* buffer; + std::string reason; }; - - { - JsonOutputSession file_session(make_json_output_target( - target_path.string(), JsonFilePathPolicy::PreserveRaw)); - ASSERT_EQ(file_session.write_final(payload, false), EXIT_SUCCESS); - } - - std::ostringstream stdout_capture; - std::ostringstream stderr_capture; - int stdout_result = EXIT_FAILURE; - { - ScopedStreamBuffers capture(stdout_capture.rdbuf(), stderr_capture.rdbuf()); - JsonOutputSession stdout_session(make_json_output_target("-")); - stdout_result = stdout_session.write_final(payload); - } - - ASSERT_EQ(stdout_result, EXIT_SUCCESS); - EXPECT_TRUE(stderr_capture.str().empty()); - EXPECT_EQ(read_text_file(target_path), payload.dump(2) + "\n"); - EXPECT_EQ(stdout_capture.str(), payload.dump(2) + "\n"); - EXPECT_EQ(nlohmann::ordered_json::parse(read_text_file(target_path)), - nlohmann::ordered_json::parse(stdout_capture.str())); -} - -TEST(JsonOutputSessionTest, FailedStdoutWriteReturnsFailureWithoutThrowing) { - WriteFailingStreamBuffer failing_stdout; - std::ostringstream stderr_capture; - int result = EXIT_SUCCESS; - - { - ScopedStreamBuffers capture(&failing_stdout, stderr_capture.rdbuf()); - JsonOutputSession session(make_json_output_target("-")); - EXPECT_NO_THROW(result = session.write_final({{"status", "complete"}})); - } - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_EQ(stderr_capture.str(), - Messages::error_prefix() + - Messages::error_json_stdout_write_failed( - Messages::json_stdout_reason_write_failed()) + - "\n"); -} - -TEST(JsonOutputSessionTest, FailedStdoutFlushReturnsFailureWithoutThrowing) { - FlushFailingStreamBuffer failing_stdout; - std::ostringstream stderr_capture; - int result = EXIT_SUCCESS; - - { - ScopedStreamBuffers capture(&failing_stdout, stderr_capture.rdbuf()); - JsonOutputSession session(make_json_output_target("-")); - EXPECT_NO_THROW(result = session.write_final({{"status", "complete"}})); - } - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_EQ(stderr_capture.str(), - Messages::error_prefix() + - Messages::error_json_stdout_write_failed( - Messages::json_stdout_reason_flush_failed()) + - "\n"); -} - -TEST(JsonOutputSessionTest, ThrowingStdoutBufferCannotEscapeBoundary) { - ThrowingStreamBuffer throwing_stdout; - std::ostringstream stderr_capture; - int result = EXIT_SUCCESS; - - { - ScopedStreamBuffers capture(&throwing_stdout, stderr_capture.rdbuf()); - JsonOutputSession session(make_json_output_target("-")); - EXPECT_NO_THROW(result = session.write_final({{"status", "complete"}})); + for (const FailureCase& entry : {FailureCase{"write", &write_failure, Messages::json_stdout_reason_write_failed()}, + FailureCase{"flush", &flush_failure, Messages::json_stdout_reason_flush_failed()}}) { + SCOPED_TRACE(entry.name); + std::ostringstream stderr_capture; + int result = EXIT_SUCCESS; + { + ScopedStreamBuffers capture(entry.buffer, stderr_capture.rdbuf()); + JsonOutputSession session(make_json_output_target("-")); + EXPECT_NO_THROW(result = session.write_final({{"status", "complete"}})); + } + EXPECT_EQ(result, EXIT_FAILURE); + EXPECT_EQ(stderr_capture.str(), + Messages::error_prefix() + Messages::error_json_stdout_write_failed(entry.reason) + "\n"); } - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_EQ(stderr_capture.str(), - Messages::error_prefix() + - Messages::error_json_stdout_write_failed( - Messages::json_stdout_reason_write_failed()) + - "\n"); } TEST(JsonOutputSessionTest, SerializationExceptionCannotEscapeBoundary) { @@ -493,14 +416,3 @@ TEST(JsonOutputSessionTest, FileObservationsCountWriterEntryAfterBuilderAndOnlyS EXPECT_EQ(failing.file_writer_attempts(), 1u); EXPECT_EQ(failing.successful_file_writes(), 0u); } - -TEST(JsonOutputSessionTest, DisabledAndStdoutNoOpsNeverReportFilePersistence) { - for (const std::string& target : {std::string{}, std::string{"-"}}) { - JsonOutputSession session(make_json_output_target(target)); - size_t builders = 0; - EXPECT_EQ(session.checkpoint([&]() { ++builders; return nlohmann::ordered_json::object(); }), EXIT_SUCCESS); - EXPECT_EQ(builders, 0u); - EXPECT_EQ(session.file_writer_attempts(), 0u); - EXPECT_EQ(session.successful_file_writes(), 0u); - } -} diff --git a/tests/test_json_schema.cpp b/tests/test_json_schema.cpp index 150ea89..8f041c4 100644 --- a/tests/test_json_schema.cpp +++ b/tests/test_json_schema.cpp @@ -136,27 +136,6 @@ TlbMeasurementRecord make_paired_tlb_record(TlbMeasurementPass pass, } // namespace -TEST(JsonSchemaTest, BenchmarkExporterIncludesBenchmarkModeAndOmitsEmptySections) { - const TemporaryJsonFile output_file("benchmark_schema"); - BenchmarkConfig config; - config.output_file = output_file.path().string(); - config.only_bandwidth = true; - config.only_latency = true; - - BenchmarkStatistics stats; - ASSERT_EQ(save_results_to_json(config, stats, 1.0), EXIT_SUCCESS); - - const nlohmann::json output_json = read_json_file(config.output_file); - EXPECT_EQ(output_json[JsonKeys::CONFIGURATION][JsonKeys::MODE], Constants::BENCHMARK_JSON_MODE_NAME); - EXPECT_EQ(output_json[JsonKeys::CONFIGURATION] - ["benchmark_schema_version"], - Constants::BENCHMARK_JSON_SCHEMA_VERSION); - EXPECT_EQ(output_json[JsonKeys::CONFIGURATION]["output_file"], config.output_file); - EXPECT_TRUE(output_json[JsonKeys::CONFIGURATION].contains(JsonKeys::LATENCY_CHAIN_MODE)); - EXPECT_FALSE(output_json.contains(JsonKeys::MAIN_MEMORY)); - EXPECT_FALSE(output_json.contains(JsonKeys::CACHE)); -} - TEST(JsonSchemaTest, BenchmarkSchemaV3IncludesCompletionAndNullableMeasurements) { const TemporaryJsonFile output_file("benchmark_v3"); BenchmarkConfig config; @@ -207,37 +186,43 @@ TEST(JsonSchemaTest, BenchmarkSchemaV3IncludesCompletionAndNullableMeasurements) stats.completed_measurements = 1; stats.loop_results.push_back(loop); - ASSERT_EQ(save_results_to_json(config, stats, 1.0), EXIT_SUCCESS); - const nlohmann::json output = read_json_file(config.output_file); - EXPECT_EQ(output["configuration"]["benchmark_schema_version"], - Constants::BENCHMARK_JSON_SCHEMA_VERSION); - EXPECT_EQ(output["configuration"]["methodology_version"], - "benchmark-v2-calibrated-seeded-balanced"); - EXPECT_DOUBLE_EQ( - output["configuration"]["latency_calibration_window_max_seconds"], - 0.300); - EXPECT_EQ(output["configuration"]["benchmark_seed"], - "18446744073709551615"); - EXPECT_EQ(output["configuration"]["output_file"], config.output_file); - EXPECT_EQ(output["status"], "partial"); - EXPECT_FALSE(output["results_complete"].get()); - EXPECT_FALSE(output["conclusions_valid"].get()); - EXPECT_EQ(output["planned_loops"], 2u); - ASSERT_EQ(output["loops"].size(), 1u); - const nlohmann::json measurements = output["loops"][0]["measurements"]; - EXPECT_EQ(measurements["main_read_bandwidth"]["status"], "measured"); - EXPECT_DOUBLE_EQ(measurements["main_read_bandwidth"]["value"].get(), - 12.5); - EXPECT_EQ(measurements["main_read_bandwidth"]["passes"], 16u); - EXPECT_EQ(measurements["main_read_bandwidth"]["qos_outcome"], - "applied-to-all-workers"); - EXPECT_EQ(measurements["main_read_bandwidth"]["qos_successful_workers"], - 4u); - EXPECT_EQ(measurements["main_read_bandwidth"]["created_workers"], 4); - EXPECT_EQ(measurements["main_write_bandwidth"]["status"], "interrupted"); - EXPECT_TRUE(measurements["main_write_bandwidth"]["value"].is_null()); - EXPECT_EQ(output["main_memory"]["bandwidth"]["read_gb_s"]["value"], 12.5); - EXPECT_TRUE(output["main_memory"]["bandwidth"]["write_gb_s"]["value"].is_null()); + for (bool populated : {true, false}) { + SCOPED_TRACE(populated); + if (!populated) { + config.only_latency = true; + stats = BenchmarkStatistics{}; + } + ASSERT_EQ(save_results_to_json(config, stats, 1.0), EXIT_SUCCESS); + const nlohmann::json output = read_json_file(config.output_file); + EXPECT_EQ(output["configuration"]["benchmark_schema_version"], Constants::BENCHMARK_JSON_SCHEMA_VERSION); + EXPECT_EQ(output["configuration"]["methodology_version"], "benchmark-v2-calibrated-seeded-balanced"); + EXPECT_DOUBLE_EQ(output["configuration"]["latency_calibration_window_max_seconds"], 0.300); + EXPECT_EQ(output["configuration"]["benchmark_seed"], "18446744073709551615"); + EXPECT_EQ(output["configuration"]["output_file"], config.output_file); + EXPECT_EQ(output[JsonKeys::CONFIGURATION][JsonKeys::MODE], Constants::BENCHMARK_JSON_MODE_NAME); + EXPECT_TRUE(output[JsonKeys::CONFIGURATION].contains(JsonKeys::LATENCY_CHAIN_MODE)); + if (!populated) { + EXPECT_FALSE(output.contains(JsonKeys::MAIN_MEMORY)); + EXPECT_FALSE(output.contains(JsonKeys::CACHE)); + continue; + } + EXPECT_EQ(output["status"], "partial"); + EXPECT_FALSE(output["results_complete"].get()); + EXPECT_FALSE(output["conclusions_valid"].get()); + EXPECT_EQ(output["planned_loops"], 2u); + ASSERT_EQ(output["loops"].size(), 1u); + const nlohmann::json measurements = output["loops"][0]["measurements"]; + EXPECT_EQ(measurements["main_read_bandwidth"]["status"], "measured"); + EXPECT_DOUBLE_EQ(measurements["main_read_bandwidth"]["value"].get(), 12.5); + EXPECT_EQ(measurements["main_read_bandwidth"]["passes"], 16u); + EXPECT_EQ(measurements["main_read_bandwidth"]["qos_outcome"], "applied-to-all-workers"); + EXPECT_EQ(measurements["main_read_bandwidth"]["qos_successful_workers"], 4u); + EXPECT_EQ(measurements["main_read_bandwidth"]["created_workers"], 4); + EXPECT_EQ(measurements["main_write_bandwidth"]["status"], "interrupted"); + EXPECT_TRUE(measurements["main_write_bandwidth"]["value"].is_null()); + EXPECT_EQ(output["main_memory"]["bandwidth"]["read_gb_s"]["value"], 12.5); + EXPECT_TRUE(output["main_memory"]["bandwidth"]["write_gb_s"]["value"].is_null()); + } } TEST(JsonSchemaTest, BenchmarkSchemaV3SerializesExactCompletionContract) { @@ -314,41 +299,6 @@ TEST(JsonSchemaTest, BenchmarkAggregateHeadlineUsesMedianAndReportsCvAndMad) { 10.0); } -TEST(JsonSchemaTest, BenchmarkCheckpointAtomicallyProgressesToComplete) { - const TemporaryJsonFile output_file("benchmark_checkpoint"); - BenchmarkConfig config; - config.output_file = output_file.path().string(); - config.only_bandwidth = true; - config.only_latency = true; - BenchmarkStatistics stats; - stats.status = BenchmarkRunStatus::Partial; - stats.status_reason = "benchmark loops remain"; - stats.planned_loops = 2; - stats.completed_loops = 1; - - ASSERT_EQ(save_results_to_json(config, stats, 0.5, false), EXIT_SUCCESS); - nlohmann::json output = read_json_file(config.output_file); - EXPECT_EQ(output["configuration"]["benchmark_schema_version"], - Constants::BENCHMARK_JSON_SCHEMA_VERSION); - EXPECT_EQ(output["configuration"]["output_file"], config.output_file); - EXPECT_EQ(output["status"], "partial"); - EXPECT_FALSE(output["results_complete"].get()); - EXPECT_FALSE(output["conclusions_valid"].get()); - - stats.status = BenchmarkRunStatus::Complete; - stats.status_reason.clear(); - stats.completed_loops = 2; - ASSERT_EQ(save_results_to_json(config, stats, 1.0, false), EXIT_SUCCESS); - output = read_json_file(config.output_file); - EXPECT_EQ(output["configuration"]["benchmark_schema_version"], - Constants::BENCHMARK_JSON_SCHEMA_VERSION); - EXPECT_EQ(output["configuration"]["output_file"], config.output_file); - EXPECT_EQ(output["status"], "complete"); - EXPECT_TRUE(output["results_complete"].get()); - EXPECT_TRUE(output["conclusions_valid"].get()); - EXPECT_FALSE(std::filesystem::exists(config.output_file + ".tmp")); -} - TEST(JsonSchemaTest, PatternExporterIncludesPatternsMode) { const TemporaryJsonFile output_file("patterns_schema"); BenchmarkConfig config; @@ -1159,110 +1109,74 @@ TEST(JsonSchemaTest, CoreToCoreV2SerializesCalibratedBalancedAuditTrail) { } TEST(JsonSchemaTest, CoreToCoreBuilderSerializesOneWayValuesAndThreadHints) { - CoreToCoreLatencyConfig config; - config.loop_count = 2; - config.latency_sample_count = 2; - - const std::string cpu_name = "test-cpu"; - ThreadHintStatus initiator_hint; - initiator_hint.qos_applied = true; - initiator_hint.qos_code = 0; - initiator_hint.affinity_requested = true; - initiator_hint.affinity_applied = false; - initiator_hint.affinity_code = 5; - initiator_hint.affinity_tag = 1; - - ThreadHintStatus responder_hint; - responder_hint.qos_applied = false; - responder_hint.qos_code = 6; - responder_hint.affinity_requested = true; - responder_hint.affinity_applied = true; - responder_hint.affinity_code = 0; - responder_hint.affinity_tag = 2; - - const std::vector scenarios = { - { - Constants::CORE_TO_CORE_SCENARIO_SAME_AFFINITY, - {12.0, 18.0}, - {15.0, 17.0}, - initiator_hint, - responder_hint, - }, - }; - - const CoreToCoreLatencyJsonContext context = { - config, - cpu_name, - 4, - 6, - 100, - 200, - 20, - scenarios, - 4.5, - }; - - const nlohmann::json output_json = build_core_to_core_latency_json(context); - - EXPECT_EQ(output_json[JsonKeys::CONFIGURATION][JsonKeys::MODE], Constants::CORE_TO_CORE_JSON_MODE_NAME); - ASSERT_TRUE(output_json["core_to_core_latency"].is_object()); - ASSERT_EQ(output_json["core_to_core_latency"]["scenarios"].size(), 1u); - const nlohmann::json scenario_json = output_json["core_to_core_latency"]["scenarios"][0]; - EXPECT_EQ(scenario_json["name"], Constants::CORE_TO_CORE_SCENARIO_SAME_AFFINITY); - const nlohmann::json samples_json = scenario_json[JsonKeys::SAMPLES_NS]; - EXPECT_EQ(samples_json[JsonKeys::VALUES], nlohmann::json::array({15.0, 17.0})); - ASSERT_TRUE(samples_json[JsonKeys::STATISTICS].is_object()); - EXPECT_DOUBLE_EQ(samples_json[JsonKeys::STATISTICS]["average"].get(), 16.0); - EXPECT_DOUBLE_EQ(samples_json[JsonKeys::STATISTICS]["median"].get(), 16.0); - - ASSERT_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES].size(), 2u); - EXPECT_DOUBLE_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES][0].get(), 6.0); - EXPECT_DOUBLE_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES][1].get(), 9.0); - - ASSERT_TRUE(scenario_json["thread_hints"].contains("initiator")); - ASSERT_TRUE(scenario_json["thread_hints"].contains("responder")); - EXPECT_EQ(scenario_json["thread_hints"]["initiator"]["affinity_tag"], 1); - EXPECT_EQ(scenario_json["thread_hints"]["responder"]["qos_code"], 6); - EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("qos_applied")); - EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("qos_code")); - EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_requested")); - EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_applied")); - EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_code")); - EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_tag")); -} + for (size_t population : {size_t{1}, size_t{2}}) { + SCOPED_TRACE(population); + CoreToCoreLatencyConfig config; + config.loop_count = static_cast(population); + config.latency_sample_count = static_cast(population); + + const std::string cpu_name = "test-cpu"; + ThreadHintStatus initiator_hint; + initiator_hint.qos_applied = true; + initiator_hint.qos_code = 0; + initiator_hint.affinity_requested = true; + initiator_hint.affinity_applied = false; + initiator_hint.affinity_code = 5; + initiator_hint.affinity_tag = 1; + + ThreadHintStatus responder_hint; + responder_hint.qos_applied = false; + responder_hint.qos_code = 6; + responder_hint.affinity_requested = true; + responder_hint.affinity_applied = true; + responder_hint.affinity_code = 0; + responder_hint.affinity_tag = 2; + + const std::vector scenarios = { + { + Constants::CORE_TO_CORE_SCENARIO_SAME_AFFINITY, + population == 2 ? std::vector{12.0, 18.0} : std::vector{10.0}, + population == 2 ? std::vector{15.0, 17.0} : std::vector{10.5}, + initiator_hint, + responder_hint, + }, + }; -TEST(JsonSchemaTest, CoreToCoreBuilderOmitsStatisticsWhenOnlySingleValueExists) { - CoreToCoreLatencyConfig config; - config.loop_count = 1; - config.latency_sample_count = 1; - - const std::string cpu_name = "test-cpu"; - const std::vector scenarios = { - { - Constants::CORE_TO_CORE_SCENARIO_NO_AFFINITY, - {10.0}, - {10.5}, - {}, - {}, - }, - }; - - const CoreToCoreLatencyJsonContext context = { - config, - cpu_name, - 4, - 6, - 100, - 200, - 20, - scenarios, - 4.5, - }; - - const nlohmann::json output_json = build_core_to_core_latency_json(context); - - const nlohmann::json scenario_json = output_json["core_to_core_latency"]["scenarios"][0]; - EXPECT_FALSE(scenario_json["round_trip_ns"].contains(JsonKeys::STATISTICS)); - EXPECT_FALSE(scenario_json["one_way_estimate_ns"].contains(JsonKeys::STATISTICS)); - EXPECT_FALSE(scenario_json[JsonKeys::SAMPLES_NS].contains(JsonKeys::STATISTICS)); + const CoreToCoreLatencyJsonContext context = { + config, cpu_name, 4, 6, 100, 200, 20, scenarios, 4.5, + }; + + const nlohmann::json output_json = build_core_to_core_latency_json(context); + + EXPECT_EQ(output_json[JsonKeys::CONFIGURATION][JsonKeys::MODE], Constants::CORE_TO_CORE_JSON_MODE_NAME); + ASSERT_TRUE(output_json["core_to_core_latency"].is_object()); + ASSERT_EQ(output_json["core_to_core_latency"]["scenarios"].size(), 1u); + const nlohmann::json scenario_json = output_json["core_to_core_latency"]["scenarios"][0]; + EXPECT_EQ(scenario_json["name"], Constants::CORE_TO_CORE_SCENARIO_SAME_AFFINITY); + if (population == 1) { + for (const char* key : {"round_trip_ns", "one_way_estimate_ns", JsonKeys::SAMPLES_NS}) { + EXPECT_FALSE(scenario_json[key].contains(JsonKeys::STATISTICS)); + } + } else { + const nlohmann::json samples_json = scenario_json[JsonKeys::SAMPLES_NS]; + EXPECT_EQ(samples_json[JsonKeys::VALUES], nlohmann::json::array({15.0, 17.0})); + ASSERT_TRUE(samples_json[JsonKeys::STATISTICS].is_object()); + EXPECT_DOUBLE_EQ(samples_json[JsonKeys::STATISTICS]["average"].get(), 16.0); + EXPECT_DOUBLE_EQ(samples_json[JsonKeys::STATISTICS]["median"].get(), 16.0); + + ASSERT_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES].size(), 2u); + EXPECT_DOUBLE_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES][0].get(), 6.0); + EXPECT_DOUBLE_EQ(scenario_json["one_way_estimate_ns"][JsonKeys::VALUES][1].get(), 9.0); + } + ASSERT_TRUE(scenario_json["thread_hints"].contains("initiator")); + ASSERT_TRUE(scenario_json["thread_hints"].contains("responder")); + EXPECT_EQ(scenario_json["thread_hints"]["initiator"]["affinity_tag"], 1); + EXPECT_EQ(scenario_json["thread_hints"]["responder"]["qos_code"], 6); + EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("qos_applied")); + EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("qos_code")); + EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_requested")); + EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_applied")); + EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_code")); + EXPECT_TRUE(scenario_json["thread_hints"]["initiator"].contains("affinity_tag")); + } } diff --git a/tests/test_json_utils.cpp b/tests/test_json_utils.cpp index a4c4c6b..ab3f5f1 100644 --- a/tests/test_json_utils.cpp +++ b/tests/test_json_utils.cpp @@ -71,7 +71,7 @@ nlohmann::json read_json(const std::filesystem::path& path) { } // namespace -TEST(JsonFileWriterTest, CreatesParentsAtomicallyReplacesAndSuppressesAnnouncement) { +TEST(JsonFileWriterTest, CreatesParentsAtomicallyReplacesAndAnnouncesOnlyWhenRequested) { TemporaryDirectory temporary("writer_replace"); const std::filesystem::path target = temporary.path() / "nested" / "result.json"; @@ -83,23 +83,15 @@ TEST(JsonFileWriterTest, CreatesParentsAtomicallyReplacesAndSuppressesAnnounceme EXPECT_EQ(read_json(target)["generation"], 1); EXPECT_FALSE(std::filesystem::exists(target.string() + ".tmp")); - ASSERT_EQ(write_json_to_file(target, {{"generation", 2}}, false), EXIT_SUCCESS); + testing::internal::CaptureStdout(); + const int second_result = write_json_to_file(target, {{"generation", 2}}, true); + const std::string second_output = testing::internal::GetCapturedStdout(); + ASSERT_EQ(second_result, EXIT_SUCCESS); + EXPECT_EQ(second_output, Messages::msg_results_saved_to(target.string()) + "\n"); EXPECT_EQ(read_json(target)["generation"], 2); EXPECT_FALSE(std::filesystem::exists(target.string() + ".tmp")); } -TEST(JsonFileWriterTest, AnnouncesSuccessfulFinalPathOnlyWhenRequested) { - TemporaryDirectory temporary("writer_announce"); - const std::filesystem::path target = temporary.path() / "result.json"; - - testing::internal::CaptureStdout(); - const int result = write_json_to_file(target, {{"ok", true}}, true); - const std::string output = testing::internal::GetCapturedStdout(); - - ASSERT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(output, Messages::msg_results_saved_to(target.string()) + "\n"); -} - TEST(JsonFileWriterTest, FailedTemporaryOpenCleansStaleTemporaryPath) { TemporaryDirectory temporary("writer_open_failure"); const std::filesystem::path target = temporary.path() / "result.json"; @@ -149,32 +141,12 @@ TEST(JsonFileWriterTest, ParentPathFilesystemErrorCannotEscapeBoundary) { EXPECT_NE(error.find("Failed to write file"), std::string::npos); } -TEST(JsonUtilsTest, ParseStringRejectsEmptyAndMalformedAndAcceptsValidJson) { - nlohmann::json parsed; - std::string error; - - EXPECT_FALSE(parse_json_from_string("", parsed, error)); - EXPECT_EQ(error, "Empty JSON string"); - - error.clear(); - EXPECT_FALSE(parse_json_from_string("{broken", parsed, error)); - EXPECT_NE(error.find("JSON parse error at position"), std::string::npos); - - EXPECT_TRUE(parse_json_from_string("{\"value\":42}", parsed, error)); - EXPECT_EQ(parsed["value"], 42); - EXPECT_TRUE(error.empty()); -} - TEST(JsonUtilsTest, UtcTimestampFormatsFixedTimeIndependentOfLocalTimezone) { const auto fixed_time = std::chrono::system_clock::from_time_t(1577934245); EXPECT_EQ(build_utc_timestamp(fixed_time), "2020-01-02T03:04:05Z"); } -TEST(JsonUtilsTest, StatisticsNeverFabricateZeroesForAnEmptyPopulation) { - EXPECT_TRUE(calculate_json_statistics({}).is_null()); -} - TEST(JsonUtilsTest, StatisticsUseSampleDeviationInterpolationCvAndMad) { const nlohmann::json statistics = calculate_json_statistics({10.0, 20.0, 30.0, 40.0}); @@ -199,36 +171,3 @@ TEST(JsonUtilsTest, SingleZeroHasNoVariationAndUndefinedCv) { EXPECT_TRUE(statistics["coefficient_of_variation_pct"].is_null()); EXPECT_DOUBLE_EQ(statistics["median_absolute_deviation"], 0.0); } - -TEST(JsonUtilsTest, ParseFileDistinguishesMissingDirectoryEmptyMalformedAndValid) { - TemporaryDirectory temporary("parse_file"); - nlohmann::json parsed; - std::string error; - - const std::filesystem::path missing = temporary.path() / "missing.json"; - EXPECT_FALSE(parse_json_from_file(missing.string(), parsed, error)); - EXPECT_EQ(error, "File does not exist: " + missing.string()); - - error.clear(); - EXPECT_FALSE(parse_json_from_file(temporary.path().string(), parsed, error)); - EXPECT_EQ(error, "Path is not a regular file: " + temporary.path().string()); - - const std::filesystem::path empty = temporary.path() / "empty.json"; - write_text(empty, ""); - error.clear(); - EXPECT_FALSE(parse_json_from_file(empty.string(), parsed, error)); - EXPECT_EQ(error, "File is empty: " + empty.string()); - - const std::filesystem::path malformed = temporary.path() / "malformed.json"; - write_text(malformed, "[invalid]"); - error.clear(); - EXPECT_FALSE(parse_json_from_file(malformed.string(), parsed, error)); - EXPECT_NE(error.find("JSON parse error at position"), std::string::npos); - - const std::filesystem::path valid = temporary.path() / "valid.json"; - write_text(valid, "{\"status\":\"complete\"}"); - error.clear(); - EXPECT_TRUE(parse_json_from_file(valid.string(), parsed, error)); - EXPECT_EQ(parsed["status"], "complete"); - EXPECT_TRUE(error.empty()); -} diff --git a/tests/test_llm_memory_config.cpp b/tests/test_llm_memory_config.cpp index 548f4d8..65c3e7f 100644 --- a/tests/test_llm_memory_config.cpp +++ b/tests/test_llm_memory_config.cpp @@ -254,48 +254,6 @@ void replace_option_value(std::vector& arguments, } // namespace -TEST(LlmMemoryConfigTest, DefaultsMatchFrozenStandaloneContract) { - const LlmMemoryConfig config; - EXPECT_EQ(config.backend, LlmMemoryBackend::Cpu); - EXPECT_EQ(config.phase, LlmPhase::Decode); - EXPECT_EQ(config.kv_layout, LlmKvLayout::Contiguous); - EXPECT_EQ(config.weight_size_mb, 0u); - EXPECT_EQ(config.layer_count, 0u); - EXPECT_EQ(config.query_head_count, 0u); - EXPECT_EQ(config.kv_head_count, 0u); - EXPECT_EQ(config.head_dimension, 0u); - EXPECT_EQ(config.kv_element_bytes, 2u); - EXPECT_EQ(config.visible_context_tokens, 0u); - EXPECT_EQ(config.prompt_tokens, 0u); - EXPECT_EQ(config.attention_query_tile_tokens, 0u); - EXPECT_EQ(config.kv_block_tokens, 0u); - EXPECT_EQ(config.batch_size, 1u); - EXPECT_EQ(config.requested_workers, 0u); - EXPECT_EQ(config.available_workers, 0u); - EXPECT_EQ(config.iterations, 0u); - EXPECT_EQ(config.loop_count, 3u); - EXPECT_EQ(config.seed, 0u); - EXPECT_FALSE(config.user_specified_backend); - EXPECT_FALSE(config.user_specified_iterations); - EXPECT_FALSE(config.user_specified_seed); - EXPECT_FALSE(config.user_specified_workers); - EXPECT_FALSE(config.user_specified_phase); - EXPECT_FALSE(config.user_specified_context_tokens); - EXPECT_FALSE(config.user_specified_prompt_tokens); - EXPECT_FALSE(config.user_specified_attention_query_tile_tokens); - EXPECT_FALSE(config.user_specified_kv_layout); - EXPECT_FALSE(config.user_specified_kv_block_tokens); - EXPECT_FALSE(config.help_printed); - EXPECT_TRUE(config.output_file.empty()); - EXPECT_TRUE(config.argv.empty()); - - const LlmMemoryConfigValidation validation = - validate_llm_memory_config(config); - EXPECT_FALSE(validation.valid); - EXPECT_EQ(validation.reason_code, - LlmMemoryConfigReason::WEIGHT_SIZE_REQUIRED); -} - TEST(LlmMemoryConfigTest, ParserResolvesRequiredInputsAndDeterministicDefaults) { LlmParserHooksScope hooks; @@ -359,26 +317,6 @@ TEST(LlmMemoryConfigTest, EXPECT_EQ(config.argv, arguments); } -TEST(LlmMemoryConfigTest, - ParserRejectsUnknownBackendWithCentralizedReason) { - LlmParserHooksScope hooks; - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.begin() + 2, - {"--llm-memory-backend", "gpu"}); - LlmMemoryConfig config; - - const CapturedLlmParse parsed = - parse_llm_arguments_capturing(arguments, config); - - EXPECT_EQ(parsed.result, EXIT_FAILURE); - EXPECT_TRUE(parsed.stdout_output.empty()); - EXPECT_EQ(first_output_line(parsed.stderr_output), - Messages::error_prefix() + - Messages::error_invalid_value( - "--llm-memory-backend", "gpu", - Messages::llm_memory_reason_backend())); -} - TEST(LlmMemoryConfigTest, ParserActivatesExplicitBackendPhaseLayoutCompatibilityMatrix) { LlmParserHooksScope hooks(8, 9); @@ -489,22 +427,7 @@ TEST(LlmMemoryConfigTest, {"--threads", "1"}); threads_first.insert(threads_first.end(), {"--llm-memory-backend", "metal"}); - std::vector prefill_backend_first = - valid_prefill_arguments(); - prefill_backend_first.insert(prefill_backend_first.begin() + 2, - {"--llm-memory-backend", "metal"}); - prefill_backend_first.insert(prefill_backend_first.end(), - {"--threads", "1"}); - std::vector prefill_threads_first = - valid_prefill_arguments(); - prefill_threads_first.insert(prefill_threads_first.begin() + 2, - {"--threads", "1"}); - prefill_threads_first.insert(prefill_threads_first.end(), - {"--llm-memory-backend", "metal"}); - - for (const std::vector& arguments : - {backend_first, threads_first, prefill_backend_first, - prefill_threads_first}) { + for (const std::vector& arguments : {backend_first, threads_first}) { SCOPED_TRACE(::testing::PrintToString(arguments)); LlmMemoryConfig config; const CapturedLlmParse parsed = @@ -523,24 +446,42 @@ TEST(LlmMemoryConfigTest, TEST(LlmMemoryConfigTest, ParserPreservesEveryExplicitFieldAndAlias) { LlmParserHooksScope hooks; const std::vector arguments = { - "memory_benchmark", "-M", - "--llm-memory-backend", "cpu", - "--weight-size-mb", "4096", - "--layers", "32", - "--query-heads", "32", - "--kv-heads", "8", - "--head-dim", "128", - "--kv-element-bytes", "4", - "--phase", "decode", - "--context-tokens", "8192", - "--kv-layout", "paged", - "--kv-block-tokens", "16", - "--batch-size", "2", - "-t", "3", - "-i", "4", - "-r", "5", - "--seed", "18446744073709551615", - "-o", "./-", + "memory_benchmark", + "-M", + "--llm-memory-backend", + "cpu", + "--weight-size-mb", + "4096", + "--layers", + "32", + "--query-heads", + "32", + "--kv-heads", + "8", + "--head-dim", + "128", + "--kv-element-bytes", + "4", + "--phase", + "decode", + "--context-tokens", + "8192", + "--kv-layout", + "paged", + "--kv-block-tokens", + "16", + "--batch-size", + "2", + "-t", + "12", + "-i", + "4", + "-r", + "5", + "--seed", + "18446744073709551615", + "-o", + "./-", }; LlmMemoryConfig config; @@ -556,7 +497,7 @@ TEST(LlmMemoryConfigTest, ParserPreservesEveryExplicitFieldAndAlias) { EXPECT_EQ(config.kv_layout, LlmKvLayout::Paged); EXPECT_EQ(config.kv_block_tokens, 16u); EXPECT_EQ(config.batch_size, 2u); - EXPECT_EQ(config.requested_workers, 3u); + EXPECT_EQ(config.requested_workers, 12u); EXPECT_EQ(config.available_workers, 8u); EXPECT_EQ(config.iterations, 4u); EXPECT_EQ(config.loop_count, 5u); @@ -575,25 +516,6 @@ TEST(LlmMemoryConfigTest, ParserPreservesEveryExplicitFieldAndAlias) { EXPECT_EQ(config.argv, arguments); } -TEST(LlmMemoryConfigTest, ParserActivatesContiguousCpuPrefillWithExactPhaseGeometry) { - LlmParserHooksScope hooks; - const std::vector arguments = valid_prefill_arguments(); - LlmMemoryConfig config; - - ASSERT_EQ(parse_llm_arguments(arguments, config), EXIT_SUCCESS); - EXPECT_EQ(config.backend, LlmMemoryBackend::Cpu); - EXPECT_EQ(config.phase, LlmPhase::Prefill); - EXPECT_EQ(config.kv_layout, LlmKvLayout::Contiguous); - EXPECT_EQ(config.visible_context_tokens, 0u); - EXPECT_EQ(config.prompt_tokens, 5u); - EXPECT_EQ(config.attention_query_tile_tokens, 2u); - EXPECT_TRUE(config.user_specified_phase); - EXPECT_FALSE(config.user_specified_context_tokens); - EXPECT_TRUE(config.user_specified_prompt_tokens); - EXPECT_TRUE(config.user_specified_attention_query_tile_tokens); - EXPECT_EQ(config.argv, arguments); -} - TEST(LlmMemoryConfigTest, ParserUsesStableOrderIndependentPrefillAndPagedLayoutRules) { LlmParserHooksScope hooks; struct InvalidCase { @@ -663,11 +585,6 @@ TEST(LlmMemoryConfigTest, ParserUsesStableOrderIndependentPrefillAndPagedLayoutR EXPECT_EQ(parsed.result, EXIT_SUCCESS); EXPECT_TRUE(parsed.stdout_output.empty()); EXPECT_TRUE(parsed.stderr_output.empty()); - EXPECT_EQ(paged_config.phase, LlmPhase::Prefill); - EXPECT_EQ(paged_config.kv_layout, LlmKvLayout::Paged); - EXPECT_EQ(paged_config.prompt_tokens, 5u); - EXPECT_EQ(paged_config.attention_query_tile_tokens, 2u); - EXPECT_EQ(paged_config.kv_block_tokens, 4u); EXPECT_TRUE(paged_config.user_specified_phase); EXPECT_TRUE(paged_config.user_specified_kv_layout); EXPECT_TRUE(paged_config.user_specified_kv_block_tokens); @@ -675,60 +592,44 @@ TEST(LlmMemoryConfigTest, ParserUsesStableOrderIndependentPrefillAndPagedLayoutR } } -TEST(LlmMemoryConfigTest, ParserRejectsCrossPhaseTokenOptionsAndUnknownPhase) { +TEST(LlmMemoryConfigTest, ParserRejectsInvalidPhaseBackendAndLayoutOptions) { LlmParserHooksScope hooks; struct InvalidCase { - std::vector arguments; - std::string reason_code; + bool prefill; + std::vector suffix; + std::string diagnostic; }; - std::vector decode_prompt = valid_llm_arguments(); - decode_prompt.insert(decode_prompt.end(), {"--prompt-tokens", "5"}); - std::vector decode_query = valid_llm_arguments(); - decode_query.insert(decode_query.end(), {"--attention-query-tile-tokens", "2"}); - std::vector prefill_context = valid_prefill_arguments(); - prefill_context.insert(prefill_context.end(), {"--context-tokens", "5"}); const std::vector cases = { - {decode_prompt, LlmMemoryConfigReason::PROMPT_TOKENS_NOT_APPLICABLE}, - {decode_query, LlmMemoryConfigReason::ATTENTION_QUERY_TILE_TOKENS_NOT_APPLICABLE}, - {prefill_context, LlmMemoryConfigReason::CONTEXT_TOKENS_NOT_APPLICABLE}, + {false, + {"--prompt-tokens", "5"}, + Messages::error_llm_memory_config_invalid(LlmMemoryConfigReason::PROMPT_TOKENS_NOT_APPLICABLE)}, + {false, + {"--attention-query-tile-tokens", "2"}, + Messages::error_llm_memory_config_invalid(LlmMemoryConfigReason::ATTENTION_QUERY_TILE_TOKENS_NOT_APPLICABLE)}, + {true, + {"--context-tokens", "5"}, + Messages::error_llm_memory_config_invalid(LlmMemoryConfigReason::CONTEXT_TOKENS_NOT_APPLICABLE)}, + {false, + {"--phase", "training"}, + Messages::error_invalid_value("--phase", "training", Messages::llm_memory_reason_phase())}, + {false, + {"--llm-memory-backend", "gpu"}, + Messages::error_invalid_value("--llm-memory-backend", "gpu", Messages::llm_memory_reason_backend())}, + {false, {"--kv-layout", "paged"}, Messages::error_llm_memory_missing_required_option("--kv-block-tokens")}, + {false, + {"--kv-layout", "sparse"}, + Messages::error_invalid_value("--kv-layout", "sparse", Messages::llm_memory_reason_kv_layout())}, }; - for (const InvalidCase& test_case : cases) { + for (const auto& test_case : cases) { + SCOPED_TRACE(::testing::PrintToString(test_case.suffix)); + auto arguments = test_case.prefill ? valid_prefill_arguments() : valid_llm_arguments(); + arguments.insert(arguments.end(), test_case.suffix.begin(), test_case.suffix.end()); LlmMemoryConfig config; - const CapturedLlmParse parsed = parse_llm_arguments_capturing(test_case.arguments, config); - EXPECT_EQ(first_output_line(parsed.stderr_output), - Messages::error_prefix() + Messages::error_llm_memory_config_invalid(test_case.reason_code)); + const auto parsed = parse_llm_arguments_capturing(arguments, config); + EXPECT_EQ(parsed.result, EXIT_FAILURE); + EXPECT_TRUE(parsed.stdout_output.empty()); + EXPECT_EQ(first_output_line(parsed.stderr_output), Messages::error_prefix() + test_case.diagnostic); } - - std::vector invalid_phase = valid_llm_arguments(); - invalid_phase.insert(invalid_phase.end(), {"--phase", "training"}); - LlmMemoryConfig config; - const CapturedLlmParse parsed = parse_llm_arguments_capturing(invalid_phase, config); - EXPECT_EQ(first_output_line(parsed.stderr_output), - Messages::error_prefix() + - Messages::error_invalid_value("--phase", "training", Messages::llm_memory_reason_phase())); -} - -TEST(LlmMemoryConfigTest, - ParserResolvesContiguousAndPagedLayoutSourcesExactly) { - LlmParserHooksScope hooks; - - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--kv-layout", "contiguous"}); - LlmMemoryConfig config; - ASSERT_EQ(parse_llm_arguments(arguments, config), EXIT_SUCCESS); - EXPECT_EQ(config.kv_layout, LlmKvLayout::Contiguous); - EXPECT_EQ(config.kv_block_tokens, 0u); - EXPECT_TRUE(config.user_specified_kv_layout); - EXPECT_FALSE(config.user_specified_kv_block_tokens); - - arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--kv-block-tokens", "16", - "--kv-layout", "paged"}); - ASSERT_EQ(parse_llm_arguments(arguments, config), EXIT_SUCCESS); - EXPECT_EQ(config.kv_layout, LlmKvLayout::Paged); - EXPECT_EQ(config.kv_block_tokens, 16u); - EXPECT_TRUE(config.user_specified_kv_layout); - EXPECT_TRUE(config.user_specified_kv_block_tokens); } TEST(LlmMemoryConfigTest, @@ -774,47 +675,6 @@ TEST(LlmMemoryConfigTest, } } -TEST(LlmMemoryConfigTest, - ParserRequiresPagedBlockTokensAndRejectsUnknownLayout) { - LlmParserHooksScope hooks; - LlmMemoryConfig config; - - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--kv-layout", "paged"}); - CapturedLlmParse parsed = - parse_llm_arguments_capturing(arguments, config); - EXPECT_EQ(parsed.result, EXIT_FAILURE); - EXPECT_TRUE(parsed.stdout_output.empty()); - EXPECT_EQ(first_output_line(parsed.stderr_output), - Messages::error_prefix() + - Messages::error_llm_memory_missing_required_option( - "--kv-block-tokens")); - - arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--kv-layout", "sparse"}); - parsed = parse_llm_arguments_capturing(arguments, config); - EXPECT_EQ(parsed.result, EXIT_FAILURE); - EXPECT_TRUE(parsed.stdout_output.empty()); - EXPECT_EQ(first_output_line(parsed.stderr_output), - Messages::error_prefix() + - Messages::error_invalid_value( - "--kv-layout", "sparse", - Messages::llm_memory_reason_kv_layout())); -} - -TEST(LlmMemoryConfigTest, - ParserRetainsExplicitWorkerRequestAboveDetectedAvailability) { - LlmParserHooksScope hooks(4, 9); - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--threads", "12", "--seed", "7"}); - LlmMemoryConfig config; - - ASSERT_EQ(parse_llm_arguments(arguments, config), EXIT_SUCCESS); - EXPECT_EQ(config.requested_workers, 12u); - EXPECT_EQ(config.available_workers, 4u); - EXPECT_TRUE(config.user_specified_workers); -} - TEST(LlmMemoryConfigTest, ParserRejectsZeroDetectedWorkersBeforeGeneratingTheSeed) { LlmParserHooksScope hooks(0, 9); @@ -950,112 +810,45 @@ TEST(LlmMemoryConfigTest, } } -TEST(LlmMemoryConfigTest, - ParserUsesStrictPositiveDecimalTokensForAllSizeFields) { +TEST(LlmMemoryConfigTest, ParserUsesStrictDecimalTokensForSizeFieldsAndUnsignedSeed) { LlmParserHooksScope hooks; - const std::vector invalid_tokens = { - "0", "-1", "+1", " 1", "1 ", "1x", "1.0", "0x10", - "18446744073709551616"}; - const std::vector options = { - "--weight-size-mb", "--layers", "--query-heads", "--kv-heads", - "--head-dim", "--kv-element-bytes", "--context-tokens", - "--kv-block-tokens", "--batch-size", "--threads", - "--iterations", "--count"}; - - for (const std::string& option : options) { - for (const std::string& token : invalid_tokens) { + // Full decimal-token grammar is owned by ConfigTest; retain each LLM dispatch and the seed's zero boundary. + const std::vector options = {"--weight-size-mb", + "--layers", + "--query-heads", + "--kv-heads", + "--head-dim", + "--kv-element-bytes", + "--context-tokens", + "--kv-block-tokens", + "--batch-size", + "--threads", + "--iterations", + "--count", + "--seed"}; + for (const auto& option : options) { + for (const std::string token : {"0", "1x", "18446744073709551616"}) { SCOPED_TRACE(option + "=" + token); - std::vector arguments = valid_llm_arguments(); - const auto position = - std::find(arguments.begin(), arguments.end(), option); - if (position == arguments.end()) { + auto arguments = valid_llm_arguments(); + const auto position = std::find(arguments.begin(), arguments.end(), option); + if (position == arguments.end()) arguments.insert(arguments.end(), {option, token}); - } else { + else *(position + 1) = token; - } LlmMemoryConfig config; - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), - EXIT_FAILURE); + const auto parsed = parse_llm_arguments_capturing(arguments, config); + const bool valid = option == "--seed" && token == "0"; + EXPECT_EQ(parsed.result, valid ? EXIT_SUCCESS : EXIT_FAILURE); + if (valid) { + EXPECT_TRUE(config.user_specified_seed); + EXPECT_EQ(config.seed, 0u); + } else { + EXPECT_FALSE(parsed.stderr_output.empty()); + } } } } -TEST(LlmMemoryConfigTest, ParserUsesStrictUnsignedSeedTokens) { - LlmParserHooksScope hooks; - for (const std::string& token : { - "-1", "+1", " 1", "1 ", "1x", "1.0", "0x10", - "18446744073709551616"}) { - SCOPED_TRACE(token); - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--seed", token}); - LlmMemoryConfig config; - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), - EXIT_FAILURE); - } - - for (const std::string& token : {"0", "18446744073709551615"}) { - SCOPED_TRACE(token); - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--seed", token}); - LlmMemoryConfig config; - ASSERT_EQ(parse_llm_arguments(arguments, config), EXIT_SUCCESS); - EXPECT_TRUE(config.user_specified_seed); - EXPECT_EQ(config.seed, token == "0" - ? 0u - : std::numeric_limits::max()); - } -} - -TEST(LlmMemoryConfigTest, - ParserAcceptsOnlyFrozenKvElementWidthsAndHeadSharing) { - LlmParserHooksScope hooks; - for (const std::string& width : {"1", "2", "4"}) { - SCOPED_TRACE(width); - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--kv-element-bytes", width}); - LlmMemoryConfig config; - ASSERT_EQ(parse_llm_arguments(arguments, config), EXIT_SUCCESS); - EXPECT_EQ(config.kv_element_bytes, - static_cast(std::stoul(width))); - } - - for (const std::string& width : {"3", "5", "8"}) { - SCOPED_TRACE(width); - std::vector arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--kv-element-bytes", width}); - LlmMemoryConfig config; - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), - EXIT_FAILURE); - } - - std::vector arguments = valid_llm_arguments(); - replace_option_value(arguments, "--query-heads", "1"); - LlmMemoryConfig config; - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), EXIT_FAILURE); - - arguments = valid_llm_arguments(); - replace_option_value(arguments, "--query-heads", "3"); - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), EXIT_FAILURE); -} - -TEST(LlmMemoryConfigTest, - ParserRejectsGeometryAndExplicitWorkBeyondFrozenGuardrails) { - LlmParserHooksScope hooks; - LlmMemoryConfig config; - - std::vector arguments = valid_llm_arguments(); - replace_option_value(arguments, "--weight-size-mb", "65537"); - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), EXIT_FAILURE); - - arguments = valid_llm_arguments(); - replace_option_value(arguments, "--layers", "18446744073709551615"); - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), EXIT_FAILURE); - - arguments = valid_llm_arguments(); - arguments.insert(arguments.end(), {"--iterations", "1000000001"}); - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), EXIT_FAILURE); -} - TEST(LlmMemoryConfigTest, ParserBracketsExactWeightAndScenarioWorkGuardrails) { LlmParserHooksScope hooks; @@ -1083,52 +876,16 @@ TEST(LlmMemoryConfigTest, 65505, 65504)); } -TEST(LlmMemoryConfigTest, ParserRejectsEveryIncompatibleOption) { +TEST(LlmMemoryConfigTest, ParserRejectsIncompatibleOptions) { LlmParserHooksScope hooks; - const std::vector> suffixes = { - {"--benchmark"}, - {"-B"}, - {"--patterns"}, - {"-P"}, - {"--analyze-tlb"}, - {"-T"}, - {"--analyze-core2core"}, - {"-C"}, - {"--gpu-bandwidth"}, - {"-G"}, - {"--only-bandwidth"}, - {"-W"}, - {"--only-latency"}, - {"-L"}, - {"--buffer-size", "1"}, - {"-b", "1"}, - {"--cache-size", "16"}, - {"-k", "16"}, - {"--latency-samples", "1"}, - {"-n", "1"}, - {"--latency-stride-bytes", "256"}, - {"-s", "256"}, - {"--latency-chain-mode", "auto"}, - {"-m", "auto"}, - {"--latency-tlb-locality-kb", "1024"}, - {"-l", "1024"}, - {"--tlb-density", "medium"}, - {"-D", "medium"}, - {"--non-cacheable"}, - {"-u"}, - {"--sweep", "threads=1"}, - {"-S", "threads=1"}, - {"--sweep-max-runs", "1"}, - {"-X", "1"}, - {"--unknown"}, - }; - for (const std::vector& suffix : suffixes) { + // All non-LLM options share the same unknown-option branch. + for (const std::vector& suffix : + {std::vector{"--benchmark"}, {"-P"}, {"--buffer-size", "1"}, {"--unknown"}}) { SCOPED_TRACE(::testing::PrintToString(suffix)); - std::vector arguments = valid_llm_arguments(); + auto arguments = valid_llm_arguments(); arguments.insert(arguments.end(), suffix.begin(), suffix.end()); LlmMemoryConfig config; - EXPECT_EQ(parse_llm_arguments_silently(arguments, config), - EXIT_FAILURE); + EXPECT_EQ(parse_llm_arguments_silently(arguments, config), EXIT_FAILURE); } } @@ -1146,18 +903,6 @@ TEST(LlmMemoryConfigTest, EXPECT_NE(parsed.stdout_output.find( "Usage: memory_benchmark --llm-memory [options]"), std::string::npos); - EXPECT_NE(parsed.stdout_output.find("memory traffic only"), - std::string::npos); - EXPECT_NE(parsed.stdout_output.find( - "--llm-memory-backend "), - std::string::npos); - EXPECT_NE(parsed.stdout_output.find("--phase "), - std::string::npos); - EXPECT_NE(parsed.stdout_output.find("--prompt-tokens "), - std::string::npos); - EXPECT_NE( - parsed.stdout_output.find("--attention-query-tile-tokens "), - std::string::npos); EXPECT_EQ(parsed.stdout_output.find(Messages::config_header(SOFTVERSION)), std::string::npos); EXPECT_TRUE(config.help_printed); @@ -1203,20 +948,12 @@ TEST(LlmMemoryConfigTest, ParserRequiresTheOwningPrimaryMode) { } TEST(LlmMemoryConfigTest, StableVocabularyAndStatusTokens) { - EXPECT_STREQ(llm_memory_backend_to_string(LlmMemoryBackend::Cpu), "cpu"); - EXPECT_STREQ(llm_memory_backend_to_string(LlmMemoryBackend::Metal), - "metal"); EXPECT_STREQ(llm_memory_backend_to_string( static_cast(99)), "unknown"); - EXPECT_STREQ(llm_phase_to_string(LlmPhase::Decode), "decode"); - EXPECT_STREQ(llm_phase_to_string(LlmPhase::Prefill), "prefill"); EXPECT_STREQ(llm_phase_to_string(static_cast(99)), "unknown"); - EXPECT_STREQ(llm_kv_layout_to_string(LlmKvLayout::Contiguous), - "contiguous"); - EXPECT_STREQ(llm_kv_layout_to_string(LlmKvLayout::Paged), "paged"); EXPECT_STREQ(llm_kv_layout_to_string(static_cast(99)), "unknown"); @@ -1224,11 +961,6 @@ TEST(LlmMemoryConfigTest, StableVocabularyAndStatusTokens) { LlmWorkUnitKind::DecodeStep); EXPECT_EQ(llm_work_unit_kind_for_phase(LlmPhase::Prefill), LlmWorkUnitKind::PrefillOperation); - EXPECT_STREQ(llm_work_unit_kind_to_string(LlmWorkUnitKind::DecodeStep), - "decode_step"); - EXPECT_STREQ( - llm_work_unit_kind_to_string(LlmWorkUnitKind::PrefillOperation), - "prefill_operation"); EXPECT_STREQ(llm_work_unit_kind_to_string( static_cast(99)), "unknown"); @@ -1240,59 +972,21 @@ TEST(LlmMemoryConfigTest, StableVocabularyAndStatusTokens) { LlmKvWriteKind::CurrentTokenAppend); EXPECT_EQ(llm_kv_write_kind_for(LlmPhase::Prefill, LlmScenario::Mixed), LlmKvWriteKind::FullPromptPopulation); - EXPECT_STREQ(llm_kv_write_kind_to_string(LlmKvWriteKind::None), "none"); - EXPECT_STREQ( - llm_kv_write_kind_to_string(LlmKvWriteKind::CurrentTokenAppend), - "current_token_append"); - EXPECT_STREQ( - llm_kv_write_kind_to_string(LlmKvWriteKind::FullPromptPopulation), - "full_prompt_population"); EXPECT_STREQ(llm_kv_write_kind_to_string( static_cast(99)), "unknown"); - EXPECT_STREQ(llm_scenario_to_string(LlmScenario::WeightsOnly), - "weights_only"); - EXPECT_STREQ(llm_scenario_to_string(LlmScenario::KvOnly), "kv_only"); - EXPECT_STREQ(llm_scenario_to_string(LlmScenario::Mixed), "mixed"); EXPECT_STREQ(llm_scenario_to_string(static_cast(99)), "unknown"); - EXPECT_STREQ(llm_attention_kind_to_string(LlmAttentionKind::Mha), "mha"); - EXPECT_STREQ(llm_attention_kind_to_string(LlmAttentionKind::Gqa), "gqa"); - EXPECT_STREQ(llm_attention_kind_to_string(LlmAttentionKind::Mqa), "mqa"); EXPECT_STREQ( llm_attention_kind_to_string(static_cast(99)), "unknown"); - EXPECT_STREQ( - llm_measurement_status_to_string(LlmMeasurementStatus::NotRun), - "not_run"); - EXPECT_STREQ( - llm_measurement_status_to_string(LlmMeasurementStatus::Measured), - "measured"); - EXPECT_STREQ( - llm_measurement_status_to_string(LlmMeasurementStatus::Interrupted), - "interrupted"); - EXPECT_STREQ( - llm_measurement_status_to_string(LlmMeasurementStatus::Invalid), - "invalid"); - EXPECT_STREQ( - llm_measurement_status_to_string(LlmMeasurementStatus::Failed), - "failed"); EXPECT_STREQ(llm_measurement_status_to_string( static_cast(99)), "invalid"); - EXPECT_STREQ(llm_run_status_to_string(LlmRunStatus::NotStarted), - "not_started"); - EXPECT_STREQ(llm_run_status_to_string(LlmRunStatus::Complete), "complete"); - EXPECT_STREQ(llm_run_status_to_string(LlmRunStatus::Partial), "partial"); - EXPECT_STREQ(llm_run_status_to_string(LlmRunStatus::Interrupted), - "interrupted"); - EXPECT_STREQ(llm_run_status_to_string(LlmRunStatus::Unsupported), - "unsupported"); - EXPECT_STREQ(llm_run_status_to_string(LlmRunStatus::Failed), "failed"); EXPECT_STREQ(llm_run_status_to_string(static_cast(99)), "failed"); } @@ -1689,25 +1383,3 @@ TEST(LlmMemoryConfigTest, ++config.loop_count; expect_invalid(config, LlmMemoryConfigReason::JSON_INTEGER_OUT_OF_RANGE); } - -TEST(LlmMemoryConfigTest, ResultFoundationKeepsUnavailableValuesAbsent) { - const LlmMeasurementState measurement; - EXPECT_EQ(measurement.status, LlmMeasurementStatus::NotRun); - EXPECT_EQ(measurement.reason_code, "not-run"); - EXPECT_EQ(measurement.plan_handle, kLlmNoTaskIndex); - EXPECT_EQ(measurement.execution.completion.completed_work_units, 0u); - EXPECT_FALSE(derive_llm_measurement_metrics(measurement).latency_seconds.has_value()); - EXPECT_FALSE(derive_llm_measurement_metrics(measurement).payload_gb_s.has_value()); - EXPECT_FALSE(measurement.execution_evidence_available); - - const LlmMemoryResult result; - EXPECT_EQ(result.status, LlmRunStatus::NotStarted); - EXPECT_EQ(result.reason_code, "not-started"); - EXPECT_FALSE(result.interruption_requested); - EXPECT_FALSE(result.results_complete); - EXPECT_FALSE(result.run_accepted); - EXPECT_FALSE(result.scenario_order_balance_complete); - EXPECT_TRUE(result.measurements.empty()); - EXPECT_EQ(result.counters.planned_measurements, 0u); - EXPECT_EQ(result.counters.completed_effective_model_payload_bytes, 0u); -} diff --git a/tests/test_llm_memory_contract.cpp b/tests/test_llm_memory_contract.cpp deleted file mode 100644 index 8ff22bd..0000000 --- a/tests/test_llm_memory_contract.cpp +++ /dev/null @@ -1,1699 +0,0 @@ -// Copyright 2026 Timo Heimonen -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. -// -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -constexpr uint64_t kAppendStepMultiplier = 0x9E3779B97F4A7C15ULL; -constexpr uint64_t kAppendLayerMultiplier = 0xBF58476D1CE4E5B9ULL; -constexpr uint64_t kAppendBatchMultiplier = 0x94D049BB133111EBULL; -constexpr uint64_t kAppendWordMultiplier = 0xD6E8FEB86659FD93ULL; -constexpr uint64_t kAppendKDomain = 0x4B4B4B4B4B4B4B4BULL; -constexpr uint64_t kAppendVDomain = 0x5656565656565656ULL; - -constexpr uint64_t kChecksumInitialA = 0x243F6A8885A308D3ULL; -constexpr uint64_t kChecksumInitialB = 0x13198A2E03707344ULL; -constexpr uint64_t kChecksumWeightDomain = 0x5745494748545F31ULL; -constexpr uint64_t kChecksumKDomain = 0x4B5F524541445F31ULL; -constexpr uint64_t kChecksumVDomain = 0x565F524541445F31ULL; -constexpr uint64_t kRunInitialA = 0x6A09E667F3BCC909ULL; -constexpr uint64_t kRunInitialB = 0xBB67AE8584CAA73BULL; - -constexpr uint64_t kKvBlockPermutationDomain = 0x4C4C4D4B56504731ULL; -constexpr uint64_t kCanonicalSegmentCapacityBytes = 256ULL * 1024ULL * 1024ULL; -constexpr size_t kCanonicalSegmentSlotsPerPool = 256; -constexpr uint64_t kCanonicalPoolCapacityBytes = - kCanonicalSegmentCapacityBytes * kCanonicalSegmentSlotsPerPool; - -enum class ContractKvLayout { - Contiguous, - Paged, -}; - -enum class ContractScenario { - WeightsOnly, - KvOnly, - Mixed, -}; - -struct PrefillPayloadContract { - std::vector tile_ends; - uint64_t attention_prefix_token_visits = 0; - uint64_t weight_passes = 0; - uint64_t weight_read_bytes = 0; - uint64_t kv_read_bytes = 0; - uint64_t kv_write_bytes = 0; - uint64_t kv_only_bytes = 0; - uint64_t mixed_bytes = 0; -}; - -struct PagedGeometryContract { - uint64_t blocks_per_sequence = 0; - uint64_t physical_blocks_per_layer = 0; - uint64_t block_bytes = 0; - uint64_t last_block_tokens = 0; - uint64_t last_block_valid_bytes = 0; - uint64_t k_logical_bytes = 0; - uint64_t k_physical_bytes = 0; - uint64_t k_layout_padding_bytes = 0; - uint64_t block_table_entries = 0; - uint64_t block_table_bytes = 0; - uint64_t decode_lookup_count = 0; - uint64_t decode_layout_metadata_bytes = 0; -}; - -struct PrefillClosedFormContract { - uint64_t tile_count = 0; - uint64_t attention_prefix_token_visits = 0; - uint64_t blocks_per_sequence = 0; - uint64_t prefix_block_visits = 0; - uint64_t lookups_per_layer_sequence = 0; -}; - -struct PrefillBlockContract { - uint64_t valid_tokens = 0; - uint64_t data_visits = 0; - uint64_t semantic_lookups = 0; - uint64_t model_payload_bytes = 0; - uint64_t layout_metadata_bytes = 0; - uint64_t accounted_bytes = 0; -}; - -struct ComponentIdentityContract { - std::string_view logical_profile_version; - std::string_view kv_layout_version; - std::optional permutation_version; - std::string_view backend_executor_version; - std::string_view resource_abi_version; - std::string_view schedule_version; - std::string_view timer_policy_version; - std::string_view buffer_pattern_version; - std::string_view write_pattern_version; - std::string_view checksum_pattern_version; - std::optional msl_revision; - std::optional msl_source_sha256; -}; - -struct ScenarioAccountingContract { - uint64_t model_payload_bytes = 0; - uint64_t layout_metadata_lookups = 0; - uint64_t layout_metadata_bytes = 0; - uint64_t accounted_bytes = 0; -}; - -enum class ChecksumComponent { - Weight, - K, - V, -}; - -struct PayloadContract { - uint64_t weight_read_per_step = 0; - uint64_t kv_read_per_step = 0; - uint64_t kv_append_per_step = 0; - uint64_t kv_only_per_step = 0; - uint64_t mixed_per_step = 0; - uint64_t weight_read_total = 0; - uint64_t kv_read_total = 0; - uint64_t kv_append_total = 0; - uint64_t kv_only_total = 0; - uint64_t mixed_total = 0; -}; - -struct ReadChecksum { - uint64_t state_a = 0; - uint64_t state_b = 0; - uint64_t exact_bytes_read = 0; - uint64_t span_count = 0; -}; - -struct RunChecksum { - uint64_t state_a = kRunInitialA; - uint64_t state_b = kRunInitialB; -}; - - -struct GenericResultIdentity { - std::string_view mode; - int schema_version; - std::string_view requested_backend; - std::string_view backend; - std::string_view requested_phase; - std::string_view phase; - std::string_view requested_kv_layout; - std::string_view kv_layout; - std::string_view methodology_version; - std::string_view status; - bool results_complete; - bool run_accepted; - bool all_planned_measurements_measured; -}; - -/** - * @brief Test-side scalar oracle for the original decode payload contract. - * - * Inputs are already resolved exact byte counts. Expansion production code - * must compare against the hard-coded values in this file instead of sharing - * this oracle. - */ -PayloadContract resolve_payload_contract(uint64_t weight_bytes, - uint64_t kv_bytes_per_token, - uint64_t visible_context_tokens, - uint64_t batch_size, - uint64_t steps) { - PayloadContract result; - result.weight_read_per_step = weight_bytes; - result.kv_read_per_step = - batch_size * visible_context_tokens * kv_bytes_per_token; - result.kv_append_per_step = batch_size * kv_bytes_per_token; - result.kv_only_per_step = - result.kv_read_per_step + result.kv_append_per_step; - result.mixed_per_step = - result.weight_read_per_step + result.kv_only_per_step; - result.weight_read_total = result.weight_read_per_step * steps; - result.kv_read_total = result.kv_read_per_step * steps; - result.kv_append_total = result.kv_append_per_step * steps; - result.kv_only_total = result.kv_only_per_step * steps; - result.mixed_total = result.mixed_per_step * steps; - return result; -} - -uint64_t ceil_divide_small(uint64_t value, uint64_t divisor) { - return value / divisor + (value % divisor != 0 ? 1 : 0); -} - -uint64_t triangular_small(uint64_t value) { - return value % 2 == 0 ? (value / 2) * (value + 1) - : value * (value / 2 + 1); -} - -std::vector enumerate_tile_ends(uint64_t prompt_tokens, - uint64_t query_tile_tokens) { - std::vector ends; - if (query_tile_tokens == 0) { - return ends; - } - uint64_t current_end = 0; - while (current_end < prompt_tokens) { - current_end += - std::min(query_tile_tokens, prompt_tokens - current_end); - ends.push_back(current_end); - } - return ends; -} - -PrefillPayloadContract resolve_prefill_payload_contract( - uint64_t weight_bytes, - uint64_t kv_bytes_per_token, - uint64_t batch_size, - uint64_t prompt_tokens, - uint64_t query_tile_tokens) { - PrefillPayloadContract result; - result.tile_ends = - enumerate_tile_ends(prompt_tokens, query_tile_tokens); - result.attention_prefix_token_visits = - std::accumulate(result.tile_ends.begin(), result.tile_ends.end(), - uint64_t{0}); - result.weight_passes = 1; - result.weight_read_bytes = weight_bytes; - result.kv_read_bytes = batch_size * - result.attention_prefix_token_visits * - kv_bytes_per_token; - result.kv_write_bytes = - batch_size * prompt_tokens * kv_bytes_per_token; - result.kv_only_bytes = result.kv_read_bytes + result.kv_write_bytes; - result.mixed_bytes = result.weight_read_bytes + result.kv_only_bytes; - return result; -} - -PagedGeometryContract resolve_paged_geometry_contract( - uint64_t sequence_tokens, - uint64_t block_tokens, - uint64_t layer_count, - uint64_t batch_size, - uint64_t k_or_v_record_bytes_per_layer) { - PagedGeometryContract result; - result.blocks_per_sequence = - ceil_divide_small(sequence_tokens, block_tokens); - result.physical_blocks_per_layer = - batch_size * result.blocks_per_sequence; - result.block_bytes = block_tokens * k_or_v_record_bytes_per_layer; - result.last_block_tokens = - sequence_tokens - (result.blocks_per_sequence - 1) * block_tokens; - result.last_block_valid_bytes = - result.last_block_tokens * k_or_v_record_bytes_per_layer; - result.k_logical_bytes = layer_count * batch_size * sequence_tokens * - k_or_v_record_bytes_per_layer; - result.k_physical_bytes = layer_count * result.physical_blocks_per_layer * - result.block_bytes; - result.k_layout_padding_bytes = - result.k_physical_bytes - result.k_logical_bytes; - result.block_table_entries = result.physical_blocks_per_layer; - result.block_table_bytes = result.block_table_entries * sizeof(uint32_t); - result.decode_lookup_count = layer_count * batch_size * - (2 * result.blocks_per_sequence + 1); - result.decode_layout_metadata_bytes = - result.decode_lookup_count * sizeof(uint32_t); - return result; -} - -ScenarioAccountingContract resolve_scenario_accounting_contract( - ContractKvLayout layout, - ContractScenario scenario, - uint64_t weight_payload_bytes, - uint64_t kv_payload_bytes, - uint64_t paged_layout_metadata_lookups) { - ScenarioAccountingContract result; - if (scenario == ContractScenario::WeightsOnly) { - result.model_payload_bytes = weight_payload_bytes; - } else { - result.model_payload_bytes = kv_payload_bytes; - if (scenario == ContractScenario::Mixed) { - result.model_payload_bytes += weight_payload_bytes; - } - if (layout == ContractKvLayout::Paged) { - result.layout_metadata_lookups = paged_layout_metadata_lookups; - } - } - result.layout_metadata_bytes = - result.layout_metadata_lookups * sizeof(uint32_t); - result.accounted_bytes = - result.model_payload_bytes + result.layout_metadata_bytes; - return result; -} - -std::vector enumerate_prefix_block_counts( - uint64_t prompt_tokens, - uint64_t query_tile_tokens, - uint64_t block_tokens) { - std::vector counts; - for (uint64_t end : - enumerate_tile_ends(prompt_tokens, query_tile_tokens)) { - counts.push_back(ceil_divide_small(end, block_tokens)); - } - return counts; -} - -uint64_t floor_sum_small(uint64_t count, - uint64_t denominator, - uint64_t slope, - uint64_t intercept) { - uint64_t result = 0; - for (uint64_t index = 0; index < count; ++index) { - result += (slope * index + intercept) / denominator; - } - return result; -} - -PrefillClosedFormContract resolve_prefill_closed_form_contract( - uint64_t prompt_tokens, - uint64_t query_tile_tokens, - uint64_t block_tokens) { - PrefillClosedFormContract result; - const uint64_t full_tile_count = prompt_tokens / query_tile_tokens; - const uint64_t remainder = prompt_tokens % query_tile_tokens; - result.tile_count = - full_tile_count + (remainder != 0 ? 1 : 0); - result.attention_prefix_token_visits = - query_tile_tokens * triangular_small(full_tile_count) + - (remainder != 0 ? prompt_tokens : 0); - result.blocks_per_sequence = - ceil_divide_small(prompt_tokens, block_tokens); - const uint64_t full_tile_block_visits = - full_tile_count + - floor_sum_small(full_tile_count, block_tokens, query_tile_tokens, - query_tile_tokens - 1); - result.prefix_block_visits = - full_tile_block_visits + - (remainder != 0 ? result.blocks_per_sequence : 0); - result.lookups_per_layer_sequence = - result.blocks_per_sequence + 2 * result.prefix_block_visits; - return result; -} - -std::vector enumerate_prefill_block_contracts( - uint64_t prompt_tokens, - uint64_t query_tile_tokens, - uint64_t block_tokens, - uint64_t k_or_v_record_bytes_per_layer) { - const std::vector tile_ends = - enumerate_tile_ends(prompt_tokens, query_tile_tokens); - const uint64_t block_count = - ceil_divide_small(prompt_tokens, block_tokens); - std::vector blocks; - blocks.reserve(static_cast(block_count)); - for (uint64_t block = 0; block < block_count; ++block) { - PrefillBlockContract contract; - const uint64_t block_start = block * block_tokens; - contract.valid_tokens = - std::min(block_tokens, prompt_tokens - block_start); - contract.data_visits = contract.valid_tokens; - uint64_t read_visits = 0; - for (uint64_t token = block_start; - token < block_start + contract.valid_tokens; ++token) { - for (uint64_t end : tile_ends) { - if (token < end) { - ++contract.data_visits; - } - } - } - for (uint64_t end : tile_ends) { - if (block_start < end) { - ++read_visits; - } - } - contract.semantic_lookups = 1 + 2 * read_visits; - contract.model_payload_bytes = - 2 * k_or_v_record_bytes_per_layer * contract.data_visits; - contract.layout_metadata_bytes = - sizeof(uint32_t) * contract.semantic_lookups; - contract.accounted_bytes = contract.model_payload_bytes + - contract.layout_metadata_bytes; - blocks.push_back(contract); - } - return blocks; -} - -size_t choose_two_way_cost_boundary( - const std::vector& blocks) { - if (blocks.size() < 2) { - return blocks.size(); - } - uint64_t total = 0; - for (const PrefillBlockContract& block : blocks) { - total += block.accounted_bytes; - } - - size_t best_boundary = 1; - uint64_t prefix = 0; - uint64_t best_scaled_distance = std::numeric_limits::max(); - for (size_t boundary = 1; boundary < blocks.size(); ++boundary) { - prefix += blocks[boundary - 1].accounted_bytes; - const uint64_t doubled_prefix = 2 * prefix; - const uint64_t distance = doubled_prefix > total - ? doubled_prefix - total - : total - doubled_prefix; - if (distance < best_scaled_distance) { - best_scaled_distance = distance; - best_boundary = boundary; - } - } - return best_boundary; -} - -std::vector> enumerate_terminal_visit_tokens( - uint64_t prompt_tokens, - uint64_t query_tile_tokens, - uint64_t block_tokens) { - const uint64_t block_count = - ceil_divide_small(prompt_tokens, block_tokens); - std::vector> visits; - for (uint64_t end : - enumerate_tile_ends(prompt_tokens, query_tile_tokens)) { - std::vector tile_visits; - tile_visits.reserve(static_cast(block_count)); - for (uint64_t block = 0; block < block_count; ++block) { - const uint64_t block_start = block * block_tokens; - const uint64_t visit_tokens = - block_start >= end - ? 0 - : std::min(block_tokens, end - block_start); - tile_visits.push_back(visit_tokens); - } - visits.push_back(std::move(tile_visits)); - } - return visits; -} - -uint64_t splitmix64_next(uint64_t& state) { - state += 0x9E3779B97F4A7C15ULL; - uint64_t value = state; - value = (value ^ (value >> 30)) * 0xBF58476D1CE4E5B9ULL; - value = (value ^ (value >> 27)) * 0x94D049BB133111EBULL; - return value ^ (value >> 31); -} - -uint64_t derive_permutation_seed(uint64_t base_seed) { - uint64_t state = base_seed ^ kKvBlockPermutationDomain; - return splitmix64_next(state); -} - -std::vector materialize_permutation(size_t entry_count, - uint64_t stream_state) { - std::vector entries(entry_count); - std::iota(entries.begin(), entries.end(), uint32_t{0}); - for (size_t index = entry_count; index > 1; --index) { - const uint64_t bound = static_cast(index); - const uint64_t threshold = (uint64_t{0} - bound) % bound; - uint64_t draw = 0; - do { - draw = splitmix64_next(stream_state); - } while (draw < threshold); - const size_t swap_index = static_cast(draw % bound); - std::swap(entries[index - 1], entries[swap_index]); - } - return entries; -} - -std::string sha256_little_endian_entries( - const std::vector& entries, - size_t entries_per_chunk) { - if (entries_per_chunk == 0) { - return {}; - } - - CC_SHA256_CTX context; - if (CC_SHA256_Init(&context) != 1) { - return {}; - } - constexpr size_t kEntriesPerUpdateCap = 1024; - std::array - bytes{}; - size_t offset = 0; - while (offset < entries.size()) { - const size_t count = - std::min({entries_per_chunk, entries.size() - offset, - kEntriesPerUpdateCap}); - for (size_t index = 0; index < count; ++index) { - const uint32_t value = entries[offset + index]; - for (size_t byte = 0; byte < sizeof(uint32_t); ++byte) { - bytes[index * sizeof(uint32_t) + byte] = - static_cast(value >> (8 * byte)); - } - } - const size_t byte_count = count * sizeof(uint32_t); - if (CC_SHA256_Update(&context, bytes.data(), - static_cast(byte_count)) != 1) { - return {}; - } - offset += count; - } - - std::array digest{}; - if (CC_SHA256_Final(digest.data(), &context) != 1) { - return {}; - } - constexpr char kLowercaseHex[] = "0123456789abcdef"; - std::string encoded(digest.size() * 2, '0'); - for (size_t index = 0; index < digest.size(); ++index) { - encoded[index * 2] = kLowercaseHex[digest[index] >> 4U]; - encoded[index * 2 + 1] = - kLowercaseHex[digest[index] & 0x0fU]; - } - return encoded; -} - -std::vector contiguous_segment_lengths(uint64_t logical_bytes) { - std::vector lengths; - uint64_t remaining = logical_bytes; - while (remaining > 0) { - const uint64_t length = - std::min(remaining, kCanonicalSegmentCapacityBytes); - lengths.push_back(length); - remaining -= length; - } - return lengths; -} - -bool paged_block_fits_canonical_segment(uint64_t block_bytes) { - return block_bytes <= kCanonicalSegmentCapacityBytes; -} - -bool segment_count_fits_pool_slots(uint64_t segment_count) { - return segment_count <= kCanonicalSegmentSlotsPerPool; -} - -void append_identity_component(std::string& identity, - std::string_view key, - std::optional value) { - identity += '|'; - identity += key; - identity += '='; - if (!value.has_value()) { - identity += "null"; - return; - } - identity += std::to_string(value->size()); - identity += ':'; - identity += *value; -} - -void append_identity_component(std::string& identity, - std::string_view key, - std::string_view value) { - identity += '|'; - identity += key; - identity += '='; - identity += std::to_string(value.size()); - identity += ':'; - identity += value; -} - -std::string serialize_component_identity( - const ComponentIdentityContract& components) { - std::string identity = "llm-memory-components-v1"; - append_identity_component(identity, "logical_profile_version", - components.logical_profile_version); - append_identity_component(identity, "kv_layout_version", - components.kv_layout_version); - append_identity_component(identity, "permutation_version", - components.permutation_version); - append_identity_component(identity, "backend_executor_version", - components.backend_executor_version); - append_identity_component(identity, "resource_abi_version", - components.resource_abi_version); - append_identity_component(identity, "schedule_version", - components.schedule_version); - append_identity_component(identity, "timer_policy_version", - components.timer_policy_version); - append_identity_component(identity, "buffer_pattern_version", - components.buffer_pattern_version); - append_identity_component(identity, "write_pattern_version", - components.write_pattern_version); - append_identity_component(identity, "checksum_pattern_version", - components.checksum_pattern_version); - append_identity_component(identity, "msl_revision", - components.msl_revision); - append_identity_component(identity, "msl_source_sha256", - components.msl_source_sha256); - return identity; -} - -uint64_t rotate_left(uint64_t value, unsigned int shift) { - return (value << shift) | (value >> (64 - shift)); -} - -uint64_t append_word(uint64_t base_seed, - uint64_t task_local_step, - uint64_t layer, - uint64_t batch_sequence, - uint64_t record_word_index, - uint64_t buffer_domain) { - return base_seed + kAppendStepMultiplier * (task_local_step + 1) + - kAppendLayerMultiplier * (layer + 1) + - kAppendBatchMultiplier * (batch_sequence + 1) + - kAppendWordMultiplier * (record_word_index + 1) + buffer_domain; -} - -std::vector append_byte_range(uint64_t base_seed, - uint64_t task_local_step, - uint64_t layer, - uint64_t batch_sequence, - uint64_t buffer_domain, - size_t record_byte_offset, - size_t byte_count) { - std::vector bytes; - bytes.reserve(byte_count); - for (size_t byte = 0; byte < byte_count; ++byte) { - const size_t canonical_byte = record_byte_offset + byte; - const uint64_t word = append_word( - base_seed, task_local_step, layer, batch_sequence, - static_cast(canonical_byte / sizeof(uint64_t)), - buffer_domain); - bytes.push_back(static_cast( - word >> (8 * (canonical_byte % sizeof(uint64_t))))); - } - return bytes; -} - -uint64_t checksum_domain(ChecksumComponent component) { - switch (component) { - case ChecksumComponent::Weight: - return kChecksumWeightDomain; - case ChecksumComponent::K: - return kChecksumKDomain; - case ChecksumComponent::V: - return kChecksumVDomain; - } - return 0; -} - -ReadChecksum initial_checksum(ChecksumComponent component) { - const uint64_t domain = checksum_domain(component); - return {kChecksumInitialA ^ domain, kChecksumInitialB + domain, 0, 0}; -} - -uint64_t load_partial_little_endian(const uint8_t* bytes, - size_t byte_count) { - uint64_t value = 0; - for (size_t index = 0; index < byte_count; ++index) { - value |= static_cast(bytes[index]) << (8 * index); - } - return value; -} - -void absorb_span(ReadChecksum& checksum, const std::vector& span) { - if (span.empty()) { - return; - } - uint64_t sum_even_words = 0; - uint64_t sum_odd_words = 0; - size_t offset = 0; - size_t word_index = 0; - while (offset < span.size()) { - const size_t remaining = span.size() - offset; - const size_t word_bytes = - remaining < sizeof(uint64_t) ? remaining : sizeof(uint64_t); - const uint64_t word = - load_partial_little_endian(span.data() + offset, word_bytes); - if (word_index % 2 == 0) { - sum_even_words += word; - } else { - sum_odd_words += word; - } - offset += word_bytes; - ++word_index; - } - - const uint64_t ordinal = checksum.span_count; - checksum.state_a = rotate_left( - checksum.state_a + sum_even_words + - kAppendStepMultiplier * (ordinal + 1), - 17); - checksum.state_b = rotate_left( - checksum.state_b + sum_odd_words + - static_cast(span.size()) + - kAppendWordMultiplier * (ordinal + 1), - 29); - checksum.exact_bytes_read += static_cast(span.size()); - ++checksum.span_count; -} - -RunChecksum fold_components( - const std::vector& canonical_components) { - RunChecksum run; - for (size_t ordinal = 0; ordinal < canonical_components.size(); - ++ordinal) { - const ReadChecksum& component = canonical_components[ordinal]; - run.state_a = rotate_left( - run.state_a + component.state_a + - kAppendStepMultiplier * (static_cast(ordinal) + 1), - 23); - run.state_b = rotate_left( - run.state_b + component.state_b + component.exact_bytes_read + - kAppendWordMultiplier * component.span_count + - kAppendBatchMultiplier * - (static_cast(ordinal) + 1), - 41); - } - return run; -} - -bool accepted_generic_result(const GenericResultIdentity& identity) { - const bool backend_is_known = - identity.backend == "cpu" || identity.backend == "metal"; - const bool phase_is_known = - identity.phase == "decode" || identity.phase == "prefill"; - const bool layout_is_known = identity.kv_layout == "contiguous" || - identity.kv_layout == "paged"; - const std::string expected_methodology = - "llm-memory-v2-" + std::string(identity.backend) + "-" + - std::string(identity.phase) + "-" + std::string(identity.kv_layout); - return identity.mode == "llm_memory" && identity.schema_version == 2 && - backend_is_known && phase_is_known && layout_is_known && - identity.backend == identity.requested_backend && - identity.phase == identity.requested_phase && - identity.kv_layout == identity.requested_kv_layout && - identity.methodology_version == expected_methodology && - identity.status == "complete" && identity.results_complete && - identity.run_accepted && - identity.all_planned_measurements_measured; -} - -} // namespace - -TEST(LlmMemoryContractTest, ResolvedTrafficFormulaGoldenVectors) { - constexpr uint64_t gibibyte = 1024ULL * 1024ULL * 1024ULL; - constexpr uint64_t weight_bytes = 4 * gibibyte; - constexpr uint64_t layer_count = 32; - constexpr uint64_t kv_head_count = 8; - constexpr uint64_t head_dimension = 128; - constexpr uint64_t kv_element_bytes = 2; - constexpr uint64_t batch_size = 1; - constexpr uint64_t visible_context_tokens = 8192; - - const uint64_t kv_vector_bytes = head_dimension * kv_element_bytes; - const uint64_t kv_record_bytes_per_layer = - 2 * kv_head_count * kv_vector_bytes; - const uint64_t kv_bytes_per_token = - layer_count * kv_record_bytes_per_layer; - const PayloadContract large = resolve_payload_contract( - weight_bytes, kv_bytes_per_token, visible_context_tokens, batch_size, - 1); - - EXPECT_EQ(kv_vector_bytes, 256u); - EXPECT_EQ(kv_record_bytes_per_layer, 4096u); - EXPECT_EQ(kv_bytes_per_token, 131072u); - EXPECT_EQ(large.kv_read_per_step, gibibyte); - EXPECT_EQ(large.kv_append_per_step, 131072u); - EXPECT_EQ(large.kv_only_per_step, 1073872896u); - EXPECT_EQ(large.mixed_per_step, 5368840192u); - EXPECT_EQ(weight_bytes, 4294967296u); - EXPECT_EQ(batch_size * kv_bytes_per_token, 131072u); - EXPECT_EQ(weight_bytes / (batch_size * kv_bytes_per_token), 32768u); - - const PayloadContract crossover = resolve_payload_contract( - weight_bytes, kv_bytes_per_token, 32768, batch_size, 1); - EXPECT_EQ(crossover.kv_read_per_step, weight_bytes); - - const PayloadContract small = - resolve_payload_contract(1024, 128, 3, 1, 4); - EXPECT_EQ(small.weight_read_per_step, 1024u); - EXPECT_EQ(small.kv_read_per_step, 384u); - EXPECT_EQ(small.kv_append_per_step, 128u); - EXPECT_EQ(small.kv_only_per_step, 512u); - EXPECT_EQ(small.mixed_per_step, 1536u); - EXPECT_EQ(small.weight_read_total, 4096u); - EXPECT_EQ(small.kv_read_total, 1536u); - EXPECT_EQ(small.kv_append_total, 512u); - EXPECT_EQ(small.kv_only_total, 2048u); - EXPECT_EQ(small.mixed_total, 6144u); - - const PayloadContract batched = - resolve_payload_contract(1024, 128, 3, 2, 4); - EXPECT_EQ(batched.weight_read_per_step, 1024u); - EXPECT_EQ(batched.kv_read_per_step, 768u); - EXPECT_EQ(batched.kv_append_per_step, 256u); - EXPECT_EQ(batched.mixed_total, 8192u); -} - -TEST(LlmMemoryContractTest, - AppendAffine64GoldenWordsDomainsWrappingAndTailBytes) { - EXPECT_EQ(append_word(0, 0, 0, 0, 0, kAppendKDomain), - 0x149454E56105BC97ULL); - EXPECT_EQ(append_word(0, 0, 0, 0, 0, kAppendVDomain), - 0x1F9F5FF06C10C7A2ULL); - EXPECT_EQ(append_word(0, 0, 0, 0, 1, kAppendKDomain), - 0xEB7D539DC75FBA2AULL); - EXPECT_EQ(append_word(std::numeric_limits::max(), 0, 0, 0, 0, - kAppendKDomain), - 0x149454E56105BC96ULL); - EXPECT_EQ(append_word(0x0123456789ABCDEFULL, 2, 3, 4, 5, - kAppendKDomain), - 0x15FD848D8C7B6F66ULL); - EXPECT_EQ(append_word(0x0123456789ABCDEFULL, 2, 3, 4, 5, - kAppendVDomain), - 0x21088F9897867A71ULL); - EXPECT_EQ(append_word(0, 1, 0, 0, 0, kAppendKDomain), - 0xB2CBCE9EE05038ACULL); - EXPECT_EQ(append_word(0, 0, 1, 0, 0, kAppendKDomain), - 0xD3EC9C527DEAA250ULL); - EXPECT_EQ(append_word(0, 0, 0, 1, 0, kAppendKDomain), - 0xA9649EA07436CE82ULL); - - const std::array expected_prefix = { - 0x97, 0xBC, 0x05, 0x61, 0xE5, 0x54, 0x94, 0x14, - 0x2A, 0xBA, 0x5F, 0xC7, 0x9D, 0x53, 0x7D}; - for (size_t length = 9; length <= expected_prefix.size(); ++length) { - const std::vector expected(expected_prefix.begin(), - expected_prefix.begin() + length); - EXPECT_EQ(append_byte_range(0, 0, 0, 0, kAppendKDomain, 0, length), - expected) - << "length=" << length; - } - - EXPECT_EQ(append_byte_range(0, 0, 0, 0, kAppendKDomain, 3, 11), - (std::vector{0x61, 0xE5, 0x54, 0x94, 0x14, 0x2A, - 0xBA, 0x5F, 0xC7, 0x9D, 0x53})); -} - -TEST(LlmMemoryContractTest, ReadChecksumGoldenStatesAndSpanParity) { - std::vector span(19); - for (size_t index = 0; index < span.size(); ++index) { - span[index] = static_cast(index); - } - - ReadChecksum weight = initial_checksum(ChecksumComponent::Weight); - EXPECT_EQ(weight.state_a, 0x737A23CFCDF757E2ULL); - EXPECT_EQ(weight.state_b, 0x6A5ED3754BC4D275ULL); - absorb_span(weight, {}); - EXPECT_EQ(weight.state_a, 0x737A23CFCDF757E2ULL); - EXPECT_EQ(weight.state_b, 0x6A5ED3754BC4D275ULL); - EXPECT_EQ(weight.exact_bytes_read, 0u); - EXPECT_EQ(weight.span_count, 0u); - absorb_span(weight, span); - EXPECT_EQ(weight.state_a, 0x451AA0ABCC0E316FULL); - EXPECT_EQ(weight.state_b, 0x37A51B246A0ABBE7ULL); - EXPECT_EQ(weight.exact_bytes_read, 19u); - EXPECT_EQ(weight.span_count, 1u); - - ReadChecksum k = initial_checksum(ChecksumComponent::K); - EXPECT_EQ(k.state_a, 0x6F6038CDC4E757E2ULL); - EXPECT_EQ(k.state_b, 0x5E78DC7344B4D275ULL); - absorb_span(k, span); - EXPECT_EQ(k.state_a, 0x6F168E8BCC0E293BULL); - EXPECT_EQ(k.state_b, 0xF6C31B24688DFD06ULL); - - ReadChecksum v = initial_checksum(ChecksumComponent::V); - EXPECT_EQ(v.state_a, 0x726038CDC4E757E2ULL); - EXPECT_EQ(v.state_b, 0x6978DC7344B4D275ULL); - absorb_span(v, span); - EXPECT_EQ(v.state_a, 0x6F168E8BCC0E2F3BULL); - EXPECT_EQ(v.state_b, 0xF6C31B2469EDFD06ULL); - - const std::vector second_span = { - 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF}; - absorb_span(weight, second_span); - EXPECT_EQ(weight.state_a, 0x24378D3C47230311ULL); - EXPECT_EQ(weight.state_b, 0xA6D7D6E2BCAEE312ULL); - EXPECT_EQ(weight.exact_bytes_read, 27u); - EXPECT_EQ(weight.span_count, 2u); -} - -TEST(LlmMemoryContractTest, ReadChecksumRepeatedSpansDoNotCancel) { - const std::vector span = {1, 2, 3, 4, 5, 6, 7, 8}; - ReadChecksum checksum = initial_checksum(ChecksumComponent::Weight); - absorb_span(checksum, span); - EXPECT_EQ(checksum.state_a, 0x471CA289ABF03371ULL); - EXPECT_EQ(checksum.state_b, 0xB643DA020828FA45ULL); - absorb_span(checksum, span); - EXPECT_EQ(checksum.state_a, 0x38035D105B391725ULL); - EXPECT_EQ(checksum.state_b, 0x5A9B9EAE6C82BAEEULL); - EXPECT_EQ(checksum.exact_bytes_read, 16u); - EXPECT_EQ(checksum.span_count, 2u); -} - -TEST(LlmMemoryContractTest, - RunChecksumFoldIncludesEmptyComponentsInCanonicalOrder) { - std::vector span(19); - for (size_t index = 0; index < span.size(); ++index) { - span[index] = static_cast(index); - } - ReadChecksum weight = initial_checksum(ChecksumComponent::Weight); - absorb_span(weight, span); - const ReadChecksum empty_k = initial_checksum(ChecksumComponent::K); - const ReadChecksum empty_v = initial_checksum(ChecksumComponent::V); - - const RunChecksum folded = fold_components({weight, empty_k, empty_v}); - EXPECT_EQ(folded.state_a, 0xBCA46801BE6585DBULL); - EXPECT_EQ(folded.state_b, 0xEAAC493C97E06CF7ULL); - - const RunChecksum omitted_empty_components = fold_components({weight}); - EXPECT_NE(omitted_empty_components.state_a, folded.state_a); - EXPECT_NE(omitted_empty_components.state_b, folded.state_b); - const RunChecksum wrong_order = fold_components({empty_k, weight, empty_v}); - EXPECT_NE(wrong_order.state_a, folded.state_a); - EXPECT_NE(wrong_order.state_b, folded.state_b); -} - -TEST(LlmMemoryContractTest, GenericV2AcceptancePredicateIsExact) { - const GenericResultIdentity accepted = { - "llm_memory", - 2, - "metal", - "metal", - "prefill", - "prefill", - "paged", - "paged", - "llm-memory-v2-metal-prefill-paged", - "complete", - true, - true, - true, - }; - EXPECT_TRUE(accepted_generic_result(accepted)); - - GenericResultIdentity candidate = accepted; - candidate.mode = "benchmark"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.schema_version = 1; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.backend = "cpu"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.phase = "decode"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.kv_layout = "contiguous"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.requested_backend = "gpu"; - candidate.backend = "gpu"; - candidate.methodology_version = "llm-memory-v2-gpu-prefill-paged"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.requested_phase = "train"; - candidate.phase = "train"; - candidate.methodology_version = "llm-memory-v2-metal-train-paged"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.requested_kv_layout = "sparse"; - candidate.kv_layout = "sparse"; - candidate.methodology_version = "llm-memory-v2-metal-prefill-sparse"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.methodology_version = - "llm-memory-v2-cpu-fixed-context-warm-layer-interleaved"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.status = "partial"; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.results_complete = false; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.run_accepted = false; - EXPECT_FALSE(accepted_generic_result(candidate)); - candidate = accepted; - candidate.all_planned_measurements_measured = false; - EXPECT_FALSE(accepted_generic_result(candidate)); -} - -TEST(LlmMemoryContractTest, - PrefillPayloadGoldenCoversTilesBoundsAndSingleWeightPass) { - const PrefillPayloadContract golden = - resolve_prefill_payload_contract(1024, 128, 2, 5, 2); - EXPECT_EQ(golden.tile_ends, - (std::vector{2, 4, 5})); - EXPECT_EQ(golden.attention_prefix_token_visits, 11u); - EXPECT_EQ(golden.weight_passes, 1u); - EXPECT_EQ(golden.weight_read_bytes, 1024u); - EXPECT_EQ(golden.kv_read_bytes, 2816u); - EXPECT_EQ(golden.kv_write_bytes, 1280u); - EXPECT_EQ(golden.kv_only_bytes, 4096u); - EXPECT_EQ(golden.mixed_bytes, 5120u); - - const PrefillPayloadContract token_tiled = - resolve_prefill_payload_contract(1024, 128, 2, 5, 1); - EXPECT_EQ(token_tiled.tile_ends, - (std::vector{1, 2, 3, 4, 5})); - EXPECT_EQ(token_tiled.attention_prefix_token_visits, 15u); - EXPECT_EQ(token_tiled.kv_read_bytes, 3840u); - EXPECT_EQ(token_tiled.kv_write_bytes, 1280u); - EXPECT_EQ(token_tiled.kv_only_bytes, 5120u); - EXPECT_EQ(token_tiled.mixed_bytes, 6144u); - - const PrefillPayloadContract full_prompt_tiled = - resolve_prefill_payload_contract(1024, 128, 2, 5, 5); - EXPECT_EQ(full_prompt_tiled.tile_ends, - (std::vector{5})); - EXPECT_EQ(full_prompt_tiled.attention_prefix_token_visits, 5u); - EXPECT_EQ(full_prompt_tiled.kv_read_bytes, 1280u); - EXPECT_EQ(full_prompt_tiled.kv_write_bytes, 1280u); - EXPECT_EQ(full_prompt_tiled.kv_only_bytes, 2560u); - EXPECT_EQ(full_prompt_tiled.mixed_bytes, 3584u); - - struct WeightPassCase { - uint64_t prompt_tokens; - uint64_t query_tile_tokens; - uint64_t batch_size; - }; - constexpr std::array cases = {{{1, 1, 1}, - {5, 1, 1}, - {5, 5, 4}, - {17, 4, 2}, - {31, 8, 7}}}; - for (const WeightPassCase& test_case : cases) { - const PrefillPayloadContract result = resolve_prefill_payload_contract( - 987, 64, test_case.batch_size, test_case.prompt_tokens, - test_case.query_tile_tokens); - EXPECT_EQ(result.weight_passes, 1u); - EXPECT_EQ(result.weight_read_bytes, 987u); - } -} - -TEST(LlmMemoryContractTest, - PagedGeometryDecodeLookupAndMetadataGoldenIsExact) { - const PagedGeometryContract paged = resolve_paged_geometry_contract( - 35, 16, 2, 2, 32); - EXPECT_EQ(paged.blocks_per_sequence, 3u); - EXPECT_EQ(paged.physical_blocks_per_layer, 6u); - EXPECT_EQ(paged.block_bytes, 512u); - EXPECT_EQ(paged.last_block_tokens, 3u); - EXPECT_EQ(paged.last_block_valid_bytes, 96u); - EXPECT_EQ(paged.k_logical_bytes, 4480u); - EXPECT_EQ(paged.k_physical_bytes, 6144u); - EXPECT_EQ(paged.k_layout_padding_bytes, 1664u); - EXPECT_EQ(paged.block_table_entries, 6u); - EXPECT_EQ(paged.block_table_bytes, 24u); - EXPECT_EQ(paged.decode_lookup_count, 28u); - EXPECT_EQ(paged.decode_layout_metadata_bytes, 112u); - - constexpr uint64_t weight_read_bytes = 1024; - constexpr uint64_t kv_bytes_per_token = 2 * 2 * 32; - constexpr uint64_t kv_read_bytes = 2 * 35 * kv_bytes_per_token; - constexpr uint64_t kv_write_bytes = 2 * kv_bytes_per_token; - constexpr uint64_t kv_only_model_payload = - kv_read_bytes + kv_write_bytes; - constexpr uint64_t mixed_model_payload = - weight_read_bytes + kv_only_model_payload; - EXPECT_EQ(kv_bytes_per_token, 128u); - EXPECT_EQ(kv_read_bytes, 8960u); - EXPECT_EQ(kv_write_bytes, 256u); - EXPECT_EQ(kv_only_model_payload, 9216u); - EXPECT_EQ(mixed_model_payload, 10240u); - - const ScenarioAccountingContract contiguous_weights = - resolve_scenario_accounting_contract( - ContractKvLayout::Contiguous, ContractScenario::WeightsOnly, - weight_read_bytes, kv_only_model_payload, - paged.decode_lookup_count); - const ScenarioAccountingContract contiguous_kv = - resolve_scenario_accounting_contract( - ContractKvLayout::Contiguous, ContractScenario::KvOnly, - weight_read_bytes, kv_only_model_payload, - paged.decode_lookup_count); - const ScenarioAccountingContract contiguous_mixed = - resolve_scenario_accounting_contract( - ContractKvLayout::Contiguous, ContractScenario::Mixed, - weight_read_bytes, kv_only_model_payload, - paged.decode_lookup_count); - const ScenarioAccountingContract paged_weights = - resolve_scenario_accounting_contract( - ContractKvLayout::Paged, ContractScenario::WeightsOnly, - weight_read_bytes, kv_only_model_payload, - paged.decode_lookup_count); - const ScenarioAccountingContract paged_kv = - resolve_scenario_accounting_contract( - ContractKvLayout::Paged, ContractScenario::KvOnly, - weight_read_bytes, kv_only_model_payload, - paged.decode_lookup_count); - const ScenarioAccountingContract paged_mixed = - resolve_scenario_accounting_contract( - ContractKvLayout::Paged, ContractScenario::Mixed, - weight_read_bytes, kv_only_model_payload, - paged.decode_lookup_count); - - EXPECT_EQ(contiguous_weights.model_payload_bytes, 1024u); - EXPECT_EQ(contiguous_weights.layout_metadata_lookups, 0u); - EXPECT_EQ(contiguous_weights.layout_metadata_bytes, 0u); - EXPECT_EQ(contiguous_weights.accounted_bytes, 1024u); - EXPECT_EQ(contiguous_kv.model_payload_bytes, 9216u); - EXPECT_EQ(contiguous_kv.layout_metadata_bytes, 0u); - EXPECT_EQ(contiguous_kv.accounted_bytes, 9216u); - EXPECT_EQ(contiguous_mixed.model_payload_bytes, 10240u); - EXPECT_EQ(contiguous_mixed.layout_metadata_bytes, 0u); - EXPECT_EQ(contiguous_mixed.accounted_bytes, 10240u); - EXPECT_EQ(paged_weights.model_payload_bytes, 1024u); - EXPECT_EQ(paged_weights.layout_metadata_lookups, 0u); - EXPECT_EQ(paged_weights.layout_metadata_bytes, 0u); - EXPECT_EQ(paged_weights.accounted_bytes, 1024u); - EXPECT_EQ(paged_kv.model_payload_bytes, 9216u); - EXPECT_EQ(paged_kv.layout_metadata_lookups, 28u); - EXPECT_EQ(paged_kv.layout_metadata_bytes, 112u); - EXPECT_EQ(paged_kv.accounted_bytes, 9328u); - EXPECT_EQ(paged_mixed.model_payload_bytes, 10240u); - EXPECT_EQ(paged_mixed.layout_metadata_lookups, 28u); - EXPECT_EQ(paged_mixed.layout_metadata_bytes, 112u); - EXPECT_EQ(paged_mixed.accounted_bytes, 10352u); - - constexpr uint64_t guardrail_bytes = - 64ULL * 1024ULL * 1024ULL * 1024ULL; - EXPECT_EQ(guardrail_bytes / contiguous_weights.accounted_bytes, - 67108864u); - EXPECT_EQ(guardrail_bytes / contiguous_kv.accounted_bytes, 7456540u); - EXPECT_EQ(guardrail_bytes / paged_kv.accounted_bytes, 7367010u); - EXPECT_EQ(guardrail_bytes / contiguous_mixed.accounted_bytes, 6710886u); - EXPECT_EQ(guardrail_bytes / paged_mixed.accounted_bytes, 6638280u); - EXPECT_NE(paged.block_table_bytes, - paged.decode_layout_metadata_bytes); -} - -TEST(LlmMemoryContractTest, - BlockPermutationGoldenVectorsAndLittleEndianHashesAreFrozen) { - const std::vector direct = materialize_permutation(8, 0); - EXPECT_EQ(direct, - (std::vector{2, 5, 0, 3, 4, 6, 1, 7})); - constexpr std::string_view direct_hash = - "9d1cfab79005723a285fec9a5716b53baa7a6c0501e3d17434bfb31ea88935d1"; - EXPECT_EQ(sha256_little_endian_entries(direct, 1), direct_hash); - EXPECT_EQ(sha256_little_endian_entries(direct, 3), direct_hash); - EXPECT_EQ(sha256_little_endian_entries(direct, 8), direct_hash); - EXPECT_EQ(sha256_little_endian_entries( - direct, std::numeric_limits::max()), - direct_hash); - - EXPECT_EQ(kKvBlockPermutationDomain, 0x4C4C4D4B56504731ULL); - EXPECT_EQ(derive_permutation_seed(42), 8109369757063363730ULL); - const std::vector derived = - materialize_permutation(8, derive_permutation_seed(42)); - EXPECT_EQ(derived, - (std::vector{0, 6, 2, 3, 7, 1, 5, 4})); - constexpr std::string_view derived_hash = - "4032b29a855010d82199c15c3f3e2b94582b86e67b3add8cb86bebc425f9c2b4"; - EXPECT_EQ(sha256_little_endian_entries(derived, 1), derived_hash); - EXPECT_EQ(sha256_little_endian_entries(derived, 3), derived_hash); - EXPECT_EQ(sha256_little_endian_entries(derived, 8), derived_hash); - EXPECT_EQ(sha256_little_endian_entries( - derived, std::numeric_limits::max()), - derived_hash); -} - -TEST(LlmMemoryContractTest, - PrefillPagedLookupAndTerminalVisitGoldensAreExact) { - const PrefillClosedFormContract aligned = - resolve_prefill_closed_form_contract(5, 2, 2); - EXPECT_EQ(enumerate_tile_ends(5, 2), - (std::vector{2, 4, 5})); - EXPECT_EQ(enumerate_prefix_block_counts(5, 2, 2), - (std::vector{1, 2, 3})); - EXPECT_EQ(aligned.tile_count, 3u); - EXPECT_EQ(aligned.attention_prefix_token_visits, 11u); - EXPECT_EQ(aligned.blocks_per_sequence, 3u); - EXPECT_EQ(aligned.prefix_block_visits, 6u); - EXPECT_EQ(aligned.lookups_per_layer_sequence, 15u); - const std::vector aligned_blocks = - enumerate_prefill_block_contracts(5, 2, 2, 32); - ASSERT_EQ(aligned_blocks.size(), 3u); - EXPECT_EQ(aligned_blocks[0].model_payload_bytes, 512u); - EXPECT_EQ(aligned_blocks[0].layout_metadata_bytes, 28u); - EXPECT_EQ(aligned_blocks[0].accounted_bytes, 540u); - EXPECT_EQ(aligned_blocks[1].model_payload_bytes, 384u); - EXPECT_EQ(aligned_blocks[1].layout_metadata_bytes, 20u); - EXPECT_EQ(aligned_blocks[1].accounted_bytes, 404u); - EXPECT_EQ(aligned_blocks[2].model_payload_bytes, 128u); - EXPECT_EQ(aligned_blocks[2].layout_metadata_bytes, 12u); - EXPECT_EQ(aligned_blocks[2].accounted_bytes, 140u); - const uint64_t aligned_model_payload = std::accumulate( - aligned_blocks.begin(), aligned_blocks.end(), uint64_t{0}, - [](uint64_t total, const PrefillBlockContract& block) { - return total + block.model_payload_bytes; - }); - const uint64_t aligned_metadata = std::accumulate( - aligned_blocks.begin(), aligned_blocks.end(), uint64_t{0}, - [](uint64_t total, const PrefillBlockContract& block) { - return total + block.layout_metadata_bytes; - }); - EXPECT_EQ(aligned_model_payload * 2 * 2, 4096u); - EXPECT_EQ(aligned_metadata * 2 * 2, 240u); - EXPECT_EQ((aligned_model_payload + aligned_metadata) * 2 * 2, 4336u); - ASSERT_EQ(choose_two_way_cost_boundary(aligned_blocks), 1u); - EXPECT_EQ(aligned_blocks[0].accounted_bytes, 540u); - EXPECT_EQ(aligned_blocks[1].accounted_bytes + - aligned_blocks[2].accounted_bytes, - 544u); - - const PrefillClosedFormContract unaligned = - resolve_prefill_closed_form_contract(7, 3, 2); - EXPECT_EQ(enumerate_tile_ends(7, 3), - (std::vector{3, 6, 7})); - EXPECT_EQ(enumerate_prefix_block_counts(7, 3, 2), - (std::vector{2, 3, 4})); - EXPECT_EQ(unaligned.tile_count, 3u); - EXPECT_EQ(unaligned.attention_prefix_token_visits, 16u); - EXPECT_EQ(unaligned.blocks_per_sequence, 4u); - EXPECT_EQ(unaligned.prefix_block_visits, 9u); - EXPECT_EQ(unaligned.lookups_per_layer_sequence, 22u); - const std::vector unaligned_blocks = - enumerate_prefill_block_contracts(7, 3, 2, 32); - ASSERT_EQ(unaligned_blocks.size(), 4u); - EXPECT_EQ(unaligned_blocks[0].accounted_bytes, 540u); - EXPECT_EQ(unaligned_blocks[1].accounted_bytes, 476u); - EXPECT_EQ(unaligned_blocks[2].accounted_bytes, 404u); - EXPECT_EQ(unaligned_blocks[3].accounted_bytes, 140u); - EXPECT_EQ(choose_two_way_cost_boundary(unaligned_blocks), 2u); - - std::vector tie_break_blocks(3); - tie_break_blocks[0].accounted_bytes = 5; - tie_break_blocks[1].accounted_bytes = 10; - tie_break_blocks[2].accounted_bytes = 5; - EXPECT_EQ(choose_two_way_cost_boundary(tie_break_blocks), 1u); - - const PrefillClosedFormContract larger_than_prompt = - resolve_prefill_closed_form_contract(5, 2, 8); - EXPECT_EQ(larger_than_prompt.blocks_per_sequence, 1u); - EXPECT_EQ(larger_than_prompt.prefix_block_visits, - larger_than_prompt.tile_count); - EXPECT_EQ(larger_than_prompt.lookups_per_layer_sequence, - 1 + 2 * larger_than_prompt.tile_count); - - const std::vector> terminal_visits = - enumerate_terminal_visit_tokens(6, 2, 4); - EXPECT_EQ(terminal_visits, - (std::vector>{{2, 0}, {4, 0}, {4, 2}})); - std::vector> terminal_visit_bytes = terminal_visits; - for (std::vector& tile : terminal_visit_bytes) { - for (uint64_t& bytes : tile) { - bytes *= 32; - } - } - EXPECT_EQ(terminal_visit_bytes, - (std::vector>{{64, 0}, - {128, 0}, - {128, 64}})); -} - -TEST(LlmMemoryContractTest, - SmallPrefillDomainEnumerationMatchesClosedFormsAndPartitionSums) { - constexpr uint64_t layer_count = 2; - constexpr uint64_t batch_size = 3; - constexpr uint64_t k_or_v_record_bytes_per_layer = 5; - constexpr uint64_t k_and_v_record_bytes_per_layer = - 2 * k_or_v_record_bytes_per_layer; - constexpr uint64_t kv_bytes_per_token = - layer_count * k_and_v_record_bytes_per_layer; - constexpr uint64_t weight_bytes = 37; - - for (uint64_t prompt_tokens = 1; prompt_tokens <= 12; ++prompt_tokens) { - for (uint64_t query_tile_tokens = 1; - query_tile_tokens <= prompt_tokens; ++query_tile_tokens) { - for (uint64_t block_tokens = 1; block_tokens <= 8; - ++block_tokens) { - const PrefillClosedFormContract closed = - resolve_prefill_closed_form_contract( - prompt_tokens, query_tile_tokens, block_tokens); - const std::vector tile_ends = enumerate_tile_ends( - prompt_tokens, query_tile_tokens); - const std::vector prefix_block_counts = - enumerate_prefix_block_counts( - prompt_tokens, query_tile_tokens, block_tokens); - const uint64_t enumerated_prefix_token_visits = - std::accumulate(tile_ends.begin(), tile_ends.end(), - uint64_t{0}); - const uint64_t enumerated_prefix_block_visits = - std::accumulate(prefix_block_counts.begin(), - prefix_block_counts.end(), uint64_t{0}); - EXPECT_EQ(tile_ends.size(), closed.tile_count) - << "P=" << prompt_tokens << " Q=" << query_tile_tokens - << " G=" << block_tokens; - EXPECT_EQ(enumerated_prefix_token_visits, - closed.attention_prefix_token_visits) - << "P=" << prompt_tokens << " Q=" << query_tile_tokens - << " G=" << block_tokens; - EXPECT_EQ(enumerated_prefix_block_visits, - closed.prefix_block_visits) - << "P=" << prompt_tokens << " Q=" << query_tile_tokens - << " G=" << block_tokens; - EXPECT_EQ(closed.lookups_per_layer_sequence, - closed.blocks_per_sequence + - 2 * enumerated_prefix_block_visits); - - const PrefillPayloadContract payload = - resolve_prefill_payload_contract( - weight_bytes, kv_bytes_per_token, batch_size, - prompt_tokens, query_tile_tokens); - const uint64_t expected_model_payload = - layer_count * batch_size * - k_and_v_record_bytes_per_layer * - (prompt_tokens + enumerated_prefix_token_visits); - EXPECT_EQ(payload.weight_passes, 1u); - EXPECT_EQ(payload.weight_read_bytes, weight_bytes); - EXPECT_EQ(payload.kv_only_bytes, expected_model_payload); - - const std::vector blocks = - enumerate_prefill_block_contracts( - prompt_tokens, query_tile_tokens, block_tokens, - k_or_v_record_bytes_per_layer); - uint64_t per_layer_sequence_model_payload = 0; - uint64_t per_layer_sequence_metadata = 0; - uint64_t per_layer_sequence_accounted = 0; - uint64_t per_layer_sequence_lookups = 0; - for (const PrefillBlockContract& block : blocks) { - per_layer_sequence_model_payload += block.model_payload_bytes; - per_layer_sequence_metadata += block.layout_metadata_bytes; - per_layer_sequence_accounted += block.accounted_bytes; - per_layer_sequence_lookups += block.semantic_lookups; - } - EXPECT_EQ(per_layer_sequence_model_payload * layer_count * batch_size, - expected_model_payload); - EXPECT_EQ(per_layer_sequence_lookups, - closed.lookups_per_layer_sequence); - EXPECT_EQ(per_layer_sequence_metadata * layer_count * batch_size, - sizeof(uint32_t) * layer_count * batch_size * - closed.lookups_per_layer_sequence); - EXPECT_EQ(per_layer_sequence_accounted * layer_count * batch_size, - expected_model_payload + - per_layer_sequence_metadata * layer_count * - batch_size); - - if (blocks.size() >= 2) { - const size_t boundary = choose_two_way_cost_boundary(blocks); - ASSERT_GT(boundary, 0u); - ASSERT_LT(boundary, blocks.size()); - uint64_t left = 0; - uint64_t right = 0; - for (size_t block = 0; block < blocks.size(); ++block) { - if (block < boundary) { - left += blocks[block].accounted_bytes; - } else { - right += blocks[block].accounted_bytes; - } - } - EXPECT_EQ(left + right, per_layer_sequence_accounted) - << "P=" << prompt_tokens << " Q=" << query_tile_tokens - << " G=" << block_tokens << " boundary=" << boundary; - const uint64_t chosen_doubled = 2 * left; - const uint64_t chosen_distance = - chosen_doubled > per_layer_sequence_accounted - ? chosen_doubled - per_layer_sequence_accounted - : per_layer_sequence_accounted - chosen_doubled; - uint64_t candidate_prefix = 0; - for (size_t candidate = 1; candidate < blocks.size(); - ++candidate) { - candidate_prefix += blocks[candidate - 1].accounted_bytes; - const uint64_t candidate_doubled = 2 * candidate_prefix; - const uint64_t candidate_distance = - candidate_doubled > per_layer_sequence_accounted - ? candidate_doubled - per_layer_sequence_accounted - : per_layer_sequence_accounted - candidate_doubled; - EXPECT_LE(chosen_distance, candidate_distance); - if (chosen_distance == candidate_distance) { - EXPECT_LE(boundary, candidate); - } - } - } - } - } - } -} - -TEST(LlmMemoryContractTest, - MetalSegmentCapacityAndPerPoolSlotCapAreFrozen) { - EXPECT_EQ(kCanonicalSegmentCapacityBytes, 268435456u); - EXPECT_EQ(kCanonicalSegmentSlotsPerPool, 256u); - EXPECT_EQ(kCanonicalPoolCapacityBytes, 68719476736ULL); - EXPECT_EQ(kCanonicalSegmentCapacityBytes / sizeof(uint32_t), - 67108864u); - constexpr std::array segmented_pools = { - "weights", "k", "v", "block_table"}; - constexpr std::array slots_per_pool = {256, 256, 256, 256}; - EXPECT_EQ(segmented_pools, - (std::array{ - "weights", "k", "v", "block_table"})); - for (size_t slots : slots_per_pool) { - EXPECT_EQ(slots, kCanonicalSegmentSlotsPerPool); - } - - EXPECT_EQ(contiguous_segment_lengths(kCanonicalSegmentCapacityBytes - 1), - (std::vector{kCanonicalSegmentCapacityBytes - 1})); - EXPECT_EQ(contiguous_segment_lengths(kCanonicalSegmentCapacityBytes), - (std::vector{kCanonicalSegmentCapacityBytes})); - EXPECT_EQ(contiguous_segment_lengths(kCanonicalSegmentCapacityBytes + 1), - (std::vector{kCanonicalSegmentCapacityBytes, 1})); - - const std::vector maximum_pool = - contiguous_segment_lengths(kCanonicalPoolCapacityBytes); - ASSERT_EQ(maximum_pool.size(), kCanonicalSegmentSlotsPerPool); - EXPECT_TRUE(std::all_of( - maximum_pool.begin(), maximum_pool.end(), [](uint64_t length) { - return length == kCanonicalSegmentCapacityBytes; - })); - EXPECT_EQ( - contiguous_segment_lengths(kCanonicalPoolCapacityBytes + 1).size(), - kCanonicalSegmentSlotsPerPool + 1); - EXPECT_TRUE(segment_count_fits_pool_slots(maximum_pool.size())); - EXPECT_FALSE(segment_count_fits_pool_slots( - contiguous_segment_lengths(kCanonicalPoolCapacityBytes + 1).size())); - - constexpr uint64_t block_bytes = 512; - constexpr uint64_t blocks_per_segment = - kCanonicalSegmentCapacityBytes / block_bytes; - constexpr uint64_t maximum_addressable_blocks = - kCanonicalSegmentSlotsPerPool * blocks_per_segment; - EXPECT_EQ(blocks_per_segment, 524288u); - EXPECT_EQ(maximum_addressable_blocks, 134217728u); - EXPECT_EQ(maximum_addressable_blocks * block_bytes, - kCanonicalPoolCapacityBytes); - EXPECT_EQ(ceil_divide_small(maximum_addressable_blocks, - blocks_per_segment), - kCanonicalSegmentSlotsPerPool); - EXPECT_EQ(ceil_divide_small(maximum_addressable_blocks + 1, - blocks_per_segment), - kCanonicalSegmentSlotsPerPool + 1); - EXPECT_TRUE(segment_count_fits_pool_slots(ceil_divide_small( - maximum_addressable_blocks, blocks_per_segment))); - EXPECT_FALSE(segment_count_fits_pool_slots(ceil_divide_small( - maximum_addressable_blocks + 1, blocks_per_segment))); - - constexpr uint64_t non_dividing_block_bytes = 96; - constexpr uint64_t non_dividing_blocks_per_segment = - kCanonicalSegmentCapacityBytes / non_dividing_block_bytes; - constexpr uint64_t non_dividing_maximum_pool = - kCanonicalSegmentSlotsPerPool * non_dividing_blocks_per_segment * - non_dividing_block_bytes; - EXPECT_LT(non_dividing_maximum_pool, kCanonicalPoolCapacityBytes); - EXPECT_EQ(kCanonicalPoolCapacityBytes - non_dividing_maximum_pool, - kCanonicalSegmentSlotsPerPool * - (kCanonicalSegmentCapacityBytes % - non_dividing_block_bytes)); - EXPECT_TRUE( - paged_block_fits_canonical_segment(kCanonicalSegmentCapacityBytes)); - EXPECT_FALSE(paged_block_fits_canonical_segment( - kCanonicalSegmentCapacityBytes + 1)); -} - -TEST(LlmMemoryContractTest, - ComponentIdentityCanonicalSerializationUsesFixedFieldOrder) { - static_assert(std::is_same_v< - decltype(ComponentIdentityContract::logical_profile_version), - std::string_view>); - static_assert(std::is_same_v< - decltype(ComponentIdentityContract::backend_executor_version), - std::string_view>); - static_assert(std::is_same_v< - decltype(ComponentIdentityContract::permutation_version), - std::optional>); - static_assert(std::is_same_v< - decltype(ComponentIdentityContract::msl_revision), - std::optional>); - const ComponentIdentityContract components = { - "l|=v", - "k=v", - "p|q", - "b", - "r", - "s", - "t", - "buf", - "write", - "sum", - "msl", - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef", - }; - EXPECT_EQ( - serialize_component_identity(components), - "llm-memory-components-v1" - "|logical_profile_version=4:l|=v" - "|kv_layout_version=3:k=v" - "|permutation_version=3:p|q" - "|backend_executor_version=1:b" - "|resource_abi_version=1:r" - "|schedule_version=1:s" - "|timer_policy_version=1:t" - "|buffer_pattern_version=3:buf" - "|write_pattern_version=5:write" - "|checksum_pattern_version=3:sum" - "|msl_revision=3:msl" - "|msl_source_sha256=64:" - "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"); - - ComponentIdentityContract cpu_components = components; - cpu_components.msl_revision.reset(); - cpu_components.msl_source_sha256.reset(); - const std::string cpu_identity = - serialize_component_identity(cpu_components); - const size_t msl_suffix = cpu_identity.find("|msl_revision="); - ASSERT_NE(msl_suffix, std::string::npos); - EXPECT_EQ(cpu_identity.substr(msl_suffix), - "|msl_revision=null|msl_source_sha256=null"); -} - -namespace { - -constexpr uint64_t kPrefillContractPhaseDomain = - 0x50524546494C4C31ULL; -constexpr uint64_t kPrefillContractOperationMultiplier = - 0x9E3779B97F4A7C15ULL; -constexpr uint64_t kPrefillContractLayerMultiplier = - 0xBF58476D1CE4E5B9ULL; -constexpr uint64_t kPrefillContractBatchMultiplier = - 0x94D049BB133111EBULL; -constexpr uint64_t kPrefillContractWordMultiplier = - 0xD6E8FEB86659FD93ULL; -constexpr uint64_t kPrefillContractKDomain = - 0x4B4B4B4B4B4B4B4BULL; -constexpr uint64_t kPrefillContractVDomain = - 0x5656565656565656ULL; - -enum class PrefillContractDomain { - K, - V, -}; - -enum class PrefillContractAccess { - Write, - Read, -}; - -struct PrefillContractTaskChecksum { - size_t exact_word_count = 0; - size_t even_word_count = 0; - size_t odd_word_count = 0; - uint64_t even_word_sum = 0; - uint64_t odd_word_sum = 0; -}; - -struct PrefillContractEvent { - PrefillContractAccess access; - PrefillContractDomain domain; - size_t tile_index; - size_t tile_end; - size_t block_index; - size_t visit_tokens; -}; - -uint64_t contract_prefill_affine_word( - uint64_t scenario_seed, uint64_t operation_ordinal, - uint64_t layer_index, uint64_t batch_sequence_index, - PrefillContractDomain domain, uint64_t logical_word_index) { - const uint64_t domain_term = - domain == PrefillContractDomain::K ? kPrefillContractKDomain - : kPrefillContractVDomain; - return scenario_seed + kPrefillContractPhaseDomain + - kPrefillContractOperationMultiplier * (operation_ordinal + 1) + - kPrefillContractLayerMultiplier * (layer_index + 1) + - kPrefillContractBatchMultiplier * - (batch_sequence_index + 1) + - domain_term + - kPrefillContractWordMultiplier * (logical_word_index + 1); -} - -PrefillContractTaskChecksum enumerate_contract_prefill_task_checksum( - uint64_t scenario_seed, size_t operation_count, - uint64_t layer_index, uint64_t batch_sequence_index, - PrefillContractDomain domain, size_t first_word, size_t word_count) { - PrefillContractTaskChecksum checksum; - for (size_t operation = 0; operation < operation_count; ++operation) { - for (size_t offset = 0; offset < word_count; ++offset) { - const size_t logical_word = first_word + offset; - const uint64_t value = contract_prefill_affine_word( - scenario_seed, operation, layer_index, batch_sequence_index, - domain, logical_word); - ++checksum.exact_word_count; - if ((logical_word & 1U) == 0) { - ++checksum.even_word_count; - checksum.even_word_sum += value; - } else { - ++checksum.odd_word_count; - checksum.odd_word_sum += value; - } - } - } - return checksum; -} - -std::vector enumerate_contract_prefill_block_trace( - size_t prompt_tokens, size_t query_tile_tokens, - size_t block_tokens, size_t block_count) { - std::vector events; - for (size_t block = 0; block < block_count; ++block) { - const size_t block_start = block * block_tokens; - const size_t valid_tokens = - std::min(block_tokens, prompt_tokens - block_start); - events.push_back({PrefillContractAccess::Write, - PrefillContractDomain::K, 0, prompt_tokens, - block, valid_tokens}); - events.push_back({PrefillContractAccess::Write, - PrefillContractDomain::V, 0, prompt_tokens, - block, valid_tokens}); - } - - size_t tile_index = 0; - size_t tile_end = 0; - while (tile_end < prompt_tokens) { - tile_end += std::min(query_tile_tokens, prompt_tokens - tile_end); - for (const PrefillContractDomain domain : - {PrefillContractDomain::K, PrefillContractDomain::V}) { - for (size_t block = 0; block < block_count; ++block) { - const size_t block_start = block * block_tokens; - if (block_start >= tile_end) { - break; - } - events.push_back({PrefillContractAccess::Read, domain, - tile_index, tile_end, block, - std::min(block_tokens, - tile_end - block_start)}); - } - } - ++tile_index; - } - return events; -} - -} // namespace - -TEST(LlmMemoryContractTest, - PrefillAffine64TwoOperationChecksumAndFinalOrdinalAreFrozen) { - constexpr uint64_t kScenarioSeed = 0x0123456789ABCDEFULL; - const PrefillContractTaskChecksum checksum = - enumerate_contract_prefill_task_checksum( - kScenarioSeed, 2, 2, 3, PrefillContractDomain::K, 1, 5); - EXPECT_EQ(checksum.exact_word_count, 10u); - EXPECT_EQ(checksum.even_word_count, 4u); - EXPECT_EQ(checksum.odd_word_count, 6u); - EXPECT_EQ(checksum.even_word_sum, 0xDC08129268383AB6ULL); - EXPECT_EQ(checksum.odd_word_sum, 0xCA0C1BDB9C545811ULL); - - EXPECT_EQ(contract_prefill_affine_word( - kScenarioSeed, 0, 2, 3, PrefillContractDomain::K, 1), - 0x7A144A570DB4D57DULL); - const uint64_t final_word = contract_prefill_affine_word( - kScenarioSeed, 1, 2, 3, PrefillContractDomain::K, 1); - EXPECT_EQ(final_word, 0x184BC4108CFF5192ULL); - constexpr std::array kFinalLittleEndianBytes = { - 0x92, 0x51, 0xFF, 0x8C, 0x10, 0xC4, 0x4B, 0x18}; - for (size_t byte = 0; byte < kFinalLittleEndianBytes.size(); ++byte) { - EXPECT_EQ(static_cast(final_word >> (byte * 8)), - kFinalLittleEndianBytes[byte]); - } -} - -TEST(LlmMemoryContractTest, - PrefillTwoBlockTwoTileOwnerLocalSemanticTraceIsFrozen) { - const std::vector trace = - enumerate_contract_prefill_block_trace(4, 2, 2, 2); - const std::array expected = {{ - {PrefillContractAccess::Write, PrefillContractDomain::K, 0, 4, 0, 2}, - {PrefillContractAccess::Write, PrefillContractDomain::V, 0, 4, 0, 2}, - {PrefillContractAccess::Write, PrefillContractDomain::K, 0, 4, 1, 2}, - {PrefillContractAccess::Write, PrefillContractDomain::V, 0, 4, 1, 2}, - {PrefillContractAccess::Read, PrefillContractDomain::K, 0, 2, 0, 2}, - {PrefillContractAccess::Read, PrefillContractDomain::V, 0, 2, 0, 2}, - {PrefillContractAccess::Read, PrefillContractDomain::K, 1, 4, 0, 2}, - {PrefillContractAccess::Read, PrefillContractDomain::K, 1, 4, 1, 2}, - {PrefillContractAccess::Read, PrefillContractDomain::V, 1, 4, 0, 2}, - {PrefillContractAccess::Read, PrefillContractDomain::V, 1, 4, 1, 2}, - }}; - ASSERT_EQ(trace.size(), expected.size()); - for (size_t index = 0; index < expected.size(); ++index) { - SCOPED_TRACE(index); - EXPECT_EQ(trace[index].access, expected[index].access); - EXPECT_EQ(trace[index].domain, expected[index].domain); - EXPECT_EQ(trace[index].tile_index, expected[index].tile_index); - EXPECT_EQ(trace[index].tile_end, expected[index].tile_end); - EXPECT_EQ(trace[index].block_index, expected[index].block_index); - EXPECT_EQ(trace[index].visit_tokens, - expected[index].visit_tokens); - } -} diff --git a/tests/test_llm_memory_executor.cpp b/tests/test_llm_memory_executor.cpp index 1d40bce..64ece49 100644 --- a/tests/test_llm_memory_executor.cpp +++ b/tests/test_llm_memory_executor.cpp @@ -540,30 +540,10 @@ bool decode_corrupting_kernel(void* opaque, const LlmKernelInvocation& invocatio bool throwing_kernel(void*, const LlmKernelInvocation&) { throw std::runtime_error("injected kernel exception"); } -enum class PagedCorruption { - Append, - Padding, -}; - -struct PagedCorruptionContext { - PagedCorruption kind = PagedCorruption::Append; - bool corrupted = false; -}; - struct PrefillCorruptionContext { std::atomic corrupted{false}; }; -enum class PagedPrefillCorruption { - FinalWrite, - Padding, -}; - -struct PagedPrefillCorruptionContext { - PagedPrefillCorruption kind = PagedPrefillCorruption::FinalWrite; - std::atomic corrupted{false}; -}; - bool prefill_corrupting_kernel(void* opaque, const LlmKernelInvocation& invocation) { auto& context = *static_cast(opaque); @@ -599,113 +579,6 @@ bool prefill_corrupting_kernel(void* opaque, return true; } -bool paged_prefill_corrupting_kernel( - void* opaque, const LlmKernelInvocation& invocation) { - auto& context = *static_cast(opaque); - if (invocation.phase != LlmPhase::Prefill || - invocation.kv_layout != LlmKvLayout::Paged || - invocation.paged_prefill_layers == nullptr || - invocation.paged_prefill_assignments == nullptr || - invocation.output == nullptr) { - return false; - } - llm_prefill_memory_paged_asm( - invocation.paged_prefill_layers, - invocation.paged_prefill_assignments, invocation.layer_count, - invocation.work_unit_count, invocation.scenario_flags, - invocation.scenario_seed, invocation.output); - - bool expected = false; - if (!context.corrupted.compare_exchange_strong( - expected, true, std::memory_order_acq_rel)) { - return true; - } - for (size_t local_layer = 0; local_layer < invocation.layer_count; - ++local_layer) { - const LlmPagedPrefillLayerDescriptor& layer = - invocation.paged_prefill_layers[local_layer]; - for (size_t local_assignment = 0; - local_assignment < layer.assignment_count; ++local_assignment) { - const LlmPagedPrefillKvAssignmentDescriptor& assignment = - invocation.paged_prefill_assignments[ - layer.first_assignment_index + local_assignment]; - if (assignment.owned_block_count == 0) { - continue; - } - size_t logical_block = assignment.first_logical_block; - size_t block_byte = assignment.block_bytes > 1 ? 1 : 0; - if (context.kind == PagedPrefillCorruption::Padding) { - const size_t terminal_block = assignment.blocks_per_sequence - 1; - const size_t assignment_end = - assignment.first_logical_block + assignment.owned_block_count; - if (terminal_block < assignment.first_logical_block || - terminal_block >= assignment_end || - assignment.last_block_valid_bytes >= assignment.block_bytes) { - continue; - } - logical_block = terminal_block; - block_byte = assignment.last_block_valid_bytes; - } - const uint32_t physical_id = - assignment.block_table_row[logical_block]; - assignment.k_layer_pool[ - static_cast(physical_id) * assignment.block_bytes + - block_byte] ^= 0x80; - return true; - } - } - context.corrupted.store(false, std::memory_order_release); - return false; -} - -bool paged_corrupting_kernel(void* opaque, - const LlmKernelInvocation& invocation) { - auto& context = *static_cast(opaque); - if (invocation.kv_layout != LlmKvLayout::Paged || - invocation.paged_layers == nullptr || - invocation.paged_assignments == nullptr || invocation.output == nullptr) { - return false; - } - llm_decode_memory_paged_asm( - invocation.paged_layers, invocation.paged_assignments, - invocation.layer_count, invocation.work_unit_count, - invocation.scenario_flags, invocation.scenario_seed, - invocation.output); - if (context.corrupted) { - return true; - } - for (size_t layer = 0; layer < invocation.layer_count; ++layer) { - const LlmPagedLayerDescriptor& layer_descriptor = - invocation.paged_layers[layer]; - for (size_t local = 0; local < layer_descriptor.assignment_count; - ++local) { - const LlmPagedKvAssignmentDescriptor& assignment = - invocation.paged_assignments[ - layer_descriptor.first_assignment_index + local]; - if (assignment.owned_block_count == 0 || - assignment.first_logical_block + assignment.owned_block_count != - assignment.blocks_per_sequence) { - continue; - } - const uint32_t physical_id = - assignment.block_table_row[assignment.blocks_per_sequence - 1]; - uint8_t* const block = - assignment.k_layer_pool + physical_id * assignment.block_bytes; - const size_t byte_offset = - context.kind == PagedCorruption::Append - ? assignment.decode_append_offset - : assignment.last_block_valid_bytes; - if (byte_offset >= assignment.block_bytes) { - return false; - } - block[byte_offset] ^= 0x80; - context.corrupted = true; - return true; - } - } - return false; -} - void observe_executor_event(void* opaque, LlmExecutorEvent event, size_t) noexcept { auto& context = *static_cast(opaque); std::lock_guard lock(context.mutex); @@ -872,9 +745,6 @@ TEST_F(LlmMemoryExecutorTest, PreparationMaterializesExactDescriptorsPatternsAnd EXPECT_EQ(actual_v.span_bytes, v_reference.span_bytes); } - EXPECT_EQ(cpu_plan.workers[0].sequences[0].append_record_byte_offset, 0u); - EXPECT_EQ(cpu_plan.workers[1].sequences[0].append_record_byte_offset, - 13u); EXPECT_EQ(resources.worker_sequences(1)[0].k_append_bytes, 6u); const size_t weight_slot = FakeMemorySystemCallState::kSlotSize; @@ -1036,32 +906,9 @@ TEST_F(LlmMemoryExecutorTest, } } -TEST_F(LlmMemoryExecutorTest, - PrefillStructuralValidationRejectsShortScenarioDescriptorSet) { - LlmMemoryWorkPlan plan = - build_executor_ready_plan(prefill_geometry(), 2); - ASSERT_TRUE(plan.valid) << plan.reason_code; - LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(plan); - ASSERT_NE(cpu_plan, nullptr); - ASSERT_TRUE(cpu_plan->prefill.has_value()); - ASSERT_FALSE(cpu_plan->workers[0].prefill_sequences.empty()); - cpu_plan->workers[0].prefill_sequences.pop_back(); - const size_t map_calls_before = state.map_calls; - - LlmExecutionResources resources; - const LlmResourcePreparationResult result = - prepare_llm_execution_resources(plan, resources); - EXPECT_FALSE(result.valid); - EXPECT_EQ(result.reason_code, LlmExecutorReason::INVALID_WORK_PLAN); - EXPECT_EQ(state.map_calls, map_calls_before); - EXPECT_FALSE(resources.valid); - EXPECT_EQ(resources.prefill_layer_descriptors, nullptr); - EXPECT_EQ(resources.prefill_sequence_descriptors, nullptr); -} - TEST_F(LlmMemoryExecutorTest, PrefillCanonicalOwnershipAndIdentityTamperingIsRejectedBeforeMmap) { - for (size_t tamper = 0; tamper < 8; ++tamper) { + for (size_t tamper = 0; tamper < 9; ++tamper) { SCOPED_TRACE(::testing::Message() << "tamper=" << tamper); LlmMemoryWorkPlan plan = build_executor_ready_plan(prefill_geometry(), 2); @@ -1132,6 +979,9 @@ TEST_F(LlmMemoryExecutorTest, cpu_plan->prefill->identity += "|tampered=1"; } else if (tamper == 5) { plan.plan_identity += "|tampered=1"; + } else if (tamper == 8) { + ASSERT_FALSE(cpu_plan->workers[0].prefill_sequences.empty()); + cpu_plan->workers[0].prefill_sequences.pop_back(); } else { const size_t scenario_base = static_cast(LlmScenario::KvOnly) * rows; @@ -1153,6 +1003,8 @@ TEST_F(LlmMemoryExecutorTest, EXPECT_EQ(result.reason_code, LlmExecutorReason::INVALID_WORK_PLAN); EXPECT_EQ(state.map_calls, map_calls_before); EXPECT_FALSE(resources.valid); + EXPECT_EQ(resources.prefill_layer_descriptors, nullptr); + EXPECT_EQ(resources.prefill_sequence_descriptors, nullptr); } } @@ -1216,16 +1068,6 @@ TEST_F(LlmMemoryExecutorTest, plan.geometry.layer_count * plan.geometry.batch_size; ASSERT_EQ(cpu_plan.sequence_descriptors_per_worker, rows * kLlmScenarioCount); - EXPECT_EQ(cpu_plan.descriptor_bytes, - cpu_plan.total_layer_descriptors * - sizeof(LlmPagedPrefillLayerDescriptor) + - cpu_plan.total_sequence_descriptors * - sizeof(LlmPagedPrefillKvAssignmentDescriptor)); - EXPECT_EQ(estimate.static_reference_bytes, - (cpu_plan.total_layer_descriptors + - 2 * paged.layout.total_physical_blocks) * - sizeof(LlmStaticSpanReference)); - LlmExecutionResources resources; const LlmResourcePreparationResult prepared = prepare_llm_execution_resources(plan, resources); @@ -1477,56 +1319,6 @@ TEST_F(LlmMemoryExecutorTest, } } -TEST_F(LlmMemoryExecutorTest, - PagedPrefillProductionAllScenariosOneAndThreeWorkersIntegration) { - for (size_t workers : {1U, 3U}) { - for (LlmScenario scenario_kind : {LlmScenario::WeightsOnly, - LlmScenario::KvOnly, - LlmScenario::Mixed}) { - SCOPED_TRACE(::testing::Message() - << "workers=" << workers - << " scenario=" << llm_scenario_to_string(scenario_kind)); - const LlmMemoryWorkPlan plan = build_executor_ready_plan( - paged_prefill_geometry(5, 2, 2, 8, 2, 2), workers); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); - ASSERT_TRUE(cpu_plan.prefill.has_value()); - ASSERT_TRUE(cpu_plan.paged.has_value()); - ASSERT_EQ(cpu_plan.effective_workers, workers); - - LlmExecutionResources resources; - const LlmResourcePreparationResult prepared = - prepare_llm_execution_resources(plan, resources); - ASSERT_TRUE(prepared.valid) << prepared.reason_code; - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, scenario_kind, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - - ScopedExecutorTimer timer_scope; - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - const LlmExecutorResult executed = - execute_llm_scenario(plan, scenario, resources, *timer); - ASSERT_TRUE(executed.valid) << executed.reason_code; - EXPECT_TRUE(executed.kernel_succeeded); - EXPECT_TRUE(executed.checksum_evaluated); - EXPECT_TRUE(executed.checksum_valid); - EXPECT_TRUE(executed.post_validation_evaluated); - EXPECT_TRUE(executed.post_validation_valid); - ASSERT_EQ(executed.actual_checksums.size(), workers); - uint64_t weight_bytes = 0; - uint64_t kv_bytes = 0; - for (const LlmWorkerChecksum& checksum : executed.actual_checksums) { - weight_bytes += checksum.weight.exact_bytes_read; - kv_bytes += checksum.k.exact_bytes_read + - checksum.v.exact_bytes_read; - } - EXPECT_EQ(weight_bytes, scenario.weight_read_bytes); - EXPECT_EQ(kv_bytes, scenario.kv_read_bytes); - } - } -} - TEST_F(LlmMemoryExecutorTest, PagedPrefillMultiTokenTerminalBlockKeepsDecodeOffsetInapplicableIntegration) { const LlmMemoryWorkPlan plan = build_executor_ready_plan( @@ -1555,48 +1347,6 @@ TEST_F(LlmMemoryExecutorTest, EXPECT_TRUE(executed.post_validation_valid); } -TEST_F(LlmMemoryExecutorTest, - PagedPrefillFinalWriteAndPaddingCorruptionFailPostValidationIntegration) { - for (PagedPrefillCorruption corruption : { - PagedPrefillCorruption::FinalWrite, - PagedPrefillCorruption::Padding}) { - SCOPED_TRACE(corruption == PagedPrefillCorruption::FinalWrite - ? "final-write-boundary-word-interior" - : "padding"); - const LlmMemoryWorkPlan plan = build_executor_ready_plan( - paged_prefill_geometry(5, 2, 4, 8, 2, 2), 1); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_LT(plan.geometry.last_block_valid_bytes, - plan.geometry.kv_block_bytes); - LlmExecutionResources resources; - const LlmResourcePreparationResult prepared = - prepare_llm_execution_resources(plan, resources); - ASSERT_TRUE(prepared.valid) << prepared.reason_code; - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - PagedPrefillCorruptionContext context; - context.kind = corruption; - ScopedExecutorTimer timer_scope; - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - - const LlmExecutorResult result = execute_llm_scenario( - plan, scenario, resources, *timer, - {paged_prefill_corrupting_kernel, &context}); - - EXPECT_TRUE(context.corrupted.load(std::memory_order_acquire)); - EXPECT_FALSE(result.valid); - EXPECT_EQ(result.reason_code, - LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED); - EXPECT_TRUE(result.kernel_succeeded); - EXPECT_TRUE(result.checksum_evaluated); - EXPECT_TRUE(result.checksum_valid); - EXPECT_TRUE(result.post_validation_evaluated); - EXPECT_FALSE(result.post_validation_valid); - } -} - TEST_F(LlmMemoryExecutorTest, PagedPrefillStructuralAndIdentityTamperingIsRejectedBeforeMmap) { for (size_t tamper = 0; tamper < 13; ++tamper) { @@ -1883,88 +1633,6 @@ TEST_F(LlmMemoryExecutorTest, } } -TEST_F(LlmMemoryExecutorTest, - PagedAppendAndPaddingCorruptionFailExcludedPostValidationIntegration) { - for (PagedCorruption corruption : {PagedCorruption::Append, - PagedCorruption::Padding}) { - SCOPED_TRACE(corruption == PagedCorruption::Append ? "append" - : "padding"); - const LlmMemoryWorkPlan plan = - build_executor_ready_plan(paged_geometry(5, 4), 1); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_LT(plan.geometry.last_block_valid_bytes, - plan.geometry.kv_block_bytes); - LlmExecutionResources resources; - const LlmResourcePreparationResult prepared = - prepare_llm_execution_resources(plan, resources); - ASSERT_TRUE(prepared.valid) << prepared.reason_code; - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::KvOnly, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - PagedCorruptionContext context{corruption, false}; - ScopedExecutorTimer timer_scope; - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - const LlmExecutorResult result = execute_llm_scenario( - plan, scenario, resources, *timer, - {paged_corrupting_kernel, &context}); - EXPECT_TRUE(context.corrupted); - EXPECT_FALSE(result.valid); - EXPECT_EQ(result.reason_code, - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - EXPECT_TRUE(result.checksum_evaluated); - EXPECT_TRUE(result.checksum_valid); - EXPECT_TRUE(result.post_validation_evaluated); - EXPECT_FALSE(result.post_validation_valid); - } -} - -TEST_F(LlmMemoryExecutorTest, - PagedWeightsOnlyRejectsUnexpectedValidKvWriteIntegration) { - const LlmMemoryWorkPlan plan = - build_executor_ready_plan(paged_geometry(5, 4), 1); - ASSERT_TRUE(plan.valid) << plan.reason_code; - LlmExecutionResources resources; - const LlmResourcePreparationResult prepared = - prepare_llm_execution_resources(plan, resources); - ASSERT_TRUE(prepared.valid) << prepared.reason_code; - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::WeightsOnly, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - PagedCorruptionContext context{PagedCorruption::Append, false}; - ScopedExecutorTimer timer_scope; - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - - const LlmExecutorResult result = execute_llm_scenario( - plan, scenario, resources, *timer, - {paged_corrupting_kernel, &context}); - - EXPECT_TRUE(context.corrupted); - EXPECT_FALSE(result.valid); - EXPECT_EQ(result.reason_code, - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - EXPECT_TRUE(result.checksum_evaluated); - EXPECT_TRUE(result.checksum_valid); - EXPECT_TRUE(result.post_validation_evaluated); - EXPECT_FALSE(result.post_validation_valid); -} - -TEST_F(LlmMemoryExecutorTest, StructuralValidationRejectsDivergentVMappingBeforeMmap) { - LlmMemoryWorkPlan plan = build_executor_ready_plan({64, 1, 1, 1, 19, 1, 2, 1}, 2); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_GT(plan.geometry.v_mapping_bytes, 1u); - --plan.geometry.v_mapping_bytes; - --plan.geometry.total_data_mapping_bytes; - - LlmBufferSet buffers; - const LlmBufferAllocationResult result = allocate_llm_buffers(plan, buffers); - EXPECT_FALSE(result.valid); - EXPECT_EQ(result.reason_code, LlmExecutorReason::INVALID_WORK_PLAN); - EXPECT_EQ(state.map_calls, 0u); - EXPECT_FALSE(buffers.complete()); -} - TEST_F(LlmMemoryExecutorTest, PagedStructuralValidationRejectsShortReferenceGeometryBeforeMmap) { LlmMemoryWorkPlan plan = @@ -2449,43 +2117,54 @@ TEST_F(LlmMemoryExecutorTest, PagedColdCheckCompletionIntegration) { *static_cast(opaque)->byte ^= 0x80; return true; }; + ScopedExecutorTimer timer_scope; for (bool prefill : {false, true}) { for (bool final_row : {false, true}) { for (bool padding : {false, true}) { - for (auto scenario : {LlmScenario::KvOnly, LlmScenario::WeightsOnly}) { - if (prefill && !padding && scenario == LlmScenario::WeightsOnly) continue; - // Prior case resources have left scope; recycle the bounded fake mapping arena. - state = {}; - const auto plan = build_executor_ready_plan(prefill ? paged_prefill_geometry() : paged_geometry(5, 4), 1); - ASSERT_TRUE(plan.valid); - LlmExecutionResources resources; - ASSERT_TRUE(prepare_llm_execution_resources(plan, resources).valid); - const auto& layout = get_llm_cpu_execution_plan(plan)->paged->layout; - const size_t layer = final_row ? layout.layer_count - 1 : 0; - const size_t batch = final_row ? layout.batch_size - 1 : 0; - const size_t logical_block = padding || !prefill ? layout.blocks_per_sequence - 1 : 0; - const size_t physical = resources.block_table[batch * layout.blocks_per_sequence + logical_block]; - const size_t offset = (layer * layout.physical_blocks_per_layer + physical) * layout.block_bytes + - (padding ? layout.last_block_valid_bytes : prefill ? 0 : layout.decode_append_offset_in_last_block); - Mutation mutation{static_cast(resources.buffers.k.get()) + offset}; - const auto task = build_llm_scenario_work_plan(plan, scenario, 1, true); - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - const auto result = execute_llm_scenario(plan, task, resources, *timer, {mutate, &mutation}); - EXPECT_FALSE(result.valid); - EXPECT_TRUE(result.checksum_valid); - const size_t failed = padding ? 2 : 1; - EXPECT_TRUE(result.cold_checks[failed].evaluated); - EXPECT_FALSE(result.cold_checks[failed].valid); - if (padding) { - EXPECT_EQ(result.cold_checks[0].evaluated, final_row); - EXPECT_EQ(result.cold_checks[1].evaluated, final_row && (!prefill || scenario != LlmScenario::WeightsOnly)); - EXPECT_EQ(result.cold_checks[0].valid, final_row); - EXPECT_EQ(result.cold_checks[1].valid, final_row && (!prefill || scenario != LlmScenario::WeightsOnly)); - } else { - EXPECT_FALSE(result.cold_checks[2].evaluated); - EXPECT_EQ(result.cold_checks[0].evaluated, !prefill && final_row); - } + for (auto scenario : {LlmScenario::KvOnly, LlmScenario::WeightsOnly, LlmScenario::Mixed}) { + if (prefill && !padding && scenario == LlmScenario::WeightsOnly) continue; + // Prior case resources have left scope; recycle the bounded fake mapping arena. + state = {}; + const auto plan = build_executor_ready_plan(prefill ? paged_prefill_geometry() : paged_geometry(5, 4), 1); + ASSERT_TRUE(plan.valid); + LlmExecutionResources resources; + ASSERT_TRUE(prepare_llm_execution_resources(plan, resources).valid); + const auto& layout = get_llm_cpu_execution_plan(plan)->paged->layout; + const size_t layer = final_row ? layout.layer_count - 1 : 0; + const size_t batch = final_row ? layout.batch_size - 1 : 0; + const size_t logical_block = padding || !prefill ? layout.blocks_per_sequence - 1 : 0; + const size_t physical = resources.block_table[batch * layout.blocks_per_sequence + logical_block]; + const size_t offset = (layer * layout.physical_blocks_per_layer + physical) * layout.block_bytes + + (padding ? layout.last_block_valid_bytes + : prefill ? 1 + : layout.decode_append_offset_in_last_block); + Mutation mutation{static_cast(resources.buffers.k.get()) + offset}; + const auto task = build_llm_scenario_work_plan(plan, scenario, 2, true); + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + const auto result = execute_llm_scenario(plan, task, resources, *timer, {mutate, &mutation}); + SCOPED_TRACE(::testing::Message() << "prefill=" << prefill << " final_row=" << final_row + << " padding=" << padding << " scenario=" << static_cast(scenario)); + EXPECT_FALSE(result.valid); + EXPECT_EQ(result.reason_code, prefill ? LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED + : LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); + EXPECT_TRUE(result.kernel_succeeded); + EXPECT_TRUE(result.checksum_evaluated); + EXPECT_TRUE(result.post_validation_evaluated); + EXPECT_FALSE(result.post_validation_valid); + EXPECT_TRUE(result.checksum_valid); + const size_t failed = padding ? 2 : 1; + EXPECT_TRUE(result.cold_checks[failed].evaluated); + EXPECT_FALSE(result.cold_checks[failed].valid); + if (padding) { + EXPECT_EQ(result.cold_checks[0].evaluated, final_row); + EXPECT_EQ(result.cold_checks[1].evaluated, final_row && (!prefill || scenario != LlmScenario::WeightsOnly)); + EXPECT_EQ(result.cold_checks[0].valid, final_row); + EXPECT_EQ(result.cold_checks[1].valid, final_row && (!prefill || scenario != LlmScenario::WeightsOnly)); + } else { + EXPECT_FALSE(result.cold_checks[2].evaluated); + EXPECT_EQ(result.cold_checks[0].evaluated, !prefill && final_row); + } } } } diff --git a/tests/test_llm_memory_json.cpp b/tests/test_llm_memory_json.cpp index ed8754e..99d15ce 100644 --- a/tests/test_llm_memory_json.cpp +++ b/tests/test_llm_memory_json.cpp @@ -791,7 +791,6 @@ TEST(LlmMemoryJsonTest, MetalDocumentPublishesSegmentationBackendTaskAndCompactE ASSERT_EQ(document["measurements"].size(), 1U); EXPECT_TRUE(document["measurements"][0]["execution"]["requested_workers"].is_null()); - EXPECT_TRUE(document["resolved_plan"]["model_work_plan"]["effective_workers"].is_null()); EXPECT_TRUE(document["measurements"][0]["execution"]["qos_successful_workers"].is_null()); EXPECT_TRUE(document["measurements"][0]["execution"]["qos_failed_workers"].is_null()); const OrderedJson& execution = document["measurements"][0]["execution"]; @@ -837,109 +836,11 @@ TEST(LlmMemoryJsonTest, MetalDocumentPublishesSegmentationBackendTaskAndCompactE EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); } -TEST(LlmMemoryJsonTest, - MetalPrefillContiguousPublishesGenericKvWriteEvidence) { - const LlmMemoryConfig config = explicit_metal_prefill_config(); - const LlmMemoryWorkPlan plan = admitted_metal_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(plan.phase, LlmPhase::Prefill); - ASSERT_EQ(plan.kv_layout, LlmKvLayout::Contiguous); - ASSERT_TRUE(plan.prefill_plan.has_value()); - const LlmMetalExecutionPlan* const metal_plan = - get_llm_metal_execution_plan(plan); - ASSERT_NE(metal_plan, nullptr); - ASSERT_TRUE(metal_plan->valid) << metal_plan->reason_code; - - const LlmBackendEvidence backend = - complete_metal_backend_evidence(plan); - const LlmMemoryResult result = - metal_result_with_measurement_and_calibration(plan); - const OrderedJson document = build_llm_memory_json( - config, plan, backend, fixed_metadata(config, plan), result); - - EXPECT_EQ(document["backend"], "metal"); - EXPECT_EQ(document["phase"], "prefill"); - EXPECT_EQ(document["kv_layout"], "contiguous"); - EXPECT_EQ(document["resolved_plan"]["work_unit_kind"], - "prefill_operation"); - EXPECT_EQ(document["methodology_version"], - "llm-memory-v2-metal-prefill-contiguous"); - EXPECT_EQ( - document["resolved_plan"]["methodology"] - ["maximum_serial_range_visits_per_lane_per_task"], - Constants::LLM_METAL_MAX_SERIAL_RANGE_VISITS_PER_LANE_PER_TASK); - EXPECT_EQ(document["configuration"]["prompt_tokens"], 5u); - EXPECT_EQ(document["configuration"]["attention_query_tile_tokens"], - 2u); - const OrderedJson& geometry = - document["resolved_plan"]["geometry"]; - EXPECT_TRUE(geometry["decode"].is_null()); - ASSERT_TRUE(geometry["prefill"].is_object()); - EXPECT_EQ(geometry["prefill"]["prompt_tokens"], 5u); - EXPECT_EQ(geometry["prefill"]["attention_query_tile_tokens"], 2u); - EXPECT_EQ(geometry["prefill"]["tile_count"], "3"); - EXPECT_EQ( - geometry["prefill"]["attention_prefix_token_visits_per_sequence"], - "11"); - - const OrderedJson& components = - document["resolved_plan"]["component_identities"]; - EXPECT_EQ(components["logical_profile_version"], - Constants::LLM_PREFILL_LOGICAL_PROFILE_VERSION); - EXPECT_EQ(components["kv_layout_version"], - Constants::LLM_CONTIGUOUS_KV_LAYOUT_VERSION); - EXPECT_TRUE(components["permutation_version"].is_null()); - EXPECT_EQ(components["backend_executor_version"], - LlmMetalPrefillContiguousVersion::EXECUTOR); - EXPECT_EQ(components["schedule_version"], - LlmMetalPrefillContiguousVersion::SCHEDULE); - EXPECT_EQ(components["write_pattern_version"], - LlmMetalPrefillContiguousVersion::WRITE_PATTERN); - EXPECT_EQ(document["resolved_plan"]["resources"]["metal"] - ["table_segments"], - nullptr); - EXPECT_TRUE(document["backend_evidence"]["cpu"].is_null()); - EXPECT_EQ(document["backend_evidence"]["metal"] - ["workload_pipelines"][2]["label"], - "membenchmark.llm-metal.pipeline.prefill-contiguous.mixed"); - - ASSERT_EQ(document["measurements"].size(), 1u); - const OrderedJson& measurement = document["measurements"][0]; - EXPECT_EQ(referenced_plan(document, measurement)["work_unit_kind"], "prefill_operation"); - EXPECT_EQ(referenced_plan(document, measurement)["kv_write_kind"], "full_prompt_population"); - const OrderedJson& task = measurement["execution"]["metal"]; - ASSERT_TRUE(task.is_object()) << task.dump(2); - EXPECT_EQ(task["pipeline"]["label"], - "membenchmark.llm-metal.pipeline.prefill-contiguous.mixed"); - EXPECT_EQ(task["grid"]["serial_range_visits_per_lane"], "34"); - EXPECT_TRUE(task["grid"]["cost_unit"].is_null()); - EXPECT_TRUE(task["grid"]["threadgroup_accounted_bytes"].empty()); - - const std::string serialized = document.dump(); - EXPECT_EQ(serialized.find("tokens/s"), std::string::npos); - EXPECT_EQ(serialized.find("tokens_per_second"), std::string::npos); - const auto& checks = document["measurements"][0]["execution"]; - EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["applicable"]); - EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["evaluated"]); - EXPECT_TRUE(named_check(checks, "kv-prefill-final-samples")["valid"]); - const auto& padding = named_check(checks, "kv-padding-canary"); - EXPECT_FALSE(padding["applicable"]); - EXPECT_TRUE(padding["evaluated"].is_null()); - EXPECT_TRUE(padding["valid"].is_null()); - EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); -} - TEST(LlmMemoryJsonTest, MetalPrefillPagedPublishesCompleteProfileAndCyclicGridEvidence) { const LlmMemoryConfig config = explicit_metal_paged_prefill_config(); const LlmMemoryWorkPlan plan = admitted_metal_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(plan.phase, LlmPhase::Prefill); - ASSERT_EQ(plan.kv_layout, LlmKvLayout::Paged); - ASSERT_TRUE(plan.prefill_plan.has_value()); - EXPECT_EQ(plan.prefill_plan->layout_metadata_lookups_per_layer_sequence, - 15U); // N + 2M = 3 + 2 * 6. - const LlmMetalExecutionPlan* const metal_plan = get_llm_metal_execution_plan(plan); ASSERT_NE(metal_plan, nullptr); @@ -1127,86 +1028,6 @@ TEST(LlmMemoryJsonTest, EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); } -TEST(LlmMemoryJsonTest, MetalPagedDocumentPublishesLayoutTableLookupAndPaddingEvidence) { - const LlmMemoryConfig config = explicit_metal_paged_config(); - const LlmMemoryWorkPlan plan = admitted_metal_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmMetalExecutionPlan* const metal_plan = get_llm_metal_execution_plan(plan); - ASSERT_NE(metal_plan, nullptr); - ASSERT_TRUE(metal_plan->valid) << metal_plan->reason_code; - ASSERT_TRUE(metal_plan->resources.paged_layout.has_value()); - ASSERT_TRUE(metal_plan->resources.table_segments.has_value()); - - const LlmBackendEvidence backend = complete_metal_backend_evidence(plan); - const LlmMemoryResult result = metal_result_with_measurement_and_calibration(plan); - const OrderedJson document = - build_llm_memory_json(config, plan, backend, fixed_metadata(config, plan), result); - - EXPECT_EQ(document["backend"], "metal"); - EXPECT_EQ(document["phase"], "decode"); - EXPECT_EQ(document["kv_layout"], "paged"); - EXPECT_EQ(document["methodology_version"], "llm-memory-v2-metal-decode-paged"); - EXPECT_EQ(document["resolved_plan"]["component_identities"]["msl_revision"], - "llm-metal-decode-prefill-contiguous-paged-msl23-v5"); - EXPECT_EQ(document["configuration"]["resolved_sources"]["kv_layout"], "explicit"); - EXPECT_EQ(document["configuration"]["resolved_sources"]["kv_block_tokens"], "explicit"); - - const OrderedJson& layout = document["resolved_plan"]["layout"]; - EXPECT_EQ(layout["kv_layout"], "paged"); - EXPECT_EQ(layout["kv_block_tokens"], 2U); - EXPECT_EQ(layout["blocks_per_sequence"], "2"); - EXPECT_EQ(layout["physical_blocks_per_layer"], "2"); - EXPECT_EQ(layout["total_physical_blocks"], "4"); - EXPECT_EQ(layout["block_bytes"], "64"); - EXPECT_EQ(layout["last_block_tokens"], "1"); - EXPECT_EQ(layout["last_block_valid_bytes"], "32"); - EXPECT_EQ(layout["block_table_entries"], "2"); - EXPECT_EQ(layout["block_table_bytes"], "8"); - EXPECT_FALSE(layout["layout_geometry_identity"].is_null()); - EXPECT_FALSE(layout["layout_identity"].is_null()); - EXPECT_EQ(layout["permutation_entry_count"], "2"); - ASSERT_TRUE(layout["permutation_sha256"].is_string()); - EXPECT_EQ(layout["permutation_sha256"].get().size(), 64U); - EXPECT_FALSE(layout["permutation_identity"].is_null()); - - const OrderedJson& resources = document["resolved_plan"]["resources"]["metal"]; - EXPECT_EQ(resources["k_segments"]["element_bytes"], "64"); - EXPECT_EQ(resources["k_segments"]["total_length_bytes"], "256"); - EXPECT_EQ(resources["v_segments"]["total_length_bytes"], "256"); - EXPECT_EQ(resources["table_segments"]["element_bytes"], "4"); - EXPECT_EQ(resources["table_segments"]["element_count"], "2"); - EXPECT_EQ(resources["table_segments"]["total_length_bytes"], "8"); - EXPECT_EQ(resources["staging_buffer_length_bytes"], "8"); - - const OrderedJson& metal_resources = document["backend_evidence"]["metal"]["resources"]; - EXPECT_EQ(document["backend_evidence"]["metal"]["capability"]["kernel_revision"], - "llm-metal-decode-prefill-contiguous-paged-msl23-v5"); - EXPECT_EQ(document["backend_evidence"]["metal"]["capability"]["kernel_source_sha256"], - canonical_llm_metal_kernel_source_sha256()); - EXPECT_TRUE(metal_resources["table_upload_completed"].get()); - EXPECT_TRUE(metal_resources["table_validation_completed"].get()); - EXPECT_EQ(metal_resources["layout_padding_bytes"], "128"); - ASSERT_TRUE(metal_resources["table_permutation"].is_object()); - EXPECT_EQ(metal_resources["table_permutation"]["sha256"], layout["permutation_sha256"]); - EXPECT_EQ(metal_resources["table_permutation"]["identity"], layout["permutation_identity"]); - - ASSERT_EQ(document["measurements"].size(), 1U); - const OrderedJson& task = document["measurements"][0]["execution"]["metal"]; - EXPECT_EQ(task["pipeline"]["label"], "membenchmark.llm-metal.pipeline.decode-paged.mixed"); - EXPECT_EQ(task["grid"]["paged_semantic_lookups"], "10"); - EXPECT_EQ(task["commands"]["timed_workload_dispatches"], 1U); - EXPECT_TRUE(task["timing"]["valid"].get()); - const auto& checks = document["measurements"][0]["execution"]; - EXPECT_TRUE(named_check(checks, "kv-append-final")["applicable"]); - EXPECT_TRUE(named_check(checks, "kv-append-final")["evaluated"]); - EXPECT_TRUE(named_check(checks, "kv-append-final")["valid"]); - const auto& padding = named_check(checks, "kv-padding-canary"); - EXPECT_TRUE(padding["applicable"]); - EXPECT_TRUE(padding["evaluated"]); - EXPECT_TRUE(padding["valid"]); - EXPECT_EQ(document["measurements"][0]["checksum"]["actual_run_checksum"]["weight"]["a_uint32_decimal"], "1"); -} - TEST(LlmMemoryJsonTest, UnsupportedMetalPagedDocumentKeepsPlannedGeometryButNotRuntimePermutation) { const LlmMemoryConfig config = explicit_metal_paged_config(); const LlmMemoryWorkPlan plan = admitted_metal_plan(config); @@ -1283,63 +1104,48 @@ TEST(LlmMemoryJsonTest, std::to_string(overage_metal->resources.known_owned_peak_bytes)); } -TEST(LlmMemoryJsonTest, UnsupportedMetalCapabilityRetainsTerminalLifecycleWithoutCpuEvidence) { - const LlmMemoryConfig config = explicit_metal_config(); - const LlmMemoryWorkPlan plan = admitted_metal_plan(config); +TEST(LlmMemoryJsonTest, TerminalMetalSetupRetainsSchemaAndLifecycleWithoutCpuFallback) { + const auto config = explicit_metal_config(); + const auto plan = admitted_metal_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; - LlmBackendEvidence backend = complete_metal_backend_evidence(plan); - backend.initialization = {LlmBackendStatus::Unsupported, LlmBackendReason::APPLE7_FAMILY_REQUIRED}; - backend.plan_resolution = {}; - backend.preparation = {}; - LlmMetalBackendEvidence* const metal = std::get_if(&backend.backend_evidence); - ASSERT_NE(metal, nullptr); - metal->capability.required_apple7_family_supported = false; - metal->timed_results_available = false; - - LlmMemoryResult result; - result.initialized = true; - result.status = LlmRunStatus::Unsupported; - result.reason_code = LlmBackendReason::APPLE7_FAMILY_REQUIRED; - const OrderedJson document = build_llm_memory_json(config, plan, backend, fixed_metadata(config, plan), result); - - EXPECT_EQ(document["status"], "unsupported"); - EXPECT_EQ(document["reason_code"], LlmBackendReason::APPLE7_FAMILY_REQUIRED); - EXPECT_TRUE(document["backend_evidence"]["cpu"].is_null()); - EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"]["initialization"]["status"], "unsupported"); - EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"]["initialization"]["reason_code"], - LlmBackendReason::APPLE7_FAMILY_REQUIRED); - EXPECT_FALSE(document["backend_evidence"]["metal"]["capability"]["required_apple7_family_supported"].get()); -} - -TEST(LlmMemoryJsonTest, FailedMetalRuntimeSetupRetainsTerminalSchemaWithoutCpuFallback) { - const LlmMemoryConfig config = explicit_metal_config(); - const LlmMemoryWorkPlan plan = admitted_metal_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - LlmBackendEvidence backend = complete_metal_backend_evidence(plan); - backend.plan_resolution = {LlmBackendStatus::Failed, LlmBackendReason::METAL_PIPELINE_CREATION_FAILED}; - backend.preparation = {}; - backend.release = {LlmBackendStatus::Ready, LlmBackendReason::VALID}; - LlmMetalBackendEvidence* const metal = std::get_if(&backend.backend_evidence); - ASSERT_NE(metal, nullptr); - metal->timed_results_available = false; - - LlmMemoryResult result; - result.initialized = true; - result.status = LlmRunStatus::Failed; - result.reason_code = LlmBackendReason::METAL_PIPELINE_CREATION_FAILED; - const OrderedJson document = build_llm_memory_json(config, plan, backend, fixed_metadata(config, plan), result); - - EXPECT_EQ(document["schema_version"], 2); - EXPECT_EQ(document["mode"], "llm_memory"); - EXPECT_EQ(document["backend"], "metal"); - EXPECT_EQ(document["status"], "failed"); - EXPECT_EQ(document["reason_code"], LlmBackendReason::METAL_PIPELINE_CREATION_FAILED); - EXPECT_FALSE(document["results_complete"].get()); - EXPECT_FALSE(document["run_accepted"].get()); - EXPECT_TRUE(document["backend_evidence"]["cpu"].is_null()); - EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"]["plan_resolution"]["status"], "failed"); - EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"]["plan_resolution"]["reason_code"], - LlmBackendReason::METAL_PIPELINE_CREATION_FAILED); + for (bool unsupported : {false, true}) { + SCOPED_TRACE(unsupported); + auto backend = complete_metal_backend_evidence(plan); + const auto* reason = + unsupported ? LlmBackendReason::APPLE7_FAMILY_REQUIRED : LlmBackendReason::METAL_PIPELINE_CREATION_FAILED; + const char* status = unsupported ? "unsupported" : "failed"; + const char* phase = unsupported ? "initialization" : "plan_resolution"; + const LlmBackendLifecycleResult terminal = {unsupported ? LlmBackendStatus::Unsupported : LlmBackendStatus::Failed, + reason}; + if (unsupported) { + backend.initialization = terminal; + backend.plan_resolution = {}; + } else + backend.plan_resolution = terminal; + backend.preparation = {}; + backend.release = {LlmBackendStatus::Ready, LlmBackendReason::VALID}; + auto* metal = std::get_if(&backend.backend_evidence); + ASSERT_NE(metal, nullptr); + metal->timed_results_available = false; + if (unsupported) metal->capability.required_apple7_family_supported = false; + LlmMemoryResult result; + result.initialized = true; + result.status = unsupported ? LlmRunStatus::Unsupported : LlmRunStatus::Failed; + result.reason_code = reason; + const auto document = build_llm_memory_json(config, plan, backend, fixed_metadata(config, plan), result); + EXPECT_EQ(document["schema_version"], 2); + EXPECT_EQ(document["mode"], "llm_memory"); + EXPECT_EQ(document["backend"], "metal"); + EXPECT_EQ(document["status"], status); + EXPECT_EQ(document["reason_code"], reason); + EXPECT_FALSE(document["results_complete"].get()); + EXPECT_FALSE(document["run_accepted"].get()); + EXPECT_TRUE(document["backend_evidence"]["cpu"].is_null()); + EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"][phase]["status"], status); + EXPECT_EQ(document["backend_evidence"]["metal"]["lifecycle"][phase]["reason_code"], reason); + if (unsupported) + EXPECT_FALSE(document["backend_evidence"]["metal"]["capability"]["required_apple7_family_supported"].get()); + } } TEST(LlmMemoryJsonTest, CompleteDocumentHasExactTopLevelIdentityAndAuditableNestedEvidence) { @@ -1865,13 +1671,6 @@ TEST(LlmMemoryJsonTest, CompletePagedPrefillPublishesLayoutAndPrefillEvidenceWit ASSERT_TRUE(cpu.prefill.has_value()); const LlmPagedCpuExecutionPlan& paged = *cpu.paged; const LlmPrefillCpuExecutionPlan& prefill_plan = *cpu.prefill; - EXPECT_EQ(paged.execution_identity, prefill_plan.identity); - for (size_t scenario_index = 1; scenario_index < kLlmScenarioCount; ++scenario_index) { - for (const LlmPrefillCpuOwnershipPlan& scope : prefill_plan.scenarios[scenario_index].ownership_scopes) { - EXPECT_EQ(scope.unit_kind, LlmPrefillPartitionUnitKind::PagedBlock); - } - } - const LlmMemoryResult result = complete_result(config, plan); ASSERT_TRUE(result.results_complete) << result.reason_code; const OrderedJson document = @@ -1945,12 +1744,6 @@ TEST(LlmMemoryJsonTest, PagedDocumentPublishesExactLayoutResourceAndOwnershipPro EXPECT_EQ(document["configuration"]["resolved_sources"]["kv_block_tokens"], "explicit"); const OrderedJson& layout = document["resolved_plan"]["layout"]; - expect_exact_keys(layout, {"kv_layout", "kv_block_tokens", "blocks_per_sequence", "physical_blocks_per_layer", - "total_physical_blocks", "block_bytes", "last_block_tokens", "last_block_valid_bytes", - "decode_append_offset_in_last_block", "block_table_entries", "block_table_bytes", - "layout_geometry_identity", "layout_identity", "permutation_domain_uint64_hex", - "permutation_seed_uint64_decimal", "permutation_algorithm_version", - "permutation_entry_count", "permutation_sha256", "permutation_identity"}); EXPECT_EQ(layout["kv_layout"], "paged"); EXPECT_EQ(layout["kv_block_tokens"], 2u); EXPECT_EQ(layout["blocks_per_sequence"], "2"); @@ -2118,6 +1911,13 @@ TEST(LlmMemoryJsonTest, PreflightAuxiliaryEstimatesExactlyMatchFinalizedPlanForA ASSERT_TRUE(preflight_executor.valid); ASSERT_TRUE(preflight_runner.valid); ASSERT_TRUE(preflight_json.valid); + EXPECT_GT(preflight_json.fixed_schema_bytes, 0u); + EXPECT_GT(preflight_json.input_string_bytes, 0u); + EXPECT_GT(preflight_json.measurement_record_bytes, 0u); + EXPECT_GT(preflight_json.worker_checksum_bytes, preflight_json.measurement_record_bytes); + EXPECT_EQ(preflight_json.total_bytes, preflight_json.fixed_schema_bytes + preflight_json.input_string_bytes + + preflight_json.measurement_record_bytes + + preflight_json.worker_checksum_bytes); size_t checksum_auxiliary_bytes = 0; size_t orchestration_auxiliary_bytes = 0; @@ -2191,38 +1991,14 @@ TEST(LlmMemoryJsonTest, CanonicalIdentityBudgetIsIndependentOfLoopCountAndReserv preflight.frozen_scenario_plan_identity_bytes = {41, 43, 47}; constexpr size_t kDomTransportExpansionFactor = 16; - const auto expected_input_string_bytes = [&](const LlmMemoryConfig& config) { - size_t argv_bytes = 0; - for (const std::string& argument : config.argv) { - argv_bytes += argument.size(); - } - size_t scenario_identity_bytes = 0; - size_t frozen_identity_bytes = preflight.frozen_reason_code_bytes + preflight.frozen_model_plan_identity_bytes + - preflight.frozen_plan_identity_bytes; - for (size_t index = 0; index < kLlmScenarioCount; ++index) { - scenario_identity_bytes += preflight.maximum_scenario_plan_identity_bytes[index]; - frozen_identity_bytes += preflight.frozen_scenario_reason_code_bytes[index] + - preflight.frozen_scenario_model_plan_identity_bytes[index] + - preflight.frozen_scenario_plan_identity_bytes[index]; - } - const size_t calibration_attempts = - config.user_specified_iterations ? 1 : 4 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS; - const size_t raw_identity_and_input_bytes = - config.output_file.size() + argv_bytes + preflight.json_identity_string_bytes + frozen_identity_bytes + - scenario_identity_bytes + calibration_attempts * scenario_identity_bytes; - return raw_identity_and_input_bytes * kDomTransportExpansionFactor; - }; - const LlmMemoryConfig one_loop = explicit_config(1); const LlmJsonPeakEstimate one_loop_estimate = calculate_llm_json_peak_estimate(one_loop, preflight); ASSERT_TRUE(one_loop_estimate.valid) << one_loop_estimate.reason_code; - EXPECT_EQ(one_loop_estimate.input_string_bytes, expected_input_string_bytes(one_loop)); LlmMemoryConfig four_loops = one_loop; four_loops.loop_count = 4; const LlmJsonPeakEstimate four_loop_estimate = calculate_llm_json_peak_estimate(four_loops, preflight); ASSERT_TRUE(four_loop_estimate.valid) << four_loop_estimate.reason_code; - EXPECT_EQ(four_loop_estimate.input_string_bytes, expected_input_string_bytes(four_loops)); constexpr size_t kScenarioIdentityBytes = 2 + 3 + 5; EXPECT_EQ(four_loop_estimate.input_string_bytes - one_loop_estimate.input_string_bytes, 0u); @@ -2231,42 +2007,44 @@ TEST(LlmMemoryJsonTest, CanonicalIdentityBudgetIsIndependentOfLoopCountAndReserv automatic.user_specified_iterations = false; const LlmJsonPeakEstimate automatic_estimate = calculate_llm_json_peak_estimate(automatic, preflight); ASSERT_TRUE(automatic_estimate.valid) << automatic_estimate.reason_code; - EXPECT_EQ(automatic_estimate.input_string_bytes, expected_input_string_bytes(automatic)); EXPECT_EQ(automatic_estimate.input_string_bytes - four_loop_estimate.input_string_bytes, (3 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS) * kScenarioIdentityBytes * kDomTransportExpansionFactor); } -TEST(LlmMemoryJsonTest, JsonPeakEstimateRejectsSyntheticIdentityReplicationOverflowBeforeDom) { - LlmMemoryConfig config = explicit_config(2); - LlmAuxiliaryPreflightView preflight; - preflight.valid = true; - preflight.backend = LlmMemoryBackend::Cpu; - preflight.effective_workers = 1; - - preflight.maximum_scenario_plan_identity_bytes[0] = std::numeric_limits::max() / 2 + 1; - LlmJsonPeakEstimate estimate = calculate_llm_json_peak_estimate(config, preflight); - EXPECT_FALSE(estimate.valid); - EXPECT_EQ(estimate.reason_code, LlmJsonReason::PEAK_BYTES_OVERFLOW); - EXPECT_EQ(estimate.total_bytes, 0u); - - config.loop_count = 1; - config.user_specified_iterations = false; - preflight.maximum_scenario_plan_identity_bytes = {}; - const size_t calibration_attempts = 4 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS; - preflight.maximum_scenario_plan_identity_bytes[1] = std::numeric_limits::max() / calibration_attempts + 1; - estimate = calculate_llm_json_peak_estimate(config, preflight); - EXPECT_FALSE(estimate.valid); - EXPECT_EQ(estimate.reason_code, LlmJsonReason::PEAK_BYTES_OVERFLOW); - EXPECT_EQ(estimate.total_bytes, 0u); - - config.user_specified_iterations = true; - preflight.maximum_scenario_plan_identity_bytes = {}; - preflight.frozen_reason_code_bytes = std::numeric_limits::max(); - preflight.frozen_model_plan_identity_bytes = 1; - estimate = calculate_llm_json_peak_estimate(config, preflight); - EXPECT_FALSE(estimate.valid); - EXPECT_EQ(estimate.reason_code, LlmJsonReason::PEAK_BYTES_OVERFLOW); - EXPECT_EQ(estimate.total_bytes, 0u); +TEST(LlmMemoryJsonTest, JsonPeakEstimateRejectsIdentityAndCountOverflowBeforeDom) { + constexpr size_t maximum = std::numeric_limits::max(); + for (size_t overflow = 0; overflow < 5; ++overflow) { + SCOPED_TRACE(overflow); + auto config = overflow == 3 ? prefill_config(1) : explicit_config(2); + auto draft = prepare_llm_memory_work_plan(plan_request(config)); + ASSERT_TRUE(draft.valid) << draft.reason_code; + auto& preflight = draft.auxiliary_preflight; + switch (overflow) { + case 0: + preflight.maximum_scenario_plan_identity_bytes[0] = maximum / 2 + 1; + break; + case 1: + config.loop_count = 1; + config.user_specified_iterations = false; + preflight.maximum_scenario_plan_identity_bytes[1] = + maximum / (4 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS) + 1; + break; + case 2: + preflight.frozen_reason_code_bytes = maximum; + preflight.frozen_model_plan_identity_bytes = 1; + break; + case 3: + preflight.json_identity_string_bytes = maximum; + break; + case 4: + config.loop_count = maximum; + break; + } + const auto estimate = calculate_llm_json_peak_estimate(config, preflight); + EXPECT_FALSE(estimate.valid); + EXPECT_EQ(estimate.reason_code, LlmJsonReason::PEAK_BYTES_OVERFLOW); + EXPECT_EQ(estimate.total_bytes, 0u); + } } TEST(LlmMemoryJsonTest, JsonPeakEstimateAccountsForEveryPrefillPartitionIdentity) { @@ -2304,43 +2082,6 @@ TEST(LlmMemoryJsonTest, JsonPeakEstimateAccountsForEveryPrefillPartitionIdentity EXPECT_EQ(expanded.total_bytes - baseline.total_bytes, added_identity_bytes * kLiveDomAndSerializedTransportFactor); } -TEST(LlmMemoryJsonTest, JsonPeakEstimateRejectsPrefillIdentityByteOverflowBeforeFinalization) { - const LlmMemoryConfig config = prefill_config(1); - LlmMemoryWorkPlanDraft draft = prepare_llm_memory_work_plan(plan_request(config)); - ASSERT_TRUE(draft.valid) << draft.reason_code; - draft.auxiliary_preflight.json_identity_string_bytes = std::numeric_limits::max(); - - const LlmJsonPeakEstimate estimate = calculate_llm_json_peak_estimate(config, draft.auxiliary_preflight); - EXPECT_FALSE(estimate.valid); - EXPECT_EQ(estimate.reason_code, LlmJsonReason::PEAK_BYTES_OVERFLOW); - EXPECT_EQ(estimate.total_bytes, 0u); -} - -TEST(LlmMemoryJsonTest, JsonPeakEstimateScalesWithMeasurementsAndWorkersAndIsAdmitted) { - const LlmMemoryConfig config = explicit_config(3); - LlmMemoryWorkPlanRequest request = plan_request(config); - const LlmMemoryWorkPlan preliminary = build_llm_memory_work_plan(request); - ASSERT_TRUE(preliminary.valid) << preliminary.reason_code; - const LlmExecutorAuxiliaryEstimate executor = calculate_llm_executor_auxiliary_estimate(preliminary); - const LlmRunnerAuxiliaryEstimate runner = calculate_llm_runner_auxiliary_estimate(config, preliminary); - const LlmJsonPeakEstimate estimate = calculate_llm_json_peak_estimate(config, preliminary); - ASSERT_TRUE(executor.valid); - ASSERT_TRUE(runner.valid); - ASSERT_TRUE(estimate.valid); - - EXPECT_GT(estimate.fixed_schema_bytes, 0u); - EXPECT_GT(estimate.input_string_bytes, 0u); - EXPECT_GT(estimate.measurement_record_bytes, 0u); - EXPECT_GT(estimate.worker_checksum_bytes, estimate.measurement_record_bytes); - EXPECT_EQ(estimate.total_bytes, estimate.fixed_schema_bytes + estimate.input_string_bytes + - estimate.measurement_record_bytes + estimate.worker_checksum_bytes); - - const LlmMemoryWorkPlan admitted = admitted_plan(config); - ASSERT_TRUE(admitted.valid) << admitted.reason_code; - EXPECT_EQ(admitted.memory_budget.request.orchestration_auxiliary_bytes, - executor.orchestration_auxiliary_bytes + runner.orchestration_auxiliary_bytes + estimate.total_bytes); -} - TEST(LlmMemoryJsonTest, JsonPeakEstimateAccountsForRetainedMetalMeasurementAndCalibrationTaskEvidence) { const LlmMemoryConfig one_loop = explicit_metal_config(); const LlmMemoryWorkPlan plan = admitted_metal_plan(one_loop); @@ -2369,18 +2110,6 @@ TEST(LlmMemoryJsonTest, JsonPeakEstimateAccountsForRetainedMetalMeasurementAndCa 2 * (3 + Constants::LLM_CALIBRATION_MAX_CORRECTIONS) * kLlmScenarioCount * metal_task_record_bytes); } -TEST(LlmMemoryJsonTest, JsonPeakEstimateRejectsCountArithmeticOverflow) { - LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - config.loop_count = std::numeric_limits::max(); - - const LlmJsonPeakEstimate estimate = calculate_llm_json_peak_estimate(config, plan); - EXPECT_FALSE(estimate.valid); - EXPECT_EQ(estimate.reason_code, LlmJsonReason::PEAK_BYTES_OVERFLOW); - EXPECT_EQ(estimate.total_bytes, 0u); -} - TEST(LlmMemoryJsonTest, ExactByteSeedAndChecksumIntegersAreCanonicalDecimalStrings) { const LlmMemoryConfig config = explicit_config(1); const LlmMemoryWorkPlan plan = admitted_plan(config); @@ -2496,66 +2225,42 @@ TEST(LlmMemoryJsonTest, InterruptedRunnerSerializesUnavailableMetricsExecutionQo } TEST(LlmMemoryJsonTest, ExecutorExceptionUsesRunnerReasonAndNullUnavailableExecutionEvidence) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend backend( - [](const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context) { - if (context.kind == LlmRunnerTaskKind::Measurement) { - throw std::runtime_error("injected JSON exception path"); - } - return successful_execution(model_plan); - }); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); - ASSERT_EQ(result.status, LlmRunStatus::Failed); - - const OrderedJson document = - build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - EXPECT_EQ(document["status"], "failed"); - EXPECT_EQ(document["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_EQ(document["diagnostic"], "injected JSON exception path"); - const OrderedJson& measurement = document["measurements"][0]; - EXPECT_TRUE(measurement["attempted"]); - EXPECT_EQ(measurement["status"], "failed"); - EXPECT_EQ(measurement["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_TRUE(measurement["execution"]["qos_successful_workers"].is_null()); - EXPECT_TRUE(measurement["execution"]["qos_failed_workers"].is_null()); - EXPECT_EQ(measurement["execution"]["status"], "unavailable"); - EXPECT_EQ(measurement["execution"]["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_TRUE(measurement["execution"]["valid"].is_null()); - EXPECT_TRUE(measurement["execution"]["requested_workers"].is_null()); - EXPECT_TRUE(measurement["execution"]["kernel_succeeded"].is_null()); - EXPECT_EQ(measurement["checksum"]["status"], "not_evaluated"); - EXPECT_EQ(measurement["checksum"]["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_TRUE(measurement["checksum"]["checksum_valid"].is_null()); -} - -TEST(LlmMemoryJsonTest, ExcludedExecutorExceptionUsesRunnerReasonAndNullUnavailableExecutionEvidence) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = admitted_plan(config); + const auto config = explicit_config(1); + const auto plan = admitted_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend backend( - [](const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context) { - if (context.kind == LlmRunnerTaskKind::Warmup) { - throw std::runtime_error("injected excluded JSON exception path"); - } - return successful_execution(model_plan); - }); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); - - const OrderedJson document = - build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - const OrderedJson& execution = document["calibration"]["attempts"]["weights_only"][0]["execution"]; - EXPECT_EQ(execution["status"], "unavailable"); - EXPECT_EQ(execution["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_TRUE(execution["valid"].is_null()); - EXPECT_TRUE(execution["requested_workers"].is_null()); - EXPECT_TRUE(execution["kernel_succeeded"].is_null()); - EXPECT_EQ(execution["checksum"]["status"], "not_evaluated"); - EXPECT_EQ(execution["checksum"]["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_TRUE(execution["checksum"]["checksum_valid"].is_null()); + for (auto phase : {LlmRunnerTaskKind::Measurement, LlmRunnerTaskKind::Warmup}) { + SCOPED_TRACE(static_cast(phase)); + FakeLlmBackend backend( + [phase](const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context) { + if (context.kind == phase) throw std::runtime_error("injected JSON exception path"); + return successful_execution(model); + }); + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); + const auto document = + build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + EXPECT_EQ(document["status"], "failed"); + EXPECT_EQ(document["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); + EXPECT_EQ(document["diagnostic"], "injected JSON exception path"); + const bool measured = phase == LlmRunnerTaskKind::Measurement; + const auto& entry = measured ? document["measurements"][0] : document["calibration"]["attempts"]["weights_only"][0]; + const auto& execution = entry["execution"]; + const auto& checksum = measured ? entry["checksum"] : execution["checksum"]; + if (measured) { + EXPECT_TRUE(entry["attempted"]); + EXPECT_EQ(entry["status"], "failed"); + EXPECT_EQ(entry["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); + EXPECT_TRUE(execution["qos_successful_workers"].is_null()); + EXPECT_TRUE(execution["qos_failed_workers"].is_null()); + } + EXPECT_EQ(execution["status"], "unavailable"); + EXPECT_EQ(execution["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); + for (const char* field : {"valid", "requested_workers", "kernel_succeeded"}) + EXPECT_TRUE(execution[field].is_null()); + EXPECT_EQ(checksum["status"], "not_evaluated"); + EXPECT_EQ(checksum["reason_code"], LlmRunnerReason::RUNNER_EXCEPTION); + EXPECT_TRUE(checksum["checksum_valid"].is_null()); + } } TEST(LlmMemoryJsonTest, InvalidElapsedMeasurementLeavesChecksumEvidenceNotEvaluatedAndNull) { @@ -2592,67 +2297,49 @@ TEST(LlmMemoryJsonTest, InvalidElapsedMeasurementLeavesChecksumEvidenceNotEvalua EXPECT_TRUE(measurement["checksum"]["actual_run_checksum"].is_null()); } -TEST(LlmMemoryJsonTest, InvalidElapsedExcludedTaskLeavesCompactChecksumEvidenceNotEvaluatedAndNull) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend backend( - [](const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context) { - LlmExecutorResult execution = successful_execution(model_plan); - if (context.kind == LlmRunnerTaskKind::Warmup) { +TEST(LlmMemoryJsonTest, MalformedExcludedChecksumCardinalityDoesNotPublishCompactChecksumEvidence) { + for (bool invalid_elapsed : {false, true}) { + const LlmMemoryConfig config = explicit_config(1); + const LlmMemoryWorkPlan plan = admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + ASSERT_GE(cpu_execution_plan(plan).effective_workers, 2u); + FakeLlmBackend backend([invalid_elapsed](const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan&, + const LlmRunnerTaskContext& context) { + LlmExecutorResult execution = successful_execution(model_plan); + if (context.kind == LlmRunnerTaskKind::Warmup) { + if (invalid_elapsed) { execution.valid = false; execution.reason_code = LlmExecutorReason::INVALID_ELAPSED_TIME; execution.elapsed_seconds = 0.0; execution.checksum_evaluated = false; execution.checksum_valid = false; - } - return execution; - }); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); - - const OrderedJson document = - build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - const OrderedJson& execution = document["calibration"]["attempts"]["weights_only"][0]["execution"]; - EXPECT_EQ(execution["status"], "invalid"); - EXPECT_EQ(execution["reason_code"], LlmExecutorReason::INVALID_ELAPSED_TIME); - EXPECT_FALSE(execution["valid"]); - EXPECT_EQ(execution["checksum"]["status"], "not_evaluated"); - EXPECT_EQ(execution["checksum"]["reason_code"], LlmExecutorReason::INVALID_ELAPSED_TIME); - EXPECT_TRUE(execution["checksum"]["checksum_valid"].is_null()); - EXPECT_TRUE(execution["checksum"]["actual_run_checksum"].is_null()); -} - -TEST(LlmMemoryJsonTest, MalformedExcludedChecksumCardinalityDoesNotPublishCompactChecksumEvidence) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_GE(cpu_execution_plan(plan).effective_workers, 2u); - FakeLlmBackend backend( - [](const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context) { - LlmExecutorResult execution = successful_execution(model_plan); - if (context.kind == LlmRunnerTaskKind::Warmup) { + } else { execution.expected_checksums.resize(1); execution.actual_checksums.resize(1); } - return execution; - }); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); - - const OrderedJson document = - build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - const OrderedJson& attempt = document["calibration"]["attempts"]["weights_only"][0]; - EXPECT_FALSE(attempt["valid"]); - EXPECT_EQ(attempt["reason_code"], LlmExecutorReason::INVALID_RESOURCES); - const OrderedJson& execution = attempt["execution"]; - EXPECT_EQ(execution["status"], "invalid"); - EXPECT_EQ(execution["reason_code"], LlmExecutorReason::INVALID_RESOURCES); - EXPECT_FALSE(execution["valid"]); - EXPECT_EQ(execution["checksum"]["status"], "not_evaluated"); - EXPECT_EQ(execution["checksum"]["reason_code"], LlmExecutorReason::INVALID_RESOURCES); - EXPECT_TRUE(execution["checksum"]["checksum_valid"].is_null()); - EXPECT_TRUE(execution["checksum"]["actual_run_checksum"].is_null()); + } + return execution; + }); + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); + + const OrderedJson document = + build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const OrderedJson& attempt = document["calibration"]["attempts"]["weights_only"][0]; + EXPECT_FALSE(attempt["valid"]); + EXPECT_EQ(attempt["reason_code"], + invalid_elapsed ? LlmExecutorReason::INVALID_ELAPSED_TIME : LlmExecutorReason::INVALID_RESOURCES); + const OrderedJson& execution = attempt["execution"]; + EXPECT_EQ(execution["status"], "invalid"); + EXPECT_EQ(execution["reason_code"], + invalid_elapsed ? LlmExecutorReason::INVALID_ELAPSED_TIME : LlmExecutorReason::INVALID_RESOURCES); + EXPECT_FALSE(execution["valid"]); + EXPECT_EQ(execution["checksum"]["status"], "not_evaluated"); + EXPECT_EQ(execution["checksum"]["reason_code"], + invalid_elapsed ? LlmExecutorReason::INVALID_ELAPSED_TIME : LlmExecutorReason::INVALID_RESOURCES); + EXPECT_TRUE(execution["checksum"]["checksum_valid"].is_null()); + EXPECT_TRUE(execution["checksum"]["actual_run_checksum"].is_null()); + } } TEST(LlmMemoryJsonTest, ChecksumMismatchSerializesEvaluatedFalseInsteadOfMissingNull) { @@ -2682,41 +2369,54 @@ TEST(LlmMemoryJsonTest, ChecksumMismatchSerializesEvaluatedFalseInsteadOfMissing EXPECT_TRUE(measurement["checksum"]["actual_worker_checksums"].is_array()); } -TEST(LlmMemoryJsonTest, PagedPostValidationFailureSerializesEvaluatedInvalidEvidence) { - const LlmMemoryConfig config = paged_config(); - const LlmMemoryWorkPlan plan = admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend backend( - [](const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context) { - LlmExecutorResult execution = successful_execution(model_plan); - if (context.kind == LlmRunnerTaskKind::Measurement) { - execution.valid = false; - execution.reason_code = LlmExecutorReason::PAGED_POST_VALIDATION_FAILED; - execution.post_validation_valid = false; - } - return execution; - }); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); - - const OrderedJson document = - build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - const OrderedJson& measurement = document["measurements"][0]; - EXPECT_EQ(measurement["status"], "invalid"); - EXPECT_EQ(measurement["reason_code"], LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - // This fixture supplies a conclusive failure for each applicable cold check. - // The serializer must retain those verdicts even though the task is excluded. - for (const char* kind : {"post-validation-structure", "kv-append-unchanged", "kv-padding-canary"}) { - const auto& check = named_check(measurement["execution"], kind); - EXPECT_TRUE(check["applicable"]); - EXPECT_TRUE(check["evaluated"]); - EXPECT_FALSE(check["valid"]); - EXPECT_EQ(check["reason_code"], LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); +TEST(LlmMemoryJsonTest, PostValidationFailureSerializesEvaluatedInvalidEvidence) { + for (bool paged : {false, true}) { + SCOPED_TRACE(paged); + const auto config = paged ? paged_config() : explicit_config(1); + const auto plan = admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const auto* reason = + paged ? LlmExecutorReason::PAGED_POST_VALIDATION_FAILED : LlmExecutorReason::DECODE_POST_VALIDATION_FAILED; + FakeLlmBackend backend([paged, reason](const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& scenario, + const LlmRunnerTaskContext& context) { + auto execution = successful_execution(model); + if (context.kind == LlmRunnerTaskKind::Measurement && (paged || scenario.scenario == LlmScenario::KvOnly)) { + execution.valid = false; + execution.reason_code = reason; + execution.post_validation_valid = false; + } + return execution; + }); + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); + const auto document = + build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); + const auto& measurement = document["measurements"][paged ? 0 : 1]; + EXPECT_EQ(measurement["status"], "invalid"); + EXPECT_EQ(measurement["reason_code"], reason); + const std::vector kinds = + paged ? std::vector{"post-validation-structure", "kv-append-unchanged", "kv-padding-canary"} + : std::vector{"kv-append-final"}; + for (const char* kind : kinds) { + const auto& check = named_check(measurement["execution"], kind); + EXPECT_TRUE(check["applicable"]); + EXPECT_TRUE(check["evaluated"]); + EXPECT_FALSE(check["valid"]); + EXPECT_EQ(check["reason_code"], reason); + } + if (!paged) { + const auto& absent = named_check(document["measurements"][0]["execution"], "kv-append-unchanged"); + EXPECT_FALSE(absent["applicable"]); + EXPECT_TRUE(absent["evaluated"].is_null()); + EXPECT_TRUE(absent["valid"].is_null()); + EXPECT_EQ(document["aggregates"]["scenarios"]["kv_only"]["effective_model_payload_gb_s"]["sample_count"], 0u); + } + EXPECT_TRUE(measurement["elapsed_seconds"].is_null()); + EXPECT_DOUBLE_EQ(measurement["execution"]["timing"]["diagnostic_elapsed_seconds"].get(), 0.150); + EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); + EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_null()); + EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); } - EXPECT_TRUE(measurement["elapsed_seconds"].is_null()); - EXPECT_DOUBLE_EQ(measurement["execution"]["timing"]["diagnostic_elapsed_seconds"].get(), 0.150); - EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); - EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); } TEST(LlmMemoryJsonTest, CommonAcceptanceFailureCannotSerializeCompleteBackendEvidenceAsValid) { @@ -2740,96 +2440,85 @@ TEST(LlmMemoryJsonTest, CommonAcceptanceFailureCannotSerializeCompleteBackendEvi } TEST(LlmMemoryJsonTest, LoopSnapshotSerializesPartialStatusAndUnavailableTailContract) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend backend; - LlmMemoryResult partial_snapshot; - bool captured_partial_snapshot = false; - LlmRunnerHooks hooks; - hooks.checkpoint = [&](const LlmMemoryResult& checkpoint_result, LlmCheckpointKind kind) { - if (!captured_partial_snapshot && kind == LlmCheckpointKind::MeasurementTerminal) { - partial_snapshot = checkpoint_result; - captured_partial_snapshot = true; - } - return EXIT_SUCCESS; - }; - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result, hooks), EXIT_SUCCESS); - ASSERT_TRUE(captured_partial_snapshot); + for (bool failed : {false, true}) { + const LlmMemoryConfig config = explicit_config(2); + const LlmMemoryWorkPlan plan = admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + FakeLlmBackend backend; + LlmMemoryResult partial_snapshot; + bool captured_partial_snapshot = false; + LlmRunnerHooks hooks; + hooks.checkpoint = [&](const LlmMemoryResult& checkpoint_result, LlmCheckpointKind kind) { + if (!captured_partial_snapshot && kind == LlmCheckpointKind::MeasurementTerminal) { + partial_snapshot = checkpoint_result; + captured_partial_snapshot = true; + } + return failed ? EXIT_FAILURE : EXIT_SUCCESS; + }; + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result, hooks), failed ? EXIT_FAILURE : EXIT_SUCCESS); + ASSERT_TRUE(captured_partial_snapshot); + + if (failed) partial_snapshot = result; + partial_snapshot.prior_file_writer_attempts = 3; + partial_snapshot.prior_successful_file_writes = 2; + const OrderedJson document = + build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), partial_snapshot); + EXPECT_EQ(document["status"], failed ? "failed" : "partial"); + EXPECT_EQ(document["reason_code"], + failed ? LlmRunnerReason::CHECKPOINT_WRITE_FAILED : LlmRunnerReason::PARTIAL_RESULTS); + EXPECT_FALSE(document["interruption_requested"]); + EXPECT_FALSE(document["results_complete"]); + EXPECT_FALSE(document["run_accepted"]); + EXPECT_FALSE(document["scenario_order_balance_complete"]); + EXPECT_EQ(document["counters"]["planned_measurements"], 6u); + EXPECT_EQ(document["counters"]["attempted_measurements"], 3u); + EXPECT_EQ(document["counters"]["terminal_measurements"], failed ? 6u : 3u); + EXPECT_EQ(document["counters"]["measured_measurements"], 3u); + if (!failed) EXPECT_EQ(document["checkpoint_lifecycle"]["current_request"], "progress"); + EXPECT_EQ(document["checkpoint_lifecycle"]["checkpoint_failed"], failed); + EXPECT_EQ(document["checkpoint_lifecycle"]["observation_point"], "before-current-snapshot-preparation"); + EXPECT_EQ(document["checkpoint_lifecycle"]["prior_file_writer_attempts"], 3u); + EXPECT_EQ(document["checkpoint_lifecycle"]["prior_successful_file_writes"], 2u); + EXPECT_EQ(document["checkpoint_lifecycle"]["checkpoint_policy"], "bounded-loop-snapshots"); + EXPECT_EQ(document["checkpoint_lifecycle"]["snapshot_interval_loops"], 1u); + EXPECT_TRUE(document["checkpoint_lifecycle"]["current_persistence_success"].is_null()); + + size_t measured_count = 0; + size_t not_run_count = 0; + for (const OrderedJson& measurement : document["measurements"]) { + if (measurement["status"] == "measured") { + ++measured_count; + EXPECT_EQ(measurement["reason_code"], "measured"); + EXPECT_TRUE(measurement["attempted"]); + EXPECT_TRUE(measurement["completed_work_units"].is_number_unsigned()); + EXPECT_TRUE(measurement["elapsed_seconds"].is_number()); + EXPECT_EQ(measurement["execution"]["status"], "valid"); + EXPECT_EQ(measurement["checksum"]["status"], "valid"); + EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); + continue; + } - const OrderedJson document = - build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), partial_snapshot); - EXPECT_EQ(document["status"], "partial"); - EXPECT_EQ(document["reason_code"], LlmRunnerReason::PARTIAL_RESULTS); - EXPECT_FALSE(document["interruption_requested"]); - EXPECT_FALSE(document["results_complete"]); - EXPECT_FALSE(document["run_accepted"]); - EXPECT_FALSE(document["scenario_order_balance_complete"]); - EXPECT_EQ(document["counters"]["planned_measurements"], 6u); - EXPECT_EQ(document["counters"]["attempted_measurements"], 3u); - EXPECT_EQ(document["counters"]["terminal_measurements"], 3u); - EXPECT_EQ(document["counters"]["measured_measurements"], 3u); - EXPECT_EQ(document["checkpoint_lifecycle"]["current_request"], "progress"); - EXPECT_EQ(document["checkpoint_lifecycle"]["snapshot_interval_loops"], 1u); - EXPECT_TRUE(document["checkpoint_lifecycle"]["current_persistence_success"].is_null()); - - size_t measured_count = 0; - size_t not_run_count = 0; - for (const OrderedJson& measurement : document["measurements"]) { - if (measurement["status"] == "measured") { - ++measured_count; - EXPECT_EQ(measurement["reason_code"], "measured"); - EXPECT_TRUE(measurement["attempted"]); - EXPECT_TRUE(measurement["completed_work_units"].is_number_unsigned()); - EXPECT_TRUE(measurement["elapsed_seconds"].is_number()); - EXPECT_EQ(measurement["execution"]["status"], "valid"); - EXPECT_EQ(measurement["checksum"]["status"], "valid"); - EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); - continue; + ASSERT_EQ(measurement["status"], failed ? "failed" : "not_run"); + ++not_run_count; + if (!failed) EXPECT_EQ(measurement["reason_code"], "not-run"); + EXPECT_FALSE(measurement["attempted"]); + EXPECT_TRUE(measurement["execution"]["qos_successful_workers"].is_null()); + EXPECT_EQ(measurement["completed_work_units"], 0u); + EXPECT_EQ(measurement["completed_effective_model_payload_bytes"], "0"); + EXPECT_TRUE(measurement["elapsed_seconds"].is_null()); + EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); + EXPECT_EQ(measurement["execution"]["status"], "not_run"); + if (!failed) EXPECT_EQ(measurement["execution"]["reason_code"], "not-run"); + EXPECT_TRUE(measurement["execution"]["valid"].is_null()); + EXPECT_TRUE(measurement["execution"]["requested_workers"].is_null()); + EXPECT_EQ(measurement["checksum"]["status"], "not_evaluated"); + if (!failed) EXPECT_EQ(measurement["checksum"]["reason_code"], "not-run"); + EXPECT_TRUE(measurement["checksum"]["checksum_valid"].is_null()); } - - ASSERT_EQ(measurement["status"], "not_run"); - ++not_run_count; - EXPECT_EQ(measurement["reason_code"], "not-run"); - EXPECT_FALSE(measurement["attempted"]); - EXPECT_TRUE(measurement["execution"]["qos_successful_workers"].is_null()); - EXPECT_EQ(measurement["completed_work_units"], 0u); - EXPECT_EQ(measurement["completed_effective_model_payload_bytes"], "0"); - EXPECT_TRUE(measurement["elapsed_seconds"].is_null()); - EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); - EXPECT_EQ(measurement["execution"]["status"], "not_run"); - EXPECT_EQ(measurement["execution"]["reason_code"], "not-run"); - EXPECT_TRUE(measurement["execution"]["valid"].is_null()); - EXPECT_TRUE(measurement["execution"]["requested_workers"].is_null()); - EXPECT_EQ(measurement["checksum"]["status"], "not_evaluated"); - EXPECT_EQ(measurement["checksum"]["reason_code"], "not-run"); - EXPECT_TRUE(measurement["checksum"]["checksum_valid"].is_null()); + EXPECT_EQ(measured_count, 3u); + EXPECT_EQ(not_run_count, 3u); } - EXPECT_EQ(measured_count, 3u); - EXPECT_EQ(not_run_count, 3u); -} - -TEST(LlmMemoryJsonTest, CheckpointFailureRetainsMeasuredPrefixAndNullFailedTailWithoutRetry) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend backend; - LlmRunnerHooks hooks; - hooks.checkpoint = [](const LlmMemoryResult&, LlmCheckpointKind) { return EXIT_FAILURE; }; - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result, hooks), EXIT_FAILURE); - - const OrderedJson document = - build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - EXPECT_EQ(document["status"], "failed"); - EXPECT_EQ(document["reason_code"], LlmRunnerReason::CHECKPOINT_WRITE_FAILED); - EXPECT_TRUE(document["checkpoint_lifecycle"]["checkpoint_failed"]); - EXPECT_EQ(document["measurements"][0]["status"], "measured"); - EXPECT_EQ(document["measurements"][3]["status"], "failed"); - EXPECT_TRUE(document["measurements"][3]["execution"]["qos_successful_workers"].is_null()); - EXPECT_TRUE(document["measurements"][3]["effective_model_payload_gb_s"].is_null()); - EXPECT_TRUE(document["measurements"][3]["checksum"]["checksum_valid"].is_null()); } TEST(LlmMemoryJsonTest, QualityWarningsMergeAndDeduplicateInStableConsoleAgreementOrder) { @@ -2874,42 +2563,6 @@ TEST(LlmMemoryJsonTest, LoopRecordsExposeOnlyRealizedPrefixAndAllMeasurementInde EXPECT_EQ(loop["measurement_indexes"], (OrderedJson::array({0, 1, 2}))); } -TEST(LlmMemoryJsonTest, DecodeWriteValidationFailureRetainsInvalidAttemptAndPopulation) { - const auto config = explicit_config(1); - const auto plan = admitted_plan(config); - ASSERT_TRUE(plan.valid); - FakeLlmBackend backend([](const LlmMemoryWorkPlan& model, const LlmScenarioWorkPlan& scenario, - const LlmRunnerTaskContext& context) { - auto execution = successful_execution(model); - if (context.kind == LlmRunnerTaskKind::Measurement && scenario.scenario == LlmScenario::KvOnly) { - execution.valid = false; - execution.reason_code = LlmExecutorReason::DECODE_POST_VALIDATION_FAILED; - execution.post_validation_valid = false; - } - return execution; - }); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); - const auto document = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - const auto& weights = document["measurements"][0]; - const auto& measurement = document["measurements"][1]; - const auto& absent_write = named_check(weights["execution"], "kv-append-unchanged"); - EXPECT_FALSE(absent_write["applicable"]); - EXPECT_TRUE(absent_write["evaluated"].is_null()); - EXPECT_TRUE(absent_write["valid"].is_null()); - const auto& failed_write = named_check(measurement["execution"], "kv-append-final"); - EXPECT_TRUE(failed_write["applicable"]); - EXPECT_TRUE(failed_write["evaluated"]); - EXPECT_FALSE(failed_write["valid"]); - - EXPECT_EQ(measurement["status"], "invalid"); - EXPECT_EQ(measurement["reason_code"], LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - EXPECT_TRUE(measurement["checksum"]["checksum_valid"]); - EXPECT_TRUE(measurement["synthetic_memory_work_units_per_second"].is_null()); - EXPECT_TRUE(measurement["effective_model_payload_gb_s"].is_null()); - EXPECT_EQ(document["aggregates"]["scenarios"]["kv_only"]["effective_model_payload_gb_s"]["sample_count"], 0u); -} - TEST(LlmMemoryJsonTest, RequiredCpuWriteValidationNotRunIsFailedWithNullVerdict) { const auto config = explicit_config(1); const auto plan = admitted_plan(config); @@ -2984,41 +2637,6 @@ TEST(LlmMemoryJsonTest, PrefillTheoreticalOverflowKeepsByteGeometryAndNullableRe EXPECT_EQ(doc["resolved_plan"]["geometry"]["kv_read_bytes_per_work_unit"], "45056"); } -TEST(LlmMemoryJsonTest, FileLifecycleReportsPriorObservationsAndUnresolvedCurrentPersistence) { - const auto config = explicit_config(1); - const auto plan = admitted_plan(config); - ASSERT_TRUE(plan.valid); - auto result = complete_result(config, plan); - result.prior_file_writer_attempts = 3; - result.prior_successful_file_writes = 2; - const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - const auto& lifecycle = doc["checkpoint_lifecycle"]; - EXPECT_EQ(lifecycle["observation_point"], "before-current-snapshot-preparation"); - EXPECT_EQ(lifecycle["prior_file_writer_attempts"], 3u); - EXPECT_EQ(lifecycle["prior_successful_file_writes"], 2u); - EXPECT_TRUE(lifecycle["current_persistence_success"].is_null()); - EXPECT_EQ(lifecycle["checkpoint_policy"], "bounded-loop-snapshots"); -} - -TEST(LlmMemoryJsonTest, MissingNamedEvidenceCannotPassThroughSuccessfulAggregateFlags) { - const auto config = explicit_config(1); - const auto plan = admitted_plan(config); - ASSERT_TRUE(plan.valid); - FakeLlmBackend backend; - backend.omit_measured_cold_checks = true; - LlmMemoryResult result; - EXPECT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_FAILURE); - const auto doc = build_llm_memory_json(config, plan, preparation_for(plan), fixed_metadata(config, plan), result); - EXPECT_FALSE(doc["run_accepted"]); - const auto& kv = doc["measurements"][1]; - EXPECT_NE(kv["status"], "measured"); - const auto& check = named_check(kv["execution"], "kv-append-final"); - EXPECT_TRUE(check["applicable"]); - EXPECT_FALSE(check["evaluated"]); - EXPECT_TRUE(check["valid"].is_null()); - EXPECT_TRUE(kv["effective_model_payload_gb_s"].is_null()); -} - TEST(LlmMemoryJsonTest, AutomaticCalibrationReferencesEveryDistinctPlanAndExpectedWitness) { auto config = explicit_config(1); config.user_specified_iterations = false; diff --git a/tests/test_llm_memory_kernels.cpp b/tests/test_llm_memory_kernels.cpp index c61f3ce..921992b 100644 --- a/tests/test_llm_memory_kernels.cpp +++ b/tests/test_llm_memory_kernels.cpp @@ -1013,29 +1013,74 @@ TEST(LlmMemoryKernelIntegrationTest, AllCpuWeightSpansDetectBoundedReadAndWorkUn } TEST(LlmMemoryKernelIntegrationTest, SafeZeroNullAndInvalidTopLevelBoundariesReturnInitialStates) { - // A null output is the sole case where no initialized result can be stored. - llm_decode_memory_asm(nullptr, nullptr, 0, 0, 0, 0, nullptr); - - alignas(16) std::array data{}; - LlmLayerDescriptor layer{data.data(), data.size(), 0, 1, 0, 0}; - LlmKvSequenceDescriptor sequence{data.data(), data.size(), data.data(), data.size(), data.data(), - 0, data.data(), 0, 0, 0}; - const LlmWorkerChecksum initial = oracle_initial_worker(); - const std::array, 6> cases = {{ - {0, 0, 0, 0, 0}, - {reinterpret_cast(&layer), 0, 1, 0, 1}, - {0, reinterpret_cast(&sequence), 1, 1, 1}, - {reinterpret_cast(&layer), 0, 1, 1, 2}, - {reinterpret_cast(&layer), reinterpret_cast(&sequence), 1, 1, 0}, - {reinterpret_cast(&layer), reinterpret_cast(&sequence), 1, 1, 4}, - }}; - for (const auto& test_case : cases) { - LlmWorkerChecksum actual; - std::memset(&actual, 0xA5, sizeof(actual)); - llm_decode_memory_asm(reinterpret_cast(test_case[0]), - reinterpret_cast(test_case[1]), test_case[2], test_case[3], - test_case[4], 123, &actual); - expect_worker_equal(actual, initial); + alignas(16) std::array weight{}; + for (size_t byte = 0; byte < weight.size(); ++byte) weight[byte] = static_cast((byte * 17 + 3) & 0xFF); + const auto weight_before = weight; + auto* const poison = reinterpret_cast(static_cast(1)); + auto* const poison_table = reinterpret_cast(static_cast(1)); + const auto verify = [&](auto kernel, const auto& layer, const auto& owner, bool prefill) { + kernel(nullptr, nullptr, 0, 0, 0, 0, nullptr); + const LlmWorkerChecksum initial = + prefill ? LlmWorkerChecksum{oracle_initial(LlmChecksumComponent::Weight), {}, {}} : oracle_initial_worker(); + struct Boundary { + bool layer; + bool owner; + uint64_t layers; + uint64_t work; + uint64_t flags; + }; + const std::array cases = {{{false, false, 0, 0, 0}, + {false, true, 1, 1, kLlmScenarioFlagWeight}, + {true, false, 0, 1, kLlmScenarioFlagWeight}, + {true, false, 1, 0, kLlmScenarioFlagWeight}, + {true, false, 1, 1, kLlmScenarioFlagKv}, + {true, true, 1, 1, 0}, + {true, true, 1, 1, 4}}}; + for (size_t index = 0; index < cases.size(); ++index) { + SCOPED_TRACE(index); + const auto& row = cases[index]; + LlmWorkerChecksum actual; + std::memset(&actual, 0xA5, sizeof(actual)); + kernel(row.layer ? &layer : nullptr, row.owner ? &owner : nullptr, row.layers, row.work, row.flags, 123, &actual); + expect_worker_equal(actual, initial); + } + // Empty ownership must be checked before any nested table, pool, or span load. + LlmWorkerChecksum zero_owned{}; + kernel(&layer, &owner, 1, 2, kLlmScenarioFlagKv, 99, &zero_owned); + expect_worker_equal(zero_owned, initial); + LlmWorkerChecksum expected = initial; + oracle_absorb(expected.weight, weight.data(), weight.size()); + oracle_absorb(expected.weight, weight.data(), weight.size()); + LlmWorkerChecksum actual{}; + kernel(&layer, reinterpret_cast(static_cast(1)), 1, 2, kLlmScenarioFlagWeight, + 0x0123456789ABCDEFULL, &actual); + expect_worker_equal(actual, expected); + EXPECT_EQ(weight, weight_before); + }; + const LlmLayerDescriptor decode{weight.data(), weight.size(), 0, 1, 0, 0}; + const LlmKvSequenceDescriptor decode_owner{poison, 0, poison, 0, poison, 0, poison, 0, 0, 0}; + const LlmPagedLayerDescriptor paged{weight.data(), weight.size(), 0, 1, 0, 0}; + const LlmPagedKvAssignmentDescriptor paged_owner{poison_table, poison, poison, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + const LlmPrefillLayerDescriptor prefill{weight.data(), weight.size(), 0, 1, 4, 0}; + const LlmPrefillKvSequenceDescriptor prefill_owner{poison, poison, 0, 0, 5, 2, 33, 4, 0, 0}; + const LlmPagedPrefillLayerDescriptor paged_prefill{weight.data(), weight.size(), 0, 1, 4, 0}; + const LlmPagedPrefillKvAssignmentDescriptor paged_prefill_owner{poison_table, poison, poison, 0, 0, 3, 2, + 66, 33, 5, 2, 33, 4, 0}; + { + SCOPED_TRACE("decode-contiguous"); + verify(llm_decode_memory_asm, decode, decode_owner, false); + } + { + SCOPED_TRACE("decode-paged"); + verify(llm_decode_memory_paged_asm, paged, paged_owner, false); + } + { + SCOPED_TRACE("prefill-contiguous"); + verify(llm_prefill_memory_asm, prefill, prefill_owner, true); + } + { + SCOPED_TRACE("prefill-paged"); + verify(llm_prefill_memory_paged_asm, paged_prefill, paged_prefill_owner, true); } } @@ -1168,149 +1213,53 @@ TEST(LlmMemoryKernelIntegrationTest, PreservesIntegerAndFullVectorCalleeSavedReg expect_worker_equal(output, expected.checksum); } -TEST(LlmMemoryKernelIntegrationTest, OneWorkerProductionExecutorRealAsmSmokeMatchesIndependentOracle) { - const LlmMemoryWorkPlan plan = build_real_executor_plan({257, 2, 4, 2, 7, 1, 3, 2}, 1); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmCpuExecutionPlan* cpu_plan = get_llm_cpu_execution_plan(plan); - ASSERT_NE(cpu_plan, nullptr); - ASSERT_EQ(cpu_plan->effective_workers, 1u); - LlmExecutionResources resources; - const LlmResourcePreparationResult prepared = prepare_llm_execution_resources(plan, resources); - ASSERT_TRUE(prepared.valid) << prepared.reason_code; - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - const OracleWorkerRun independent = - oracle_worker_run(resources.worker_layers(0), - cpu_plan->layer_descriptors_per_worker, - resources.worker_sequences(0), scenario.work_units, - kLlmScenarioFlagMixed, scenario.scenario_seed); - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - const LlmExecutorResult result = - execute_llm_scenario(plan, scenario, resources, *timer, production_llm_kernel_adapter()); - ASSERT_TRUE(result.valid) << result.reason_code; - EXPECT_EQ(result.reason_code, LlmExecutorReason::VALID); - EXPECT_TRUE(result.kernel_succeeded); - EXPECT_TRUE(result.checksum_valid); - EXPECT_TRUE(std::isfinite(result.elapsed_seconds)); - EXPECT_GT(result.elapsed_seconds, 0.0); - EXPECT_EQ(result.created_workers, 1u); - EXPECT_EQ(result.completed_workers, 1u); - ASSERT_EQ(result.actual_checksums.size(), 1u); - expect_worker_equal(result.actual_checksums[0], independent.checksum); - EXPECT_EQ(result.actual_checksums[0].weight.exact_bytes_read, scenario.weight_read_bytes); - EXPECT_EQ(result.actual_checksums[0].k.exact_bytes_read + result.actual_checksums[0].v.exact_bytes_read, - scenario.kv_read_bytes); -} - TEST(LlmMemoryKernelIntegrationTest, MultiWorkerProductionExecutorRealAsmCoversExactPlannerRanges) { - const LlmMemoryWorkPlan plan = build_real_executor_plan({513, 2, 4, 2, 7, 1, 3, 2}, 3); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmCpuExecutionPlan* cpu_plan = get_llm_cpu_execution_plan(plan); - ASSERT_NE(cpu_plan, nullptr); - ASSERT_EQ(cpu_plan->effective_workers, 3u); - LlmExecutionResources resources; - const LlmResourcePreparationResult prepared = prepare_llm_execution_resources(plan, resources); - ASSERT_TRUE(prepared.valid) << prepared.reason_code; - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - - std::vector independent; - independent.reserve(cpu_plan->effective_workers); - for (size_t worker = 0; worker < cpu_plan->effective_workers; ++worker) { - independent.push_back(oracle_worker_run( - resources.worker_layers(worker), - cpu_plan->layer_descriptors_per_worker, - resources.worker_sequences(worker), scenario.work_units, - kLlmScenarioFlagMixed, scenario.scenario_seed)); - } - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - const LlmExecutorResult result = - execute_llm_scenario(plan, scenario, resources, *timer, production_llm_kernel_adapter()); - ASSERT_TRUE(result.valid) << result.reason_code; - EXPECT_TRUE(result.kernel_succeeded); - EXPECT_TRUE(result.checksum_valid); - EXPECT_EQ(result.created_workers, cpu_plan->effective_workers); - EXPECT_EQ(result.completed_workers, cpu_plan->effective_workers); - ASSERT_EQ(result.actual_checksums.size(), independent.size()); - - uint64_t weight_bytes = 0; - uint64_t kv_read_bytes = 0; - for (size_t worker = 0; worker < independent.size(); ++worker) { - SCOPED_TRACE(::testing::Message() << "worker=" << worker); - expect_worker_equal(result.actual_checksums[worker], independent[worker].checksum); - weight_bytes += result.actual_checksums[worker].weight.exact_bytes_read; - kv_read_bytes += - result.actual_checksums[worker].k.exact_bytes_read + result.actual_checksums[worker].v.exact_bytes_read; - } - EXPECT_EQ(weight_bytes, scenario.weight_read_bytes); - EXPECT_EQ(kv_read_bytes, scenario.kv_read_bytes); -} - -TEST(LlmMemoryKernelIntegrationTest, - PagedSafeZeroNullAndWeightsOnlyBoundariesDoNotTouchKvResources) { - // A null output must return before constructing a frame or touching inputs. - llm_decode_memory_paged_asm(nullptr, nullptr, 0, 0, 0, 0, nullptr); - - alignas(16) std::array weight{}; - for (size_t byte = 0; byte < weight.size(); ++byte) { - weight[byte] = static_cast((byte * 17 + 3) & 0xFF); + for (size_t workers : {1u, 3u}) { + SCOPED_TRACE(workers); + const LlmMemoryWorkPlan plan = build_real_executor_plan({workers == 1 ? 257u : 513u, 2, 4, 2, 7, 1, 3, 2}, workers); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const LlmCpuExecutionPlan* cpu_plan = get_llm_cpu_execution_plan(plan); + ASSERT_NE(cpu_plan, nullptr); + ASSERT_EQ(cpu_plan->effective_workers, workers); + LlmExecutionResources resources; + const LlmResourcePreparationResult prepared = prepare_llm_execution_resources(plan, resources); + ASSERT_TRUE(prepared.valid) << prepared.reason_code; + const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 2, true); + ASSERT_TRUE(scenario.valid) << scenario.reason_code; + + std::vector independent; + independent.reserve(cpu_plan->effective_workers); + for (size_t worker = 0; worker < cpu_plan->effective_workers; ++worker) { + independent.push_back(oracle_worker_run(resources.worker_layers(worker), cpu_plan->layer_descriptors_per_worker, + resources.worker_sequences(worker), scenario.work_units, + kLlmScenarioFlagMixed, scenario.scenario_seed)); + } + auto timer = HighResTimer::create(); + ASSERT_TRUE(timer.has_value()); + const LlmExecutorResult result = + execute_llm_scenario(plan, scenario, resources, *timer, production_llm_kernel_adapter()); + ASSERT_TRUE(result.valid) << result.reason_code; + EXPECT_TRUE(result.kernel_succeeded); + EXPECT_EQ(result.reason_code, LlmExecutorReason::VALID); + EXPECT_TRUE(std::isfinite(result.elapsed_seconds)); + EXPECT_GT(result.elapsed_seconds, 0.0); + EXPECT_TRUE(result.checksum_valid); + EXPECT_EQ(result.created_workers, cpu_plan->effective_workers); + EXPECT_EQ(result.completed_workers, cpu_plan->effective_workers); + ASSERT_EQ(result.actual_checksums.size(), independent.size()); + + uint64_t weight_bytes = 0; + uint64_t kv_read_bytes = 0; + for (size_t worker = 0; worker < independent.size(); ++worker) { + SCOPED_TRACE(::testing::Message() << "worker=" << worker); + expect_worker_equal(result.actual_checksums[worker], independent[worker].checksum); + weight_bytes += result.actual_checksums[worker].weight.exact_bytes_read; + kv_read_bytes += + result.actual_checksums[worker].k.exact_bytes_read + result.actual_checksums[worker].v.exact_bytes_read; + } + EXPECT_EQ(weight_bytes, scenario.weight_read_bytes); + EXPECT_EQ(kv_read_bytes, scenario.kv_read_bytes); } - LlmPagedLayerDescriptor layer{weight.data(), weight.size(), 0, 1, 0, 0}; - LlmPagedKvAssignmentDescriptor zero_assignment{ - reinterpret_cast(static_cast(1)), - reinterpret_cast(static_cast(1)), - reinterpret_cast(static_cast(1)), - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - 0, - }; - const LlmWorkerChecksum initial = oracle_initial_worker(); - const auto expect_initial = [&](const LlmPagedLayerDescriptor* layers, - const LlmPagedKvAssignmentDescriptor* assignments, - uint64_t layer_count, uint64_t work_units, - uint64_t flags) { - LlmWorkerChecksum actual; - std::memset(&actual, 0xA5, sizeof(actual)); - llm_decode_memory_paged_asm(layers, assignments, layer_count, work_units, - flags, 123, &actual); - expect_worker_equal(actual, initial); - }; - - expect_initial(nullptr, &zero_assignment, 1, 1, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 0, 1, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 1, 0, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 1, 1, kLlmScenarioFlagKv); - expect_initial(&layer, &zero_assignment, 1, 1, 0); - expect_initial(&layer, &zero_assignment, 1, 1, 4); - - // owned_block_count==0 must be checked before any nested table or pool load. - LlmWorkerChecksum zero_owned{}; - llm_decode_memory_paged_asm(&layer, &zero_assignment, 1, 1, - kLlmScenarioFlagKv, 0, &zero_owned); - expect_worker_equal(zero_owned, initial); - - // An invalid assignment base makes any accidental weights-only KV access - // fault, while the weight component must still absorb both work units. - const auto weight_before = weight; - LlmWorkerChecksum expected_weights_only = initial; - oracle_absorb(expected_weights_only.weight, weight.data(), weight.size()); - oracle_absorb(expected_weights_only.weight, weight.data(), weight.size()); - LlmWorkerChecksum weights_only{}; - llm_decode_memory_paged_asm( - &layer, - reinterpret_cast( - static_cast(1)), - 1, 2, kLlmScenarioFlagWeight, 0x0123456789ABCDEFULL, &weights_only); - expect_worker_equal(weights_only, expected_weights_only); - EXPECT_EQ(weight, weight_before); } TEST(LlmMemoryKernelIntegrationTest, @@ -1499,58 +1448,6 @@ TEST(LlmMemoryKernelIntegrationTest, EXPECT_EQ(fixture.v_pools, expected_v); } -TEST(LlmMemoryKernelIntegrationTest, - PrefillSafeBoundariesAndWeightsOnlyNeverTouchKvDescriptors) { - llm_prefill_memory_asm(nullptr, nullptr, 0, 0, 0, 0, nullptr); - - alignas(16) std::array weight{}; - for (size_t byte = 0; byte < weight.size(); ++byte) { - weight[byte] = static_cast((byte * 17 + 3) & 0xFF); - } - LlmPrefillLayerDescriptor layer{weight.data(), weight.size(), 0, 1, 4, 0}; - LlmPrefillKvSequenceDescriptor zero_owner{ - reinterpret_cast(static_cast(1)), - reinterpret_cast(static_cast(1)), - 0, 0, 5, 2, 33, 4, 0, 0}; - const LlmWorkerChecksum initial{ - oracle_initial(LlmChecksumComponent::Weight), {}, {}}; - const auto expect_initial = [&](const LlmPrefillLayerDescriptor* layers, - const LlmPrefillKvSequenceDescriptor* owners, - uint64_t layer_count, - uint64_t operation_count, - uint64_t flags) { - LlmWorkerChecksum actual; - std::memset(&actual, 0xA5, sizeof(actual)); - llm_prefill_memory_asm(layers, owners, layer_count, operation_count, - flags, 123, &actual); - expect_worker_equal(actual, initial); - }; - - expect_initial(nullptr, &zero_owner, 1, 1, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 0, 1, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 1, 0, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 1, 1, kLlmScenarioFlagKv); - expect_initial(&layer, &zero_owner, 1, 1, 0); - expect_initial(&layer, &zero_owner, 1, 1, 4); - - LlmWorkerChecksum zero_owned{}; - llm_prefill_memory_asm(&layer, &zero_owner, 1, 2, - kLlmScenarioFlagKv, 99, &zero_owned); - expect_worker_equal(zero_owned, initial); - - LlmWorkerChecksum expected_weights_only = initial; - oracle_absorb(expected_weights_only.weight, weight.data(), weight.size()); - oracle_absorb(expected_weights_only.weight, weight.data(), weight.size()); - LlmWorkerChecksum weights_only{}; - llm_prefill_memory_asm( - &layer, - reinterpret_cast( - static_cast(1)), - 1, 2, kLlmScenarioFlagWeight, 0x0123456789ABCDEFULL, - &weights_only); - expect_worker_equal(weights_only, expected_weights_only); -} - TEST(LlmMemoryKernelIntegrationTest, PrefillQBoundariesAndExact31_32_33RecordsMatchByteOracle) { struct Case { @@ -1664,71 +1561,14 @@ TEST(LlmMemoryKernelIntegrationTest, expect_worker_equal(output, expected); } -TEST(LlmMemoryKernelsIntegrationTest, - PagedPrefillSafeBoundariesAndWeightsOnlyNeverTouchKvDescriptors) { - llm_prefill_memory_paged_asm(nullptr, nullptr, 0, 0, 0, 0, nullptr); - - alignas(16) std::array weight{}; - for (size_t byte = 0; byte < weight.size(); ++byte) { - weight[byte] = static_cast((byte * 17 + 3) & 0xFF); - } - LlmPagedPrefillLayerDescriptor layer{ - weight.data(), weight.size(), 0, 1, 4, 0}; - LlmPagedPrefillKvAssignmentDescriptor zero_owner{ - reinterpret_cast(static_cast(1)), - reinterpret_cast(static_cast(1)), - reinterpret_cast(static_cast(1)), - 0, 0, 3, 2, 66, 33, 5, 2, 33, 4, 0}; - const LlmWorkerChecksum initial{ - oracle_initial(LlmChecksumComponent::Weight), {}, {}}; - const auto expect_initial = [&](const LlmPagedPrefillLayerDescriptor* layers, - const LlmPagedPrefillKvAssignmentDescriptor* - assignments, - uint64_t layer_count, - uint64_t operation_count, uint64_t flags) { - LlmWorkerChecksum actual; - std::memset(&actual, 0xA5, sizeof(actual)); - llm_prefill_memory_paged_asm(layers, assignments, layer_count, - operation_count, flags, 123, &actual); - expect_worker_equal(actual, initial); - }; - - expect_initial(nullptr, &zero_owner, 1, 1, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 0, 1, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 1, 0, kLlmScenarioFlagWeight); - expect_initial(&layer, nullptr, 1, 1, kLlmScenarioFlagKv); - expect_initial(&layer, &zero_owner, 1, 1, 0); - expect_initial(&layer, &zero_owner, 1, 1, 4); - - // owned_block_count==0 must be checked before any inner pointer load. - LlmWorkerChecksum zero_owned{}; - llm_prefill_memory_paged_asm(&layer, &zero_owner, 1, 2, - kLlmScenarioFlagKv, 99, &zero_owned); - expect_worker_equal(zero_owned, initial); - - LlmWorkerChecksum expected_weights_only = initial; - oracle_absorb(expected_weights_only.weight, weight.data(), weight.size()); - oracle_absorb(expected_weights_only.weight, weight.data(), weight.size()); - LlmWorkerChecksum weights_only{}; - llm_prefill_memory_paged_asm( - &layer, - reinterpret_cast( - static_cast(1)), - 1, 2, kLlmScenarioFlagWeight, 0x0123456789ABCDEFULL, - &weights_only); - expect_worker_equal(weights_only, expected_weights_only); -} - -TEST(LlmMemoryKernelsIntegrationTest, - PagedPrefillGoldenGeometryAndExactRecordTailsMatchIndependentOracle) { +TEST(LlmMemoryKernelIntegrationTest, PagedPrefillGoldenGeometryAndExactRecordTailsMatchIndependentOracle) { struct Case { size_t prompt_tokens; size_t query_tile_tokens; size_t block_tokens; size_t record_bytes; }; - constexpr std::array kCases = { - {{5, 2, 2, 31}, {7, 3, 2, 32}, {6, 2, 4, 33}}}; + constexpr std::array kCases = {{{5, 2, 2, 31}, {7, 3, 2, 32}, {6, 2, 4, 33}, {5, 1, 2, 31}, {5, 5, 2, 33}}}; constexpr size_t kOperations = 2; constexpr size_t kLayer = 7; constexpr size_t kBatch = 3; @@ -1758,28 +1598,33 @@ TEST(LlmMemoryKernelsIntegrationTest, table[logical_block] = static_cast( physical_block_count - 1 - logical_block); } + const bool query_boundary = + test_case.query_tile_tokens == 1 || test_case.query_tile_tokens == test_case.prompt_tokens; + if (query_boundary) table = {3, 1, 4}; const std::vector table_before = table; - std::array weight{}; - for (size_t byte = 0; byte < weight.size(); ++byte) { - weight[byte] = static_cast((byte * 17 + 3) & 0xFF); - } - const auto weight_before = weight; - std::vector k_pool( - physical_block_count * block_bytes + kCanaryBytes); - std::vector v_pool( - physical_block_count * block_bytes + kCanaryBytes); - for (size_t byte = 0; byte < k_pool.size(); ++byte) { - k_pool[byte] = static_cast((byte * 19 + 5) & 0xFF); - v_pool[byte] = static_cast((byte * 29 + 7) & 0xFF); + const size_t pool_bytes = physical_block_count * block_bytes + (query_boundary ? 0 : kCanaryBytes); + GuardedMapping weight(33); + GuardedMapping k_pool(pool_bytes); + GuardedMapping v_pool(pool_bytes); + ASSERT_TRUE(weight.valid()); + ASSERT_TRUE(k_pool.valid()); + ASSERT_TRUE(v_pool.valid()); + for (size_t byte = 0; byte < 33; ++byte) weight.payload()[byte] = static_cast((byte * 17 + 3) & 0xFF); + for (size_t byte = 0; byte < pool_bytes; ++byte) { + k_pool.payload()[byte] = static_cast((byte * 19 + 5) & 0xFF); + v_pool.payload()[byte] = static_cast((byte * 29 + 7) & 0xFF); } - const std::vector initial_k = k_pool; - const std::vector initial_v = v_pool; - std::vector expected_k = k_pool; - std::vector expected_v = v_pool; - - LlmPagedPrefillLayerDescriptor layer{ - weight.data() + 1, 33, 0, 1, kLayer, 0}; + const std::vector weight_before(weight.payload(), weight.payload() + 33); + const std::vector weight_prefix(weight.accessible_begin(), weight.payload()); + const std::vector k_prefix(k_pool.accessible_begin(), k_pool.payload()); + const std::vector v_prefix(v_pool.accessible_begin(), v_pool.payload()); + const std::vector initial_k(k_pool.payload(), k_pool.payload() + pool_bytes); + const std::vector initial_v(v_pool.payload(), v_pool.payload() + pool_bytes); + auto expected_k = initial_k; + auto expected_v = initial_v; + + LlmPagedPrefillLayerDescriptor layer{weight.payload(), 33, 0, 1, kLayer, 0}; LlmPagedPrefillKvAssignmentDescriptor expected_assignment{ table.data(), expected_k.data(), expected_v.data(), 0, block_count, block_count, test_case.block_tokens, block_bytes, @@ -1790,18 +1635,21 @@ TEST(LlmMemoryKernelsIntegrationTest, kLlmScenarioFlagMixed, kSeed); LlmPagedPrefillKvAssignmentDescriptor actual_assignment = expected_assignment; - actual_assignment.k_layer_pool = k_pool.data(); - actual_assignment.v_layer_pool = v_pool.data(); + actual_assignment.k_layer_pool = k_pool.payload(); + actual_assignment.v_layer_pool = v_pool.payload(); LlmWorkerChecksum actual{}; llm_prefill_memory_paged_asm( &layer, &actual_assignment, 1, kOperations, kLlmScenarioFlagMixed, kSeed, &actual); expect_worker_equal(actual, expected); - EXPECT_EQ(weight, weight_before); + EXPECT_TRUE(std::equal(weight_before.begin(), weight_before.end(), weight.payload())); + EXPECT_TRUE(std::equal(weight_prefix.begin(), weight_prefix.end(), weight.accessible_begin())); + EXPECT_TRUE(std::equal(k_prefix.begin(), k_prefix.end(), k_pool.accessible_begin())); + EXPECT_TRUE(std::equal(v_prefix.begin(), v_prefix.end(), v_pool.accessible_begin())); EXPECT_EQ(table, table_before); - EXPECT_EQ(k_pool, expected_k); - EXPECT_EQ(v_pool, expected_v); + EXPECT_TRUE(std::equal(expected_k.begin(), expected_k.end(), k_pool.payload())); + EXPECT_TRUE(std::equal(expected_v.begin(), expected_v.end(), v_pool.payload())); size_t read_bytes_per_operation = 0; size_t spans_per_operation = 0; @@ -1833,21 +1681,17 @@ TEST(LlmMemoryKernelsIntegrationTest, for (size_t byte = 0; byte < valid_bytes; ++byte) { const size_t logical_byte = logical_block * block_bytes + byte; const size_t physical_byte = physical_block * block_bytes + byte; - EXPECT_EQ(k_pool[physical_byte], - oracle_prefill_byte( - kSeed, kOperations - 1, kLayer, kBatch, - LlmPrefillKvDomain::K, logical_byte)); - EXPECT_EQ(v_pool[physical_byte], - oracle_prefill_byte( - kSeed, kOperations - 1, kLayer, kBatch, - LlmPrefillKvDomain::V, logical_byte)); + EXPECT_EQ(k_pool.payload()[physical_byte], + oracle_prefill_byte(kSeed, kOperations - 1, kLayer, kBatch, LlmPrefillKvDomain::K, logical_byte)); + EXPECT_EQ(v_pool.payload()[physical_byte], + oracle_prefill_byte(kSeed, kOperations - 1, kLayer, kBatch, LlmPrefillKvDomain::V, logical_byte)); } } const size_t terminal_physical = table.back(); for (size_t byte = last_block_valid_bytes; byte < block_bytes; ++byte) { const size_t physical_byte = terminal_physical * block_bytes + byte; - EXPECT_EQ(k_pool[physical_byte], initial_k[physical_byte]); - EXPECT_EQ(v_pool[physical_byte], initial_v[physical_byte]); + EXPECT_EQ(k_pool.payload()[physical_byte], initial_k[physical_byte]); + EXPECT_EQ(v_pool.payload()[physical_byte], initial_v[physical_byte]); } for (size_t physical_block = 0; physical_block < physical_block_count; ++physical_block) { @@ -1855,146 +1699,21 @@ TEST(LlmMemoryKernelsIntegrationTest, continue; } const size_t first_byte = physical_block * block_bytes; - EXPECT_TRUE(std::equal( - k_pool.begin() + first_byte, - k_pool.begin() + first_byte + block_bytes, - initial_k.begin() + first_byte)); - EXPECT_TRUE(std::equal( - v_pool.begin() + first_byte, - v_pool.begin() + first_byte + block_bytes, - initial_v.begin() + first_byte)); - } - EXPECT_TRUE(std::equal( - k_pool.end() - kCanaryBytes, k_pool.end(), - initial_k.end() - kCanaryBytes)); - EXPECT_TRUE(std::equal( - v_pool.end() - kCanaryBytes, v_pool.end(), - initial_v.end() - kCanaryBytes)); - } -} - -TEST(LlmMemoryKernelsIntegrationTest, - PagedPrefillQOneAndQEqualsPromptUseExactGuardedRanges) { - struct Case { - size_t query_tile_tokens; - size_t record_bytes; - }; - constexpr size_t kPromptTokens = 5; - constexpr size_t kBlockTokens = 2; - constexpr size_t kBlockCount = 3; - constexpr size_t kPhysicalBlocks = 5; - constexpr size_t kOperations = 2; - constexpr size_t kLayer = 6; - constexpr size_t kBatch = 2; - constexpr uint64_t kSeed = 0xD1B54A32D192ED03ULL; - constexpr std::array kCases = {{{1, 31}, {kPromptTokens, 33}}}; - const std::array table = {3, 1, 4}; - - for (const Case& test_case : kCases) { - SCOPED_TRACE(::testing::Message() - << "P=" << kPromptTokens - << " Q=" << test_case.query_tile_tokens - << " R=" << test_case.record_bytes); - const size_t block_bytes = kBlockTokens * test_case.record_bytes; - const size_t last_block_valid_bytes = test_case.record_bytes; - const size_t pool_bytes = kPhysicalBlocks * block_bytes; - GuardedMapping weight(33); - GuardedMapping k_pool(pool_bytes); - GuardedMapping v_pool(pool_bytes); - ASSERT_TRUE(weight.valid()); - ASSERT_TRUE(k_pool.valid()); - ASSERT_TRUE(v_pool.valid()); - for (size_t byte = 0; byte < 33; ++byte) { - weight.payload()[byte] = - static_cast((byte * 17 + 3) & 0xFF); - } - for (size_t byte = 0; byte < pool_bytes; ++byte) { - k_pool.payload()[byte] = - static_cast((byte * 19 + 5) & 0xFF); - v_pool.payload()[byte] = - static_cast((byte * 29 + 7) & 0xFF); - } - const std::vector weight_prefix( - weight.accessible_begin(), weight.payload()); - const std::vector k_prefix( - k_pool.accessible_begin(), k_pool.payload()); - const std::vector v_prefix( - v_pool.accessible_begin(), v_pool.payload()); - const std::vector weight_before( - weight.payload(), weight.payload() + 33); - const std::vector initial_k( - k_pool.payload(), k_pool.payload() + pool_bytes); - const std::vector initial_v( - v_pool.payload(), v_pool.payload() + pool_bytes); - std::vector expected_k = initial_k; - std::vector expected_v = initial_v; - - LlmPagedPrefillLayerDescriptor layer{ - weight.payload(), 33, 0, 1, kLayer, 0}; - LlmPagedPrefillKvAssignmentDescriptor expected_assignment{ - table.data(), expected_k.data(), expected_v.data(), - 0, kBlockCount, kBlockCount, kBlockTokens, block_bytes, - last_block_valid_bytes, kPromptTokens, - test_case.query_tile_tokens, test_case.record_bytes, kLayer, kBatch}; - const LlmWorkerChecksum expected = oracle_paged_prefill_worker_run( - &layer, 1, &expected_assignment, kOperations, - kLlmScenarioFlagMixed, kSeed); - LlmPagedPrefillKvAssignmentDescriptor actual_assignment = - expected_assignment; - actual_assignment.k_layer_pool = k_pool.payload(); - actual_assignment.v_layer_pool = v_pool.payload(); - - LlmWorkerChecksum actual{}; - llm_prefill_memory_paged_asm( - &layer, &actual_assignment, 1, kOperations, - kLlmScenarioFlagMixed, kSeed, &actual); - expect_worker_equal(actual, expected); - EXPECT_TRUE(std::equal( - expected_k.begin(), expected_k.end(), k_pool.payload())); - EXPECT_TRUE(std::equal( - expected_v.begin(), expected_v.end(), v_pool.payload())); - EXPECT_TRUE(std::equal( - weight_before.begin(), weight_before.end(), weight.payload())); - EXPECT_TRUE(std::equal( - weight_prefix.begin(), weight_prefix.end(), - weight.accessible_begin())); - EXPECT_TRUE(std::equal( - k_prefix.begin(), k_prefix.end(), k_pool.accessible_begin())); - EXPECT_TRUE(std::equal( - v_prefix.begin(), v_prefix.end(), v_pool.accessible_begin())); - - size_t read_bytes_per_operation = 0; - size_t spans_per_operation = 0; - for (size_t tile_end = 0; tile_end < kPromptTokens;) { - tile_end += std::min(test_case.query_tile_tokens, - kPromptTokens - tile_end); - read_bytes_per_operation += tile_end * test_case.record_bytes; - spans_per_operation += - tile_end / kBlockTokens + - (tile_end % kBlockTokens == 0 ? 0 : 1); + EXPECT_TRUE(std::equal(k_pool.payload() + first_byte, k_pool.payload() + first_byte + block_bytes, + initial_k.begin() + first_byte)); + EXPECT_TRUE(std::equal(v_pool.payload() + first_byte, v_pool.payload() + first_byte + block_bytes, + initial_v.begin() + first_byte)); } - EXPECT_EQ(actual.k.exact_bytes_read, - read_bytes_per_operation * kOperations); - EXPECT_EQ(actual.v.exact_bytes_read, - read_bytes_per_operation * kOperations); - EXPECT_EQ(actual.k.span_count, spans_per_operation * kOperations); - EXPECT_EQ(actual.v.span_count, spans_per_operation * kOperations); - - // The terminal logical block is mapped to the last physical block, whose - // end directly borders PROT_NONE. Its invalid suffix must remain untouched. - const size_t terminal_base = - static_cast(table.back()) * block_bytes; - for (size_t byte = last_block_valid_bytes; byte < block_bytes; ++byte) { - EXPECT_EQ(k_pool.payload()[terminal_base + byte], - initial_k[terminal_base + byte]); - EXPECT_EQ(v_pool.payload()[terminal_base + byte], - initial_v[terminal_base + byte]); + if (!query_boundary) { + EXPECT_TRUE( + std::equal(initial_k.end() - kCanaryBytes, initial_k.end(), k_pool.payload() + pool_bytes - kCanaryBytes)); + EXPECT_TRUE( + std::equal(initial_v.end() - kCanaryBytes, initial_v.end(), v_pool.payload() + pool_bytes - kCanaryBytes)); } } } -TEST(LlmMemoryKernelsIntegrationTest, - PagedPrefillMultiLayerBatchTileAndOperationTraversalMatchesOracle) { +TEST(LlmMemoryKernelIntegrationTest, PagedPrefillMultiLayerBatchTileAndOperationTraversalMatchesOracle) { constexpr size_t kLayerCount = 2; constexpr size_t kBatchCount = 2; constexpr size_t kPromptTokens = 7; @@ -2083,8 +1802,7 @@ TEST(LlmMemoryKernelsIntegrationTest, EXPECT_EQ(actual.v.span_count, actual.k.span_count); } -TEST(LlmMemoryKernelsIntegrationTest, - PagedPrefillWrongSameMultiplicityBlockTableDoesNotMatchOracle) { +TEST(LlmMemoryKernelIntegrationTest, PagedPrefillWrongSameMultiplicityBlockTableDoesNotMatchOracle) { constexpr size_t kPromptTokens = 5; constexpr size_t kQueryTileTokens = 2; constexpr size_t kBlockTokens = 2; @@ -2144,8 +1862,7 @@ TEST(LlmMemoryKernelsIntegrationTest, actual.v.state_b != expected.v.state_b); } -TEST(LlmMemoryKernelsIntegrationTest, - PagedPrefillPreservesIntegerAndFullVectorCalleeSavedRegisters) { +TEST(LlmMemoryKernelIntegrationTest, PagedPrefillPreservesIntegerAndFullVectorCalleeSavedRegisters) { constexpr size_t kPromptTokens = 5; constexpr size_t kQueryTileTokens = 2; constexpr size_t kBlockTokens = 2; diff --git a/tests/test_llm_memory_output.cpp b/tests/test_llm_memory_output.cpp index e5bcd1c..e200e95 100644 --- a/tests/test_llm_memory_output.cpp +++ b/tests/test_llm_memory_output.cpp @@ -22,22 +22,12 @@ #include #include "core/config/constants.h" -#include "llm_memory/llm_cpu_backend.h" #include "llm_memory/llm_json.h" #include "llm_memory/llm_output.h" #include "output/console/messages/messages_api.h" -#include "utils/numeric_utils.h" namespace { -const LlmCpuExecutionPlan& cpu_execution_plan(const LlmMemoryWorkPlan& plan) { - const LlmCpuExecutionPlan* const cpu_plan = get_llm_cpu_execution_plan(plan); - if (cpu_plan == nullptr) { - throw std::logic_error("expected CPU execution plan"); - } - return *cpu_plan; -} - void set_headline(LlmMemoryResult& result, LlmScenario scenario, double latency_seconds, double work_units_per_second, double bandwidth_gb_s) { LlmScenarioAggregate& aggregate = result.aggregates[static_cast(scenario)]; @@ -63,6 +53,8 @@ LlmMemoryWorkPlan make_console_plan() { plan.geometry.kv_read_bytes_per_work_unit = 768; plan.geometry.kv_write_bytes_per_work_unit = 256; plan.geometry.kv_capacity_bytes = 2048; + plan.geometry.kv_only_effective_model_payload_bytes_per_work_unit = 1024; + plan.geometry.mixed_effective_model_payload_bytes_per_work_unit = 2048; plan.geometry.traffic_crossover_context_tokens = 4.0; return plan; } @@ -71,6 +63,9 @@ LlmMemoryWorkPlan make_paged_console_plan() { LlmMemoryWorkPlan plan = make_console_plan(); plan.kv_layout = LlmKvLayout::Paged; plan.geometry.kv_layout = LlmKvLayout::Paged; + plan.geometry.layer_count = 2; + plan.geometry.batch_size = 1; + plan.geometry.layout_metadata_lookups_per_layer_sequence_per_work_unit = 10; plan.geometry.kv_block_tokens = 4; plan.geometry.kv_blocks_per_sequence = 2; plan.geometry.physical_blocks_per_layer = 2; @@ -139,26 +134,6 @@ LlmMemoryWorkPlan make_metal_console_plan() { return plan; } -LlmMemoryWorkPlan make_metal_prefill_console_plan() { - LlmMemoryWorkPlan plan = make_prefill_console_plan(); - plan.backend = LlmMemoryBackend::Metal; - plan.geometry.kv_read_bytes_per_work_unit = 1408; - plan.geometry.kv_write_bytes_per_work_unit = 640; - plan.geometry.kv_only_effective_model_payload_bytes_per_work_unit = 2048; - plan.geometry.mixed_effective_model_payload_bytes_per_work_unit = 3072; - LlmMetalExecutionPlan execution; - execution.valid = true; - execution.reason_code = LlmMetalPlanReason::VALID; - execution.resources.valid = true; - execution.resources.reason_code = LlmMetalPlanReason::VALID; - execution.resources.weight_segments.segment_count = 1; - execution.resources.k_segments.segment_count = 1; - execution.resources.v_segments.segment_count = 1; - execution.resources.argument_buffer_encoded_length = 8192; - plan.backend_execution_plan = std::move(execution); - return plan; -} - LlmMemoryWorkPlan make_metal_paged_console_plan() { LlmGeometryRequest geometry_request; geometry_request.active_weight_bytes = 1024; @@ -303,159 +278,6 @@ LlmMemoryWorkPlan make_paged_prefill_console_plan() { return plan; } -LlmMemoryConfig fake_runner_config() { - LlmMemoryConfig config; - config.weight_size_mb = 1; - config.layer_count = 1; - config.query_head_count = 1; - config.kv_head_count = 1; - config.head_dimension = 16; - config.kv_element_bytes = 1; - config.visible_context_tokens = 2; - config.batch_size = 1; - config.requested_workers = 1; - config.available_workers = 1; - config.iterations = 4; - config.loop_count = 3; - config.seed = 42; - config.user_specified_iterations = true; - config.user_specified_seed = true; - config.user_specified_workers = true; - return config; -} - -LlmMemoryWorkPlanRequest fake_runner_plan_request(const LlmMemoryConfig& config) { - LlmMemoryWorkPlanRequest request; - request.geometry.active_weight_bytes = config.weight_size_mb * Constants::BYTES_PER_MB; - request.geometry.layer_count = config.layer_count; - request.geometry.query_head_count = config.query_head_count; - request.geometry.kv_head_count = config.kv_head_count; - request.geometry.head_dimension = config.head_dimension; - request.geometry.kv_element_bytes = config.kv_element_bytes; - request.geometry.visible_context_tokens = config.visible_context_tokens; - request.geometry.batch_size = config.batch_size; - request.requested_workers = config.requested_workers; - request.available_workers = config.available_workers; - request.available_memory_bytes = 8ULL * 1024ULL * Constants::BYTES_PER_MB; - request.mapping_granularity_bytes = 1; - request.base_seed = config.seed; - return request; -} - -LlmMemoryWorkPlan make_fake_runner_plan(const LlmMemoryConfig& config) { - LlmMemoryWorkPlanRequest request = fake_runner_plan_request(config); - const LlmMemoryWorkPlan preliminary = build_llm_memory_work_plan(request); - if (!preliminary.valid) { - return build_llm_memory_work_plan(request); - } - - const LlmExecutorAuxiliaryEstimate executor = calculate_llm_executor_auxiliary_estimate(preliminary); - const LlmRunnerAuxiliaryEstimate runner = calculate_llm_runner_auxiliary_estimate(config, preliminary); - if (!executor.valid || !runner.valid || - !NumericUtils::checked_add(executor.checksum_auxiliary_bytes, runner.checksum_auxiliary_bytes, - request.checksum_auxiliary_bytes) || - !NumericUtils::checked_add(executor.orchestration_auxiliary_bytes, runner.orchestration_auxiliary_bytes, - request.orchestration_auxiliary_bytes)) { - return build_llm_memory_work_plan(request); - } - return build_llm_memory_work_plan(request); -} - -LlmExecutorResult successful_fake_execution(const LlmMemoryWorkPlan& plan) { - const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); - LlmExecutorResult execution; - execution.valid = true; - execution.reason_code = LlmExecutorReason::VALID; - execution.elapsed_seconds = 0.150; - execution.requested_workers = cpu_plan.effective_workers; - execution.created_workers = cpu_plan.effective_workers; - execution.completed_workers = cpu_plan.effective_workers; - execution.qos_successful_workers = cpu_plan.effective_workers; - execution.kernel_succeeded = true; - execution.timer_started = true; - execution.timer_stopped = true; - execution.checksum_evaluated = true; - execution.checksum_valid = true; - execution.post_validation_evaluated = true; - execution.post_validation_valid = true; - execution.expected_checksums.resize(cpu_plan.effective_workers); - execution.actual_checksums = execution.expected_checksums; - execution.expected_run_checksum = {11, 22}; - execution.actual_run_checksum = execution.expected_run_checksum; - return execution; -} - -class FakeLlmBackend final : public LlmBackend { - public: - FakeLlmBackend() { evidence_.backend = LlmMemoryBackend::Cpu; } - - LlmMemoryBackend kind() const noexcept override { return LlmMemoryBackend::Cpu; } - - LlmBackendAuxiliaryEstimate calculate_auxiliary_estimate( - const LlmMemoryWorkPlan& model_plan) const noexcept override { - const LlmExecutorAuxiliaryEstimate cpu = calculate_llm_executor_auxiliary_estimate(model_plan); - LlmBackendAuxiliaryEstimate estimate; - estimate.valid = cpu.valid; - estimate.reason_code = cpu.reason_code; - estimate.checksum_auxiliary_bytes = cpu.checksum_auxiliary_bytes; - estimate.orchestration_auxiliary_bytes = cpu.orchestration_auxiliary_bytes; - estimate.total_auxiliary_bytes = cpu.total_auxiliary_bytes; - estimate.backend_evidence = cpu; - return estimate; - } - - LlmBackendLifecycleResult initialize(const LlmMemoryConfig&) noexcept override { - evidence_ = LlmBackendEvidence{}; - evidence_.backend = LlmMemoryBackend::Cpu; - evidence_.initialization = {LlmBackendStatus::Ready, LlmBackendReason::VALID}; - return evidence_.initialization; - } - - LlmBackendLifecycleResult resolve_execution_plan(const LlmMemoryWorkPlan&) noexcept override { - evidence_.plan_resolution = {LlmBackendStatus::Ready, LlmBackendReason::VALID}; - return evidence_.plan_resolution; - } - - LlmBackendLifecycleResult prepare_resources(const LlmMemoryWorkPlan&) noexcept override { - evidence_.backend_evidence = LlmCpuBackendEvidence{}; - evidence_.preparation = {LlmBackendStatus::Ready, LlmBackendReason::VALID}; - return evidence_.preparation; - } - - LlmExpectedChecksumResult expected_cpu_checksum(const LlmMemoryWorkPlan& model_plan, - const LlmScenarioWorkPlan&) const noexcept override { - LlmExpectedChecksumResult expected; - expected.valid = true; - expected.reason_code = LlmExecutorReason::VALID; - expected.workers.resize(cpu_execution_plan(model_plan).effective_workers); - expected.run_checksum = {11, 22}; - return expected; - } - - LlmTaskExecutionResult execute_task(const LlmMemoryWorkPlan& model_plan, const LlmScenarioWorkPlan& scenario_plan, - const LlmRunnerTaskContext& context) override { - auto execution = successful_fake_execution(model_plan); - execution.cold_checks = required_llm_cold_checks(model_plan, scenario_plan.scenario); - for (size_t slot = 0; slot < execution.cold_checks.size(); ++slot) { - resolve_llm_cold_check(execution.cold_checks, slot, true); - } - execution.kv_write_validation_applicable = scenario_plan.scenario != LlmScenario::WeightsOnly; - execution.kv_write_validation_evaluated = execution.kv_write_validation_applicable; - execution.kv_write_validation_valid = execution.kv_write_validation_applicable; - return adapt_llm_cpu_executor_result(model_plan, scenario_plan, context, std::move(execution)); - } - - const LlmBackendEvidence& evidence() const noexcept override { return evidence_; } - - LlmBackendLifecycleResult release_resources() noexcept override { - evidence_.release = {LlmBackendStatus::Ready, LlmBackendReason::VALID}; - return evidence_.release; - } - - private: - LlmBackendEvidence evidence_; -}; - size_t count_substrings(const std::string& text, const std::string& needle) { size_t count = 0; size_t position = 0; @@ -676,65 +498,6 @@ TEST(LlmMemoryOutputTest, MetalPagedReportPrintsTableLookupPaddingAndCanaryEvide EXPECT_NE(output.find("Metal validation: checksum=valid, kv_write=valid, canary=valid"), std::string::npos); } -TEST(LlmMemoryOutputTest, - MetalPrefillReportUsesKvWriteTerminologyWithoutTokenRate) { - const LlmMemoryWorkPlan plan = make_metal_prefill_console_plan(); - LlmBackendEvidence backend; - backend.backend = LlmMemoryBackend::Metal; - LlmMetalBackendEvidence metal_backend; - metal_backend.capability.device_name = "Test Metal Device"; - metal_backend.capability.argument_buffer_encoded_length = 8192; - backend.backend_evidence = std::move(metal_backend); - - LlmMemoryResult result; - set_headline(result, LlmScenario::Mixed, 0.004, 250.0, 75.0); - LlmMeasurementState measurement; - measurement.scenario = LlmScenario::Mixed; - LlmMetalTaskEvidence task; - task.pipeline_label = - "membenchmark.llm-metal.pipeline.prefill-contiguous.mixed"; - task.grid_plan_available = true; - task.grid_plan.actual_threadgroups = 2; - task.grid_plan.threads_per_threadgroup = 64; - task.timing_evaluated = true; - task.timing_valid = true; - measurement.execution.timing = {true, true, 0.004}; - task.checksum_evaluated = true; - task.checksum_valid = true; - task.kv_write_validation_evaluated = true; - task.kv_write_validation_valid = true; - measurement.execution.backend_evidence = std::move(task); - result.measurements.push_back(std::move(measurement)); - - LlmResultMetadata metadata; - metadata.main_thread_qos = {true, true, 0}; - metadata.environment_start.thermal_state = "nominal"; - metadata.environment_end = metadata.environment_start; - - testing::internal::CaptureStdout(); - testing::internal::CaptureStderr(); - print_llm_memory_console_report(plan, backend, metadata, result); - const std::string errors = testing::internal::GetCapturedStderr(); - const std::string output = testing::internal::GetCapturedStdout(); - - EXPECT_TRUE(errors.empty()) << errors; - EXPECT_NE(output.find("backend=metal, phase=prefill, " - "work_unit=prefill_operation, " - "kv_layout=contiguous"), - std::string::npos); - EXPECT_NE(output.find( - "pipeline=membenchmark.llm-metal.pipeline.prefill-contiguous.mixed"), - std::string::npos); - EXPECT_NE(output.find("Metal validation: checksum=valid, " - "kv_write=valid, canary=not-applicable"), - std::string::npos); - EXPECT_NE(output.find("250.00 synthetic prefill operations/s"), - std::string::npos); - EXPECT_EQ(output.find("Metal owner grid:"), std::string::npos); - EXPECT_EQ(output.find("append="), std::string::npos); - EXPECT_EQ(output.find("tokens/s"), std::string::npos); -} - TEST(LlmMemoryOutputTest, MetalPagedPrefillReportPublishesCompleteLayoutAndFullPromptEvidence) { const LlmMemoryWorkPlan plan = make_metal_paged_prefill_console_plan(); @@ -809,14 +572,6 @@ TEST(LlmMemoryOutputTest, EXPECT_NE(output.find("backend=metal, phase=prefill, " "work_unit=prefill_operation, kv_layout=paged"), std::string::npos); - EXPECT_NE(output.find(" Prompt tokens (P): 5\n"), - std::string::npos); - EXPECT_NE(output.find(" Attention query tile tokens (Q): 2\n"), - std::string::npos); - EXPECT_NE(output.find(" Attention query tiles (C): 3\n"), - std::string::npos); - EXPECT_NE(output.find(" Prefix token visits / sequence: 11\n"), - std::string::npos); EXPECT_NE(output.find("Metal segments: weights=1, K=1, V=1"), std::string::npos); EXPECT_NE(output.find("Paged KV block tokens (G): 2"), @@ -1062,51 +817,3 @@ TEST(LlmMemoryOutputTest, EmitsDeduplicatedWarningsInContractOrder) { "Warning: LLM Mixed duration quality is above-target-single-work-unit\n" "Warning: LLM KV only duration quality is above-target-window\n"); } - -TEST(LlmMemoryOutputTest, ConsoleHeadlinesAgreeExactlyWithJsonFromSameFakeRunnerResult) { - const LlmMemoryConfig config = fake_runner_config(); - const LlmMemoryWorkPlan plan = make_fake_runner_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - - FakeLlmBackend backend; - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS); - ASSERT_TRUE(result.results_complete); - - LlmResultMetadata metadata; - metadata.main_thread_qos = {true, true, 0}; - metadata.environment_start.thermal_state = "nominal"; - metadata.environment_end = metadata.environment_start; - const nlohmann::ordered_json document = - build_llm_memory_json(config, plan, LlmResourcePreparationResult{}, metadata, result); - - testing::internal::CaptureStdout(); - testing::internal::CaptureStderr(); - print_llm_memory_console_report(plan, backend.evidence(), metadata, result); - const std::string errors = testing::internal::GetCapturedStderr(); - const std::string output = testing::internal::GetCapturedStdout(); - EXPECT_TRUE(errors.empty()) << errors; - - for (LlmScenario scenario : {LlmScenario::WeightsOnly, LlmScenario::KvOnly, LlmScenario::Mixed}) { - const size_t index = static_cast(scenario); - const LlmScenarioAggregate& aggregate = result.aggregates[index]; - ASSERT_TRUE(aggregate.work_unit_latency_seconds.headline.has_value()); - ASSERT_TRUE(aggregate.synthetic_memory_work_units_per_second.headline.has_value()); - ASSERT_TRUE(aggregate.effective_model_payload_gb_s.headline.has_value()); - - const std::string scenario_token = llm_scenario_to_string(scenario); - const nlohmann::ordered_json& json_aggregate = document["aggregates"]["scenarios"][scenario_token]; - const double json_latency = json_aggregate["synthetic_work_unit_latency_seconds"]["headline"].get(); - const double json_work_units_per_second = - json_aggregate["synthetic_memory_work_units_per_second"]["headline"].get(); - const double json_bandwidth = json_aggregate["effective_model_payload_gb_s"]["headline"].get(); - EXPECT_DOUBLE_EQ(json_latency, *aggregate.work_unit_latency_seconds.headline); - EXPECT_DOUBLE_EQ(json_work_units_per_second, *aggregate.synthetic_memory_work_units_per_second.headline); - EXPECT_DOUBLE_EQ(json_bandwidth, *aggregate.effective_model_payload_gb_s.headline); - - const std::string expected_line = Messages::report_llm_memory_scenario_headline( - Messages::report_llm_memory_scenario_name(scenario_token), "decode step", "decode steps", json_latency * 1000.0, - json_work_units_per_second, json_bandwidth, scenario == LlmScenario::Mixed); - EXPECT_EQ(count_substrings(output, expected_line), 1u) << expected_line << "\n" << output; - } -} diff --git a/tests/test_llm_memory_runner.cpp b/tests/test_llm_memory_runner.cpp index e2d8a69..fcf3433 100644 --- a/tests/test_llm_memory_runner.cpp +++ b/tests/test_llm_memory_runner.cpp @@ -676,29 +676,6 @@ TEST(LlmMemoryRunnerTest, } } - size_t call_index = kLlmScenarioCount; - for (size_t loop_index = 0; loop_index < config.loop_count; ++loop_index) { - const std::array expected_order = - build_llm_scenario_order(loop_index); - for (size_t position = 0; position < kLlmScenarioCount; ++position) { - ASSERT_LT(call_index, executor.calls.size()); - const TaskRecord& measurement = executor.calls[call_index++]; - EXPECT_EQ(measurement.context.kind, - LlmRunnerTaskKind::Measurement); - EXPECT_EQ(measurement.context.purpose, "measurement"); - EXPECT_EQ(measurement.context.scenario, expected_order[position]); - EXPECT_EQ(measurement.context.loop_index, loop_index); - EXPECT_EQ(measurement.context.order_position, position); - const LlmScenarioWorkPlan& frozen = - frozen_plan(result, static_cast(expected_order[position])); - EXPECT_EQ(measurement.work_units, frozen.work_units); - EXPECT_EQ(measurement.payload_bytes, - frozen.effective_model_payload_bytes); - EXPECT_EQ(measurement.plan_identity, frozen.plan_identity); - } - } - EXPECT_EQ(call_index, executor.calls.size()); - ASSERT_EQ(result.loops.size(), 3u); EXPECT_EQ(result.loops[0].planned_order, (std::array{LlmScenario::WeightsOnly, LlmScenario::KvOnly, LlmScenario::Mixed})); @@ -727,19 +704,6 @@ TEST(LlmMemoryRunnerTest, size_t expected_payload_per_loop = 0; for (size_t handle : result.frozen_plan_handles) { const auto& scenario = result.scenario_plans.at(handle).plan; - EXPECT_EQ(scenario.work_unit_kind, LlmWorkUnitKind::DecodeStep); - EXPECT_EQ(scenario.kv_write_kind, - scenario.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::CurrentTokenAppend); - EXPECT_EQ(scenario.layout_metadata_lookup_count_per_work_unit, 0u); - EXPECT_EQ(scenario.layout_metadata_read_bytes_per_work_unit, 0u); - EXPECT_EQ(scenario.accounted_bytes_per_work_unit, - scenario.effective_model_payload_bytes_per_work_unit); - EXPECT_EQ(scenario.layout_metadata_lookup_count, 0u); - EXPECT_EQ(scenario.layout_metadata_read_bytes, 0u); - EXPECT_EQ(scenario.task_accounted_bytes, - scenario.effective_model_payload_bytes); expected_payload_per_loop += scenario.effective_model_payload_bytes; } EXPECT_EQ(result.counters.planned_effective_model_payload_bytes, expected_payload_per_loop * 3); @@ -758,12 +722,6 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(measurement_plan(result, measurement).effective_model_payload_bytes, frozen.effective_model_payload_bytes); EXPECT_EQ(measurement.execution.completion.completed_effective_model_payload_bytes, frozen.effective_model_payload_bytes); - EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count_per_work_unit, 0u); - EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_read_bytes_per_work_unit, 0u); - EXPECT_EQ(measurement_plan(result, measurement).accounted_bytes_per_work_unit, - frozen.effective_model_payload_bytes_per_work_unit); - EXPECT_EQ(measurement_plan(result, measurement).task_accounted_bytes, - frozen.effective_model_payload_bytes); EXPECT_EQ(measurement.execution.completion.completed_task_accounted_bytes, frozen.effective_model_payload_bytes); EXPECT_EQ(measurement.execution.status, LlmTaskExecutionStatus::Complete); @@ -832,6 +790,19 @@ TEST(LlmMemoryRunnerTest, PositionBalanceRetainsDirectedCarryoverAndPartialBlock LlmMemoryResult result; ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS); EXPECT_TRUE(result.run_accepted); + EXPECT_EQ(result.status, LlmRunStatus::Complete); + EXPECT_TRUE(result.results_complete); + if (count == 1) { + EXPECT_EQ(result.quality_warnings, (std::vector{"scenario-order-not-balanced"})); + EXPECT_EQ(result.logical_checkpoint_attempts, 2u); + for (const auto& aggregate : result.aggregates) { + EXPECT_EQ(aggregate.observed_cv_classification, "insufficient-samples"); + ASSERT_EQ(aggregate.accepted_measurement_ids.size(), 1u); + EXPECT_EQ(aggregate.effective_model_payload_gb_s.headline, + derive_llm_measurement_metrics(result.measurements.at(aggregate.accepted_measurement_ids.front())) + .payload_gb_s); + } + } EXPECT_EQ(result.scenario_order_balance_complete, count % 3 == 0); ASSERT_EQ(backend.calls.size(), 3 + 3 * count); std::array, 3> transitions{}; @@ -865,313 +836,92 @@ TEST(LlmMemoryRunnerTest, PositionBalanceRetainsDirectedCarryoverAndPartialBlock } } -TEST(LlmMemoryRunnerTest, CompleteSingleLoopIsAcceptedWithSeparateBalanceAndSampleQuality) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - LlmMemoryResult result; - - ASSERT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_SUCCESS); - EXPECT_EQ(result.status, LlmRunStatus::Complete); - EXPECT_TRUE(result.results_complete); - EXPECT_FALSE(result.scenario_order_balance_complete); - EXPECT_TRUE(result.run_accepted); - EXPECT_EQ(result.quality_warnings, (std::vector{"scenario-order-not-balanced"})); - EXPECT_EQ(result.logical_checkpoint_attempts, 2u); - for (const LlmScenarioAggregate& aggregate : result.aggregates) { - EXPECT_EQ(aggregate.observed_cv_classification, "insufficient-samples"); - ASSERT_EQ(aggregate.accepted_measurement_ids.size(), 1u); - EXPECT_EQ(aggregate.effective_model_payload_gb_s.headline, derive_llm_measurement_metrics(result.measurements.at(aggregate.accepted_measurement_ids.front())).payload_gb_s); - } -} - -TEST(LlmMemoryRunnerTest, - PreinitializedMetalReadyPlanIsReusedAndRetainsTaggedTaskEvidence) { - const LlmMemoryConfig config = explicit_metal_config(); - const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmMetalExecutionPlan* const execution_plan = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution_plan, nullptr); - ASSERT_TRUE(execution_plan->valid) << execution_plan->reason_code; - const LlmRunnerAuxiliaryEstimate auxiliary = - calculate_llm_runner_auxiliary_estimate(config, plan); - ASSERT_TRUE(auxiliary.valid) << auxiliary.reason_code; - EXPECT_EQ(auxiliary.checksum_auxiliary_bytes, 0u); - EXPECT_GT(auxiliary.orchestration_auxiliary_bytes, 0u); - - FakeLlmBackend backend(LlmMemoryBackend::Metal); - ASSERT_EQ(backend.initialize(config).status, LlmBackendStatus::Ready); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_SUCCESS); - EXPECT_EQ(backend.initialize_calls, 1u); - EXPECT_EQ(backend.plan_resolution_calls, 1u); - EXPECT_EQ(backend.preparation_calls, 1u); - EXPECT_EQ(backend.release_calls, 1u); - EXPECT_EQ(result.status, LlmRunStatus::Complete); - EXPECT_TRUE(result.results_complete); - ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); - for (const LlmMeasurementState& measurement : result.measurements) { - const auto* const metal = std::get_if(&measurement.execution.backend_evidence); - ASSERT_NE(metal, nullptr); - EXPECT_TRUE(metal->timed_pipeline_available); - EXPECT_TRUE(metal->grid_plan_available); - EXPECT_TRUE(metal->grid_plan.valid); - EXPECT_TRUE(metal->grid_plan.threadgroup_accounted_bytes.empty()); - EXPECT_TRUE(metal->checksum_evaluated); - EXPECT_TRUE(metal->checksum_valid); - EXPECT_TRUE(metal->kv_write_validation_valid); - EXPECT_TRUE(metal->post_validation_valid); - EXPECT_EQ((get_llm_cpu_task_evidence(measurement.execution) ? get_llm_cpu_task_evidence(measurement.execution)->requested_workers : 0), 0u); - EXPECT_EQ((get_llm_cpu_task_evidence(measurement.execution) ? get_llm_cpu_task_evidence(measurement.execution)->requested_workers : 0), 0u); - } - for (size_t index = 0; index < kLlmScenarioCount; ++index) { - ASSERT_EQ(result.calibration_attempt_counts[index], 1u); - const LlmTaskExecutionEvidence& compact = - result.calibration_attempts[index][0].execution; - EXPECT_FALSE(compact.cpu_evidence_available); - EXPECT_TRUE(compact.metal_evidence_available); - ASSERT_TRUE(compact.metal.has_value()); - EXPECT_EQ(compact.metal->pipeline_label, - "membenchmark.llm-metal.pipeline.decode-contiguous.fake"); - EXPECT_TRUE(compact.metal->checksum_valid); - } -} - -TEST(LlmMemoryRunnerTest, - PreinitializedPagedMetalPlanPreservesLookupAccounting) { - const LlmMemoryConfig config = explicit_metal_paged_config(); - const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.methodology_version, - "llm-memory-v2-metal-decode-paged"); - const LlmMetalExecutionPlan* const execution_plan = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution_plan, nullptr); - ASSERT_TRUE(execution_plan->valid) << execution_plan->reason_code; - ASSERT_TRUE(execution_plan->resources.paged_layout.has_value()); - ASSERT_TRUE(execution_plan->resources.table_segments.has_value()); - const LlmRunnerAuxiliaryEstimate auxiliary = - calculate_llm_runner_auxiliary_estimate(config, plan); - ASSERT_TRUE(auxiliary.valid) << auxiliary.reason_code; - - FakeLlmBackend backend(LlmMemoryBackend::Metal); - ASSERT_EQ(backend.initialize(config).status, - LlmBackendStatus::Ready); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_SUCCESS); - EXPECT_EQ(result.status, LlmRunStatus::Complete); - EXPECT_TRUE(result.results_complete); - ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); - size_t expected_task_lookups = 0; - for (const LlmMeasurementState& measurement : result.measurements) { - const size_t expected_per_work_unit = - measurement.scenario == LlmScenario::WeightsOnly ? 0 : 3; - EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count_per_work_unit, - expected_per_work_unit); - EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_read_bytes_per_work_unit, - expected_per_work_unit * sizeof(uint32_t)); - EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count, - expected_per_work_unit * config.iterations); - EXPECT_EQ(measurement.execution.completion.completed_layout_metadata_lookup_count, - expected_per_work_unit * config.iterations); - expected_task_lookups += expected_per_work_unit * config.iterations; - } - EXPECT_EQ(result.counters.planned_layout_metadata_lookup_count, - expected_task_lookups); - EXPECT_EQ(result.counters.completed_layout_metadata_lookup_count, - expected_task_lookups); -} - -TEST(LlmMemoryRunnerTest, - PreinitializedMetalPrefillRunsSingleOperationTasks) { - const LlmMemoryConfig config = explicit_metal_prefill_config(); - ASSERT_EQ(config.iterations, 1u); - const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.methodology_version, - "llm-memory-v2-metal-prefill-contiguous"); - EXPECT_EQ(plan.phase, LlmPhase::Prefill); - EXPECT_EQ(plan.kv_layout, LlmKvLayout::Contiguous); - EXPECT_EQ(plan.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - ASSERT_TRUE(plan.geometry.prefill.has_value()); - EXPECT_EQ(plan.geometry.prefill->prompt_tokens, 5u); - EXPECT_EQ(plan.geometry.prefill->attention_query_tile_tokens, 2u); - const LlmMetalExecutionPlan* const execution_plan = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution_plan, nullptr); - ASSERT_TRUE(execution_plan->valid) << execution_plan->reason_code; - - FakeLlmBackend backend(LlmMemoryBackend::Metal); - ASSERT_EQ(backend.initialize(config).status, - LlmBackendStatus::Ready); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_SUCCESS) - << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(frozen_plans_valid(result)); - ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); - ASSERT_EQ(backend.calls.size(), 2 * kLlmScenarioCount); - - for (size_t index = 0; index < kLlmScenarioCount; ++index) { - const LlmScenarioWorkPlan& frozen = - frozen_plan(result, index); - EXPECT_EQ(frozen.work_units, 1u); - EXPECT_EQ(frozen.work_unit_kind, - LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(frozen.kv_write_kind, - frozen.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation); - ASSERT_EQ(result.calibration_attempt_counts[index], 1u); - } - for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_EQ(measurement_plan(result, measurement).work_units, 1u); - EXPECT_EQ(measurement.execution.completion.completed_work_units, 1u); - EXPECT_EQ(measurement_plan(result, measurement).work_unit_kind, - LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(measurement_plan(result, measurement).kv_write_kind, - measurement.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation); - const LlmMetalRuntimeEvidence* const metal = - get_llm_metal_task_evidence(measurement.execution); - ASSERT_NE(metal, nullptr); - EXPECT_EQ( - metal->pipeline_label, - "membenchmark.llm-metal.pipeline.prefill-contiguous.fake"); - EXPECT_EQ(metal->kv_write_validation_evaluated, - measurement.scenario != LlmScenario::WeightsOnly); - EXPECT_TRUE(metal->kv_write_validation_valid); - } -} - -TEST(LlmMemoryRunnerTest, - PreinitializedMetalPagedPrefillPreservesFullPromptLookupEvidence) { - const LlmMemoryConfig config = explicit_metal_paged_prefill_config(); - const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.methodology_version, - "llm-memory-v2-metal-prefill-paged"); - EXPECT_EQ(plan.phase, LlmPhase::Prefill); - EXPECT_EQ(plan.kv_layout, LlmKvLayout::Paged); - ASSERT_TRUE(plan.prefill_plan.has_value()); - EXPECT_EQ(plan.prefill_plan->layout_metadata_lookups_per_layer_sequence, - 15U); - const LlmMetalExecutionPlan* const execution_plan = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution_plan, nullptr); - ASSERT_TRUE(execution_plan->valid) << execution_plan->reason_code; - ASSERT_TRUE(execution_plan->resources.paged_layout.has_value()); - ASSERT_TRUE(execution_plan->resources.table_segments.has_value()); - - FakeLlmBackend backend(LlmMemoryBackend::Metal); - ASSERT_EQ(backend.initialize(config).status, - LlmBackendStatus::Ready); - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_SUCCESS) - << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(frozen_plans_valid(result)); - ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); - - size_t expected_total_lookups = 0; - for (const LlmMeasurementState& measurement : result.measurements) { - const bool weights_only = - measurement.scenario == LlmScenario::WeightsOnly; - const size_t expected_lookups = weights_only ? 0 : 15; - EXPECT_EQ(measurement_plan(result, measurement).work_unit_kind, - LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(measurement_plan(result, measurement).kv_write_kind, - weights_only ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_lookup_count_per_work_unit, - expected_lookups); - EXPECT_EQ(measurement_plan(result, measurement).layout_metadata_read_bytes_per_work_unit, - expected_lookups * sizeof(uint32_t)); - EXPECT_EQ(measurement.execution.completion.completed_layout_metadata_lookup_count, - expected_lookups); - expected_total_lookups += expected_lookups; - - const LlmMetalRuntimeEvidence* const metal = - get_llm_metal_task_evidence(measurement.execution); - ASSERT_NE(metal, nullptr); - EXPECT_EQ(metal->pipeline_label, - "membenchmark.llm-metal.pipeline.prefill-paged.fake"); - EXPECT_EQ(plan.component_identities.checksum_pattern_version, - LlmMetalPrefillPagedVersion::CHECKSUM); - EXPECT_EQ(metal->grid_plan.paged_semantic_lookups, - expected_lookups); - EXPECT_EQ(metal->grid_plan.serial_range_visits_per_lane, - weights_only ? 1U : 0U); - EXPECT_EQ(metal->kv_write_validation_evaluated, !weights_only); - EXPECT_TRUE(metal->kv_write_validation_valid); - } - EXPECT_EQ(result.counters.planned_layout_metadata_lookup_count, - expected_total_lookups); - EXPECT_EQ(result.counters.completed_layout_metadata_lookup_count, - expected_total_lookups); -} - -TEST(LlmMemoryRunnerTest, - AutomaticMetalPrefillConfirmsAndFreezesIrreducibleSingleOperation) { - LlmMemoryConfig config = explicit_metal_prefill_config(); - config.iterations = 0; - config.user_specified_iterations = false; - config.weight_size_mb = 8; - const LlmMemoryWorkPlan plan = build_metal_runner_plan(config, true); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(plan.backend, LlmMemoryBackend::Metal); - ASSERT_EQ(plan.phase, LlmPhase::Prefill); - - FakeLlmBackend backend(LlmMemoryBackend::Metal); - backend.mutate = [](const LlmMemoryWorkPlan&, - const LlmScenarioWorkPlan& task_plan, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult& result) { - double elapsed_seconds = result.timing.elapsed_seconds; - if (context.purpose == "pilot") { - elapsed_seconds = static_cast(task_plan.work_units) * 0.300; - } else if (context.purpose == "single_unit_confirmation") { - elapsed_seconds = 0.300; +TEST(LlmMemoryRunnerTest, ReadyProfilesReuseInitializationAndRetainTaggedTaskEvidence) { + LlmMemoryConfig cpu_paged = explicit_config(1); + cpu_paged.kv_layout = LlmKvLayout::Paged; + cpu_paged.kv_block_tokens = 2; + cpu_paged.user_specified_kv_layout = true; + cpu_paged.user_specified_kv_block_tokens = true; + for (const auto& config : {explicit_metal_config(), explicit_metal_paged_config(), explicit_metal_prefill_config(), + explicit_metal_paged_prefill_config(), cpu_paged}) { + SCOPED_TRACE(::testing::Message() << "backend=" << static_cast(config.backend) + << " phase=" << static_cast(config.phase) + << " layout=" << static_cast(config.kv_layout)); + const bool metal_backend = config.backend == LlmMemoryBackend::Metal; + const bool prefill = config.phase == LlmPhase::Prefill; + const bool paged = config.kv_layout == LlmKvLayout::Paged; + const auto plan = metal_backend ? build_metal_runner_plan(config, true) : build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + FakeLlmBackend backend(config.backend); + ASSERT_EQ(backend.initialize(config).status, LlmBackendStatus::Ready); + LlmMemoryResult result; + ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), EXIT_SUCCESS); + EXPECT_EQ(backend.initialize_calls, 1u); + EXPECT_EQ(backend.plan_resolution_calls, 1u); + EXPECT_EQ(backend.preparation_calls, 1u); + EXPECT_EQ(backend.release_calls, 1u); + EXPECT_EQ(result.status, LlmRunStatus::Complete); + EXPECT_TRUE(result.results_complete); + ASSERT_TRUE(frozen_plans_valid(result)); + ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); + size_t total_lookups = 0; + for (const auto& measurement : result.measurements) { + const bool weights_only = measurement.scenario == LlmScenario::WeightsOnly; + const auto& task = measurement_plan(result, measurement); + const size_t lookups = !paged || weights_only ? 0 : prefill ? 15 : 3; + EXPECT_EQ(measurement.status, LlmMeasurementStatus::Measured); + EXPECT_TRUE(measurement.execution_evidence_available); + EXPECT_EQ(task.work_units, config.iterations); + EXPECT_EQ(measurement.execution.completion.completed_work_units, config.iterations); + EXPECT_EQ(task.work_unit_kind, prefill ? LlmWorkUnitKind::PrefillOperation : LlmWorkUnitKind::DecodeStep); + EXPECT_EQ(task.kv_write_kind, weights_only ? LlmKvWriteKind::None + : prefill ? LlmKvWriteKind::FullPromptPopulation + : LlmKvWriteKind::CurrentTokenAppend); + if (metal_backend) { + EXPECT_EQ(task.layout_metadata_lookup_count_per_work_unit, lookups); + EXPECT_EQ(task.layout_metadata_read_bytes_per_work_unit, lookups * sizeof(uint32_t)); + EXPECT_EQ(task.layout_metadata_lookup_count, lookups * config.iterations); + EXPECT_EQ(measurement.execution.completion.completed_layout_metadata_lookup_count, lookups * config.iterations); + total_lookups += lookups * config.iterations; + const auto* metal = get_llm_metal_task_evidence(measurement.execution); + ASSERT_NE(metal, nullptr); + EXPECT_EQ(get_llm_cpu_task_evidence(measurement.execution), nullptr); + EXPECT_TRUE(metal->timed_pipeline_available); + EXPECT_TRUE(metal->grid_plan_available); + EXPECT_TRUE(metal->grid_plan.valid); + EXPECT_TRUE(metal->grid_plan.threadgroup_accounted_bytes.empty()); + EXPECT_TRUE(metal->checksum_evaluated); + EXPECT_TRUE(metal->checksum_valid); + EXPECT_TRUE(metal->kv_write_validation_valid); + EXPECT_TRUE(metal->post_validation_valid); + const std::string label = std::string("membenchmark.llm-metal.pipeline.") + (prefill ? "prefill-" : "decode-") + + (paged ? "paged.fake" : "contiguous.fake"); + EXPECT_EQ(metal->pipeline_label, label); + if (prefill) EXPECT_EQ(metal->kv_write_validation_evaluated, !weights_only); + if (prefill && paged) { + EXPECT_EQ(metal->grid_plan.paged_semantic_lookups, lookups); + EXPECT_EQ(metal->grid_plan.serial_range_visits_per_lane, weights_only ? 1u : 0u); + } + } else { + const auto* cpu = std::get_if(&measurement.execution.backend_evidence); + ASSERT_NE(cpu, nullptr); + EXPECT_EQ(cpu->executor.reason_code, LlmExecutorReason::VALID); + EXPECT_TRUE(cpu->executor.post_validation_evaluated); + EXPECT_TRUE(cpu->executor.post_validation_valid); + } + } + if (metal_backend) { + EXPECT_EQ(result.counters.planned_layout_metadata_lookup_count, total_lookups); + EXPECT_EQ(result.counters.completed_layout_metadata_lookup_count, total_lookups); + for (size_t index = 0; index < kLlmScenarioCount; ++index) { + ASSERT_EQ(result.calibration_attempt_counts[index], 1u); + const auto& compact = result.calibration_attempts[index][0].execution; + EXPECT_FALSE(compact.cpu_evidence_available); + EXPECT_TRUE(compact.metal_evidence_available); + ASSERT_TRUE(compact.metal.has_value()); + EXPECT_TRUE(compact.metal->checksum_valid); + } } - result.timing.elapsed_seconds = elapsed_seconds; - LlmMetalTaskEvidence* const metal = - std::get_if(&result.backend_evidence); - ASSERT_NE(metal, nullptr); - metal->gpu_end_seconds = metal->gpu_start_seconds + elapsed_seconds; - metal->gpu_elapsed_seconds = elapsed_seconds; - }; - LlmMemoryResult result; - - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_SUCCESS) - << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(frozen_plans_valid(result)); - for (size_t handle : result.frozen_plan_handles) { - const auto& frozen = result.scenario_plans.at(handle).plan; - EXPECT_EQ(frozen.work_units, 1u); - EXPECT_EQ(frozen.work_unit_kind, - LlmWorkUnitKind::PrefillOperation); - } - - const std::vector kv = records_for_scenario( - backend.calls, LlmScenario::KvOnly, false); - const auto confirmation = std::find_if( - kv.begin(), kv.end(), [](const TaskRecord& record) { - return record.context.purpose == "single_unit_confirmation"; - }); - ASSERT_NE(confirmation, kv.end()); - EXPECT_EQ(confirmation->work_units, 1u); - ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); - for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_EQ(measurement_plan(result, measurement).work_units, 1u); - EXPECT_EQ(measurement.execution.completion.completed_work_units, 1u); - EXPECT_NE(get_llm_metal_task_evidence(measurement.execution), nullptr); - EXPECT_EQ(get_llm_cpu_task_evidence(measurement.execution), nullptr); } } @@ -1255,91 +1005,52 @@ TEST(LlmMemoryRunnerTest, } } -TEST(LlmMemoryRunnerTest, - PreinitializedMetalUnsupportedTerminatesUnresolvedPlanBeforeBudgetChecks) { - const LlmMemoryConfig config = explicit_metal_config(); - LlmMemoryWorkPlan unresolved = build_metal_runner_plan(config, false); - ASSERT_TRUE(unresolved.valid) << unresolved.reason_code; - const LlmMetalExecutionPlan* const unresolved_execution = - get_llm_metal_execution_plan(unresolved); - ASSERT_NE(unresolved_execution, nullptr); - ASSERT_FALSE(unresolved_execution->valid); - unresolved.memory_budget.request.checksum_auxiliary_bytes = 0; - unresolved.memory_budget.request.orchestration_auxiliary_bytes = 0; - - FakeLlmBackend backend(LlmMemoryBackend::Metal); - backend.initialization = {LlmBackendStatus::Unsupported, - LlmBackendReason::METAL_DEVICE_UNAVAILABLE}; - ASSERT_EQ(backend.initialize(config).status, - LlmBackendStatus::Unsupported); - std::vector checkpoints; - LlmMemoryResult result; - EXPECT_EQ(run_llm_memory_suite( - config, unresolved, backend, result, - recording_checkpoints(checkpoints)), - EXIT_FAILURE); - ASSERT_TRUE(result.initialized); - EXPECT_EQ(result.status, LlmRunStatus::Unsupported); - EXPECT_EQ(result.reason_code, - LlmBackendReason::METAL_DEVICE_UNAVAILABLE); - EXPECT_EQ(backend.initialize_calls, 1u); - EXPECT_EQ(backend.plan_resolution_calls, 0u); - EXPECT_EQ(backend.preparation_calls, 0u); - EXPECT_TRUE(backend.calls.empty()); - EXPECT_EQ(backend.release_calls, 1u); - ASSERT_EQ(checkpoints.size(), 1u); - EXPECT_EQ(checkpoints.front().kind, LlmCheckpointKind::CommandTerminal); - EXPECT_EQ(checkpoints.front().status, LlmRunStatus::Unsupported); - - const LlmMemoryWorkPlan not_preinitialized = - build_metal_runner_plan(config, false); - ASSERT_TRUE(not_preinitialized.valid) << not_preinitialized.reason_code; - FakeLlmBackend fresh_backend(LlmMemoryBackend::Metal); - LlmMemoryResult rejected; - EXPECT_EQ(run_llm_memory_suite( - config, not_preinitialized, fresh_backend, rejected), - EXIT_FAILURE); - EXPECT_FALSE(rejected.initialized); - EXPECT_EQ(rejected.reason_code, - LlmRunnerReason::INVALID_MODEL_WORK_PLAN); - EXPECT_EQ(fresh_backend.initialize_calls, 0u); - EXPECT_EQ(fresh_backend.release_calls, 0u); -} - -TEST(LlmMemoryRunnerTest, - PreinitializedMetalFailureProducesOneTerminalCheckpoint) { - const LlmMemoryConfig config = explicit_metal_config(); - const LlmMemoryWorkPlan logical_plan = - build_metal_runner_plan(config, false); - ASSERT_TRUE(logical_plan.valid) << logical_plan.reason_code; +TEST(LlmMemoryRunnerTest, PreinitializedMetalTerminalOutcomesPrecedeBudgetChecks) { + for (bool failed : {false, true}) { + SCOPED_TRACE(::testing::Message() << "failed=" << failed); + const LlmMemoryConfig config = explicit_metal_config(); + LlmMemoryWorkPlan unresolved = build_metal_runner_plan(config, false); + ASSERT_TRUE(unresolved.valid) << unresolved.reason_code; + const LlmMetalExecutionPlan* const unresolved_execution = get_llm_metal_execution_plan(unresolved); + ASSERT_NE(unresolved_execution, nullptr); + ASSERT_FALSE(unresolved_execution->valid); + unresolved.memory_budget.request.checksum_auxiliary_bytes = 0; + unresolved.memory_budget.request.orchestration_auxiliary_bytes = 0; + + FakeLlmBackend backend(LlmMemoryBackend::Metal); + backend.initialization = { + failed ? LlmBackendStatus::Failed : LlmBackendStatus::Unsupported, + failed ? LlmBackendReason::METAL_KERNEL_COMPILATION_FAILED : LlmBackendReason::METAL_DEVICE_UNAVAILABLE}; + ASSERT_EQ(backend.initialize(config).status, failed ? LlmBackendStatus::Failed : LlmBackendStatus::Unsupported); + std::vector checkpoints; + LlmMemoryResult result; + EXPECT_EQ(run_llm_memory_suite(config, unresolved, backend, result, recording_checkpoints(checkpoints)), + EXIT_FAILURE); + ASSERT_TRUE(result.initialized); + EXPECT_EQ(result.status, failed ? LlmRunStatus::Failed : LlmRunStatus::Unsupported); + EXPECT_EQ(result.reason_code, + failed ? LlmBackendReason::METAL_KERNEL_COMPILATION_FAILED : LlmBackendReason::METAL_DEVICE_UNAVAILABLE); + EXPECT_EQ(backend.initialize_calls, 1u); + EXPECT_EQ(backend.plan_resolution_calls, 0u); + EXPECT_EQ(backend.preparation_calls, 0u); + EXPECT_TRUE(backend.calls.empty()); + EXPECT_EQ(backend.release_calls, 1u); + ASSERT_EQ(checkpoints.size(), 1u); + EXPECT_EQ(checkpoints.front().kind, LlmCheckpointKind::CommandTerminal); + EXPECT_EQ(checkpoints.front().status, failed ? LlmRunStatus::Failed : LlmRunStatus::Unsupported); - FakeLlmBackend backend(LlmMemoryBackend::Metal); - backend.initialization = { - LlmBackendStatus::Failed, - LlmBackendReason::METAL_KERNEL_COMPILATION_FAILED}; - ASSERT_EQ(backend.initialize(config).status, - LlmBackendStatus::Failed); - std::vector checkpoints; - LlmMemoryResult result; + EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); - EXPECT_EQ(run_llm_memory_suite( - config, logical_plan, backend, result, - recording_checkpoints(checkpoints)), - EXIT_FAILURE); - ASSERT_TRUE(result.initialized); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, - LlmBackendReason::METAL_KERNEL_COMPILATION_FAILED); - EXPECT_EQ(backend.initialize_calls, 1u); - EXPECT_EQ(backend.plan_resolution_calls, 0u); - EXPECT_EQ(backend.preparation_calls, 0u); - EXPECT_TRUE(backend.calls.empty()); - EXPECT_EQ(backend.release_calls, 1u); - ASSERT_EQ(checkpoints.size(), 1u); - EXPECT_EQ(checkpoints.front().kind, - LlmCheckpointKind::CommandTerminal); - EXPECT_EQ(checkpoints.front().status, LlmRunStatus::Failed); - EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); + const LlmMemoryWorkPlan not_preinitialized = build_metal_runner_plan(config, false); + ASSERT_TRUE(not_preinitialized.valid) << not_preinitialized.reason_code; + FakeLlmBackend fresh_backend(LlmMemoryBackend::Metal); + LlmMemoryResult rejected; + EXPECT_EQ(run_llm_memory_suite(config, not_preinitialized, fresh_backend, rejected), EXIT_FAILURE); + EXPECT_FALSE(rejected.initialized); + EXPECT_EQ(rejected.reason_code, LlmRunnerReason::INVALID_MODEL_WORK_PLAN); + EXPECT_EQ(fresh_backend.initialize_calls, 0u); + EXPECT_EQ(fresh_backend.release_calls, 0u); + } } TEST(LlmMemoryRunnerTest, @@ -1404,132 +1115,48 @@ TEST(LlmMemoryRunnerTest, EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); } -TEST(LlmMemoryRunnerTest, InitializationUnsupportedAndFailedDoNotExecuteOrFallback) { - struct Case { - LlmBackendStatus backend_status; - LlmRunStatus run_status; - const char* reason_code; - }; - constexpr std::array cases = {{ - {LlmBackendStatus::Unsupported, LlmRunStatus::Unsupported, - LlmBackendReason::TASK_UNSUPPORTED}, - {LlmBackendStatus::Failed, LlmRunStatus::Failed, - LlmBackendReason::BACKEND_INITIALIZATION_FAILED}, - }}; - - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - for (const Case& test_case : cases) { - SCOPED_TRACE(test_case.reason_code); - FakeLlmBackend backend; - backend.initialization = {test_case.backend_status, - test_case.reason_code}; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_FAILURE); - EXPECT_TRUE(result.initialized); - EXPECT_EQ(result.status, test_case.run_status); - EXPECT_EQ(result.reason_code, test_case.reason_code); - EXPECT_EQ(backend.kind(), LlmMemoryBackend::Cpu); - EXPECT_EQ(backend.initialize_calls, 1u); - EXPECT_EQ(backend.plan_resolution_calls, 0u); - EXPECT_EQ(backend.preparation_calls, 0u); - EXPECT_TRUE(backend.calls.empty()); - EXPECT_EQ(backend.release_calls, 1u); - } -} - -TEST(LlmMemoryRunnerTest, - UnsupportedLifecycleIsTerminalAtEveryPhaseAndCanonicalBeforeRelease) { - enum class UnsupportedPhase { Initialization, PlanResolution, Preparation }; - constexpr std::array phases = { - UnsupportedPhase::Initialization, UnsupportedPhase::PlanResolution, - UnsupportedPhase::Preparation}; - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - - for (UnsupportedPhase phase : phases) { - FakeLlmBackend backend; - backend.initialization_reason_storage = "backend-private-unsupported"; - const LlmBackendLifecycleResult unsupported = { - LlmBackendStatus::Unsupported, - backend.initialization_reason_storage}; - if (phase == UnsupportedPhase::Initialization) { - backend.initialization = unsupported; - backend.clear_initialization_reason_on_release = true; - } else if (phase == UnsupportedPhase::PlanResolution) { - backend.plan_resolution = unsupported; - } else { - backend.preparation = unsupported; - } - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Unsupported); - EXPECT_EQ(result.reason_code, LlmBackendReason::TASK_UNSUPPORTED); - EXPECT_TRUE(backend.calls.empty()); - EXPECT_EQ(backend.initialize_calls, 1u); - EXPECT_EQ(backend.plan_resolution_calls, - phase == UnsupportedPhase::Initialization ? 0u : 1u); - EXPECT_EQ(backend.preparation_calls, - phase == UnsupportedPhase::Preparation ? 1u : 0u); - EXPECT_EQ(backend.release_calls, 1u); - } -} - -TEST(LlmMemoryRunnerTest, - LifecycleFailuresBeforeTasksProduceOneTerminalCheckpoint) { - enum class FailedPhase { Initialization, PlanResolution, Preparation }; - constexpr std::array phases = { - FailedPhase::Initialization, FailedPhase::PlanResolution, - FailedPhase::Preparation}; - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); +TEST(LlmMemoryRunnerTest, TerminalLifecycleOutcomesAreCanonicalAndCheckpointedAtEveryPhase) { + const auto config = explicit_config(1); + const auto plan = build_runner_admitted_plan(config); ASSERT_TRUE(plan.valid) << plan.reason_code; - - for (FailedPhase phase : phases) { - FakeLlmBackend backend; - const LlmBackendLifecycleResult failure = { - LlmBackendStatus::Failed, - LlmBackendReason::BACKEND_INITIALIZATION_FAILED}; - if (phase == FailedPhase::Initialization) { - backend.initialization = failure; - } else if (phase == FailedPhase::PlanResolution) { - backend.plan_resolution = failure; - } else { - backend.preparation = failure; + for (bool failed : {false, true}) { + SCOPED_TRACE(::testing::Message() << "failed=" << failed); + for (size_t phase = 0; phase < 3; ++phase) { + SCOPED_TRACE(::testing::Message() << "failed=" << failed << " phase=" << phase); + FakeLlmBackend backend; + backend.initialization_reason_storage = "backend-private-unsupported"; + const LlmBackendLifecycleResult terminal = { + failed ? LlmBackendStatus::Failed : LlmBackendStatus::Unsupported, + failed ? std::string_view(LlmBackendReason::BACKEND_INITIALIZATION_FAILED) + : std::string_view(backend.initialization_reason_storage)}; + if (phase == 0) { + backend.initialization = terminal; + backend.clear_initialization_reason_on_release = true; + } else if (phase == 1) + backend.plan_resolution = terminal; + else + backend.preparation = terminal; + std::vector checkpoints; + LlmMemoryResult result; + EXPECT_EQ(run_llm_memory_suite(config, plan, backend, result, recording_checkpoints(checkpoints)), EXIT_FAILURE); + EXPECT_TRUE(result.initialized); + EXPECT_EQ(result.status, failed ? LlmRunStatus::Failed : LlmRunStatus::Unsupported); + EXPECT_EQ(result.reason_code, + failed ? LlmBackendReason::BACKEND_INITIALIZATION_FAILED : LlmBackendReason::TASK_UNSUPPORTED); + EXPECT_TRUE(backend.calls.empty()); + EXPECT_EQ(backend.initialize_calls, 1u); + EXPECT_EQ(backend.plan_resolution_calls, phase == 0 ? 0u : 1u); + EXPECT_EQ(backend.preparation_calls, phase == 2 ? 1u : 0u); + EXPECT_EQ(backend.release_calls, 1u); + ASSERT_EQ(checkpoints.size(), 1u); + EXPECT_EQ(checkpoints.front().kind, LlmCheckpointKind::CommandTerminal); + EXPECT_EQ(checkpoints.front().status, result.status); + EXPECT_TRUE(checkpoints.front().terminal_checkpoint_attempted); + EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); + EXPECT_EQ(result.logical_checkpoint_attempts, 1u); + EXPECT_EQ(result.successful_logical_checkpoints, 1u); + EXPECT_TRUE(result.terminal_checkpoint_completed); } - std::vector checkpoints; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite( - config, plan, backend, result, - recording_checkpoints(checkpoints)), - EXIT_FAILURE); - EXPECT_TRUE(result.initialized); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, - LlmBackendReason::BACKEND_INITIALIZATION_FAILED); - EXPECT_TRUE(backend.calls.empty()); - ASSERT_EQ(checkpoints.size(), 1u); - EXPECT_EQ(checkpoints.front().kind, - LlmCheckpointKind::CommandTerminal); - EXPECT_EQ(checkpoints.front().status, LlmRunStatus::Failed); - EXPECT_TRUE(checkpoints.front().terminal_checkpoint_attempted); - EXPECT_FALSE(checkpoints.front().terminal_checkpoint_completed); - EXPECT_EQ(result.logical_checkpoint_attempts, 1u); - EXPECT_EQ(result.successful_logical_checkpoints, 1u); - EXPECT_TRUE(result.terminal_checkpoint_completed); - EXPECT_EQ(backend.initialize_calls, 1u); - EXPECT_EQ(backend.plan_resolution_calls, - phase == FailedPhase::Initialization ? 0u : 1u); - EXPECT_EQ(backend.preparation_calls, - phase == FailedPhase::Preparation ? 1u : 0u); - EXPECT_EQ(backend.release_calls, 1u); } } @@ -1562,8 +1189,6 @@ TEST(LlmMemoryRunnerTest, OversizedGenericReasonIsCanonicalizedIntoFrozenRunnerC EXPECT_EQ(result.statistics_workspace.absolute_deviations.capacity(), config.loop_count); for (const LlmScenarioAggregate& aggregate : result.aggregates) { EXPECT_EQ(aggregate.accepted_measurement_ids.capacity(), config.loop_count); - EXPECT_EQ(aggregate.accepted_measurement_ids.capacity(), config.loop_count); - EXPECT_EQ(aggregate.accepted_measurement_ids.capacity(), config.loop_count); } } @@ -1592,9 +1217,6 @@ TEST(LlmMemoryRunnerTest, size_t call_index = 0; for (LlmScenario scenario : kScenarios) { const size_t index = static_cast(scenario); - const LlmScenarioLimits limits = - calculate_llm_scenario_limits(plan.geometry, scenario); - ASSERT_TRUE(limits.valid); const TaskRecord& shape_warmup = executor.calls[call_index++]; const TaskRecord& pilot = executor.calls[call_index++]; const TaskRecord& correction = executor.calls[call_index++]; @@ -1608,16 +1230,10 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(shape_warmup.work_units, pilot.work_units); EXPECT_EQ(shape_warmup.payload_bytes, pilot.payload_bytes); EXPECT_EQ(shape_warmup.plan_identity, pilot.plan_identity); - EXPECT_EQ(pilot.work_units, calculate_llm_pilot_work_units(limits)); EXPECT_EQ(correction.context.kind, LlmRunnerTaskKind::Calibration); EXPECT_EQ(correction.context.purpose, "correction"); EXPECT_EQ(correction.context.scenario, scenario); - const size_t expected_work_units = - calculate_llm_calibrated_work_units( - 0.010 * (index + 1), pilot.work_units, limits); - EXPECT_EQ(correction.work_units, expected_work_units); - EXPECT_EQ(frozen_plan(result, index).work_units, - expected_work_units); + EXPECT_EQ(frozen_plan(result, index).work_units, correction.work_units); EXPECT_EQ(correction.payload_bytes, frozen_plan(result, index) .effective_model_payload_bytes); @@ -1721,309 +1337,132 @@ TEST(LlmMemoryRunnerTest, TEST(LlmMemoryRunnerTest, SingleUnitConfirmationWarmsOnlyNewShapeAndFreezesAboveTargetEvidence) { - LlmMemoryConfig config = automatic_prefill_config(); - config.weight_size_mb = 8; - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(plan.phase, LlmPhase::Prefill); - ASSERT_EQ(plan.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - const LlmScenarioLimits weights_limits = calculate_llm_scenario_limits( - plan.geometry, LlmScenario::WeightsOnly); - const LlmScenarioLimits kv_limits = - calculate_llm_scenario_limits(plan.geometry, LlmScenario::KvOnly); - const LlmScenarioLimits mixed_limits = - calculate_llm_scenario_limits(plan.geometry, LlmScenario::Mixed); - ASSERT_TRUE(weights_limits.valid); - ASSERT_TRUE(kv_limits.valid); - ASSERT_TRUE(mixed_limits.valid); - EXPECT_EQ(calculate_llm_pilot_work_units(weights_limits), 1u); - EXPECT_GT(calculate_llm_pilot_work_units(kv_limits), 1u); - EXPECT_EQ(calculate_llm_pilot_work_units(mixed_limits), 1u); + for (bool metal : {false, true}) { + SCOPED_TRACE(::testing::Message() << "metal=" << metal); + LlmMemoryConfig config = metal ? explicit_metal_prefill_config() : automatic_prefill_config(); + config.iterations = 0; + config.user_specified_iterations = false; + config.weight_size_mb = 8; + const LlmMemoryWorkPlan plan = metal ? build_metal_runner_plan(config, true) : build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + ASSERT_EQ(plan.phase, LlmPhase::Prefill); + ASSERT_EQ(plan.work_unit_kind, LlmWorkUnitKind::PrefillOperation); + const LlmScenarioLimits weights_limits = calculate_llm_scenario_limits(plan.geometry, LlmScenario::WeightsOnly); + const LlmScenarioLimits kv_limits = calculate_llm_scenario_limits(plan.geometry, LlmScenario::KvOnly); + const LlmScenarioLimits mixed_limits = calculate_llm_scenario_limits(plan.geometry, LlmScenario::Mixed); + ASSERT_TRUE(weights_limits.valid); + ASSERT_TRUE(kv_limits.valid); + ASSERT_TRUE(mixed_limits.valid); + EXPECT_EQ(calculate_llm_pilot_work_units(weights_limits), 1u); + EXPECT_GT(calculate_llm_pilot_work_units(kv_limits), 1u); + EXPECT_EQ(calculate_llm_pilot_work_units(mixed_limits), 1u); + + FakeLlmBackend executor(metal ? LlmMemoryBackend::Metal : LlmMemoryBackend::Cpu); + executor.mutate = [](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan& task_plan, + const LlmRunnerTaskContext& context, size_t, LlmTaskExecutionResult& result) { + if (context.purpose == "pilot") { + result.timing.elapsed_seconds = static_cast(task_plan.work_units) * 0.300; + } else if (context.purpose == "single_unit_confirmation") { + result.timing.elapsed_seconds = 0.300; + } + if (auto* evidence = std::get_if(&result.backend_evidence)) { + evidence->gpu_end_seconds = evidence->gpu_start_seconds + result.timing.elapsed_seconds; + evidence->gpu_elapsed_seconds = result.timing.elapsed_seconds; + } + }; + LlmMemoryResult result; - FakeLlmBackend executor; - executor.mutate = [](const LlmMemoryWorkPlan&, - const LlmScenarioWorkPlan& task_plan, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult& result) { - if (context.purpose == "pilot") { - result.timing.elapsed_seconds = - static_cast(task_plan.work_units) * 0.300; - } else if (context.purpose == "single_unit_confirmation") { - result.timing.elapsed_seconds = 0.300; + ASSERT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_SUCCESS); + ASSERT_TRUE(frozen_plans_valid(result)); + for (size_t handle : result.frozen_plan_handles) { + const auto& frozen = result.scenario_plans.at(handle).plan; + EXPECT_EQ(frozen.work_units, 1u); + EXPECT_EQ(frozen.work_unit_kind, LlmWorkUnitKind::PrefillOperation); + const LlmKvWriteKind expected_write = + frozen.scenario == LlmScenario::WeightsOnly ? LlmKvWriteKind::None : LlmKvWriteKind::FullPromptPopulation; + EXPECT_EQ(frozen.kv_write_kind, expected_write); } - }; - LlmMemoryResult result; - - ASSERT_EQ(run_llm_memory_suite(config, plan, executor, result), - EXIT_SUCCESS); - ASSERT_TRUE(frozen_plans_valid(result)); - for (size_t handle : result.frozen_plan_handles) { - const auto& frozen = result.scenario_plans.at(handle).plan; - EXPECT_EQ(frozen.work_units, 1u); - EXPECT_EQ(frozen.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - const LlmKvWriteKind expected_write = - frozen.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation; - EXPECT_EQ(frozen.kv_write_kind, expected_write); - } - - const std::vector weights = records_for_scenario( - executor.calls, LlmScenario::WeightsOnly, false); - const std::vector kv = - records_for_scenario(executor.calls, LlmScenario::KvOnly, false); - const std::vector mixed = - records_for_scenario(executor.calls, LlmScenario::Mixed, false); - ASSERT_EQ(weights.size(), 3u); - ASSERT_EQ(kv.size(), 5u); - ASSERT_EQ(mixed.size(), 3u); - EXPECT_EQ(weights[0].context.purpose, "calibration_shape_warmup"); - EXPECT_EQ(weights[1].context.purpose, "pilot"); - EXPECT_EQ(weights[2].context.purpose, "frozen_measurement_warmup"); - EXPECT_EQ(mixed[0].context.purpose, "calibration_shape_warmup"); - EXPECT_EQ(mixed[1].context.purpose, "pilot"); - EXPECT_EQ(mixed[2].context.purpose, "frozen_measurement_warmup"); - EXPECT_EQ(kv[0].context.purpose, "calibration_shape_warmup"); - EXPECT_EQ(kv[1].context.purpose, "pilot"); - EXPECT_EQ(kv[2].context.purpose, - "single_unit_confirmation_warmup"); - EXPECT_EQ(kv[2].context.kind, LlmRunnerTaskKind::Warmup); - EXPECT_EQ(kv[2].work_units, 1u); - EXPECT_EQ(kv[3].context.purpose, "single_unit_confirmation"); - EXPECT_EQ(kv[3].context.kind, LlmRunnerTaskKind::Calibration); - EXPECT_EQ(kv[3].work_units, 1u); - EXPECT_EQ(kv[4].context.purpose, "frozen_measurement_warmup"); - - ASSERT_EQ(result.calibration_attempts[0].size(), 3u); - ASSERT_EQ(result.calibration_attempts[1].size(), 5u); - ASSERT_EQ(result.calibration_attempts[2].size(), 3u); - EXPECT_EQ(result.calibration_attempts[0][1].duration_quality, - "above-target-single-work-unit"); - EXPECT_EQ(result.calibration_attempts[1][3].duration_quality, - "above-target-single-work-unit"); - EXPECT_EQ(result.calibration_attempts[2][1].duration_quality, - "above-target-single-work-unit"); - for (const TaskRecord& call : executor.calls) { - EXPECT_EQ(call.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - const LlmKvWriteKind expected_write = - call.context.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation; - EXPECT_EQ(call.kv_write_kind, expected_write); - } -} - -TEST(LlmMemoryRunnerTest, - FakeBackendRunsGenericPrefillWorkUnitsAndAccountsFullPromptWrites) { - const LlmMemoryConfig config = explicit_prefill_config(); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(plan.phase, LlmPhase::Prefill); - ASSERT_EQ(plan.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - ASSERT_TRUE(plan.geometry.prefill.has_value()); - EXPECT_FALSE(plan.geometry.decode.has_value()); - EXPECT_EQ(plan.geometry.prefill->prompt_tokens, config.prompt_tokens); - EXPECT_EQ(plan.geometry.prefill->attention_query_tile_tokens, - config.attention_query_tile_tokens); - FakeLlmBackend backend; - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, plan, backend, result), - EXIT_SUCCESS) - << result.reason_code << ": " << result.diagnostic; - ASSERT_TRUE(frozen_plans_valid(result)); - ASSERT_EQ(result.measurements.size(), - config.loop_count * kLlmScenarioCount); - ASSERT_EQ(backend.calls.size(), - kLlmScenarioCount + result.measurements.size()); - - size_t payload_bytes_per_loop = 0; - for (size_t index = 0; index < kLlmScenarioCount; ++index) { - const LlmScenarioWorkPlan& frozen = - frozen_plan(result, index); - const LlmKvWriteKind expected_write = - frozen.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation; - EXPECT_EQ(frozen.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(frozen.kv_write_kind, expected_write); - EXPECT_EQ(frozen.work_units, config.iterations); - if (frozen.scenario == LlmScenario::WeightsOnly) { - EXPECT_EQ(frozen.kv_write_bytes, 0u); - } else { - EXPECT_GT(frozen.kv_write_bytes, 0u); + const std::vector weights = records_for_scenario(executor.calls, LlmScenario::WeightsOnly, false); + const std::vector kv = records_for_scenario(executor.calls, LlmScenario::KvOnly, false); + const std::vector mixed = records_for_scenario(executor.calls, LlmScenario::Mixed, false); + ASSERT_EQ(weights.size(), 3u); + ASSERT_EQ(kv.size(), 5u); + ASSERT_EQ(mixed.size(), 3u); + EXPECT_EQ(weights[0].context.purpose, "calibration_shape_warmup"); + EXPECT_EQ(weights[1].context.purpose, "pilot"); + EXPECT_EQ(weights[2].context.purpose, "frozen_measurement_warmup"); + EXPECT_EQ(mixed[0].context.purpose, "calibration_shape_warmup"); + EXPECT_EQ(mixed[1].context.purpose, "pilot"); + EXPECT_EQ(mixed[2].context.purpose, "frozen_measurement_warmup"); + EXPECT_EQ(kv[0].context.purpose, "calibration_shape_warmup"); + EXPECT_EQ(kv[1].context.purpose, "pilot"); + EXPECT_EQ(kv[2].context.purpose, "single_unit_confirmation_warmup"); + EXPECT_EQ(kv[2].context.kind, LlmRunnerTaskKind::Warmup); + EXPECT_EQ(kv[2].work_units, 1u); + EXPECT_EQ(kv[3].context.purpose, "single_unit_confirmation"); + EXPECT_EQ(kv[3].context.kind, LlmRunnerTaskKind::Calibration); + EXPECT_EQ(kv[3].work_units, 1u); + EXPECT_EQ(kv[4].context.purpose, "frozen_measurement_warmup"); + + ASSERT_EQ(result.calibration_attempts[0].size(), 3u); + ASSERT_EQ(result.calibration_attempts[1].size(), 5u); + ASSERT_EQ(result.calibration_attempts[2].size(), 3u); + EXPECT_EQ(result.calibration_attempts[0][1].duration_quality, "above-target-single-work-unit"); + EXPECT_EQ(result.calibration_attempts[1][3].duration_quality, "above-target-single-work-unit"); + EXPECT_EQ(result.calibration_attempts[2][1].duration_quality, "above-target-single-work-unit"); + for (const TaskRecord& call : executor.calls) { + EXPECT_EQ(call.work_unit_kind, LlmWorkUnitKind::PrefillOperation); + const LlmKvWriteKind expected_write = call.context.scenario == LlmScenario::WeightsOnly + ? LlmKvWriteKind::None + : LlmKvWriteKind::FullPromptPopulation; + EXPECT_EQ(call.kv_write_kind, expected_write); } - payload_bytes_per_loop += frozen.effective_model_payload_bytes; - } - - const size_t expected_work_units = - config.loop_count * kLlmScenarioCount * config.iterations; - EXPECT_EQ(result.counters.planned_work_units, expected_work_units); - EXPECT_EQ(result.counters.completed_work_units, expected_work_units); - EXPECT_EQ(result.counters.planned_effective_model_payload_bytes, - payload_bytes_per_loop * config.loop_count); - EXPECT_EQ(result.counters.completed_effective_model_payload_bytes, - payload_bytes_per_loop * config.loop_count); - EXPECT_EQ(result.counters.planned_layout_metadata_lookup_count, 0u); - EXPECT_EQ(result.counters.completed_layout_metadata_lookup_count, 0u); - - for (const TaskRecord& call : backend.calls) { - const LlmKvWriteKind expected_write = - call.context.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation; - EXPECT_EQ(call.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(call.kv_write_kind, expected_write); - } - for (const LlmMeasurementState& measurement : result.measurements) { - const LlmKvWriteKind expected_write = - measurement.scenario == LlmScenario::WeightsOnly - ? LlmKvWriteKind::None - : LlmKvWriteKind::FullPromptPopulation; - EXPECT_EQ(measurement_plan(result, measurement).work_unit_kind, - LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(measurement_plan(result, measurement).kv_write_kind, expected_write); - EXPECT_EQ(measurement_plan(result, measurement).work_units, config.iterations); - EXPECT_EQ(measurement.execution.completion.completed_work_units, config.iterations); } } -TEST(LlmMemoryRunnerTest, - ProductionCpuBackendExecutesContiguousAndPagedPrefillIntegration) { - const LlmMemoryConfig config = explicit_prefill_config(1, 1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - - std::unique_ptr backend = create_llm_cpu_backend(); - ASSERT_NE(backend, nullptr); - const LlmBackendAuxiliaryEstimate auxiliary = - backend->calculate_auxiliary_estimate(plan); - ASSERT_TRUE(auxiliary.valid) << auxiliary.reason_code; - EXPECT_EQ(auxiliary.reason_code, LlmBackendReason::VALID); - EXPECT_GT(auxiliary.checksum_auxiliary_bytes, 0u); - EXPECT_GT(auxiliary.orchestration_auxiliary_bytes, 0u); - EXPECT_EQ(auxiliary.total_auxiliary_bytes, - auxiliary.checksum_auxiliary_bytes + - auxiliary.orchestration_auxiliary_bytes); - - const LlmBackendLifecycleResult initialization = - backend->initialize(config); - ASSERT_EQ(initialization.status, LlmBackendStatus::Ready); - EXPECT_EQ(initialization.reason_code, LlmBackendReason::VALID); - const LlmBackendLifecycleResult resolution = - backend->resolve_execution_plan(plan); - EXPECT_EQ(resolution.status, LlmBackendStatus::Ready); - EXPECT_EQ(resolution.reason_code, LlmBackendReason::VALID); - const LlmBackendLifecycleResult preparation = - backend->prepare_resources(plan); - ASSERT_EQ(preparation.status, LlmBackendStatus::Ready); - EXPECT_EQ(preparation.reason_code, LlmBackendReason::VALID); - - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::Mixed, 1, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - LlmRunnerTaskContext context; - context.kind = LlmRunnerTaskKind::Warmup; - context.purpose = "contiguous_prefill_production_boundary"; - context.scenario = LlmScenario::Mixed; - const LlmTaskExecutionResult execution = - backend->execute_task(plan, scenario, context); - EXPECT_EQ(execution.status, LlmTaskExecutionStatus::Complete); - EXPECT_EQ(execution.reason_code, LlmExecutorReason::VALID); - EXPECT_TRUE(execution.timing.evaluated); - EXPECT_TRUE(execution.timing.valid); - EXPECT_TRUE(execution.validation.evaluated); - EXPECT_TRUE(execution.validation.valid); - EXPECT_EQ(execution.completion.completed_work_units, 1u); - const LlmExecutorResult* const cpu_evidence = - get_llm_cpu_task_evidence(execution); - ASSERT_NE(cpu_evidence, nullptr); - EXPECT_TRUE(cpu_evidence->checksum_valid); - EXPECT_TRUE(cpu_evidence->post_validation_valid); - const LlmBackendLifecycleResult release = backend->release_resources(); - EXPECT_EQ(release.status, LlmBackendStatus::Ready); - EXPECT_EQ(release.reason_code, LlmBackendReason::VALID); - - LlmMemoryConfig paged_config = explicit_prefill_config(1, 1); - paged_config.kv_layout = LlmKvLayout::Paged; - paged_config.kv_block_tokens = 2; - paged_config.user_specified_kv_layout = true; - paged_config.user_specified_kv_block_tokens = true; - const LlmMemoryWorkPlan paged_plan = - build_runner_admitted_plan(paged_config); - ASSERT_TRUE(paged_plan.valid) << paged_plan.reason_code; - const LlmBackendAuxiliaryEstimate paged_auxiliary = - backend->calculate_auxiliary_estimate(paged_plan); - ASSERT_TRUE(paged_auxiliary.valid) << paged_auxiliary.reason_code; - EXPECT_GT(paged_auxiliary.checksum_auxiliary_bytes, 0u); - EXPECT_GT(paged_auxiliary.orchestration_auxiliary_bytes, 0u); - ASSERT_EQ(backend->initialize(paged_config).status, - LlmBackendStatus::Ready); - const LlmBackendLifecycleResult paged_resolution = - backend->resolve_execution_plan(paged_plan); - ASSERT_EQ(paged_resolution.status, LlmBackendStatus::Ready); - EXPECT_EQ(paged_resolution.reason_code, LlmBackendReason::VALID); - const LlmBackendLifecycleResult paged_preparation = - backend->prepare_resources(paged_plan); - ASSERT_EQ(paged_preparation.status, LlmBackendStatus::Ready); - EXPECT_EQ(paged_preparation.reason_code, LlmBackendReason::VALID); - - const LlmScenarioWorkPlan paged_scenario = - build_llm_scenario_work_plan(paged_plan, LlmScenario::Mixed, 1, true); - ASSERT_TRUE(paged_scenario.valid) << paged_scenario.reason_code; - EXPECT_GT(paged_scenario.layout_metadata_lookup_count, 0u); - LlmRunnerTaskContext paged_context; - paged_context.kind = LlmRunnerTaskKind::Warmup; - paged_context.purpose = "paged_prefill_production_boundary"; - paged_context.scenario = LlmScenario::Mixed; - const LlmTaskExecutionResult paged_execution = - backend->execute_task(paged_plan, paged_scenario, paged_context); - EXPECT_EQ(paged_execution.status, LlmTaskExecutionStatus::Complete); - EXPECT_EQ(paged_execution.reason_code, LlmExecutorReason::VALID); - EXPECT_TRUE(paged_execution.timing.evaluated); - EXPECT_TRUE(paged_execution.timing.valid); - EXPECT_TRUE(paged_execution.validation.evaluated); - EXPECT_TRUE(paged_execution.validation.valid); - EXPECT_EQ(paged_execution.completion.completed_work_units, 1u); - EXPECT_EQ(paged_execution.completion.completed_layout_metadata_lookup_count, - paged_scenario.layout_metadata_lookup_count); - const LlmExecutorResult* const paged_cpu_evidence = - get_llm_cpu_task_evidence(paged_execution); - ASSERT_NE(paged_cpu_evidence, nullptr); - EXPECT_TRUE(paged_cpu_evidence->checksum_valid); - EXPECT_TRUE(paged_cpu_evidence->post_validation_valid); - EXPECT_EQ(backend->release_resources().status, - LlmBackendStatus::Ready); -} - -TEST(LlmMemoryRunnerTest, StopBeforeFirstTaskInterruptsAllSlotsWithoutExecutorWork) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - std::vector checkpoints; - LlmRunnerHooks hooks = recording_checkpoints(checkpoints); - hooks.stop_requested = []() { return true; }; - LlmMemoryResult result; +TEST(LlmMemoryRunnerTest, StopBeforeMeasurementsInterruptsAllSlotsAndRetainsCompletedWarmup) { + for (bool after_warmup : {false, true}) { + SCOPED_TRACE(::testing::Message() << "after_warmup=" << after_warmup); + const LlmMemoryConfig config = explicit_config(2); + const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + bool stop = !after_warmup; + FakeLlmBackend executor; + executor.mutate = [&](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context, + size_t, LlmTaskExecutionResult&) { + if (context.kind == LlmRunnerTaskKind::Warmup) stop = true; + }; + std::vector checkpoints; + LlmRunnerHooks hooks = recording_checkpoints(checkpoints); + hooks.stop_requested = [&]() { return stop; }; + LlmMemoryResult result; - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_SUCCESS); - EXPECT_TRUE(executor.calls.empty()); - EXPECT_EQ(result.status, LlmRunStatus::Interrupted); - EXPECT_TRUE(result.interruption_requested); - EXPECT_EQ(result.counters.attempted_loops, 0u); - EXPECT_EQ(result.counters.attempted_measurements, 0u); - EXPECT_EQ(result.counters.terminal_measurements, 6u); - EXPECT_TRUE(frozen_plans_valid(result)); - EXPECT_GT(result.counters.planned_work_units, 0u); - EXPECT_GT(result.counters.planned_effective_model_payload_bytes, 0u); - for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_LT(static_cast(measurement.scenario), kLlmScenarioCount); - EXPECT_EQ(measurement_plan(result, measurement).work_units, config.iterations); - EXPECT_GT(measurement_plan(result, measurement).effective_model_payload_bytes, 0u); + EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_SUCCESS); + EXPECT_EQ(executor.calls.size(), after_warmup ? 1u : 0u); + if (after_warmup) { + ASSERT_EQ(result.calibration_attempts[0].size(), 1u); + EXPECT_TRUE(result.calibration_attempts[0][0].terminal); + EXPECT_TRUE(result.calibration_attempts[0][0].valid); + EXPECT_EQ(result.calibration_attempts[0][0].purpose, "frozen_measurement_warmup"); + } + EXPECT_EQ(result.status, LlmRunStatus::Interrupted); + EXPECT_TRUE(result.interruption_requested); + EXPECT_EQ(result.counters.attempted_loops, 0u); + EXPECT_EQ(result.counters.attempted_measurements, 0u); + EXPECT_EQ(result.counters.terminal_measurements, 6u); + EXPECT_TRUE(frozen_plans_valid(result)); + EXPECT_GT(result.counters.planned_work_units, 0u); + EXPECT_GT(result.counters.planned_effective_model_payload_bytes, 0u); + for (const LlmMeasurementState& measurement : result.measurements) { + EXPECT_LT(static_cast(measurement.scenario), kLlmScenarioCount); + EXPECT_EQ(measurement_plan(result, measurement).work_units, config.iterations); + EXPECT_GT(measurement_plan(result, measurement).effective_model_payload_bytes, 0u); + } + expect_interrupted_tail(result, 0); + ASSERT_EQ(checkpoints.size(), 1u); + EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); } - expect_interrupted_tail(result, 0); - ASSERT_EQ(checkpoints.size(), 1u); - EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); } TEST(LlmMemoryRunnerTest, StopDuringStartedMeasurementKeepsCurrentTaskAndInterruptsTail) { @@ -2139,310 +1578,137 @@ TEST(LlmMemoryRunnerTest, StopHookExceptionAfterLastMeasurementInvalidatesOtherw }; hooks.checkpoint = [&](const LlmMemoryResult& snapshot, LlmCheckpointKind kind) { checkpoints.push_back(capture_checkpoint(snapshot, kind)); - if (kind == LlmCheckpointKind::MeasurementTerminal && snapshot.counters.measured_measurements == 9) { - throw_from_stop = true; - } - return EXIT_SUCCESS; - }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_EQ(result.diagnostic, "injected stop failure"); - EXPECT_TRUE(result.results_complete); - EXPECT_TRUE(result.scenario_order_balance_complete); - EXPECT_FALSE(result.run_accepted); - EXPECT_EQ(result.counters.measured_measurements, 9u); - ASSERT_EQ(checkpoints.size(), 4u); - EXPECT_EQ(checkpoints[2].kind, LlmCheckpointKind::MeasurementTerminal); - EXPECT_EQ(checkpoints[2].measured_measurements, 9u); - EXPECT_EQ(checkpoints[2].status, LlmRunStatus::Complete); - EXPECT_EQ(checkpoints.back().kind, LlmCheckpointKind::CommandTerminal); - EXPECT_EQ(checkpoints.back().status, LlmRunStatus::Failed); -} - -TEST(LlmMemoryRunnerTest, BackendFailureWinsSimultaneousStopAndPreservesInterruptedTail) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - bool stop = false; - FakeLlmBackend executor; - executor.mutate = [&](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context, - size_t, LlmTaskExecutionResult& execution) { - if (context.kind == LlmRunnerTaskKind::Measurement) { - stop = true; - fail_execution(execution); - } - }; - LlmRunnerHooks hooks; - hooks.stop_requested = [&]() { return stop; }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, LlmBackendReason::RESOURCES_NOT_PREPARED); - EXPECT_TRUE(result.interruption_requested); - ASSERT_EQ(result.measurements[0].status, LlmMeasurementStatus::Failed); - EXPECT_EQ(result.measurements[0].reason_code, - LlmBackendReason::RESOURCES_NOT_PREPARED); - for (size_t index = 1; index < result.measurements.size(); ++index) { - EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Interrupted); - } -} - -TEST(LlmMemoryRunnerTest, ValidationFailureWinsSimultaneousStopAndIsExcludedFromAggregates) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - bool stop = false; - FakeLlmBackend executor; - executor.mutate = [&](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context, - size_t, LlmTaskExecutionResult& execution) { - if (context.kind == LlmRunnerTaskKind::Measurement) { - stop = true; - invalidate_execution(execution); - } - }; - LlmRunnerHooks hooks; - hooks.stop_requested = [&]() { return stop; }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, LlmBackendReason::VALIDATION_FAILED); - EXPECT_TRUE(result.interruption_requested); - EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Invalid); - EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); - EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); -} - -TEST(LlmMemoryRunnerTest, MeasurementCheckpointFailureIsTerminalAndNeverRetried) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - std::vector kinds; - LlmRunnerHooks hooks; - hooks.stop_requested = []() { return false; }; - hooks.checkpoint = [&](const LlmMemoryResult&, LlmCheckpointKind kind) { - kinds.push_back(kind); - return EXIT_FAILURE; - }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::MeasurementTerminal})); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, LlmRunnerReason::CHECKPOINT_WRITE_FAILED); - EXPECT_TRUE(result.checkpoint_failed); - EXPECT_FALSE(result.terminal_checkpoint_attempted); - EXPECT_EQ(result.counters.attempted_measurements, 3u); - EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Measured); - EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); - EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); - EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).work_units_per_second.has_value()); - EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).payload_gb_s.has_value()); - EXPECT_TRUE(get_llm_cpu_task_evidence(result.measurements[0].execution)->checksum_valid); - EXPECT_EQ(result.measurements[0].execution.status, - LlmTaskExecutionStatus::Complete); - EXPECT_TRUE(result.measurements[0].execution.timing.valid); - EXPECT_TRUE(result.measurements[0].execution.validation.valid); - EXPECT_TRUE(std::holds_alternative( - result.measurements[0].execution.backend_evidence)); - EXPECT_EQ(result.aggregates[0].accepted_measurement_ids.size(), 1u); - EXPECT_EQ(result.counters.planned_loops, 2u); - EXPECT_EQ(result.counters.attempted_loops, 1u); - EXPECT_EQ(result.counters.completed_loops, 1u); - EXPECT_EQ(result.counters.planned_measurements, 6u); - EXPECT_EQ(result.counters.terminal_measurements, 6u); - EXPECT_EQ(result.counters.measured_measurements, 3u); - EXPECT_EQ(result.counters.completed_work_units, 3 * config.iterations); - EXPECT_EQ(result.counters.completed_effective_model_payload_bytes, - measurement_plan(result, result.measurements[0]).effective_model_payload_bytes + - measurement_plan(result, result.measurements[1]).effective_model_payload_bytes + - measurement_plan(result, result.measurements[2]).effective_model_payload_bytes); - for (size_t index = 3; index < result.measurements.size(); ++index) { - EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Failed); - } - -} - -TEST(LlmMemoryRunnerTest, CanonicalResultReasonsDetachFromEveryOwningDomain) { - std::string runner_reason = LlmRunnerReason::CHECKPOINT_WRITE_FAILED; - std::string work_plan_reason = LlmWorkPlanReason::TASK_ACCOUNTED_BYTES_CAP_EXCEEDED; - std::string executor_reason = LlmExecutorReason::KERNEL_FAILED; - std::string paged_option_reason = - LlmWorkPlanReason::KV_BLOCK_TOKENS_NOT_APPLICABLE; - - const std::string_view canonical_runner = canonicalize_llm_result_reason_code(runner_reason); - const std::string_view canonical_work_plan = canonicalize_llm_result_reason_code(work_plan_reason); - const std::string_view canonical_executor = canonicalize_llm_result_reason_code(executor_reason); - const std::string_view canonical_paged_option = - canonicalize_llm_result_reason_code(paged_option_reason); - std::fill(runner_reason.begin(), runner_reason.end(), 'x'); - std::fill(work_plan_reason.begin(), work_plan_reason.end(), 'x'); - std::fill(executor_reason.begin(), executor_reason.end(), 'x'); - std::fill(paged_option_reason.begin(), paged_option_reason.end(), 'x'); - - EXPECT_EQ(canonical_runner, LlmRunnerReason::CHECKPOINT_WRITE_FAILED); - EXPECT_EQ(canonical_runner.data(), - canonicalize_llm_result_reason_code(LlmRunnerReason::CHECKPOINT_WRITE_FAILED).data()); - EXPECT_EQ(canonical_work_plan, LlmWorkPlanReason::TASK_ACCOUNTED_BYTES_CAP_EXCEEDED); - EXPECT_EQ(canonical_work_plan.data(), - canonicalize_llm_result_reason_code(LlmWorkPlanReason::TASK_ACCOUNTED_BYTES_CAP_EXCEEDED).data()); - EXPECT_EQ(canonical_executor, LlmExecutorReason::KERNEL_FAILED); - EXPECT_EQ(canonical_executor.data(), - canonicalize_llm_result_reason_code(LlmExecutorReason::KERNEL_FAILED).data()); - EXPECT_EQ(canonical_paged_option, - LlmWorkPlanReason::KV_BLOCK_TOKENS_NOT_APPLICABLE); - EXPECT_EQ(canonical_paged_option.data(), - canonicalize_llm_result_reason_code( - LlmWorkPlanReason::KV_BLOCK_TOKENS_NOT_APPLICABLE) - .data()); - EXPECT_EQ(canonicalize_llm_result_reason_code("not-a-reason"), LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); -} - -TEST(LlmMemoryRunnerTest, - PagedPostValidationFailureCanonicalizesExactlyThroughRunnerFailure) { - std::string owned_reason = - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED; - const std::string_view canonical = - canonicalize_llm_result_reason_code(owned_reason); - std::fill(owned_reason.begin(), owned_reason.end(), 'x'); - EXPECT_EQ(canonical, LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - EXPECT_NE(canonical, LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); - EXPECT_EQ( - canonical.data(), - canonicalize_llm_result_reason_code( - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED) - .data()); - - LlmMemoryConfig config = explicit_config(1); - config.kv_layout = LlmKvLayout::Paged; - config.kv_block_tokens = 2; - config.user_specified_kv_layout = true; - config.user_specified_kv_block_tokens = true; - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - executor.mutate = [](const LlmMemoryWorkPlan&, - const LlmScenarioWorkPlan&, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult& execution) { - if (context.kind != LlmRunnerTaskKind::Measurement) { - return; + if (kind == LlmCheckpointKind::MeasurementTerminal && snapshot.counters.measured_measurements == 9) { + throw_from_stop = true; } - execution.status = LlmTaskExecutionStatus::Invalid; - execution.reason_code = - std::string(LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - execution.validation.evaluated = true; - execution.validation.valid = false; - LlmCpuTaskEvidence evidence; - evidence.executor.valid = false; - evidence.executor.reason_code = - std::string(LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - evidence.executor.post_validation_evaluated = true; - evidence.executor.post_validation_valid = false; - execution.backend_evidence = std::move(evidence); + return EXIT_SUCCESS; }; LlmMemoryResult result; - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result), - EXIT_FAILURE); + EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - EXPECT_NE(result.reason_code, LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); - ASSERT_FALSE(result.measurements.empty()); - const LlmMeasurementState& measurement = result.measurements.front(); - EXPECT_EQ(measurement.status, LlmMeasurementStatus::Invalid); - EXPECT_EQ(measurement.reason_code, - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - EXPECT_EQ(measurement.execution.reason_code, - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - const auto* retained = - std::get_if(&measurement.execution.backend_evidence); - ASSERT_NE(retained, nullptr); - EXPECT_EQ(retained->executor.reason_code, - LlmExecutorReason::PAGED_POST_VALIDATION_FAILED); - EXPECT_TRUE(retained->executor.post_validation_evaluated); - EXPECT_FALSE(retained->executor.post_validation_valid); + EXPECT_EQ(result.reason_code, LlmRunnerReason::RUNNER_EXCEPTION); + EXPECT_EQ(result.diagnostic, "injected stop failure"); + EXPECT_TRUE(result.results_complete); + EXPECT_TRUE(result.scenario_order_balance_complete); + EXPECT_FALSE(result.run_accepted); + EXPECT_EQ(result.counters.measured_measurements, 9u); + ASSERT_EQ(checkpoints.size(), 4u); + EXPECT_EQ(checkpoints[2].kind, LlmCheckpointKind::MeasurementTerminal); + EXPECT_EQ(checkpoints[2].measured_measurements, 9u); + EXPECT_EQ(checkpoints[2].status, LlmRunStatus::Complete); + EXPECT_EQ(checkpoints.back().kind, LlmCheckpointKind::CommandTerminal); + EXPECT_EQ(checkpoints.back().status, LlmRunStatus::Failed); } -TEST(LlmMemoryRunnerTest, - PrefillPostValidationFailureCanonicalizesExactly) { - std::string owned_reason = - LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED; - const std::string_view canonical = - canonicalize_llm_result_reason_code(owned_reason); - std::fill(owned_reason.begin(), owned_reason.end(), 'x'); - - EXPECT_EQ(canonical, LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED); - EXPECT_NE(canonical, LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); - EXPECT_EQ( - canonical.data(), - canonicalize_llm_result_reason_code( - LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED) - .data()); +TEST(LlmMemoryRunnerTest, BackendAndValidationFailuresWinSimultaneousStopAndPreserveInterruptedTail) { + for (bool validation_failure : {false, true}) { + SCOPED_TRACE(::testing::Message() << "validation_failure=" << validation_failure); + const LlmMemoryConfig config = explicit_config(2); + const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + bool stop = false; + FakeLlmBackend executor; + executor.mutate = [&](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context, + size_t, LlmTaskExecutionResult& execution) { + if (context.kind == LlmRunnerTaskKind::Measurement) { + stop = true; + if (validation_failure) + invalidate_execution(execution); + else + fail_execution(execution); + } + }; + LlmRunnerHooks hooks; + hooks.stop_requested = [&]() { return stop; }; + LlmMemoryResult result; + + EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); + EXPECT_EQ(result.status, LlmRunStatus::Failed); + EXPECT_EQ(result.reason_code, + validation_failure ? LlmBackendReason::VALIDATION_FAILED : LlmBackendReason::RESOURCES_NOT_PREPARED); + EXPECT_TRUE(result.interruption_requested); + ASSERT_EQ(result.measurements[0].status, + validation_failure ? LlmMeasurementStatus::Invalid : LlmMeasurementStatus::Failed); + EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); + EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); + EXPECT_EQ(result.measurements[0].reason_code, + validation_failure ? LlmBackendReason::VALIDATION_FAILED : LlmBackendReason::RESOURCES_NOT_PREPARED); + for (size_t index = 1; index < result.measurements.size(); ++index) { + EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Interrupted); + } + } } -TEST(LlmMemoryRunnerTest, - AcceptedCpuMeasurementRetainsPagedPostValidationEvidence) { - LlmMemoryConfig config = explicit_config(1); - config.kv_layout = LlmKvLayout::Paged; - config.kv_block_tokens = 2; - config.user_specified_kv_layout = true; - config.user_specified_kv_block_tokens = true; - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(plan.kv_layout, LlmKvLayout::Paged); - const LlmCpuExecutionPlan* cpu_plan = get_llm_cpu_execution_plan(plan); - ASSERT_NE(cpu_plan, nullptr); - const size_t effective_workers = cpu_plan->effective_workers; +TEST(LlmMemoryRunnerTest, MeasurementCheckpointFailureIsTerminalAndNeverRetried) { + for (bool request_stop : {false, true}) { + SCOPED_TRACE(::testing::Message() << "request_stop=" << request_stop); + const LlmMemoryConfig config = explicit_config(2); + const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + bool stop = false; + FakeLlmBackend executor; + std::vector kinds; + LlmRunnerHooks hooks; + hooks.stop_requested = [&]() { return stop; }; + hooks.checkpoint = [&](const LlmMemoryResult&, LlmCheckpointKind kind) { + kinds.push_back(kind); + stop = request_stop; + return EXIT_FAILURE; + }; + LlmMemoryResult result; - FakeLlmBackend executor; - executor.mutate = [effective_workers]( - const LlmMemoryWorkPlan&, - const LlmScenarioWorkPlan&, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult& execution) { - if (context.kind != LlmRunnerTaskKind::Measurement) { - return; + EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); + EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::MeasurementTerminal})); + EXPECT_EQ(result.status, LlmRunStatus::Failed); + EXPECT_EQ(result.reason_code, LlmRunnerReason::CHECKPOINT_WRITE_FAILED); + EXPECT_TRUE(result.checkpoint_failed); + EXPECT_EQ(result.interruption_requested, request_stop); + EXPECT_FALSE(result.terminal_checkpoint_attempted); + EXPECT_EQ(result.counters.attempted_measurements, 3u); + EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Measured); + EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); + EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).work_units_per_second.has_value()); + EXPECT_TRUE(derive_llm_measurement_metrics(result.measurements[0]).payload_gb_s.has_value()); + EXPECT_TRUE(get_llm_cpu_task_evidence(result.measurements[0].execution)->checksum_valid); + EXPECT_EQ(result.measurements[0].execution.status, LlmTaskExecutionStatus::Complete); + EXPECT_TRUE(result.measurements[0].execution.timing.valid); + EXPECT_TRUE(result.measurements[0].execution.validation.valid); + EXPECT_TRUE(std::holds_alternative(result.measurements[0].execution.backend_evidence)); + EXPECT_EQ(result.aggregates[0].accepted_measurement_ids.size(), 1u); + EXPECT_EQ(result.counters.planned_loops, 2u); + EXPECT_EQ(result.counters.attempted_loops, 1u); + EXPECT_EQ(result.counters.completed_loops, 1u); + EXPECT_EQ(result.counters.planned_measurements, 6u); + EXPECT_EQ(result.counters.terminal_measurements, 6u); + EXPECT_EQ(result.counters.measured_measurements, 3u); + EXPECT_EQ(result.counters.completed_work_units, 3 * config.iterations); + EXPECT_EQ(result.counters.completed_effective_model_payload_bytes, + measurement_plan(result, result.measurements[0]).effective_model_payload_bytes + + measurement_plan(result, result.measurements[1]).effective_model_payload_bytes + + measurement_plan(result, result.measurements[2]).effective_model_payload_bytes); + for (size_t index = 3; index < result.measurements.size(); ++index) { + EXPECT_EQ(result.measurements[index].status, + request_stop ? LlmMeasurementStatus::Interrupted : LlmMeasurementStatus::Failed); + if (request_stop) EXPECT_EQ(result.measurements[index].reason_code, LlmRunnerReason::INTERRUPTION_BEFORE_TASK); } - LlmCpuTaskEvidence evidence = std::get(execution.backend_evidence); - evidence.executor.valid = true; - evidence.executor.reason_code = LlmExecutorReason::VALID; - evidence.executor.elapsed_seconds = execution.timing.elapsed_seconds; - evidence.executor.requested_workers = effective_workers; - evidence.executor.created_workers = effective_workers; - evidence.executor.completed_workers = effective_workers; - evidence.executor.qos_successful_workers = effective_workers; - evidence.executor.kernel_succeeded = true; - evidence.executor.timer_started = true; - evidence.executor.timer_stopped = true; - evidence.executor.checksum_evaluated = true; - evidence.executor.checksum_valid = true; - evidence.executor.post_validation_evaluated = true; - evidence.executor.post_validation_valid = true; - execution.backend_evidence = std::move(evidence); - }; - LlmMemoryResult result; + } +} - ASSERT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_SUCCESS); - ASSERT_TRUE(result.results_complete); - ASSERT_EQ(result.measurements.size(), kLlmScenarioCount); - for (const LlmMeasurementState& measurement : result.measurements) { - SCOPED_TRACE(::testing::Message() - << "scenario=" << static_cast(measurement.scenario)); - ASSERT_EQ(measurement.status, LlmMeasurementStatus::Measured); - ASSERT_TRUE(measurement.execution_evidence_available); - const auto* retained = std::get_if(&measurement.execution.backend_evidence); - ASSERT_NE(retained, nullptr); - EXPECT_EQ(retained->executor.reason_code, LlmExecutorReason::VALID); - EXPECT_TRUE(retained->executor.post_validation_evaluated); - EXPECT_TRUE(retained->executor.post_validation_valid); +TEST(LlmMemoryRunnerTest, CanonicalResultReasonsDetachFromEveryOwningDomain) { + for (const auto* expected : + {LlmRunnerReason::CHECKPOINT_WRITE_FAILED, LlmWorkPlanReason::TASK_ACCOUNTED_BYTES_CAP_EXCEEDED, + LlmExecutorReason::KERNEL_FAILED, LlmWorkPlanReason::KV_BLOCK_TOKENS_NOT_APPLICABLE, + LlmExecutorReason::PREFILL_POST_VALIDATION_FAILED, LlmExecutorReason::PAGED_POST_VALIDATION_FAILED, + LlmExecutorReason::DECODE_POST_VALIDATION_FAILED, LlmBackendReason::KV_WRITE_VALIDATION_MISMATCH}) { + SCOPED_TRACE(expected); + std::string owned(expected); + const auto canonical = canonicalize_llm_result_reason_code(owned); + std::fill(owned.begin(), owned.end(), 'x'); + EXPECT_EQ(canonical, expected); + EXPECT_EQ(canonical.data(), canonicalize_llm_result_reason_code(expected).data()); } + EXPECT_EQ(canonicalize_llm_result_reason_code("not-a-reason"), LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); } TEST(LlmMemoryRunnerTest, ResultCopiesAndMovesRetainStaticRunnerFailureAfterSourcesReset) { @@ -2486,42 +1752,6 @@ TEST(LlmMemoryRunnerTest, ResultCopiesAndMovesRetainStaticRunnerFailureAfterSour } } -TEST(LlmMemoryRunnerTest, CheckpointFailureWinsPendingStopAndInterruptsOnlyUnstartedTail) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - bool stop = false; - FakeLlmBackend executor; - std::vector kinds; - LlmRunnerHooks hooks; - hooks.stop_requested = [&]() { return stop; }; - hooks.checkpoint = [&](const LlmMemoryResult&, LlmCheckpointKind kind) { - kinds.push_back(kind); - stop = true; - return EXIT_FAILURE; - }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::MeasurementTerminal})); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, LlmRunnerReason::CHECKPOINT_WRITE_FAILED); - EXPECT_TRUE(result.checkpoint_failed); - EXPECT_TRUE(result.interruption_requested); - EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Measured); - EXPECT_EQ(result.counters.planned_loops, 2u); - EXPECT_EQ(result.counters.attempted_loops, 1u); - EXPECT_EQ(result.counters.completed_loops, 1u); - EXPECT_EQ(result.counters.planned_measurements, 6u); - EXPECT_EQ(result.counters.attempted_measurements, 3u); - EXPECT_EQ(result.counters.terminal_measurements, 6u); - EXPECT_EQ(result.counters.measured_measurements, 3u); - for (size_t index = 3; index < result.measurements.size(); ++index) { - EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Interrupted) << index; - EXPECT_EQ(result.measurements[index].reason_code, LlmRunnerReason::INTERRUPTION_BEFORE_TASK) << index; - } -} - TEST(LlmMemoryRunnerTest, TypedAndUnknownCheckpointExceptionsAreTerminalAndNeverRetried) { for (bool typed : {true, false}) { SCOPED_TRACE(typed ? "typed" : "unknown"); @@ -2689,58 +1919,42 @@ TEST(LlmMemoryRunnerTest, PartialAggregatesRetainEarlierMeasuredValueAndExcludeL EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[5]).payload_gb_s.has_value()); } -TEST(LlmMemoryRunnerTest, InvalidAuthoritativeElapsedIsRejectedByCommonAcceptance) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - executor.mutate = [](const LlmMemoryWorkPlan&, - const LlmScenarioWorkPlan&, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult& execution) { - if (context.kind == LlmRunnerTaskKind::Measurement) { - execution.timing.valid = false; - execution.timing.elapsed_seconds = 0.0; - } - }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, - LlmBackendReason::INVALID_AUTHORITATIVE_ELAPSED); - ASSERT_FALSE(result.measurements.empty()); - EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Invalid); - EXPECT_EQ(result.measurements[0].reason_code, - LlmBackendReason::INVALID_AUTHORITATIVE_ELAPSED); - EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); - EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); -} - -TEST(LlmMemoryRunnerTest, MeasurementCompletionMismatchIsRejectedWithoutAggregate) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - executor.mutate = [](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context, - size_t, LlmTaskExecutionResult& execution) { - if (context.kind == LlmRunnerTaskKind::Measurement) { - --execution.completion.completed_work_units; - } - }; - LlmMemoryResult result; +TEST(LlmMemoryRunnerTest, InvalidTimingAndCompletionAreRejectedWithoutAggregates) { + for (bool completion_mismatch : {false, true}) { + SCOPED_TRACE(::testing::Message() << "completion_mismatch=" << completion_mismatch); + const LlmMemoryConfig config = explicit_config(1); + const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + FakeLlmBackend executor; + executor.mutate = [completion_mismatch](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, + const LlmRunnerTaskContext& context, size_t, + LlmTaskExecutionResult& execution) { + if (context.kind == LlmRunnerTaskKind::Measurement) { + if (completion_mismatch) + --execution.completion.completed_work_units; + else { + execution.timing.valid = false; + execution.timing.elapsed_seconds = 0.0; + } + } + }; + LlmMemoryResult result; - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, LlmBackendReason::TASK_COMPLETION_MISMATCH); - ASSERT_FALSE(result.measurements.empty()); - EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Failed); - EXPECT_EQ(result.measurements[0].reason_code, - LlmBackendReason::TASK_COMPLETION_MISMATCH); - EXPECT_TRUE(std::holds_alternative( - result.measurements[0].execution.backend_evidence)); - EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); - EXPECT_EQ(result.counters.measured_measurements, 0u); + EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result), EXIT_FAILURE); + EXPECT_EQ(result.status, LlmRunStatus::Failed); + EXPECT_EQ(result.reason_code, completion_mismatch ? LlmBackendReason::TASK_COMPLETION_MISMATCH + : LlmBackendReason::INVALID_AUTHORITATIVE_ELAPSED); + ASSERT_FALSE(result.measurements.empty()); + EXPECT_EQ(result.measurements[0].status, + completion_mismatch ? LlmMeasurementStatus::Failed : LlmMeasurementStatus::Invalid); + EXPECT_EQ(result.measurements[0].reason_code, completion_mismatch + ? LlmBackendReason::TASK_COMPLETION_MISMATCH + : LlmBackendReason::INVALID_AUTHORITATIVE_ELAPSED); + EXPECT_FALSE(derive_llm_measurement_metrics(result.measurements[0]).latency_seconds.has_value()); + EXPECT_TRUE(result.aggregates[0].accepted_measurement_ids.empty()); + EXPECT_TRUE(std::holds_alternative(result.measurements[0].execution.backend_evidence)); + EXPECT_EQ(result.counters.measured_measurements, 0u); + } } TEST(LlmMemoryRunnerTest, ExcludedCompletionMismatchRetainsGenericReason) { @@ -2807,38 +2021,6 @@ TEST(LlmMemoryRunnerTest, CalibrationFailureRetainsAttemptAndFinalizesMeasuremen EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); } -TEST(LlmMemoryRunnerTest, SuccessfulExcludedTaskWinsStopAndInterruptsBeforeMeasurements) { - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - bool stop = false; - FakeLlmBackend executor; - executor.mutate = [&](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, const LlmRunnerTaskContext& context, - size_t, LlmTaskExecutionResult&) { - if (context.kind == LlmRunnerTaskKind::Warmup) { - stop = true; - } - }; - std::vector checkpoints; - LlmRunnerHooks hooks = recording_checkpoints(checkpoints); - hooks.stop_requested = [&]() { return stop; }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_SUCCESS); - EXPECT_EQ(result.status, LlmRunStatus::Interrupted); - EXPECT_TRUE(result.interruption_requested); - ASSERT_EQ(result.calibration_attempts[0].size(), 1u); - EXPECT_TRUE(result.calibration_attempts[0][0].terminal); - EXPECT_TRUE(result.calibration_attempts[0][0].valid); - EXPECT_EQ(result.calibration_attempts[0][0].purpose, - "frozen_measurement_warmup"); - EXPECT_EQ(result.counters.attempted_measurements, 0u); - EXPECT_EQ(result.counters.terminal_measurements, 3u); - expect_interrupted_tail(result, 0); - ASSERT_EQ(checkpoints.size(), 1u); - EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); -} - TEST(LlmMemoryRunnerTest, ExcludedTaskFailureWinsSimultaneousStopAndRetainsAttemptEvidence) { const LlmMemoryConfig config = explicit_config(1); const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); @@ -2909,84 +2091,54 @@ TEST(LlmMemoryRunnerTest, ExcludedExecutorExceptionRetainsTerminalAttemptBeforeC } TEST(LlmMemoryRunnerTest, TypedAndUnknownExecutorExceptionsRemainInsideRunnerBoundary) { - for (bool typed : {true, false}) { - SCOPED_TRACE(typed ? "typed" : "unknown"); - const LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - executor.mutate = [typed](const LlmMemoryWorkPlan&, - const LlmScenarioWorkPlan&, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult&) { - if (context.kind == LlmRunnerTaskKind::Measurement) { - if (typed) { - throw std::runtime_error("injected executor failure"); + for (bool request_stop : {false, true}) { + SCOPED_TRACE(::testing::Message() << "request_stop=" << request_stop); + for (bool typed : {true, false}) { + SCOPED_TRACE(typed ? "typed" : "unknown"); + const LlmMemoryConfig config = explicit_config(1); + const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); + ASSERT_TRUE(plan.valid) << plan.reason_code; + bool stop = false; + FakeLlmBackend executor; + executor.mutate = [typed, request_stop, &stop](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, + const LlmRunnerTaskContext& context, size_t, + LlmTaskExecutionResult&) { + if (context.kind == LlmRunnerTaskKind::Measurement) { + stop = request_stop; + if (typed) { + throw std::runtime_error("injected executor failure"); + } + throw 17; } - throw 17; + }; + std::vector checkpoints; + LlmRunnerHooks hooks = recording_checkpoints(checkpoints); + hooks.stop_requested = [&]() { return stop; }; + LlmMemoryResult result; + EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); + EXPECT_EQ(result.status, LlmRunStatus::Failed); + EXPECT_EQ(result.reason_code, + typed ? LlmRunnerReason::RUNNER_EXCEPTION : LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); + if (typed) { + EXPECT_EQ(result.diagnostic, "injected executor failure"); } - }; - std::vector checkpoints; - LlmMemoryResult result; - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, recording_checkpoints(checkpoints)), - EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, - typed ? LlmRunnerReason::RUNNER_EXCEPTION - : LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); - if (typed) { - EXPECT_EQ(result.diagnostic, "injected executor failure"); - } - EXPECT_EQ(result.counters.attempted_measurements, 1u); - EXPECT_EQ(result.counters.terminal_measurements, 3u); - for (const LlmMeasurementState& measurement : result.measurements) { - EXPECT_EQ(measurement.status, LlmMeasurementStatus::Failed); - } - ASSERT_EQ(checkpoints.size(), 1u); - EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); - EXPECT_EQ(checkpoints[0].attempted_measurements, 1u); - EXPECT_EQ(checkpoints[0].terminal_measurements, 3u); - EXPECT_EQ(checkpoints[0].measured_measurements, 0u); - EXPECT_EQ(result.logical_checkpoint_attempts, 1u); - EXPECT_EQ(result.successful_logical_checkpoints, 1u); - EXPECT_TRUE(result.terminal_checkpoint_completed); - } -} - -TEST(LlmMemoryRunnerTest, ExecutorExceptionObservesStopAtTaskBoundaryWithoutProgressSnapshot) { - const LlmMemoryConfig config = explicit_config(2); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - bool stop = false; - FakeLlmBackend executor; - executor.mutate = [&stop](const LlmMemoryWorkPlan&, const LlmScenarioWorkPlan&, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult&) { - if (context.kind == LlmRunnerTaskKind::Measurement) { - stop = true; - throw std::runtime_error("injected executor failure"); - } - }; - std::vector kinds; - LlmRunnerHooks hooks; - hooks.stop_requested = [&]() { return stop; }; - hooks.checkpoint = [&](const LlmMemoryResult&, LlmCheckpointKind kind) { - kinds.push_back(kind); - if (kind == LlmCheckpointKind::MeasurementTerminal) { - stop = true; + EXPECT_EQ(result.counters.attempted_measurements, 1u); + EXPECT_EQ(result.counters.terminal_measurements, 3u); + EXPECT_EQ(result.interruption_requested, request_stop); + EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Failed); + for (size_t index = 1; index < result.measurements.size(); ++index) { + EXPECT_EQ(result.measurements[index].status, + request_stop ? LlmMeasurementStatus::Interrupted : LlmMeasurementStatus::Failed); + } + ASSERT_EQ(checkpoints.size(), 1u); + EXPECT_EQ(checkpoints[0].kind, LlmCheckpointKind::CommandTerminal); + EXPECT_EQ(checkpoints[0].attempted_measurements, 1u); + EXPECT_EQ(checkpoints[0].terminal_measurements, 3u); + EXPECT_EQ(checkpoints[0].measured_measurements, 0u); + EXPECT_EQ(result.logical_checkpoint_attempts, 1u); + EXPECT_EQ(result.successful_logical_checkpoints, 1u); + EXPECT_TRUE(result.terminal_checkpoint_completed); } - return EXIT_SUCCESS; - }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result, hooks), EXIT_FAILURE); - EXPECT_EQ(kinds, (std::vector{LlmCheckpointKind::CommandTerminal})); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, LlmRunnerReason::RUNNER_EXCEPTION); - EXPECT_TRUE(result.interruption_requested); - EXPECT_EQ(result.measurements[0].status, LlmMeasurementStatus::Failed); - for (size_t index = 1; index < result.measurements.size(); ++index) { - EXPECT_EQ(result.measurements[index].status, LlmMeasurementStatus::Interrupted) << index; } } @@ -3104,7 +2256,7 @@ TEST(LlmMemoryRunnerTest, EXPECT_EQ(result.logical_checkpoint_attempts, 0u); } -TEST(LlmMemoryRunnerTest, RunnerAuxiliaryEstimateHasExactCategoryBreakdownForExplicitAndAutomaticModes) { +TEST(LlmMemoryRunnerTest, RunnerAuxiliaryEstimateAccountsCategoriesForExplicitAndAutomaticModes) { std::array estimates; for (size_t mode = 0; mode < estimates.size(); ++mode) { const LlmMemoryConfig config = mode == 0 ? explicit_config(2) : automatic_config(2); @@ -3123,33 +2275,6 @@ TEST(LlmMemoryRunnerTest, RunnerAuxiliaryEstimateHasExactCategoryBreakdownForExp EXPECT_EQ(estimate.loop_record_bytes, config.loop_count * sizeof(LlmLoopRecord)); EXPECT_EQ(estimate.calibration_record_bytes, attempts_per_scenario * kLlmScenarioCount * sizeof(LlmCalibrationAttempt)); - // Independently enumerate the bounded plan shapes. The estimate's category - // values are not used to derive the expected identity/storage budget. - std::array max_work{}; - size_t expected_identity_bytes = 0; - size_t largest_identity = 0; - for (size_t index = 0; index < 3; ++index) { - const auto scenario = static_cast(index); - max_work[index] = calculate_llm_scenario_limits(plan.geometry, scenario, plan.backend).effective_maximum_work_units; - const auto maximum = build_llm_scenario_work_plan(plan, scenario, max_work[index], config.user_specified_iterations); - ASSERT_TRUE(maximum.valid); - largest_identity = std::max(largest_identity, maximum.plan_identity.size()); - expected_identity_bytes += 2 * (maximum.plan_identity.size() + 1) * (attempts_per_scenario + 1); - } - const size_t canonical_count = 3 * (attempts_per_scenario + 1); - expected_identity_bytes += 2 * (plan.plan_identity.size() + 1) * canonical_count; - EXPECT_EQ(estimate.calibration_identity_bytes, expected_identity_bytes); - const auto maximum_frozen = freeze_llm_scenario_work_plans(plan, max_work, config.user_specified_iterations); - ASSERT_TRUE(maximum_frozen.valid); - size_t frozen_strings = maximum_frozen.reason_code.size() + maximum_frozen.model_plan_identity.size() + - maximum_frozen.plan_identity.size() + 3; - for (const auto& scenario : maximum_frozen.scenarios) - frozen_strings += scenario.reason_code.size() + scenario.model_plan_identity.size() + scenario.plan_identity.size() + 3; - const size_t expected_fixed_bytes = 2 * frozen_strings + - 4 * (largest_identity + plan.plan_identity.size() + 2) + - 2 * 65 * (planned_measurements + 1) + 2 * 513 + - canonical_count * (sizeof(LlmCanonicalScenarioPlan) + 2 * sizeof(size_t)); - EXPECT_EQ(estimate.fixed_metadata_bytes, expected_fixed_bytes); EXPECT_EQ(estimate.aggregate_value_bytes, planned_measurements * sizeof(size_t)); EXPECT_EQ(estimate.statistics_workspace_bytes, config.loop_count * 3 * sizeof(double)); EXPECT_EQ(estimate.warning_record_bytes, (kLlmScenarioCount + 1) * sizeof(std::string_view)); @@ -3225,99 +2350,8 @@ TEST(LlmMemoryRunnerTest, RunnerAuxiliaryBudgetAcceptsExactBoundaryAndRejectsOne } } -TEST(LlmMemoryRunnerTest, StableTaskAndCheckpointTokensCoverUnknownValues) { - EXPECT_STREQ(LlmBackendReason::KV_WRITE_VALIDATION_MISMATCH, - "kv-write-validation-mismatch"); - EXPECT_EQ(canonicalize_llm_result_reason_code( - LlmBackendReason::KV_WRITE_VALIDATION_MISMATCH), - LlmBackendReason::KV_WRITE_VALIDATION_MISMATCH); - EXPECT_STREQ(llm_runner_task_kind_to_string(LlmRunnerTaskKind::Warmup), "warmup"); - EXPECT_STREQ(llm_runner_task_kind_to_string(LlmRunnerTaskKind::Calibration), "calibration"); - EXPECT_STREQ(llm_runner_task_kind_to_string(LlmRunnerTaskKind::Measurement), "measurement"); - EXPECT_STREQ(llm_runner_task_kind_to_string(static_cast(99)), "unknown"); - EXPECT_STREQ(llm_checkpoint_kind_to_string(LlmCheckpointKind::MeasurementTerminal), "measurement-terminal"); - EXPECT_STREQ(llm_checkpoint_kind_to_string(LlmCheckpointKind::CommandTerminal), "command-terminal"); - EXPECT_STREQ(llm_checkpoint_kind_to_string(static_cast(99)), "unknown"); -} - } // namespace -TEST(LlmMemoryRunnerTest, - DecodePostValidationFailureCanonicalizesExactlyThroughRunnerFailure) { - std::string owned_reason = - LlmExecutorReason::DECODE_POST_VALIDATION_FAILED; - const std::string_view canonical = - canonicalize_llm_result_reason_code(owned_reason); - std::fill(owned_reason.begin(), owned_reason.end(), 'x'); - EXPECT_EQ(canonical, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - EXPECT_NE(canonical, LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); - EXPECT_EQ( - canonical.data(), - canonicalize_llm_result_reason_code( - LlmExecutorReason::DECODE_POST_VALIDATION_FAILED) - .data()); - - LlmMemoryConfig config = explicit_config(1); - const LlmMemoryWorkPlan plan = build_runner_admitted_plan(config); - ASSERT_TRUE(plan.valid) << plan.reason_code; - FakeLlmBackend executor; - executor.mutate = [](const LlmMemoryWorkPlan&, - const LlmScenarioWorkPlan&, - const LlmRunnerTaskContext& context, size_t, - LlmTaskExecutionResult& execution) { - if (context.kind != LlmRunnerTaskKind::Measurement || context.scenario != LlmScenario::KvOnly) { - return; - } - execution.status = LlmTaskExecutionStatus::Invalid; - execution.reason_code = - std::string(LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - execution.validation.evaluated = true; - execution.validation.valid = false; - LlmCpuTaskEvidence evidence; - evidence.executor.valid = false; - evidence.executor.reason_code = - std::string(LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - evidence.executor.post_validation_evaluated = true; - evidence.executor.post_validation_valid = false; - evidence.executor.kv_write_validation_applicable = true; - evidence.executor.kv_write_validation_evaluated = true; - evidence.executor.kv_write_validation_valid = false; - evidence.executor.cold_checks = make_llm_cold_checks(true, LlmColdCheckKind::KvAppendFinal, true, false); - resolve_llm_cold_check(evidence.executor.cold_checks, 0, true); - resolve_llm_cold_check(evidence.executor.cold_checks, 1, false, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - execution.backend_evidence = std::move(evidence); - }; - LlmMemoryResult result; - - EXPECT_EQ(run_llm_memory_suite(config, plan, executor, result), - EXIT_FAILURE); - EXPECT_EQ(result.status, LlmRunStatus::Failed); - EXPECT_EQ(result.reason_code, - LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - EXPECT_NE(result.reason_code, LlmRunnerReason::RUNNER_UNKNOWN_EXCEPTION); - ASSERT_FALSE(result.measurements.empty()); - const LlmMeasurementState& measurement = result.measurements[1]; - EXPECT_EQ(measurement.status, LlmMeasurementStatus::Invalid); - EXPECT_EQ(measurement.reason_code, - LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - EXPECT_EQ(measurement.execution.reason_code, - LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - const auto* retained = - std::get_if(&measurement.execution.backend_evidence); - ASSERT_NE(retained, nullptr); - EXPECT_EQ(retained->executor.reason_code, - LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); - EXPECT_TRUE(retained->executor.post_validation_evaluated); - EXPECT_FALSE(retained->executor.post_validation_valid); - EXPECT_TRUE(retained->executor.kv_write_validation_applicable); - EXPECT_TRUE(retained->executor.kv_write_validation_evaluated); - EXPECT_FALSE(retained->executor.kv_write_validation_valid); - EXPECT_TRUE(retained->executor.cold_checks[0].valid); - EXPECT_TRUE(retained->executor.cold_checks[1].evaluated); - EXPECT_FALSE(retained->executor.cold_checks[1].valid); - EXPECT_EQ(retained->executor.cold_checks[1].reason_code, LlmExecutorReason::DECODE_POST_VALIDATION_FAILED); -} - TEST(LlmMemoryRunnerTest, CanonicalInsertionReusesExactPlanAndSnapshotRemapsStableHandles) { const auto config = explicit_config(1, 8); const auto model = build_runner_admitted_plan(config); @@ -3427,20 +2461,6 @@ TEST(LlmMemoryRunnerTest, BoundedSnapshotsObservePriorWritersAndDoNotSortBetween } } -TEST(LlmMemoryRunnerTest, SnapshotRejectsUnresolvedPlanAndContradictoryCompletedWork) { - const auto config = explicit_config(1, 1); - const auto model = build_runner_admitted_plan(config); - FakeLlmBackend backend; - LlmMemoryResult result; - ASSERT_EQ(run_llm_memory_suite(config, model, backend, result), EXIT_SUCCESS); - auto invalid = result; - invalid.measurements[0].plan_handle = kLlmNoTaskIndex; - EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); - invalid = result; - ++invalid.measurements[0].execution.completion.completed_work_units; - EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); -} - TEST(LlmMemoryRunnerTest, CanonicalCpuExpectationRequiresPreparedMatchingBackendResources) { const auto config = explicit_config(1, 1); const auto model = build_runner_admitted_plan(config); @@ -3455,20 +2475,33 @@ TEST(LlmMemoryRunnerTest, CanonicalCpuExpectationRequiresPreparedMatchingBackend EXPECT_TRUE(result.scenario_plans.at(handle).expected.cpu_workers.empty()); } -TEST(LlmMemoryRunnerTest, SnapshotRejectsSwappedFrozenAndCalibrationScenarioReferences) { +TEST(LlmMemoryRunnerTest, SnapshotRejectsContradictoryPlanReferencesAndCompletedWork) { const auto config = explicit_config(1, 1); const auto model = build_runner_admitted_plan(config); FakeLlmBackend backend; LlmMemoryResult result; ASSERT_EQ(run_llm_memory_suite(config, model, backend, result), EXIT_SUCCESS); - auto invalid = result; - std::swap(invalid.frozen_plan_handles[0], invalid.frozen_plan_handles[1]); - EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); - invalid = result; - ASSERT_GT(invalid.calibration_attempt_counts[0], 0u); - invalid.calibration_attempts[0][0].plan_handle = invalid.frozen_plan_handles[1]; - EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); - invalid = result; - invalid.calibration_attempts[0][0].scenario = LlmScenario::KvOnly; - EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); + ASSERT_GT(result.calibration_attempt_counts[0], 0u); + for (size_t mutation = 0; mutation < 5; ++mutation) { + SCOPED_TRACE(mutation); + auto invalid = result; + switch (mutation) { + case 0: + std::swap(invalid.frozen_plan_handles[0], invalid.frozen_plan_handles[1]); + break; + case 1: + invalid.calibration_attempts[0][0].plan_handle = invalid.frozen_plan_handles[1]; + break; + case 2: + invalid.calibration_attempts[0][0].scenario = LlmScenario::KvOnly; + break; + case 3: + invalid.measurements[0].plan_handle = kLlmNoTaskIndex; + break; + case 4: + ++invalid.measurements[0].execution.completion.completed_work_units; + break; + } + EXPECT_THROW(prepare_llm_result_snapshot(invalid), std::invalid_argument); + } } diff --git a/tests/test_llm_memory_work_plan.cpp b/tests/test_llm_memory_work_plan.cpp index d29ca81..276c019 100644 --- a/tests/test_llm_memory_work_plan.cpp +++ b/tests/test_llm_memory_work_plan.cpp @@ -592,107 +592,6 @@ class LlmMemoryWorkPlanSystemCallsTest } // namespace TEST(LlmMemoryWorkPlanTest, ConstantsAndIdentitiesMatchFrozenContract) { - EXPECT_EQ(Constants::LLM_DEFAULT_KV_ELEMENT_BYTES, 2u); - EXPECT_EQ(Constants::LLM_DEFAULT_BATCH_SIZE, 1u); - EXPECT_EQ(Constants::LLM_DEFAULT_LOOP_COUNT, 3u); - EXPECT_EQ(Constants::LLM_RANGE_ALIGNMENT_BYTES, 32u); - EXPECT_DOUBLE_EQ(Constants::LLM_CALIBRATION_TARGET_SECONDS, 0.150); - EXPECT_DOUBLE_EQ(Constants::LLM_CALIBRATION_MIN_SECONDS, 0.100); - EXPECT_DOUBLE_EQ(Constants::LLM_CALIBRATION_MAX_SECONDS, 0.250); - EXPECT_EQ(Constants::LLM_CALIBRATION_MAX_CORRECTIONS, 2u); - EXPECT_EQ(Constants::LLM_CALIBRATION_MIN_PILOT_BYTES, - 8 * Constants::BYTES_PER_MB); - EXPECT_EQ(Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT, 1000000000u); - EXPECT_EQ(Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK, - 64ULL * 1024ULL * Constants::BYTES_PER_MB); - EXPECT_DOUBLE_EQ(Constants::LLM_STREAMING_CV_WARNING_PCT, 5.0); - EXPECT_EQ(Constants::LLM_JSON_SCHEMA_VERSION, 2); - EXPECT_STREQ(Constants::LLM_JSON_MODE_NAME, "llm_memory"); - EXPECT_EQ(Constants::LLM_WEIGHT_PASSES_PER_WORK_UNIT, 1u); - EXPECT_EQ(Constants::LLM_KV_REPLAY_FACTOR, 1u); - EXPECT_STREQ(Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION, - "llm-memory-v2-cpu-decode-contiguous"); - EXPECT_STREQ(Constants::LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION, - "llm-memory-v2-cpu-prefill-paged"); - EXPECT_STREQ(Constants::LLM_COMPONENT_IDENTITY_VERSION, - "llm-memory-components-v1"); - EXPECT_STREQ(Constants::LLM_LOGICAL_PROFILE_VERSION, - "decode_steady_fixed_context"); - EXPECT_STREQ(Constants::LLM_CONTIGUOUS_KV_LAYOUT_VERSION, - "contiguous_layer_batch_token_head_dimension"); - EXPECT_STREQ(Constants::LLM_CPU_EXECUTOR_VERSION, - "llm-cpu-executor-v1-arm64-decode-contiguous"); - EXPECT_STREQ(Constants::LLM_DESCRIPTOR_ABI_VERSION, - "llm-memory-descriptor-abi-v1"); - EXPECT_STREQ(Constants::LLM_CPU_SCHEDULE_VERSION, - "worker-local-layer-order-no-per-layer-global-barrier"); - EXPECT_STREQ(Constants::LLM_CPU_TIMER_POLICY_VERSION, - "synchronized-start-to-last-worker-completion-per-scenario-task"); - EXPECT_STREQ(Constants::LLM_BUFFER_PATTERN_VERSION, - "llm-buffer-pattern-v1"); - EXPECT_STREQ(Constants::LLM_APPEND_PATTERN_VERSION, - "llm-kv-append-affine64-v1"); - EXPECT_STREQ(Constants::LLM_READ_CHECKSUM_VERSION, - "llm-read-checksum-v1"); - EXPECT_EQ(Constants::LLM_KV_BLOCK_TABLE_ENTRY_BYTES, sizeof(uint32_t)); - EXPECT_STREQ(Constants::LLM_PAGED_KV_LAYOUT_VERSION, - "paged-uint32-block-table-full-blocks-v1"); - EXPECT_STREQ(Constants::LLM_KV_BLOCK_PERMUTATION_VERSION, - "splitmix64-fisher-yates-rejection-v1"); - EXPECT_STREQ(Constants::LLM_PAGED_CPU_EXECUTION_IDENTITY_VERSION, - "llm-paged-cpu-execution-v1"); - EXPECT_STREQ(Constants::LLM_PAGED_CPU_SCHEDULE_VERSION, - "decode-kv-accounted-prefix-balanced-rotating-v1"); - EXPECT_STREQ(Constants::LLM_PAGED_CPU_EXECUTOR_VERSION, - "llm-cpu-executor-v1-arm64-decode-paged"); - EXPECT_STREQ(Constants::LLM_PAGED_DESCRIPTOR_ABI_VERSION, - "llm-memory-paged-descriptor-abi-v1"); - EXPECT_STREQ(Constants::LLM_PAGED_BUFFER_PATTERN_VERSION, - "llm-paged-physical-buffer-pattern-v1"); - EXPECT_STREQ(Constants::LLM_PAGED_READ_CHECKSUM_VERSION, - "llm-paged-read-checksum-v1"); - EXPECT_STREQ(Constants::LLM_PREFILL_PAGED_CPU_EXECUTOR_VERSION, - "llm-cpu-executor-v1-arm64-prefill-paged"); - EXPECT_STREQ(Constants::LLM_PREFILL_PAGED_DESCRIPTOR_ABI_VERSION, - "llm-memory-prefill-paged-descriptor-abi-v1"); - EXPECT_STREQ(LlmPrefillVersion::PAGED_CHECKSUM_ORACLE, - "llm-prefill-paged-affine64-lookup-v1"); - EXPECT_STREQ(LlmMetalDecodePagedVersion::EXECUTOR, - "llm-metal-executor-v1-decode-paged"); - EXPECT_STREQ(LlmMetalDecodePagedVersion::SCHEDULE, - "llm-metal-decode-paged-block-owner-grid-stride-v1"); - EXPECT_STREQ(LlmMetalDecodePagedVersion::BUFFER_PATTERN, - "llm-paged-physical-buffer-pattern-v1"); - EXPECT_STREQ(LlmMetalDecodePagedVersion::WRITE_PATTERN, - "llm-metal-decode-paged-append-affine32-v1"); - EXPECT_STREQ(LlmMetalDecodePagedVersion::CHECKSUM, - "llm-metal-paged-dual-mod32-lookup-mix-v1"); - EXPECT_STREQ(LlmMetalPrefillContiguousVersion::EXECUTOR, - "llm-metal-executor-v1-prefill-contiguous"); - EXPECT_STREQ(LlmMetalPrefillContiguousVersion::SCHEDULE, - "llm-metal-prefill-contiguous-lane-local-vector-stripe-v1"); - EXPECT_STREQ(LlmMetalPrefillContiguousVersion::TIMER, - "metal-command-buffer-gpu-start-end-v1"); - EXPECT_STREQ(LlmMetalPrefillContiguousVersion::BUFFER_PATTERN, - "llm-metal-contiguous-affine32-v1"); - EXPECT_STREQ(LlmMetalPrefillContiguousVersion::WRITE_PATTERN, - "llm-metal-prefill-contiguous-full-prompt-affine32-v1"); - EXPECT_STREQ(LlmMetalPrefillContiguousVersion::CHECKSUM, - "llm-metal-dual-mod32-v1"); - EXPECT_STREQ(LlmMetalPrefillPagedVersion::EXECUTOR, - "llm-metal-executor-v1-prefill-paged"); - EXPECT_STREQ( - LlmMetalPrefillPagedVersion::SCHEDULE, - "llm-metal-prefill-paged-cyclic-block-owner-grid-stride-v1"); - EXPECT_STREQ(LlmMetalPrefillPagedVersion::TIMER, - "metal-command-buffer-gpu-start-end-v1"); - EXPECT_STREQ(LlmMetalPrefillPagedVersion::BUFFER_PATTERN, - "llm-paged-physical-buffer-pattern-v1"); - EXPECT_STREQ(LlmMetalPrefillPagedVersion::WRITE_PATTERN, - "llm-metal-prefill-paged-full-prompt-affine32-v1"); - EXPECT_STREQ( - LlmMetalPrefillPagedVersion::CHECKSUM, - "llm-metal-paged-prefill-dual-mod32-lookup-address-mix-v1"); EXPECT_EQ(build_llm_methodology_version( LlmMemoryBackend::Cpu, LlmPhase::Decode, LlmKvLayout::Contiguous), @@ -886,27 +785,18 @@ TEST(LlmMemoryWorkPlanTest, LlmGeometryRequest request; const char* reason_code; }; - const std::array invalid_cases = {{ - {{0, 2, 4, 2, 8, 2, 3, 1}, - LlmWorkPlanReason::ACTIVE_WEIGHT_BYTES_ZERO}, - {{1024, 0, 4, 2, 8, 2, 3, 1}, - LlmWorkPlanReason::LAYER_COUNT_ZERO}, - {{1024, 2, 0, 2, 8, 2, 3, 1}, - LlmWorkPlanReason::QUERY_HEAD_COUNT_ZERO}, - {{1024, 2, 4, 0, 8, 2, 3, 1}, - LlmWorkPlanReason::KV_HEAD_COUNT_ZERO}, - {{1024, 2, 4, 2, 0, 2, 3, 1}, - LlmWorkPlanReason::HEAD_DIMENSION_ZERO}, - {{1024, 2, 4, 2, 8, 3, 3, 1}, - LlmWorkPlanReason::INVALID_KV_ELEMENT_BYTES}, - {{1024, 2, 4, 2, 8, 2, 0, 1}, - LlmWorkPlanReason::CONTEXT_TOKENS_ZERO}, - {{1024, 2, 4, 2, 8, 2, 3, 0}, - LlmWorkPlanReason::BATCH_SIZE_ZERO}, - {{1024, 2, 1, 2, 8, 2, 3, 1}, - LlmWorkPlanReason::QUERY_HEADS_BELOW_KV_HEADS}, - {{1024, 2, 3, 2, 8, 2, 3, 1}, - LlmWorkPlanReason::QUERY_HEADS_NOT_DIVISIBLE_BY_KV_HEADS}, + const std::array invalid_cases = {{ + {{0, 2, 4, 2, 8, 2, 3, 1}, LlmWorkPlanReason::ACTIVE_WEIGHT_BYTES_ZERO}, + {{1024, 0, 4, 2, 8, 2, 3, 1}, LlmWorkPlanReason::LAYER_COUNT_ZERO}, + {{1024, 2, 0, 2, 8, 2, 3, 1}, LlmWorkPlanReason::QUERY_HEAD_COUNT_ZERO}, + {{1024, 2, 4, 0, 8, 2, 3, 1}, LlmWorkPlanReason::KV_HEAD_COUNT_ZERO}, + {{1024, 2, 4, 2, 0, 2, 3, 1}, LlmWorkPlanReason::HEAD_DIMENSION_ZERO}, + {{1024, 2, 4, 2, 8, 3, 3, 1}, LlmWorkPlanReason::INVALID_KV_ELEMENT_BYTES}, + {{1024, 2, 4, 2, 8, 2, 0, 1}, LlmWorkPlanReason::CONTEXT_TOKENS_ZERO}, + {{1024, 2, 4, 2, 8, 2, 3, 0}, LlmWorkPlanReason::BATCH_SIZE_ZERO}, + {{1024, 2, 1, 2, 8, 2, 3, 1}, LlmWorkPlanReason::QUERY_HEADS_BELOW_KV_HEADS}, + {{1024, 2, 3, 2, 8, 2, 3, 1}, LlmWorkPlanReason::QUERY_HEADS_NOT_DIVISIBLE_BY_KV_HEADS}, + {{1024, 2, 4, 2, 8, 2, 3, 1, 16}, LlmWorkPlanReason::KV_BLOCK_TOKENS_NOT_APPLICABLE}, }}; for (const InvalidCase& invalid_case : invalid_cases) { @@ -914,13 +804,10 @@ TEST(LlmMemoryWorkPlanTest, const LlmGeometry geometry = resolve_llm_geometry(invalid_case.request); EXPECT_FALSE(geometry.valid); EXPECT_EQ(geometry.reason_code, invalid_case.reason_code); + expect_invalid_plan(build_llm_memory_work_plan(work_plan_request(invalid_case.request, 2, 2)), + invalid_case.reason_code); } - expect_invalid_plan( - build_llm_memory_work_plan( - work_plan_request(invalid_cases[5].request, 2, 2)), - LlmWorkPlanReason::INVALID_KV_ELEMENT_BYTES); - LlmGeometryRequest incompatible_geometry = small_geometry_request(); incompatible_geometry.phase = LlmPhase::Prefill; EXPECT_EQ(resolve_llm_geometry(incompatible_geometry).reason_code, @@ -1032,6 +919,27 @@ TEST(LlmMemoryWorkPlanTest, LayerBatchTokenLayoutOffsetsAreExact) { } EXPECT_EQ(plan.geometry.k_mapping_bytes, 384u); EXPECT_EQ(plan.geometry.v_mapping_bytes, 384u); + const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); + EXPECT_EQ(cpu_plan.layer_descriptors_per_worker, 2u); + EXPECT_EQ(cpu_plan.sequence_descriptors_per_worker, 4u); + EXPECT_EQ(cpu_plan.total_layer_descriptors, 4u); + EXPECT_EQ(cpu_plan.total_sequence_descriptors, 8u); + for (const LlmWorkerWorkPlan& worker : cpu_plan.workers) { + ASSERT_EQ(worker.layers.size(), 2u); + ASSERT_EQ(worker.sequences.size(), 4u); + for (size_t layer = 0; layer < 2; ++layer) { + EXPECT_EQ(worker.layers[layer].first_sequence_index, layer * 2); + EXPECT_EQ(worker.layers[layer].sequence_count, 2u); + EXPECT_EQ(worker.layers[layer].layer_index, layer); + for (size_t batch = 0; batch < 2; ++batch) { + const LlmKvSequenceRangeTemplate& sequence = worker.sequences[layer * 2 + batch]; + EXPECT_EQ(sequence.layer_index, layer); + EXPECT_EQ(sequence.batch_sequence_index, batch); + } + } + const size_t batch_zero_end = worker.sequences[0].k_visible.offset_bytes + worker.sequences[0].k_visible.span_bytes; + EXPECT_LE(batch_zero_end, worker.sequences[1].k_visible.offset_bytes); + } } TEST(LlmMemoryWorkPlanTest, @@ -1122,157 +1030,6 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_EQ(second.v_append.offset_bytes, second.k_append.offset_bytes); } -TEST(LlmMemoryWorkPlanTest, - BatchDescriptorsStaySeparateAndLayerLinksRemainLocal) { - const LlmGeometryRequest geometry = {1024, 2, 4, 2, 8, 2, 3, 2}; - const LlmMemoryWorkPlan plan = - build_llm_memory_work_plan(work_plan_request(geometry, 2, 2)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); - EXPECT_EQ(cpu_plan.layer_descriptors_per_worker, 2u); - EXPECT_EQ(cpu_plan.sequence_descriptors_per_worker, 4u); - EXPECT_EQ(cpu_plan.total_layer_descriptors, 4u); - EXPECT_EQ(cpu_plan.total_sequence_descriptors, 8u); - for (const LlmWorkerWorkPlan& worker : cpu_plan.workers) { - ASSERT_EQ(worker.layers.size(), 2u); - ASSERT_EQ(worker.sequences.size(), 4u); - for (size_t layer = 0; layer < 2; ++layer) { - EXPECT_EQ(worker.layers[layer].first_sequence_index, layer * 2); - EXPECT_EQ(worker.layers[layer].sequence_count, 2u); - EXPECT_EQ(worker.layers[layer].layer_index, layer); - for (size_t batch = 0; batch < 2; ++batch) { - const LlmKvSequenceRangeTemplate& sequence = - worker.sequences[layer * 2 + batch]; - EXPECT_EQ(sequence.layer_index, layer); - EXPECT_EQ(sequence.batch_sequence_index, batch); - } - } - const size_t batch_zero_end = - worker.sequences[0].k_visible.offset_bytes + - worker.sequences[0].k_visible.span_bytes; - EXPECT_LE(batch_zero_end, worker.sequences[1].k_visible.offset_bytes); - } -} - -TEST(LlmMemoryWorkPlanTest, - PlannerStorageUsesActualRetainedCapacitiesAndEntersPeakBudget) { - LlmMemoryWorkPlanRequest request = - work_plan_request(small_geometry_request(2), 3, 3); - request.checksum_auxiliary_bytes = 7; - request.orchestration_auxiliary_bytes = 9; - const LlmMemoryWorkPlan plan = build_llm_memory_work_plan(request); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); - - size_t actual_capacity_bytes = - plan.weight_layers.capacity() * sizeof(LlmByteRange) + - cpu_plan.workers.capacity() * sizeof(LlmWorkerWorkPlan); - for (const LlmWorkerWorkPlan& worker : cpu_plan.workers) { - actual_capacity_bytes += - worker.layers.capacity() * sizeof(LlmLayerRangeTemplate); - actual_capacity_bytes += - worker.sequences.capacity() * sizeof(LlmKvSequenceRangeTemplate); - } - const size_t requested_element_bytes = - plan.geometry.layer_count * sizeof(LlmByteRange) + - cpu_plan.effective_workers * sizeof(LlmWorkerWorkPlan) + - cpu_plan.effective_workers * plan.geometry.layer_count * - sizeof(LlmLayerRangeTemplate) + - cpu_plan.effective_workers * plan.geometry.layer_count * - plan.geometry.batch_size * sizeof(LlmKvSequenceRangeTemplate); - EXPECT_GE(actual_capacity_bytes, requested_element_bytes); - EXPECT_EQ(cpu_plan.planner_storage_bytes, actual_capacity_bytes); - EXPECT_EQ(plan.memory_budget.request.planner_storage_bytes, - actual_capacity_bytes); - EXPECT_EQ(plan.memory_budget.request.auxiliary_bytes, - cpu_plan.descriptor_bytes + actual_capacity_bytes + 7 + 9); - EXPECT_EQ(plan.memory_budget.request.required_total_bytes, - plan.memory_budget.request.committed_data_bytes + - plan.memory_budget.request.auxiliary_bytes); -} - -TEST(LlmMemoryWorkPlanTest, - ConfigAdapterForwardsResolvedFieldsBudgetInputsAndValidation) { - LlmMemoryConfig config; - config.weight_size_mb = 1; - config.layer_count = 2; - config.query_head_count = 4; - config.kv_head_count = 2; - config.head_dimension = 8; - config.kv_element_bytes = 4; - config.visible_context_tokens = 3; - config.batch_size = 2; - config.requested_workers = 3; - config.seed = 99; - - const LlmMemoryWorkPlan plan = build_llm_memory_work_plan( - config, 5, 16 * kGiB, 4096, 7, 9); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.geometry.active_weight_bytes_per_work_unit, - Constants::BYTES_PER_MB); - EXPECT_EQ(plan.geometry.layer_count, 2u); - EXPECT_EQ(plan.geometry.query_head_count, 4u); - EXPECT_EQ(plan.geometry.kv_head_count, 2u); - EXPECT_EQ(plan.geometry.head_dimension, 8u); - EXPECT_EQ(plan.geometry.kv_element_bytes, 4u); - ASSERT_TRUE(plan.geometry.decode.has_value()); - EXPECT_EQ(plan.geometry.decode->visible_context_tokens, 3u); - EXPECT_FALSE(plan.geometry.prefill.has_value()); - EXPECT_EQ(plan.geometry.batch_size, 2u); - EXPECT_EQ(plan.backend, LlmMemoryBackend::Cpu); - EXPECT_EQ(plan.phase, LlmPhase::Decode); - EXPECT_EQ(plan.kv_layout, LlmKvLayout::Contiguous); - EXPECT_EQ(plan.work_unit_kind, LlmWorkUnitKind::DecodeStep); - EXPECT_EQ(cpu_execution_plan(plan).requested_workers, 3u); - EXPECT_EQ(cpu_execution_plan(plan).available_workers, 5u); - EXPECT_EQ(cpu_execution_plan(plan).effective_workers, 3u); - EXPECT_EQ(plan.base_seed, 99u); - EXPECT_EQ(plan.memory_budget.available_memory_bytes, 16 * kGiB); - EXPECT_FALSE(plan.memory_budget.used_fallback); - EXPECT_EQ(plan.memory_budget.request.mapping_granularity_bytes, 4096u); - EXPECT_EQ(plan.memory_budget.request.checksum_auxiliary_bytes, 7u); - EXPECT_EQ(plan.memory_budget.request.orchestration_auxiliary_bytes, 9u); - - config.weight_size_mb = 0; - expect_invalid_plan( - build_llm_memory_work_plan(config, 5, 16 * kGiB, 4096, 7, 9), - LlmMemoryConfigReason::WEIGHT_SIZE_REQUIRED); -} - -TEST(LlmMemoryWorkPlanTest, - RequestedWorkersReduceUntilEveryWorkerHasExecutableWork) { - const LlmGeometryRequest geometry = {2, 2, 1, 1, 1, 1, 1, 1}; - const LlmMemoryWorkPlan plan = - build_llm_memory_work_plan(work_plan_request(geometry, 100, 100)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); - EXPECT_EQ(cpu_plan.requested_workers, 100u); - EXPECT_EQ(cpu_plan.available_workers, 100u); - EXPECT_EQ(cpu_plan.effective_workers, 1u); - EXPECT_EQ(plan.geometry.active_weight_bytes_per_work_unit, 2u); - EXPECT_EQ(plan.geometry.kv_capacity_bytes, 4u); - ASSERT_EQ(cpu_plan.workers.size(), 1u); - EXPECT_FALSE(cpu_plan.workers[0].layers.empty()); - EXPECT_FALSE(cpu_plan.workers[0].sequences.empty()); -} - -TEST(LlmMemoryWorkPlanTest, - AvailableWorkersCapAnOtherwiseExecutableRequestedTeam) { - const LlmGeometryRequest geometry = {100, 1, 1, 1, 8, 1, 1, 1}; - const LlmMemoryWorkPlan plan = - build_llm_memory_work_plan(work_plan_request(geometry, 5, 2)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); - EXPECT_EQ(cpu_plan.requested_workers, 5u); - EXPECT_EQ(cpu_plan.available_workers, 2u); - EXPECT_EQ(cpu_plan.effective_workers, 2u); - ASSERT_EQ(cpu_plan.workers.size(), 2u); - for (const LlmWorkerWorkPlan& worker : cpu_plan.workers) { - EXPECT_GT(worker.layers[0].weight.span_bytes, 0u); - EXPECT_GT(worker.sequences[0].k_visible.span_bytes, 0u); - } -} - TEST(LlmMemoryWorkPlanTest, SmallerLayerSpansRemainExactWithoutScenarioEmptyWorkers) { // The three-worker team has KV work for every worker. The two-byte second @@ -1322,426 +1079,166 @@ TEST(LlmMemoryWorkPlanTest, } } -TEST(LlmMemoryWorkPlanTest, DescriptorAbiMatchesPhaseZeroGoldenOffsets) { - static_assert(!std::is_copy_constructible_v); - static_assert(!std::is_copy_assignable_v); - static_assert(std::is_nothrow_move_constructible_v); - static_assert(std::is_nothrow_move_assignable_v); - static_assert(std::is_standard_layout_v); - static_assert(std::is_standard_layout_v); - static_assert(std::is_standard_layout_v); - static_assert( - std::is_standard_layout_v); - EXPECT_EQ(alignof(LlmLayerDescriptor), 16u); - EXPECT_EQ(sizeof(LlmLayerDescriptor), 48u); - EXPECT_EQ(offsetof(LlmLayerDescriptor, weight_ptr), 0u); - EXPECT_EQ(offsetof(LlmLayerDescriptor, weight_bytes), 8u); - EXPECT_EQ(offsetof(LlmLayerDescriptor, first_sequence_index), 16u); - EXPECT_EQ(offsetof(LlmLayerDescriptor, sequence_count), 24u); - EXPECT_EQ(offsetof(LlmLayerDescriptor, layer_index), 32u); - EXPECT_EQ(offsetof(LlmLayerDescriptor, reserved_zero), 40u); - EXPECT_EQ(alignof(LlmKvSequenceDescriptor), 16u); - EXPECT_EQ(sizeof(LlmKvSequenceDescriptor), 80u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, k_visible_ptr), 0u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, k_visible_bytes), 8u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, v_visible_ptr), 16u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, v_visible_bytes), 24u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, k_append_ptr), 32u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, k_append_bytes), 40u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, v_append_ptr), 48u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, v_append_bytes), 56u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, batch_sequence_index), 64u); - EXPECT_EQ(offsetof(LlmKvSequenceDescriptor, append_record_byte_offset), - 72u); - EXPECT_EQ(alignof(LlmPagedLayerDescriptor), 16u); - EXPECT_EQ(sizeof(LlmPagedLayerDescriptor), 48u); - EXPECT_EQ(offsetof(LlmPagedLayerDescriptor, weight_ptr), 0u); - EXPECT_EQ(offsetof(LlmPagedLayerDescriptor, weight_bytes), 8u); - EXPECT_EQ(offsetof(LlmPagedLayerDescriptor, first_assignment_index), - 16u); - EXPECT_EQ(offsetof(LlmPagedLayerDescriptor, assignment_count), 24u); - EXPECT_EQ(offsetof(LlmPagedLayerDescriptor, layer_index), 32u); - EXPECT_EQ(offsetof(LlmPagedLayerDescriptor, reserved_zero), 40u); - EXPECT_EQ(alignof(LlmPagedKvAssignmentDescriptor), 16u); - EXPECT_EQ(sizeof(LlmPagedKvAssignmentDescriptor), 96u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, block_table_row), - 0u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, k_layer_pool), 8u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, v_layer_pool), 16u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, - first_logical_block), - 24u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, owned_block_count), - 32u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, blocks_per_sequence), - 40u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, block_bytes), 48u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, - last_block_valid_bytes), - 56u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, - decode_append_offset), - 64u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, append_record_bytes), - 72u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, layer_index), 80u); - EXPECT_EQ(offsetof(LlmPagedKvAssignmentDescriptor, - batch_sequence_index), - 88u); -} - -TEST(LlmMemoryWorkPlanTest, - CpuExecutionPlanAccessorsRequireMatchingBackendAndVariant) { - LlmMemoryWorkPlan plan; - EXPECT_EQ(get_llm_cpu_execution_plan(plan), - &cpu_execution_plan(plan)); - const LlmMemoryWorkPlan& const_plan = plan; - EXPECT_EQ(get_llm_cpu_execution_plan(const_plan), - &cpu_execution_plan(const_plan)); - - plan.backend_execution_plan = LlmMetalExecutionPlan{}; - EXPECT_EQ(get_llm_cpu_execution_plan(plan), nullptr); - EXPECT_EQ(get_llm_cpu_execution_plan(const_plan), nullptr); - - plan.backend_execution_plan = LlmCpuExecutionPlan{}; - plan.backend = LlmMemoryBackend::Metal; - EXPECT_EQ(get_llm_cpu_execution_plan(plan), nullptr); - EXPECT_EQ(get_llm_cpu_execution_plan(const_plan), nullptr); -} - -TEST(LlmMemoryWorkPlanTest, - MetalDecodeContiguousDraftAndFinalPlanStayWorkerFree) { - LlmMemoryWorkPlanDraft draft = - prepare_llm_memory_work_plan(metal_work_plan_request()); - ASSERT_TRUE(draft.valid) << draft.reason_code; - EXPECT_FALSE(draft.candidate.valid); - EXPECT_EQ(draft.candidate.reason_code, LlmWorkPlanReason::VALID); - ASSERT_TRUE(draft.candidate.geometry.valid); - ASSERT_EQ(draft.candidate.weight_layers.size(), - draft.candidate.geometry.layer_count); - ASSERT_TRUE(draft.auxiliary_preflight.valid); - EXPECT_EQ(draft.auxiliary_preflight.backend, - LlmMemoryBackend::Metal); - EXPECT_EQ(draft.auxiliary_preflight.effective_workers, 0u); - EXPECT_EQ(draft.candidate.methodology_version, - "llm-memory-v2-metal-decode-contiguous"); - EXPECT_EQ(draft.candidate.component_identities.backend_executor_version, - LlmMetalDecodeContiguousVersion::EXECUTOR); - EXPECT_EQ(draft.candidate.component_identities.schedule_version, - LlmMetalDecodeContiguousVersion::SCHEDULE); - EXPECT_EQ(draft.candidate.component_identities.buffer_pattern_version, - LlmMetalDecodeContiguousVersion::BUFFER_PATTERN); - EXPECT_EQ(draft.candidate.component_identities.write_pattern_version, - LlmMetalDecodeContiguousVersion::WRITE_PATTERN); - EXPECT_EQ(draft.candidate.component_identities.checksum_pattern_version, - LlmMetalDecodeContiguousVersion::CHECKSUM); - EXPECT_FALSE(draft.candidate.component_identities.msl_revision.has_value()); - const LlmMetalExecutionPlan* unresolved = - get_llm_metal_execution_plan(draft.candidate); - ASSERT_NE(unresolved, nullptr); - EXPECT_FALSE(unresolved->valid); - - LlmMemoryWorkPlan plan = finalize_llm_memory_work_plan( - std::move(draft), 19, 23); - ASSERT_TRUE(plan.valid) << plan.reason_code; - unresolved = get_llm_metal_execution_plan(plan); - ASSERT_NE(unresolved, nullptr); - EXPECT_FALSE(unresolved->valid); - EXPECT_EQ(plan.memory_budget.request.checksum_auxiliary_bytes, 19u); - EXPECT_EQ(plan.memory_budget.request.orchestration_auxiliary_bytes, 23u); - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - EXPECT_EQ(scenario.work_unit_kind, LlmWorkUnitKind::DecodeStep); -} - -TEST(LlmMemoryWorkPlanTest, - MetalDecodePagedDraftCarriesLayoutIntoExactRuntimePlanning) { - LlmMemoryWorkPlanDraft draft = - prepare_llm_memory_work_plan(metal_paged_work_plan_request()); - ASSERT_TRUE(draft.valid) << draft.reason_code; - EXPECT_FALSE(draft.candidate.valid); - ASSERT_TRUE(draft.candidate.geometry.valid); - EXPECT_EQ(draft.candidate.methodology_version, - "llm-memory-v2-metal-decode-paged"); - ASSERT_TRUE( - draft.candidate.component_identities.permutation_version.has_value()); - EXPECT_EQ( - *draft.candidate.component_identities.permutation_version, - Constants::LLM_KV_BLOCK_PERMUTATION_VERSION); - EXPECT_EQ( - draft.candidate.component_identities.backend_executor_version, - LlmMetalDecodePagedVersion::EXECUTOR); - EXPECT_EQ(draft.candidate.component_identities.schedule_version, - LlmMetalDecodePagedVersion::SCHEDULE); - EXPECT_EQ(draft.candidate.component_identities.buffer_pattern_version, - LlmMetalDecodePagedVersion::BUFFER_PATTERN); - EXPECT_EQ(draft.candidate.component_identities.write_pattern_version, - LlmMetalDecodePagedVersion::WRITE_PATTERN); - EXPECT_EQ(draft.candidate.component_identities.checksum_pattern_version, - LlmMetalDecodePagedVersion::CHECKSUM); - EXPECT_EQ( - draft.candidate.memory_budget.request - .requested_block_table_mapping_bytes, - draft.candidate.geometry.block_table_bytes); - EXPECT_EQ( - draft.candidate.memory_budget.request - .committed_block_table_mapping_bytes, - draft.candidate.geometry.block_table_bytes); - ASSERT_TRUE(draft.auxiliary_preflight.valid); - EXPECT_EQ(draft.auxiliary_preflight.backend, - LlmMemoryBackend::Metal); - EXPECT_EQ(draft.auxiliary_preflight.kv_layout, - LlmKvLayout::Paged); - EXPECT_EQ(draft.auxiliary_preflight.effective_workers, 0u); - - LlmMetalExecutionPlan provisional = build_llm_metal_execution_plan( - metal_resource_request(draft.candidate)); - ASSERT_TRUE(provisional.valid) << provisional.reason_code; - ASSERT_TRUE(provisional.resources.paged_layout.has_value()); - ASSERT_TRUE(provisional.resources.table_segments.has_value()); - EXPECT_EQ(provisional.resources.paged_layout->geometry_identity, - build_llm_kv_layout_plan( - {draft.candidate.geometry.decode->visible_context_tokens, - draft.candidate.geometry.kv_block_tokens, - draft.candidate.geometry.layer_count, - draft.candidate.geometry.batch_size, - draft.candidate.geometry - .k_or_v_record_bytes_per_layer}) - .geometry_identity); - ASSERT_TRUE(attach_llm_metal_execution_plan( - draft, std::move(provisional))); - ASSERT_TRUE(draft.auxiliary_preflight.valid); - EXPECT_GT(draft.auxiliary_preflight.metal_planned_resource_count, 0u); - - constexpr size_t kChecksumAuxiliaryBytes = 7; - constexpr size_t kOrchestrationAuxiliaryBytes = 11; - constexpr size_t kCommandAuxiliaryBytes = - kChecksumAuxiliaryBytes + kOrchestrationAuxiliaryBytes; - LlmMetalExecutionPlan exact = build_llm_metal_execution_plan( - metal_resource_request(draft.candidate, - kCommandAuxiliaryBytes)); - ASSERT_TRUE(exact.valid) << exact.reason_code; - const size_t expected_transient_peak_bytes = - exact.resources.transient_peak_bytes; - LlmMemoryWorkPlan plan = finalize_llm_memory_work_plan( - std::move(draft), std::move(exact), kChecksumAuxiliaryBytes, - kOrchestrationAuxiliaryBytes); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmMetalExecutionPlan* const execution = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution, nullptr); - ASSERT_TRUE(execution->valid) << execution->reason_code; - ASSERT_TRUE(execution->resources.paged_layout.has_value()); - ASSERT_TRUE(execution->resources.table_segments.has_value()); - EXPECT_EQ(segment_length_sum(*execution->resources.table_segments), - plan.geometry.block_table_bytes); - EXPECT_EQ( - plan.memory_budget.request.requested_block_table_mapping_bytes, - plan.geometry.block_table_bytes); - EXPECT_EQ( - plan.memory_budget.request.committed_block_table_mapping_bytes, - plan.geometry.block_table_bytes); - EXPECT_EQ(plan.memory_budget.request.layout_transient_bytes, - expected_transient_peak_bytes); - EXPECT_EQ(plan.memory_budget.request.required_total_bytes, - execution->resources.known_owned_peak_bytes); - - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::KvOnly, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - EXPECT_EQ(scenario.layout_metadata_lookup_count_per_work_unit, 28u); - EXPECT_EQ(scenario.layout_metadata_read_bytes_per_work_unit, 112u); - EXPECT_EQ(scenario.layout_metadata_lookup_count, 56u); - EXPECT_EQ(scenario.layout_metadata_read_bytes, 224u); -} - -TEST(LlmMemoryWorkPlanTest, - MetalPrefillContiguousFreezesExactPayloadsAndIdentities) { - LlmMemoryWorkPlanDraft draft = - prepare_llm_memory_work_plan(metal_prefill_work_plan_request()); - ASSERT_TRUE(draft.valid) << draft.reason_code; - const LlmMemoryWorkPlan& candidate = draft.candidate; - EXPECT_FALSE(candidate.valid); - EXPECT_EQ(candidate.reason_code, LlmWorkPlanReason::VALID); - EXPECT_EQ(candidate.backend, LlmMemoryBackend::Metal); - EXPECT_EQ(candidate.phase, LlmPhase::Prefill); - EXPECT_EQ(candidate.kv_layout, LlmKvLayout::Contiguous); - EXPECT_EQ(candidate.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(get_llm_cpu_execution_plan(candidate), nullptr); - ASSERT_TRUE(candidate.geometry.valid) << candidate.geometry.reason_code; - EXPECT_FALSE(candidate.geometry.decode.has_value()); - ASSERT_TRUE(candidate.geometry.prefill.has_value()); - EXPECT_EQ(candidate.geometry.prefill->prompt_tokens, 5u); - EXPECT_EQ(candidate.geometry.prefill->attention_query_tile_tokens, 2u); - EXPECT_EQ(candidate.geometry.prefill->tile_count, 3u); - EXPECT_EQ( - candidate.geometry.prefill->attention_prefix_token_visits_per_sequence, - 11u); - - ASSERT_TRUE(candidate.prefill_plan.has_value()); - const LlmPrefillPlan& logical = *candidate.prefill_plan; - ASSERT_TRUE(logical.valid) << logical.reason_code; - EXPECT_EQ(logical.prompt_tokens, 5u); - EXPECT_EQ(logical.attention_query_tile_tokens, 2u); - EXPECT_EQ(logical.full_query_tile_count, 2u); - EXPECT_EQ(logical.final_query_tile_tokens, 1u); - EXPECT_EQ(logical.tile_count, 3u); - EXPECT_EQ(logical.attention_prefix_token_visits_per_sequence, 11u); - EXPECT_EQ(logical.weight_read_bytes_per_work_unit, 1024u); - EXPECT_EQ(logical.kv_read_bytes_per_work_unit, 2816u); - EXPECT_EQ(logical.kv_write_bytes_per_work_unit, 1280u); - EXPECT_EQ(logical.kv_only_payload_bytes_per_work_unit, 4096u); - EXPECT_EQ(logical.mixed_payload_bytes_per_work_unit, 5120u); - EXPECT_FALSE(logical.paged); - EXPECT_EQ(logical.layout_metadata_lookups_per_work_unit, 0u); - EXPECT_EQ(logical.layout_metadata_read_bytes_per_work_unit, 0u); - - EXPECT_EQ(candidate.methodology_version, - "llm-memory-v2-metal-prefill-contiguous"); - EXPECT_EQ(candidate.component_identities.logical_profile_version, - Constants::LLM_PREFILL_LOGICAL_PROFILE_VERSION); - EXPECT_EQ(candidate.component_identities.kv_layout_version, - Constants::LLM_CONTIGUOUS_KV_LAYOUT_VERSION); - EXPECT_FALSE( - candidate.component_identities.permutation_version.has_value()); - EXPECT_EQ(candidate.component_identities.backend_executor_version, - LlmMetalPrefillContiguousVersion::EXECUTOR); - EXPECT_EQ(candidate.component_identities.resource_abi_version, - Constants::LLM_METAL_ARGUMENT_BUFFER_ABI_VERSION); - EXPECT_EQ(candidate.component_identities.schedule_version, - LlmMetalPrefillContiguousVersion::SCHEDULE); - EXPECT_EQ(candidate.component_identities.timer_policy_version, - LlmMetalPrefillContiguousVersion::TIMER); - EXPECT_EQ(candidate.component_identities.buffer_pattern_version, - LlmMetalPrefillContiguousVersion::BUFFER_PATTERN); - EXPECT_EQ(candidate.component_identities.write_pattern_version, - LlmMetalPrefillContiguousVersion::WRITE_PATTERN); - EXPECT_EQ(candidate.component_identities.checksum_pattern_version, - LlmMetalPrefillContiguousVersion::CHECKSUM); - EXPECT_FALSE(candidate.component_identities.msl_revision.has_value()); - EXPECT_FALSE(candidate.component_identities.msl_source_sha256.has_value()); - EXPECT_FALSE(candidate.component_identities.identity.empty()); - EXPECT_FALSE(candidate.plan_identity.empty()); - ASSERT_TRUE(draft.auxiliary_preflight.valid); - EXPECT_EQ(draft.auxiliary_preflight.backend, - LlmMemoryBackend::Metal); - EXPECT_EQ(draft.auxiliary_preflight.kv_layout, - LlmKvLayout::Contiguous); - EXPECT_EQ(draft.auxiliary_preflight.effective_workers, 0u); - - LlmMetalExecutionPlan provisional = build_llm_metal_execution_plan( - metal_resource_request(candidate)); - ASSERT_TRUE(provisional.valid) << provisional.reason_code; - EXPECT_FALSE(provisional.resources.paged_layout.has_value()); - EXPECT_FALSE(provisional.resources.table_segments.has_value()); - EXPECT_EQ(segment_length_sum(provisional.resources.weight_segments), - 1024u); - EXPECT_EQ(segment_length_sum(provisional.resources.k_segments), 640u); - EXPECT_EQ(segment_length_sum(provisional.resources.v_segments), 640u); - ASSERT_TRUE( - attach_llm_metal_execution_plan(draft, std::move(provisional))); - - constexpr size_t kChecksumAuxiliaryBytes = 7; - constexpr size_t kOrchestrationAuxiliaryBytes = 11; - constexpr size_t kCommandAuxiliaryBytes = - kChecksumAuxiliaryBytes + kOrchestrationAuxiliaryBytes; - LlmMetalExecutionPlan exact = build_llm_metal_execution_plan( - metal_resource_request(draft.candidate, kCommandAuxiliaryBytes)); - ASSERT_TRUE(exact.valid) << exact.reason_code; - LlmMemoryWorkPlan plan = finalize_llm_memory_work_plan( - std::move(draft), std::move(exact), kChecksumAuxiliaryBytes, - kOrchestrationAuxiliaryBytes); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmMetalExecutionPlan* const execution = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution, nullptr); - ASSERT_TRUE(execution->valid) << execution->reason_code; - EXPECT_FALSE(execution->resources.paged_layout.has_value()); - EXPECT_FALSE(execution->resources.table_segments.has_value()); - EXPECT_EQ(segment_length_sum(execution->resources.weight_segments), - 1024u); - EXPECT_EQ(segment_length_sum(execution->resources.k_segments), 640u); - EXPECT_EQ(segment_length_sum(execution->resources.v_segments), 640u); - ASSERT_TRUE(plan.component_identities.msl_revision.has_value()); - ASSERT_TRUE(plan.component_identities.msl_source_sha256.has_value()); - EXPECT_EQ(*plan.component_identities.msl_revision, - execution->msl_revision); - EXPECT_EQ(*plan.component_identities.msl_source_sha256, - execution->msl_source_sha256); - - const LlmScenarioWorkPlan weights = build_llm_scenario_work_plan( - plan, LlmScenario::WeightsOnly, 2, true); - const LlmScenarioWorkPlan kv = build_llm_scenario_work_plan( - plan, LlmScenario::KvOnly, 2, true); - const LlmScenarioWorkPlan mixed = build_llm_scenario_work_plan( - plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(weights.valid) << weights.reason_code; - ASSERT_TRUE(kv.valid) << kv.reason_code; - ASSERT_TRUE(mixed.valid) << mixed.reason_code; - EXPECT_EQ(weights.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(weights.kv_write_kind, LlmKvWriteKind::None); - EXPECT_EQ(weights.weight_read_bytes_per_work_unit, 1024u); - EXPECT_EQ(weights.kv_read_bytes_per_work_unit, 0u); - EXPECT_EQ(weights.kv_write_bytes_per_work_unit, 0u); - EXPECT_EQ(weights.effective_model_payload_bytes_per_work_unit, 1024u); - EXPECT_EQ(weights.weight_read_bytes, 2048u); - EXPECT_EQ(weights.effective_model_payload_bytes, 2048u); - EXPECT_EQ(kv.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(kv.kv_write_kind, LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(kv.weight_read_bytes_per_work_unit, 0u); - EXPECT_EQ(kv.kv_read_bytes_per_work_unit, 2816u); - EXPECT_EQ(kv.kv_write_bytes_per_work_unit, 1280u); - EXPECT_EQ(kv.effective_model_payload_bytes_per_work_unit, 4096u); - EXPECT_EQ(kv.kv_read_bytes, 5632u); - EXPECT_EQ(kv.kv_write_bytes, 2560u); - EXPECT_EQ(kv.effective_model_payload_bytes, 8192u); - EXPECT_EQ(mixed.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(mixed.kv_write_kind, - LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(mixed.weight_read_bytes_per_work_unit, 1024u); - EXPECT_EQ(mixed.kv_read_bytes_per_work_unit, 2816u); - EXPECT_EQ(mixed.kv_write_bytes_per_work_unit, 1280u); - EXPECT_EQ(mixed.effective_model_payload_bytes_per_work_unit, 5120u); - EXPECT_EQ(mixed.weight_read_bytes, 2048u); - EXPECT_EQ(mixed.kv_read_bytes, 5632u); - EXPECT_EQ(mixed.kv_write_bytes, 2560u); - EXPECT_EQ(mixed.effective_model_payload_bytes, 10240u); - for (const LlmScenarioWorkPlan* scenario : {&weights, &kv, &mixed}) { - EXPECT_EQ(scenario->layout_metadata_lookup_count_per_work_unit, 0u); - EXPECT_EQ(scenario->layout_metadata_read_bytes_per_work_unit, 0u); - EXPECT_EQ(scenario->accounted_bytes_per_work_unit, - scenario->effective_model_payload_bytes_per_work_unit); - EXPECT_EQ(scenario->layout_metadata_lookup_count, 0u); - EXPECT_EQ(scenario->layout_metadata_read_bytes, 0u); - EXPECT_EQ(scenario->task_accounted_bytes, - scenario->effective_model_payload_bytes); +TEST(LlmMemoryWorkPlanTest, ExecutionPlanAccessorsRequireMatchingBackendAndVariant) { + for (LlmMemoryBackend backend : {LlmMemoryBackend::Cpu, LlmMemoryBackend::Metal}) { + for (bool metal_variant : {false, true}) { + SCOPED_TRACE(static_cast(backend)); + SCOPED_TRACE(metal_variant); + LlmMemoryWorkPlan plan; + plan.backend = backend; + if (metal_variant) plan.backend_execution_plan = LlmMetalExecutionPlan{}; + const LlmMemoryWorkPlan& const_plan = plan; + const bool cpu_match = backend == LlmMemoryBackend::Cpu && !metal_variant; + const bool metal_match = backend == LlmMemoryBackend::Metal && metal_variant; + EXPECT_EQ(get_llm_cpu_execution_plan(plan) != nullptr, cpu_match); + EXPECT_EQ(get_llm_cpu_execution_plan(const_plan), get_llm_cpu_execution_plan(plan)); + EXPECT_EQ(get_llm_metal_execution_plan(plan) != nullptr, metal_match); + EXPECT_EQ(get_llm_metal_execution_plan(const_plan), get_llm_metal_execution_plan(plan)); + } } } -TEST(LlmMemoryWorkPlanTest, - MetalDecodePagedRejectsMismatchedRuntimeLayoutEvidence) { - LlmMemoryWorkPlanDraft draft = - prepare_llm_memory_work_plan(metal_paged_work_plan_request()); - ASSERT_TRUE(draft.valid) << draft.reason_code; - ASSERT_TRUE(attach_llm_metal_execution_plan( - draft, build_llm_metal_execution_plan( - metal_resource_request(draft.candidate)))); - - LlmMetalExecutionPlan mismatched = build_llm_metal_execution_plan( - metal_resource_request(draft.candidate)); - ASSERT_TRUE(mismatched.valid) << mismatched.reason_code; - ASSERT_TRUE(mismatched.resources.paged_layout.has_value()); - ++mismatched.resources.paged_layout->block_bytes; - LlmMemoryWorkPlan plan = finalize_llm_memory_work_plan( - std::move(draft), std::move(mismatched), 0, 0); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmMetalExecutionPlan* const rejected = - get_llm_metal_execution_plan(plan); - ASSERT_NE(rejected, nullptr); - EXPECT_FALSE(rejected->valid); - EXPECT_EQ(rejected->reason_code, - LlmWorkPlanReason::INVALID_MODEL_WORK_PLAN); +TEST(LlmMemoryWorkPlanTest, MetalProfilesCarryExactRuntimePlanningAndIdentities) { + struct Profile { + LlmMemoryWorkPlanRequest request; + const char* methodology; + const char* executor; + const char* schedule; + const char* buffer_pattern; + const char* write_pattern; + const char* checksum; + size_t kv_lookups_per_work_unit; + }; + const std::array profiles = {{ + {metal_work_plan_request(), "llm-memory-v2-metal-decode-contiguous", LlmMetalDecodeContiguousVersion::EXECUTOR, + LlmMetalDecodeContiguousVersion::SCHEDULE, LlmMetalDecodeContiguousVersion::BUFFER_PATTERN, + LlmMetalDecodeContiguousVersion::WRITE_PATTERN, LlmMetalDecodeContiguousVersion::CHECKSUM, 0}, + {metal_paged_work_plan_request(), "llm-memory-v2-metal-decode-paged", LlmMetalDecodePagedVersion::EXECUTOR, + LlmMetalDecodePagedVersion::SCHEDULE, LlmMetalDecodePagedVersion::BUFFER_PATTERN, + LlmMetalDecodePagedVersion::WRITE_PATTERN, LlmMetalDecodePagedVersion::CHECKSUM, 28}, + {metal_prefill_work_plan_request(), "llm-memory-v2-metal-prefill-contiguous", + LlmMetalPrefillContiguousVersion::EXECUTOR, LlmMetalPrefillContiguousVersion::SCHEDULE, + LlmMetalPrefillContiguousVersion::BUFFER_PATTERN, LlmMetalPrefillContiguousVersion::WRITE_PATTERN, + LlmMetalPrefillContiguousVersion::CHECKSUM, 0}, + {metal_prefill_work_plan_request(LlmKvLayout::Paged), "llm-memory-v2-metal-prefill-paged", + LlmMetalPrefillPagedVersion::EXECUTOR, LlmMetalPrefillPagedVersion::SCHEDULE, + LlmMetalPrefillPagedVersion::BUFFER_PATTERN, LlmMetalPrefillPagedVersion::WRITE_PATTERN, + LlmMetalPrefillPagedVersion::CHECKSUM, 60}, + }}; + for (const Profile& profile : profiles) { + SCOPED_TRACE(profile.methodology); + LlmMemoryWorkPlanDraft unresolved_draft = prepare_llm_memory_work_plan(profile.request); + ASSERT_TRUE(unresolved_draft.valid) << unresolved_draft.reason_code; + LlmMemoryWorkPlan unresolved_plan = finalize_llm_memory_work_plan(std::move(unresolved_draft), 19, 23); + ASSERT_TRUE(unresolved_plan.valid) << unresolved_plan.reason_code; + const auto* unresolved_execution = get_llm_metal_execution_plan(unresolved_plan); + ASSERT_NE(unresolved_execution, nullptr); + EXPECT_FALSE(unresolved_execution->valid); + EXPECT_EQ(unresolved_plan.memory_budget.request.checksum_auxiliary_bytes, 19U); + EXPECT_EQ(unresolved_plan.memory_budget.request.orchestration_auxiliary_bytes, 23U); + LlmMemoryWorkPlanDraft draft = prepare_llm_memory_work_plan(profile.request); + ASSERT_TRUE(draft.valid) << draft.reason_code; + const auto& candidate = draft.candidate; + EXPECT_FALSE(candidate.valid); + EXPECT_EQ(candidate.reason_code, LlmWorkPlanReason::VALID); + EXPECT_EQ(candidate.backend, LlmMemoryBackend::Metal); + EXPECT_EQ(candidate.phase, profile.request.geometry.phase); + EXPECT_EQ(candidate.kv_layout, profile.request.geometry.kv_layout); + EXPECT_EQ(get_llm_cpu_execution_plan(candidate), nullptr); + ASSERT_TRUE(candidate.geometry.valid) << candidate.geometry.reason_code; + EXPECT_EQ(candidate.weight_layers.size(), candidate.geometry.layer_count); + EXPECT_EQ(candidate.methodology_version, profile.methodology); + EXPECT_EQ(candidate.component_identities.backend_executor_version, profile.executor); + EXPECT_EQ(candidate.component_identities.schedule_version, profile.schedule); + EXPECT_EQ(candidate.component_identities.buffer_pattern_version, profile.buffer_pattern); + EXPECT_EQ(candidate.component_identities.write_pattern_version, profile.write_pattern); + EXPECT_EQ(candidate.component_identities.checksum_pattern_version, profile.checksum); + EXPECT_FALSE(candidate.component_identities.msl_revision.has_value()); + EXPECT_FALSE(candidate.component_identities.msl_source_sha256.has_value()); + EXPECT_FALSE(candidate.component_identities.identity.empty()); + EXPECT_FALSE(candidate.plan_identity.empty()); + const bool paged = candidate.kv_layout == LlmKvLayout::Paged; + const bool prefill = candidate.phase == LlmPhase::Prefill; + ASSERT_EQ(candidate.component_identities.permutation_version.has_value(), paged); + if (paged) + EXPECT_EQ(*candidate.component_identities.permutation_version, Constants::LLM_KV_BLOCK_PERMUTATION_VERSION); + ASSERT_EQ(candidate.geometry.decode.has_value(), !prefill); + ASSERT_EQ(candidate.geometry.prefill.has_value(), prefill); + ASSERT_EQ(candidate.prefill_plan.has_value(), prefill); + if (prefill) { + ASSERT_TRUE(candidate.prefill_plan->valid) << candidate.prefill_plan->reason_code; + EXPECT_EQ(candidate.prefill_plan->paged, paged); + EXPECT_EQ(candidate.component_identities.logical_profile_version, Constants::LLM_PREFILL_LOGICAL_PROFILE_VERSION); + EXPECT_EQ(candidate.component_identities.resource_abi_version, Constants::LLM_METAL_ARGUMENT_BUFFER_ABI_VERSION); + EXPECT_EQ(candidate.component_identities.timer_policy_version, LlmMetalPrefillContiguousVersion::TIMER); + } + EXPECT_EQ(candidate.component_identities.kv_layout_version, + paged ? Constants::LLM_PAGED_KV_LAYOUT_VERSION : Constants::LLM_CONTIGUOUS_KV_LAYOUT_VERSION); + ASSERT_TRUE(draft.auxiliary_preflight.valid); + EXPECT_EQ(draft.auxiliary_preflight.backend, LlmMemoryBackend::Metal); + EXPECT_EQ(draft.auxiliary_preflight.kv_layout, candidate.kv_layout); + EXPECT_EQ(draft.auxiliary_preflight.effective_workers, 0U); + const auto* unresolved = get_llm_metal_execution_plan(candidate); + ASSERT_NE(unresolved, nullptr); + EXPECT_FALSE(unresolved->valid); + + LlmMetalExecutionPlan provisional = build_llm_metal_execution_plan(metal_resource_request(candidate)); + ASSERT_TRUE(provisional.valid) << provisional.reason_code; + ASSERT_EQ(provisional.resources.paged_layout.has_value(), paged); + ASSERT_EQ(provisional.resources.table_segments.has_value(), paged); + if (paged) { + const size_t tokens = + prefill ? candidate.geometry.prefill->prompt_tokens : candidate.geometry.decode->visible_context_tokens; + EXPECT_EQ( + provisional.resources.paged_layout->geometry_identity, + build_llm_kv_layout_plan({tokens, candidate.geometry.kv_block_tokens, candidate.geometry.layer_count, + candidate.geometry.batch_size, candidate.geometry.k_or_v_record_bytes_per_layer}) + .geometry_identity); + } + ASSERT_TRUE(attach_llm_metal_execution_plan(draft, std::move(provisional))); + ASSERT_TRUE(draft.auxiliary_preflight.valid); + EXPECT_GT(draft.auxiliary_preflight.metal_planned_resource_count, 0U); + LlmMetalExecutionPlan exact = build_llm_metal_execution_plan(metal_resource_request(draft.candidate, 18)); + ASSERT_TRUE(exact.valid) << exact.reason_code; + const size_t transient_peak = exact.resources.transient_peak_bytes; + LlmMemoryWorkPlan plan = finalize_llm_memory_work_plan(std::move(draft), std::move(exact), 7, 11); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const auto* execution = get_llm_metal_execution_plan(plan); + ASSERT_NE(execution, nullptr); + ASSERT_TRUE(execution->valid) << execution->reason_code; + ASSERT_EQ(execution->resources.paged_layout.has_value(), paged); + ASSERT_EQ(execution->resources.table_segments.has_value(), paged); + EXPECT_EQ(segment_length_sum(execution->resources.weight_segments), + plan.geometry.active_weight_bytes_per_work_unit); + EXPECT_EQ(segment_length_sum(execution->resources.k_segments), plan.geometry.k_mapping_bytes); + EXPECT_EQ(segment_length_sum(execution->resources.v_segments), plan.geometry.v_mapping_bytes); + if (paged) EXPECT_EQ(segment_length_sum(*execution->resources.table_segments), plan.geometry.block_table_bytes); + EXPECT_EQ(plan.memory_budget.request.requested_block_table_mapping_bytes, plan.geometry.block_table_bytes); + EXPECT_EQ(plan.memory_budget.request.committed_block_table_mapping_bytes, plan.geometry.block_table_bytes); + EXPECT_EQ(plan.memory_budget.request.layout_transient_bytes, transient_peak); + EXPECT_EQ(plan.memory_budget.request.required_total_bytes, execution->resources.known_owned_peak_bytes); + EXPECT_EQ(plan.memory_budget.request.checksum_auxiliary_bytes, 7U); + EXPECT_EQ(plan.memory_budget.request.orchestration_auxiliary_bytes, 11U); + ASSERT_TRUE(plan.component_identities.msl_revision.has_value()); + ASSERT_TRUE(plan.component_identities.msl_source_sha256.has_value()); + EXPECT_EQ(*plan.component_identities.msl_revision, execution->msl_revision); + EXPECT_EQ(*plan.component_identities.msl_source_sha256, execution->msl_source_sha256); + for (auto scenario : {LlmScenario::WeightsOnly, LlmScenario::KvOnly, LlmScenario::Mixed}) { + const auto work = build_llm_scenario_work_plan(plan, scenario, 2, true); + ASSERT_TRUE(work.valid) << work.reason_code; + EXPECT_EQ(work.work_unit_kind, prefill ? LlmWorkUnitKind::PrefillOperation : LlmWorkUnitKind::DecodeStep); + const size_t lookups = scenario == LlmScenario::WeightsOnly ? 0 : profile.kv_lookups_per_work_unit; + EXPECT_EQ(work.layout_metadata_lookup_count_per_work_unit, lookups); + EXPECT_EQ(work.layout_metadata_read_bytes_per_work_unit, lookups * 4); + EXPECT_EQ(work.layout_metadata_lookup_count, lookups * 2); + EXPECT_EQ(work.layout_metadata_read_bytes, lookups * 8); + EXPECT_EQ(work.accounted_bytes_per_work_unit, + work.effective_model_payload_bytes_per_work_unit + work.layout_metadata_read_bytes_per_work_unit); + if (prefill) + EXPECT_EQ(work.kv_write_kind, + scenario == LlmScenario::WeightsOnly ? LlmKvWriteKind::None : LlmKvWriteKind::FullPromptPopulation); + } + } } TEST(LlmMemoryWorkPlanTest, @@ -1818,49 +1315,6 @@ TEST(LlmMemoryWorkPlanTest, LlmMetalPlanReason::SEMANTIC_VISIT_CAP_EXCEEDED); } -TEST(LlmMemoryWorkPlanTest, - MetalPagedPrefillWeightsOnlySerialVisitLimitAcceptsBoundaryAndRejectsOneOver) { - LlmMemoryWorkPlanRequest request = - metal_prefill_work_plan_request(LlmKvLayout::Paged); - request.geometry.active_weight_bytes = 17; - request.geometry.layer_count = 17; - LlmMemoryWorkPlanDraft draft = prepare_llm_memory_work_plan(request); - ASSERT_TRUE(draft.valid) << draft.reason_code; - LlmMemoryWorkPlan model = - finalize_llm_memory_work_plan(std::move(draft), 0, 0); - ASSERT_TRUE(model.valid) << model.reason_code; - - size_t visits_per_work_unit = 0; - ASSERT_TRUE(calculate_llm_metal_prefill_serial_range_visits_per_work_unit( - model.geometry, LlmScenario::WeightsOnly, visits_per_work_unit)); - ASSERT_EQ(visits_per_work_unit, 17U); - const size_t expected_cap = - Constants::LLM_METAL_MAX_SERIAL_RANGE_VISITS_PER_LANE_PER_TASK / - visits_per_work_unit; - const LlmScenarioLimits limits = calculate_llm_scenario_limits( - model.geometry, LlmScenario::WeightsOnly, LlmMemoryBackend::Metal); - ASSERT_TRUE(limits.valid) << limits.reason_code; - EXPECT_EQ(limits.maximum_work_units_by_work_unit_cap, expected_cap); - - const LlmScenarioWorkPlan boundary = build_llm_scenario_work_plan( - model, LlmScenario::WeightsOnly, expected_cap, true); - ASSERT_TRUE(boundary.valid) << boundary.reason_code; - const LlmScenarioWorkPlan exceeded = build_llm_scenario_work_plan( - model, LlmScenario::WeightsOnly, expected_cap + 1, true); - EXPECT_FALSE(exceeded.valid); - EXPECT_EQ(exceeded.reason_code, - LlmWorkPlanReason::WORK_UNIT_CAP_EXCEEDED); - - LlmGeometry over_cap = model.geometry; - over_cap.layer_count = - Constants::LLM_METAL_MAX_SERIAL_RANGE_VISITS_PER_LANE_PER_TASK + 1; - const LlmScenarioLimits rejected = calculate_llm_scenario_limits( - over_cap, LlmScenario::WeightsOnly, LlmMemoryBackend::Metal); - EXPECT_FALSE(rejected.valid); - EXPECT_EQ(rejected.reason_code, - LlmMetalPlanReason::SERIAL_RANGE_VISIT_CAP_EXCEEDED); -} - TEST(LlmMemoryWorkPlanTest, MetalRuntimePlanFinalizationBindsExactResourcesAndAuxiliaryBytes) { LlmMemoryWorkPlanDraft draft = @@ -2037,144 +1491,6 @@ TEST(LlmMemoryWorkPlanTest, } } -TEST(LlmMemoryWorkPlanTest, - MetalPrefillPagedFreezesExactPayloadsLookupsAndIdentities) { - LlmMemoryWorkPlanDraft draft = prepare_llm_memory_work_plan( - metal_prefill_work_plan_request(LlmKvLayout::Paged)); - ASSERT_TRUE(draft.valid) << draft.reason_code; - const LlmMemoryWorkPlan& candidate = draft.candidate; - EXPECT_FALSE(candidate.valid); - EXPECT_EQ(candidate.reason_code, LlmWorkPlanReason::VALID); - EXPECT_EQ(candidate.backend, LlmMemoryBackend::Metal); - EXPECT_EQ(candidate.phase, LlmPhase::Prefill); - EXPECT_EQ(candidate.kv_layout, LlmKvLayout::Paged); - EXPECT_EQ(candidate.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - ASSERT_TRUE(candidate.geometry.valid) << candidate.geometry.reason_code; - EXPECT_FALSE(candidate.geometry.decode.has_value()); - ASSERT_TRUE(candidate.geometry.prefill.has_value()); - EXPECT_EQ(candidate.geometry.prefill->prompt_tokens, 5u); - EXPECT_EQ(candidate.geometry.prefill->attention_query_tile_tokens, 2u); - EXPECT_EQ(candidate.geometry.prefill->tile_count, 3u); - EXPECT_EQ(candidate.geometry.kv_blocks_per_sequence, 3u); - EXPECT_EQ(candidate.geometry.last_block_tokens, 1u); - EXPECT_EQ(candidate.geometry.last_block_valid_bytes, 32u); - EXPECT_EQ(candidate.geometry.k_mapping_bytes, 768u); - EXPECT_EQ(candidate.geometry.v_mapping_bytes, 768u); - EXPECT_EQ(candidate.geometry.k_layout_padding_bytes, 128u); - EXPECT_EQ(candidate.geometry.v_layout_padding_bytes, 128u); - EXPECT_EQ(candidate.geometry.block_table_entries, 6u); - EXPECT_EQ(candidate.geometry.block_table_bytes, 24u); - EXPECT_EQ( - candidate.geometry - .layout_metadata_lookups_per_layer_sequence_per_work_unit, - 15u); - - ASSERT_TRUE(candidate.prefill_plan.has_value()); - const LlmPrefillPlan& logical = *candidate.prefill_plan; - ASSERT_TRUE(logical.valid) << logical.reason_code; - EXPECT_TRUE(logical.paged); - EXPECT_EQ(logical.blocks_per_sequence, 3u); - EXPECT_EQ(logical.prefix_block_visits_per_sequence, 6u); - EXPECT_EQ(logical.layout_metadata_lookups_per_layer_sequence, 15u); - EXPECT_EQ(logical.layout_metadata_lookups_per_work_unit, 60u); - EXPECT_EQ(logical.layout_metadata_read_bytes_per_work_unit, 240u); - - EXPECT_EQ(candidate.methodology_version, - "llm-memory-v2-metal-prefill-paged"); - EXPECT_EQ(candidate.component_identities.logical_profile_version, - Constants::LLM_PREFILL_LOGICAL_PROFILE_VERSION); - EXPECT_EQ(candidate.component_identities.kv_layout_version, - Constants::LLM_PAGED_KV_LAYOUT_VERSION); - ASSERT_TRUE( - candidate.component_identities.permutation_version.has_value()); - EXPECT_EQ(*candidate.component_identities.permutation_version, - Constants::LLM_KV_BLOCK_PERMUTATION_VERSION); - EXPECT_EQ(candidate.component_identities.backend_executor_version, - LlmMetalPrefillPagedVersion::EXECUTOR); - EXPECT_EQ(candidate.component_identities.resource_abi_version, - Constants::LLM_METAL_ARGUMENT_BUFFER_ABI_VERSION); - EXPECT_EQ(candidate.component_identities.schedule_version, - LlmMetalPrefillPagedVersion::SCHEDULE); - EXPECT_EQ(candidate.component_identities.timer_policy_version, - LlmMetalPrefillPagedVersion::TIMER); - EXPECT_EQ(candidate.component_identities.buffer_pattern_version, - LlmMetalPrefillPagedVersion::BUFFER_PATTERN); - EXPECT_EQ(candidate.component_identities.write_pattern_version, - LlmMetalPrefillPagedVersion::WRITE_PATTERN); - EXPECT_EQ(candidate.component_identities.checksum_pattern_version, - LlmMetalPrefillPagedVersion::CHECKSUM); - EXPECT_FALSE(candidate.component_identities.msl_revision.has_value()); - EXPECT_FALSE(candidate.component_identities.msl_source_sha256.has_value()); - - LlmMetalExecutionPlan provisional = build_llm_metal_execution_plan( - metal_resource_request(candidate)); - ASSERT_TRUE(provisional.valid) << provisional.reason_code; - ASSERT_TRUE(provisional.resources.paged_layout.has_value()); - ASSERT_TRUE(provisional.resources.table_segments.has_value()); - EXPECT_EQ(segment_length_sum(provisional.resources.weight_segments), - 1024u); - EXPECT_EQ(segment_length_sum(provisional.resources.k_segments), 768u); - EXPECT_EQ(segment_length_sum(provisional.resources.v_segments), 768u); - EXPECT_EQ(segment_length_sum(*provisional.resources.table_segments), - 24u); - ASSERT_TRUE( - attach_llm_metal_execution_plan(draft, std::move(provisional))); - - LlmMetalExecutionPlan exact = build_llm_metal_execution_plan( - metal_resource_request(draft.candidate)); - ASSERT_TRUE(exact.valid) << exact.reason_code; - LlmMemoryWorkPlan plan = finalize_llm_memory_work_plan( - std::move(draft), std::move(exact), 0, 0); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmMetalExecutionPlan* const execution = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution, nullptr); - ASSERT_TRUE(execution->valid) << execution->reason_code; - ASSERT_TRUE(execution->resources.paged_layout.has_value()); - ASSERT_TRUE(execution->resources.table_segments.has_value()); - ASSERT_TRUE(plan.component_identities.msl_revision.has_value()); - ASSERT_TRUE(plan.component_identities.msl_source_sha256.has_value()); - EXPECT_EQ(*plan.component_identities.msl_revision, - execution->msl_revision); - EXPECT_EQ(*plan.component_identities.msl_source_sha256, - execution->msl_source_sha256); - EXPECT_EQ(plan.component_identities.backend_executor_version, - LlmMetalPrefillPagedVersion::EXECUTOR); - EXPECT_EQ(plan.component_identities.schedule_version, - LlmMetalPrefillPagedVersion::SCHEDULE); - EXPECT_EQ(plan.component_identities.buffer_pattern_version, - LlmMetalPrefillPagedVersion::BUFFER_PATTERN); - EXPECT_EQ(plan.component_identities.write_pattern_version, - LlmMetalPrefillPagedVersion::WRITE_PATTERN); - EXPECT_EQ(plan.component_identities.checksum_pattern_version, - LlmMetalPrefillPagedVersion::CHECKSUM); - - const LlmScenarioWorkPlan weights = build_llm_scenario_work_plan( - plan, LlmScenario::WeightsOnly, 2, true); - const LlmScenarioWorkPlan kv = build_llm_scenario_work_plan( - plan, LlmScenario::KvOnly, 2, true); - const LlmScenarioWorkPlan mixed = build_llm_scenario_work_plan( - plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(weights.valid) << weights.reason_code; - ASSERT_TRUE(kv.valid) << kv.reason_code; - ASSERT_TRUE(mixed.valid) << mixed.reason_code; - EXPECT_EQ(weights.layout_metadata_lookup_count_per_work_unit, 0u); - EXPECT_EQ(weights.layout_metadata_read_bytes_per_work_unit, 0u); - EXPECT_EQ(weights.accounted_bytes_per_work_unit, 1024u); - EXPECT_EQ(kv.kv_write_kind, LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(kv.layout_metadata_lookup_count_per_work_unit, 60u); - EXPECT_EQ(kv.layout_metadata_read_bytes_per_work_unit, 240u); - EXPECT_EQ(kv.layout_metadata_lookup_count, 120u); - EXPECT_EQ(kv.layout_metadata_read_bytes, 480u); - EXPECT_EQ(kv.accounted_bytes_per_work_unit, 4336u); - EXPECT_EQ(kv.task_accounted_bytes, 8672u); - EXPECT_EQ(mixed.kv_write_kind, LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(mixed.layout_metadata_lookup_count_per_work_unit, 60u); - EXPECT_EQ(mixed.layout_metadata_read_bytes_per_work_unit, 240u); - EXPECT_EQ(mixed.accounted_bytes_per_work_unit, 5360u); - EXPECT_EQ(mixed.task_accounted_bytes, 10720u); -} - TEST(LlmMemoryWorkPlanTest, MetalPlanningRejectsWorkersAndMismatchedExactAuxiliary) { LlmMemoryWorkPlanRequest workers = metal_work_plan_request(); @@ -2182,25 +1498,29 @@ TEST(LlmMemoryWorkPlanTest, expect_invalid_plan(build_llm_memory_work_plan(workers), LlmWorkPlanReason::METAL_WORKERS_NOT_APPLICABLE); - LlmMemoryWorkPlanDraft draft = - prepare_llm_memory_work_plan(metal_work_plan_request()); - ASSERT_TRUE(draft.valid) << draft.reason_code; - ASSERT_TRUE(attach_llm_metal_execution_plan( - draft, build_llm_metal_execution_plan( - metal_resource_request(draft.candidate)))); - LlmMetalExecutionPlan mismatched = build_llm_metal_execution_plan( - metal_resource_request(draft.candidate, 44)); - ASSERT_TRUE(mismatched.valid) << mismatched.reason_code; - LlmMemoryWorkPlan invalid = finalize_llm_memory_work_plan( - std::move(draft), std::move(mismatched), 21, 22); - ASSERT_TRUE(invalid.valid) << invalid.reason_code; - const LlmMetalExecutionPlan* const rejected_runtime = - get_llm_metal_execution_plan(invalid); - ASSERT_NE(rejected_runtime, nullptr); - EXPECT_FALSE(rejected_runtime->valid); - EXPECT_EQ(rejected_runtime->reason_code, - LlmWorkPlanReason::INVALID_MODEL_WORK_PLAN); - EXPECT_EQ(invalid.reason_code, LlmWorkPlanReason::VALID); + for (bool paged_layout_mismatch : {false, true}) { + SCOPED_TRACE(paged_layout_mismatch); + LlmMemoryWorkPlanDraft draft = prepare_llm_memory_work_plan(paged_layout_mismatch ? metal_paged_work_plan_request() + : metal_work_plan_request()); + ASSERT_TRUE(draft.valid) << draft.reason_code; + ASSERT_TRUE(attach_llm_metal_execution_plan( + draft, build_llm_metal_execution_plan(metal_resource_request(draft.candidate)))); + LlmMetalExecutionPlan mismatched = + build_llm_metal_execution_plan(metal_resource_request(draft.candidate, paged_layout_mismatch ? 0 : 44)); + ASSERT_TRUE(mismatched.valid) << mismatched.reason_code; + if (paged_layout_mismatch) { + ASSERT_TRUE(mismatched.resources.paged_layout.has_value()); + ++mismatched.resources.paged_layout->block_bytes; + } + LlmMemoryWorkPlan plan = finalize_llm_memory_work_plan( + std::move(draft), std::move(mismatched), paged_layout_mismatch ? 0 : 21, paged_layout_mismatch ? 0 : 22); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const auto* rejected = get_llm_metal_execution_plan(plan); + ASSERT_NE(rejected, nullptr); + EXPECT_FALSE(rejected->valid); + EXPECT_EQ(rejected->reason_code, LlmWorkPlanReason::INVALID_MODEL_WORK_PLAN); + EXPECT_EQ(plan.reason_code, LlmWorkPlanReason::VALID); + } } TEST(LlmMemoryWorkPlanTest, @@ -2250,6 +1570,10 @@ TEST(LlmMemoryWorkPlanTest, TEST(LlmMemoryWorkPlanTest, MoveOperationsPreserveDestinationAndInvalidateSource) { + static_assert(!std::is_copy_constructible_v); + static_assert(!std::is_copy_assignable_v); + static_assert(std::is_nothrow_move_constructible_v); + static_assert(std::is_nothrow_move_assignable_v); const LlmMemoryWorkPlan constructor_expected = build_llm_memory_work_plan( work_plan_request(small_geometry_request(), 2, 2)); ASSERT_TRUE(constructor_expected.valid) << constructor_expected.reason_code; @@ -2403,46 +1727,66 @@ TEST(LlmMemoryWorkPlanTest, "guardrail-limited-below-target"); } -TEST(LlmMemoryWorkPlanTest, - MetalPrefillSerialRangeLimitAcceptsBoundaryAndRejectsOneVisitOver) { - LlmGeometryRequest request = metal_prefill_work_plan_request().geometry; - request.active_weight_bytes = 1; - request.layer_count = 1; - request.query_head_count = 1; - request.kv_head_count = 1; - request.head_dimension = 1; - request.kv_element_bytes = 1; - request.batch_size = 1; - request.prompt_tokens = - Constants::LLM_METAL_MAX_SERIAL_RANGE_VISITS_PER_LANE_PER_TASK / 2 - 1; - request.attention_query_tile_tokens = request.prompt_tokens; - const LlmGeometry geometry = resolve_llm_geometry(request); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_TRUE(geometry.prefill.has_value()); - ASSERT_EQ(geometry.prefill->tile_count, 1U); - - size_t boundary_visits = 0; - ASSERT_TRUE(calculate_llm_metal_prefill_serial_range_visits_per_work_unit( - geometry, LlmScenario::KvOnly, boundary_visits)); - EXPECT_EQ(boundary_visits, - Constants::LLM_METAL_MAX_SERIAL_RANGE_VISITS_PER_LANE_PER_TASK); - const LlmScenarioLimits boundary = calculate_llm_scenario_limits( - geometry, LlmScenario::KvOnly, LlmMemoryBackend::Metal); - ASSERT_TRUE(boundary.valid) << boundary.reason_code; - EXPECT_EQ(boundary.maximum_work_units_by_work_unit_cap, 1U); - EXPECT_EQ(boundary.effective_maximum_work_units, 1U); - - size_t over_cap_visits = 0; - ASSERT_TRUE(calculate_llm_metal_prefill_serial_range_visits_per_work_unit( - geometry, LlmScenario::Mixed, over_cap_visits)); - EXPECT_EQ( - over_cap_visits, - Constants::LLM_METAL_MAX_SERIAL_RANGE_VISITS_PER_LANE_PER_TASK + 1); - const LlmScenarioLimits rejected = calculate_llm_scenario_limits( - geometry, LlmScenario::Mixed, LlmMemoryBackend::Metal); - EXPECT_FALSE(rejected.valid); - EXPECT_EQ(rejected.reason_code, - LlmMetalPlanReason::SERIAL_RANGE_VISIT_CAP_EXCEEDED); +TEST(LlmMemoryWorkPlanTest, MetalPrefillSerialRangeLimitAcceptsBoundaryAndRejectsOneVisitOver) { + const size_t cap = Constants::LLM_METAL_MAX_SERIAL_RANGE_VISITS_PER_LANE_PER_TASK; + for (bool paged_weights : {false, true}) { + SCOPED_TRACE(paged_weights); + LlmMemoryWorkPlanRequest request = + metal_prefill_work_plan_request(paged_weights ? LlmKvLayout::Paged : LlmKvLayout::Contiguous); + auto& geometry_request = request.geometry; + if (paged_weights) { + geometry_request.active_weight_bytes = 17; + geometry_request.layer_count = 17; + } else { + geometry_request.active_weight_bytes = 1; + geometry_request.layer_count = 1; + geometry_request.query_head_count = 1; + geometry_request.kv_head_count = 1; + geometry_request.head_dimension = 1; + geometry_request.kv_element_bytes = 1; + geometry_request.batch_size = 1; + geometry_request.prompt_tokens = cap / 2 - 1; + geometry_request.attention_query_tile_tokens = geometry_request.prompt_tokens; + } + LlmGeometry geometry = resolve_llm_geometry(geometry_request); + ASSERT_TRUE(geometry.valid) << geometry.reason_code; + LlmMemoryWorkPlan model; + if (paged_weights) { + LlmMemoryWorkPlanDraft draft = prepare_llm_memory_work_plan(request); + ASSERT_TRUE(draft.valid) << draft.reason_code; + model = finalize_llm_memory_work_plan(std::move(draft), 0, 0); + ASSERT_TRUE(model.valid) << model.reason_code; + geometry = model.geometry; + } else { + ASSERT_TRUE(geometry.prefill.has_value()); + ASSERT_EQ(geometry.prefill->tile_count, 1U); + } + const auto scenario = paged_weights ? LlmScenario::WeightsOnly : LlmScenario::KvOnly; + const size_t expected_visits = paged_weights ? 17 : cap; + const size_t expected_work_units = paged_weights ? cap / 17 : 1; + size_t visits = 0; + ASSERT_TRUE(calculate_llm_metal_prefill_serial_range_visits_per_work_unit(geometry, scenario, visits)); + EXPECT_EQ(visits, expected_visits); + const auto limits = calculate_llm_scenario_limits(geometry, scenario, LlmMemoryBackend::Metal); + ASSERT_TRUE(limits.valid) << limits.reason_code; + EXPECT_EQ(limits.maximum_work_units_by_work_unit_cap, expected_work_units); + if (!paged_weights) EXPECT_EQ(limits.effective_maximum_work_units, 1U); + if (paged_weights) { + const auto boundary = build_llm_scenario_work_plan(model, scenario, expected_work_units, true); + ASSERT_TRUE(boundary.valid) << boundary.reason_code; + const auto exceeded = build_llm_scenario_work_plan(model, scenario, expected_work_units + 1, true); + EXPECT_FALSE(exceeded.valid); + EXPECT_EQ(exceeded.reason_code, LlmWorkPlanReason::WORK_UNIT_CAP_EXCEEDED); + } + LlmGeometry over_cap = geometry; + if (paged_weights) over_cap.layer_count = cap + 1; + const auto over_cap_scenario = paged_weights ? LlmScenario::WeightsOnly : LlmScenario::Mixed; + ASSERT_TRUE(calculate_llm_metal_prefill_serial_range_visits_per_work_unit(over_cap, over_cap_scenario, visits)); + EXPECT_EQ(visits, cap + 1); + const auto rejected = calculate_llm_scenario_limits(over_cap, over_cap_scenario, LlmMemoryBackend::Metal); + EXPECT_FALSE(rejected.valid); + EXPECT_EQ(rejected.reason_code, LlmMetalPlanReason::SERIAL_RANGE_VISIT_CAP_EXCEEDED); + } } TEST(LlmMemoryWorkPlanTest, @@ -2755,27 +2099,6 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_EQ(first.kv_replay_factor, 1u); EXPECT_EQ(first.methodology_version, Constants::LLM_CPU_DECODE_CONTIGUOUS_METHODOLOGY_VERSION); - EXPECT_EQ(first.component_identities.logical_profile_version, - Constants::LLM_LOGICAL_PROFILE_VERSION); - EXPECT_EQ(first.component_identities.kv_layout_version, - Constants::LLM_CONTIGUOUS_KV_LAYOUT_VERSION); - EXPECT_FALSE(first.component_identities.permutation_version.has_value()); - EXPECT_EQ(first.component_identities.backend_executor_version, - Constants::LLM_CPU_EXECUTOR_VERSION); - EXPECT_EQ(first.component_identities.resource_abi_version, - Constants::LLM_DESCRIPTOR_ABI_VERSION); - EXPECT_EQ(first.component_identities.schedule_version, - Constants::LLM_CPU_SCHEDULE_VERSION); - EXPECT_EQ(first.component_identities.timer_policy_version, - Constants::LLM_CPU_TIMER_POLICY_VERSION); - EXPECT_EQ(first.component_identities.buffer_pattern_version, - Constants::LLM_BUFFER_PATTERN_VERSION); - EXPECT_EQ(first.component_identities.write_pattern_version, - Constants::LLM_APPEND_PATTERN_VERSION); - EXPECT_EQ(first.component_identities.checksum_pattern_version, - Constants::LLM_READ_CHECKSUM_VERSION); - EXPECT_FALSE(first.component_identities.msl_revision.has_value()); - EXPECT_FALSE(first.component_identities.msl_source_sha256.has_value()); const std::string expected_component_identity = "llm-memory-components-v1" "|logical_profile_version=27:decode_steady_fixed_context" @@ -2944,26 +2267,6 @@ TEST(LlmMemoryWorkPlanTest, const LlmGeometry& geometry = plan.geometry; ASSERT_TRUE(geometry.valid) << geometry.reason_code; ASSERT_TRUE(geometry.decode.has_value()); - EXPECT_EQ(geometry.decode->visible_context_tokens, 35u); - EXPECT_EQ(geometry.kv_block_tokens, 16u); - EXPECT_EQ(geometry.kv_blocks_per_sequence, 3u); - EXPECT_EQ(geometry.physical_blocks_per_layer, 6u); - EXPECT_EQ(geometry.total_physical_blocks, 12u); - EXPECT_EQ(geometry.kv_block_bytes, 512u); - EXPECT_EQ(geometry.last_block_tokens, 3u); - EXPECT_EQ(geometry.last_block_valid_bytes, 96u); - EXPECT_EQ(geometry.decode_append_offset_in_last_block, 64u); - EXPECT_EQ(geometry.k_logical_bytes, 4480u); - EXPECT_EQ(geometry.v_logical_bytes, 4480u); - EXPECT_EQ(geometry.k_mapping_bytes, 6144u); - EXPECT_EQ(geometry.v_mapping_bytes, 6144u); - EXPECT_EQ(geometry.k_layout_padding_bytes, 1664u); - EXPECT_EQ(geometry.v_layout_padding_bytes, 1664u); - EXPECT_EQ(geometry.block_table_entries, 6u); - EXPECT_EQ(geometry.block_table_bytes, 24u); - EXPECT_EQ(geometry.kv_capacity_bytes, 12288u); - EXPECT_EQ(geometry.total_data_mapping_bytes, 13312u); - const LlmCpuExecutionPlan& cpu_plan = cpu_execution_plan(plan); EXPECT_EQ(cpu_plan.requested_workers, 5u); EXPECT_EQ(cpu_plan.available_workers, 5u); @@ -3162,23 +2465,6 @@ TEST(LlmMemoryWorkPlanTest, ASSERT_EQ(cpu_plan.effective_workers, 5u); ASSERT_EQ(paged.layout.blocks_per_sequence, 3u); EXPECT_TRUE(ownership.valid); - EXPECT_EQ(ownership.worker_count, 5u); - EXPECT_EQ(ownership.layer_sequence_count, 4u); - EXPECT_EQ(ownership.total_owned_blocks, 12u); - EXPECT_EQ(ownership.assignments.size(), 12u); - EXPECT_EQ(ownership.total_model_payload_bytes_per_work_unit, 9216u); - EXPECT_EQ(ownership.total_layout_metadata_lookup_count_per_work_unit, - 28u); - EXPECT_EQ(ownership.total_layout_metadata_read_bytes_per_work_unit, - 112u); - EXPECT_EQ(ownership.total_accounted_bytes_per_work_unit, 9328u); - EXPECT_EQ(ownership.worker_accounted_bytes_per_work_unit, - (std::vector{1300, 2064, 2332, 2332, 1300})); - EXPECT_EQ(ownership.minimum_worker_accounted_bytes_per_work_unit, 1300u); - EXPECT_EQ(ownership.maximum_worker_accounted_bytes_per_work_unit, 2332u); - EXPECT_EQ(ownership.worker_accounted_imbalance_bytes_per_work_unit, - 1032u); - size_t source_index = 0; for (size_t ordinal = 0; ordinal < 4; ++ordinal) { const size_t layer = ordinal / 2; @@ -3383,20 +2669,6 @@ TEST(LlmMemoryWorkPlanTest, } } -TEST(LlmMemoryWorkPlanTest, - ContiguousGeometryRejectsNonzeroPagedBlockSize) { - LlmGeometryRequest request = small_geometry_request(); - request.kv_block_tokens = 16; - - const LlmGeometry geometry = resolve_llm_geometry(request); - EXPECT_FALSE(geometry.valid); - EXPECT_EQ(geometry.reason_code, - LlmWorkPlanReason::KV_BLOCK_TOKENS_NOT_APPLICABLE); - expect_invalid_plan( - build_llm_memory_work_plan(work_plan_request(request)), - LlmWorkPlanReason::KV_BLOCK_TOKENS_NOT_APPLICABLE); -} - TEST(LlmMemoryWorkPlanTest, PagedActivePlanRejectsRawInvalidBlockSizesWithStableReasons) { struct InvalidCase { @@ -3593,57 +2865,28 @@ TEST(LlmMemoryWorkPlanTest, LlmKvLayoutRequest request; const char* reason_code; }; - const std::array zero_cases = {{ + const size_t uint32_boundary = static_cast(std::numeric_limits::max()); + const size_t maximum = std::numeric_limits::max(); + const size_t maximum_block_tokens = size_t{1} << 31; + const std::array invalid_cases = {{ {{0, 16, 2, 2, 32}, LlmKvLayoutReason::SEQUENCE_TOKENS_ZERO}, {{35, 0, 2, 2, 32}, LlmKvLayoutReason::BLOCK_TOKENS_ZERO}, {{35, 16, 0, 2, 32}, LlmKvLayoutReason::LAYER_COUNT_ZERO}, {{35, 16, 2, 0, 32}, LlmKvLayoutReason::BATCH_SIZE_ZERO}, {{35, 16, 2, 2, 0}, LlmKvLayoutReason::RECORD_BYTES_ZERO}, - }}; - for (const InvalidCase& test_case : zero_cases) { - SCOPED_TRACE(test_case.reason_code); - const LlmKvLayoutPlan plan = - build_llm_kv_layout_plan(test_case.request); - EXPECT_FALSE(plan.valid); - EXPECT_EQ(plan.reason_code, test_case.reason_code); - EXPECT_TRUE(plan.geometry_identity.empty()); - } - const size_t uint32_boundary = - static_cast(std::numeric_limits::max()); - const std::array block_token_cases = {{ - {{35, 3, 2, 2, 32}, - LlmKvLayoutReason::BLOCK_TOKENS_NOT_POWER_OF_TWO}, - {{35, uint32_boundary, 2, 2, 32}, - LlmKvLayoutReason::BLOCK_TOKENS_NOT_POWER_OF_TWO}, - {{35, uint32_boundary + 1, 2, 2, 32}, - LlmKvLayoutReason::BLOCK_TOKENS_EXCEEDS_UINT32}, - }}; - for (const InvalidCase& test_case : block_token_cases) { - SCOPED_TRACE(test_case.reason_code); - const LlmKvLayoutPlan plan = - build_llm_kv_layout_plan(test_case.request); - EXPECT_FALSE(plan.valid); - EXPECT_EQ(plan.reason_code, test_case.reason_code); - EXPECT_TRUE(plan.geometry_identity.empty()); - } + {{35, 3, 2, 2, 32}, LlmKvLayoutReason::BLOCK_TOKENS_NOT_POWER_OF_TWO}, + {{35, uint32_boundary, 2, 2, 32}, LlmKvLayoutReason::BLOCK_TOKENS_NOT_POWER_OF_TWO}, + {{35, uint32_boundary + 1, 2, 2, 32}, LlmKvLayoutReason::BLOCK_TOKENS_EXCEEDS_UINT32}, - const size_t maximum = std::numeric_limits::max(); - const size_t maximum_block_tokens = size_t{1} << 31; - const std::array overflow_cases = {{ - {{2, 1, 1, maximum, 1}, - LlmKvLayoutReason::PHYSICAL_BLOCK_COUNT_OVERFLOW}, + {{2, 1, 1, maximum, 1}, LlmKvLayoutReason::PHYSICAL_BLOCK_COUNT_OVERFLOW}, {{1, 2, 1, 1, maximum}, LlmKvLayoutReason::BLOCK_BYTES_OVERFLOW}, - {{2, 1, maximum, 1, 1}, - LlmKvLayoutReason::LOGICAL_BYTES_OVERFLOW}, - {{1, maximum_block_tokens, 2, 1, - maximum / maximum_block_tokens}, - LlmKvLayoutReason::PHYSICAL_BYTES_OVERFLOW}, + {{2, 1, maximum, 1, 1}, LlmKvLayoutReason::LOGICAL_BYTES_OVERFLOW}, + {{1, maximum_block_tokens, 2, 1, maximum / maximum_block_tokens}, LlmKvLayoutReason::PHYSICAL_BYTES_OVERFLOW}, }}; - for (const InvalidCase& test_case : overflow_cases) { + for (const InvalidCase& test_case : invalid_cases) { SCOPED_TRACE(test_case.reason_code); - const LlmKvLayoutPlan plan = - build_llm_kv_layout_plan(test_case.request); + const LlmKvLayoutPlan plan = build_llm_kv_layout_plan(test_case.request); EXPECT_FALSE(plan.valid); EXPECT_EQ(plan.reason_code, test_case.reason_code); EXPECT_TRUE(plan.geometry_identity.empty()); @@ -3892,134 +3135,6 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_EQ(entries.back(), std::numeric_limits::max()); } -TEST(LlmMemoryWorkPlanTest, - PagedDecodeScenariosAccountModelMetadataAndCheckedTaskTotals) { - const LlmKvLayoutPlan layout = - build_llm_kv_layout_plan(paged_golden_request()); - ASSERT_TRUE(layout.valid) << layout.reason_code; - const LlmKvBlockTable table = materialize_llm_kv_block_table( - layout, derive_llm_kv_permutation_seed(42)); - ASSERT_TRUE(table.valid) << table.reason_code; - const std::string layout_identity = - serialize_llm_kv_layout_identity(layout, table.permutation); - ASSERT_FALSE(layout_identity.empty()); - - struct ScenarioCase { - LlmScenario scenario; - size_t payload_per_work_unit; - size_t lookups_per_work_unit; - size_t metadata_bytes_per_work_unit; - size_t accounted_bytes_per_work_unit; - }; - const std::array cases = {{ - {LlmScenario::WeightsOnly, 1024, 0, 0, 1024}, - {LlmScenario::KvOnly, 9216, 28, 112, 9328}, - {LlmScenario::Mixed, 10240, 28, 112, 10352}, - }}; - constexpr size_t kWorkUnits = 3; - for (const ScenarioCase& test_case : cases) { - SCOPED_TRACE(llm_scenario_to_string(test_case.scenario)); - const LlmPagedDecodeWorkloadPlan plan = - build_llm_paged_decode_workload_plan( - layout, test_case.scenario, kWorkUnits, - test_case.payload_per_work_unit, table.permutation); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.layout_identity, layout_identity); - EXPECT_EQ(plan.layout_metadata_lookup_count_per_layer_sequence, - test_case.lookups_per_work_unit == 0 ? 0u : 7u); - EXPECT_EQ(plan.layout_metadata_lookup_count_per_work_unit, - test_case.lookups_per_work_unit); - EXPECT_EQ(plan.layout_metadata_read_bytes_per_work_unit, - test_case.metadata_bytes_per_work_unit); - EXPECT_EQ(plan.accounted_bytes_per_work_unit, - test_case.accounted_bytes_per_work_unit); - EXPECT_EQ(plan.maximum_work_units_by_guardrail, - Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK / - test_case.accounted_bytes_per_work_unit); - EXPECT_EQ(plan.maximum_work_units_by_work_unit_cap, - Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT); - EXPECT_EQ(plan.effective_maximum_work_units, - std::min(Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT, - plan.maximum_work_units_by_guardrail)); - EXPECT_EQ(plan.effective_model_payload_bytes, - kWorkUnits * test_case.payload_per_work_unit); - EXPECT_EQ(plan.layout_metadata_lookup_count, - kWorkUnits * test_case.lookups_per_work_unit); - EXPECT_EQ(plan.layout_metadata_read_bytes, - kWorkUnits * test_case.metadata_bytes_per_work_unit); - EXPECT_EQ(plan.task_accounted_bytes, - kWorkUnits * test_case.accounted_bytes_per_work_unit); - EXPECT_FALSE(plan.identity.empty()); - } - - const size_t maximum = std::numeric_limits::max(); - const LlmPagedDecodeWorkloadPlan add_overflow = - build_llm_paged_decode_workload_plan( - layout, LlmScenario::KvOnly, 1, maximum, table.permutation); - EXPECT_FALSE(add_overflow.valid); - EXPECT_EQ(add_overflow.reason_code, - LlmKvLayoutReason::TASK_TOTAL_OVERFLOW); - EXPECT_TRUE(add_overflow.identity.empty()); - - const LlmPagedDecodeWorkloadPlan work_unit_cap_excess = - build_llm_paged_decode_workload_plan( - layout, LlmScenario::WeightsOnly, - Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT + 1, 1, - table.permutation); - EXPECT_FALSE(work_unit_cap_excess.valid); - EXPECT_EQ(work_unit_cap_excess.reason_code, - LlmKvLayoutReason::WORK_UNIT_CAP_EXCEEDED); - EXPECT_TRUE(work_unit_cap_excess.identity.empty()); - - const LlmPagedDecodeWorkloadPlan exact_work_unit_cap = - build_llm_paged_decode_workload_plan( - layout, LlmScenario::WeightsOnly, - Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT, 1, - table.permutation); - ASSERT_TRUE(exact_work_unit_cap.valid) - << exact_work_unit_cap.reason_code; - EXPECT_EQ(exact_work_unit_cap.effective_maximum_work_units, - Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT); - EXPECT_EQ(exact_work_unit_cap.task_accounted_bytes, - Constants::LLM_MAX_WORK_UNITS_PER_MEASUREMENT); - - const size_t exact_work_units = - Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK / 1024; - const LlmPagedDecodeWorkloadPlan exact_guardrail = - build_llm_paged_decode_workload_plan( - layout, LlmScenario::WeightsOnly, exact_work_units, 1024, - table.permutation); - ASSERT_TRUE(exact_guardrail.valid) << exact_guardrail.reason_code; - EXPECT_EQ(exact_guardrail.maximum_work_units_by_guardrail, - exact_work_units); - EXPECT_EQ(exact_guardrail.task_accounted_bytes, - Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK); - const LlmPagedDecodeWorkloadPlan excess_guardrail = - build_llm_paged_decode_workload_plan( - layout, LlmScenario::WeightsOnly, exact_work_units + 1, 1024, - table.permutation); - EXPECT_FALSE(excess_guardrail.valid); - EXPECT_EQ(excess_guardrail.reason_code, - LlmKvLayoutReason::TASK_ACCOUNTED_BYTES_CAP_EXCEEDED); - - const LlmPagedDecodeWorkloadPlan irreducible = - build_llm_paged_decode_workload_plan( - layout, LlmScenario::WeightsOnly, 1, - Constants::LLM_MAX_ACCOUNTED_BYTES_PER_TASK + 1, - table.permutation); - EXPECT_FALSE(irreducible.valid); - EXPECT_EQ(irreducible.reason_code, - LlmKvLayoutReason::GUARDRAIL_BELOW_ONE_WORK_UNIT); - - const LlmPagedDecodeWorkloadPlan invalid_identity = - build_llm_paged_decode_workload_plan( - layout, LlmScenario::KvOnly, 1, 9216, - LlmKvPermutationIdentity{}); - EXPECT_FALSE(invalid_identity.valid); - EXPECT_EQ(invalid_identity.reason_code, - LlmKvLayoutReason::INVALID_LAYOUT_IDENTITY); -} - TEST(LlmMemoryWorkPlanTest, PagedCpuOwnershipIsBlockExclusiveRotatingAndLookupInvariant) { const LlmKvLayoutPlan layout = @@ -4290,109 +3405,6 @@ TEST(LlmMemoryWorkPlanTest, serialized_layout_a, "permutation_sha256", permutation_a.permutation.sha256); - const LlmPagedDecodeWorkloadPlan workload_a = - build_llm_paged_decode_workload_plan( - layout_a, LlmScenario::KvOnly, 3, 9216, - permutation_a.permutation); - const LlmPagedDecodeWorkloadPlan workload_b = - build_llm_paged_decode_workload_plan( - layout_b, LlmScenario::KvOnly, 3, 9216, - permutation_b.permutation); - const LlmPagedDecodeWorkloadPlan workload_changed = - build_llm_paged_decode_workload_plan( - layout_a, LlmScenario::KvOnly, 4, 9216, - permutation_a.permutation); - const LlmPagedDecodeWorkloadPlan workload_seed_changed = - build_llm_paged_decode_workload_plan( - layout_a, LlmScenario::KvOnly, 3, 9216, - permutation_changed.permutation); - ASSERT_TRUE(workload_a.valid) << workload_a.reason_code; - ASSERT_TRUE(workload_b.valid) << workload_b.reason_code; - ASSERT_TRUE(workload_changed.valid) << workload_changed.reason_code; - ASSERT_TRUE(workload_seed_changed.valid) - << workload_seed_changed.reason_code; - EXPECT_EQ(workload_a.identity, workload_b.identity); - EXPECT_NE(workload_a.identity, workload_changed.identity); - EXPECT_NE(workload_a.identity, workload_seed_changed.identity); - EXPECT_EQ(workload_a.identity.rfind( - Constants::LLM_PAGED_DECODE_WORKLOAD_IDENTITY_VERSION, 0), - 0u); - expect_length_prefixed_identity_field( - workload_a.identity, "layout_identity", serialized_layout_a); - - const LlmKvCpuOwnershipPlan ownership_a = - build_llm_paged_decode_kv_cpu_ownership_plan(layout_a, 5); - const LlmKvCpuOwnershipPlan ownership_b = - build_llm_paged_decode_kv_cpu_ownership_plan(layout_b, 5); - const LlmKvCpuOwnershipPlan ownership_changed = - build_llm_paged_decode_kv_cpu_ownership_plan(layout_a, 4); - ASSERT_TRUE(ownership_a.valid) << ownership_a.reason_code; - ASSERT_TRUE(ownership_b.valid) << ownership_b.reason_code; - ASSERT_TRUE(ownership_changed.valid) << ownership_changed.reason_code; - const std::string cpu_a = serialize_llm_kv_cpu_execution_identity( - workload_a, ownership_a); - const std::string cpu_b = serialize_llm_kv_cpu_execution_identity( - workload_b, ownership_b); - const std::string cpu_changed = serialize_llm_kv_cpu_execution_identity( - workload_a, ownership_changed); - EXPECT_EQ(cpu_a, cpu_b); - EXPECT_NE(cpu_a, cpu_changed); - EXPECT_EQ(cpu_a.rfind( - Constants::LLM_PAGED_CPU_EXECUTION_IDENTITY_VERSION, 0), - 0u); - expect_length_prefixed_identity_field( - cpu_a, "workload_identity", workload_a.identity); - expect_length_prefixed_identity_field( - cpu_a, "ownership_identity", ownership_a.identity); - const LlmKvCpuOwnershipPlan cross_layout_ownership = - build_llm_paged_decode_kv_cpu_ownership_plan(layout_changed, 5); - ASSERT_TRUE(cross_layout_ownership.valid) - << cross_layout_ownership.reason_code; - EXPECT_TRUE(serialize_llm_kv_cpu_execution_identity( - workload_a, cross_layout_ownership) - .empty()); - LlmKvCpuOwnershipPlan missing_ownership_identity = ownership_a; - missing_ownership_identity.identity.clear(); - EXPECT_TRUE(serialize_llm_kv_cpu_execution_identity( - workload_a, missing_ownership_identity) - .empty()); - - const LlmKvMetalSegmentPlan segments_a = - build_llm_kv_metal_segment_plan(layout_a, {1024, 16}); - const LlmKvMetalSegmentPlan segments_b = - build_llm_kv_metal_segment_plan(layout_b, {1024, 16}); - const LlmKvMetalSegmentPlan segments_changed = - build_llm_kv_metal_segment_plan(layout_a, {1536, 16}); - ASSERT_TRUE(segments_a.valid) << segments_a.reason_code; - ASSERT_TRUE(segments_b.valid) << segments_b.reason_code; - ASSERT_TRUE(segments_changed.valid) << segments_changed.reason_code; - const std::string metal_a = serialize_llm_kv_metal_execution_identity( - workload_a, segments_a); - const std::string metal_b = serialize_llm_kv_metal_execution_identity( - workload_b, segments_b); - const std::string metal_changed = serialize_llm_kv_metal_execution_identity( - workload_a, segments_changed); - EXPECT_EQ(metal_a, metal_b); - EXPECT_NE(metal_a, metal_changed); - EXPECT_EQ(metal_a.rfind( - Constants::LLM_PAGED_METAL_EXECUTION_IDENTITY_VERSION, 0), - 0u); - expect_length_prefixed_identity_field( - metal_a, "workload_identity", workload_a.identity); - expect_length_prefixed_identity_field( - metal_a, "segment_identity", segments_a.identity); - const LlmKvMetalSegmentPlan cross_layout_segments = - build_llm_kv_metal_segment_plan(layout_changed, {1024, 16}); - ASSERT_TRUE(cross_layout_segments.valid) - << cross_layout_segments.reason_code; - EXPECT_TRUE(serialize_llm_kv_metal_execution_identity( - workload_a, cross_layout_segments) - .empty()); - LlmPagedDecodeWorkloadPlan missing_workload_identity = workload_a; - missing_workload_identity.identity.clear(); - EXPECT_TRUE(serialize_llm_kv_metal_execution_identity( - missing_workload_identity, segments_a) - .empty()); } namespace { @@ -4640,86 +3652,48 @@ LlmPrefillAffine64Checksum enumerate_prefill_task_checksum( } // namespace -TEST(LlmMemoryWorkPlanTest, - PrefillProductionGoldenPayloadAndAttentionMathAreExact) { - const LlmPrefillPlan plan = - resolve_llm_prefill_plan(exact_prefill_request()); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.full_query_tile_count, 2u); - EXPECT_EQ(plan.final_query_tile_tokens, 1u); - EXPECT_EQ(plan.tile_count, 3u); - EXPECT_EQ(plan.attention_prefix_token_visits_per_sequence, 11u); - EXPECT_EQ(plan.causal_token_pairs_per_sequence, 15u); - EXPECT_EQ(plan.logical_attention_pairs, 240u); - EXPECT_EQ(plan.logical_attention_fma_terms, 1920u); - EXPECT_EQ(plan.kv_bytes_per_token, 128u); - EXPECT_EQ(plan.weight_read_bytes_per_work_unit, 1024u); - EXPECT_EQ(plan.kv_read_bytes_per_work_unit, 2816u); - EXPECT_EQ(plan.kv_write_bytes_per_work_unit, 1280u); - EXPECT_EQ(plan.kv_only_payload_bytes_per_work_unit, 4096u); - EXPECT_EQ(plan.mixed_payload_bytes_per_work_unit, 5120u); -} - -TEST(LlmMemoryWorkPlanTest, - PrefillBoundaryTilesAndRemaindersUseExactClosedForms) { - const LlmPrefillPlan token_tiles = - resolve_llm_prefill_plan(exact_prefill_request(5, 1)); - const LlmPrefillPlan whole_prompt_tile = - resolve_llm_prefill_plan(exact_prefill_request(5, 5)); - const LlmPrefillPlan remainder_tiles = - resolve_llm_prefill_plan(exact_prefill_request(7, 3)); - ASSERT_TRUE(token_tiles.valid) << token_tiles.reason_code; - ASSERT_TRUE(whole_prompt_tile.valid) << whole_prompt_tile.reason_code; - ASSERT_TRUE(remainder_tiles.valid) << remainder_tiles.reason_code; - - EXPECT_EQ(token_tiles.tile_count, 5u); - EXPECT_EQ(token_tiles.attention_prefix_token_visits_per_sequence, 15u); - EXPECT_EQ(token_tiles.weight_read_bytes_per_work_unit, 1024u); - EXPECT_EQ(whole_prompt_tile.tile_count, 1u); - EXPECT_EQ(whole_prompt_tile.attention_prefix_token_visits_per_sequence, 5u); - EXPECT_EQ(whole_prompt_tile.weight_read_bytes_per_work_unit, 1024u); - EXPECT_EQ(remainder_tiles.full_query_tile_count, 2u); - EXPECT_EQ(remainder_tiles.final_query_tile_tokens, 1u); - EXPECT_EQ(remainder_tiles.tile_count, 3u); - EXPECT_EQ(remainder_tiles.attention_prefix_token_visits_per_sequence, 16u); - EXPECT_EQ(remainder_tiles.causal_token_pairs_per_sequence, 28u); - - EXPECT_EQ(resolve_llm_prefill_plan(exact_prefill_request(0, 1)).reason_code, - LlmPrefillReason::PROMPT_TOKENS_ZERO); - EXPECT_EQ(resolve_llm_prefill_plan(exact_prefill_request(5, 0)).reason_code, - LlmPrefillReason::QUERY_TILE_TOKENS_ZERO); - EXPECT_EQ(resolve_llm_prefill_plan(exact_prefill_request(5, 6)).reason_code, - LlmPrefillReason::QUERY_TILE_TOKENS_EXCEEDS_PROMPT); -} - -TEST(LlmMemoryWorkPlanTest, - PrefillPagedClosedFormsAndTerminalBlockBytesAreExact) { - const LlmPrefillPlan first = - resolve_llm_prefill_plan(exact_prefill_request(5, 2, 2)); - const LlmPrefillPlan second = - resolve_llm_prefill_plan(exact_prefill_request(7, 3, 2)); - const LlmPrefillPlan terminal = - resolve_llm_prefill_plan(exact_prefill_request(6, 2, 4)); - ASSERT_TRUE(first.valid) << first.reason_code; - ASSERT_TRUE(second.valid) << second.reason_code; - ASSERT_TRUE(terminal.valid) << terminal.reason_code; - - EXPECT_EQ(first.blocks_per_sequence, 3u); - EXPECT_EQ(first.prefix_block_visits_per_sequence, 6u); - EXPECT_EQ(first.layout_metadata_lookups_per_layer_sequence, 15u); - EXPECT_EQ(second.blocks_per_sequence, 4u); - EXPECT_EQ(second.prefix_block_visits_per_sequence, 9u); - EXPECT_EQ(second.layout_metadata_lookups_per_layer_sequence, 22u); - - const LlmPrefillUnitRangeCost terminal_block = - calculate_llm_prefill_paged_block_cost(terminal, 1); - ASSERT_TRUE(terminal_block.valid) << terminal_block.reason_code; - EXPECT_EQ(terminal_block.valid_token_count, 2u); - EXPECT_EQ(terminal_block.data_visit_count, 4u); - EXPECT_EQ(terminal_block.model_payload_bytes, 256u); - EXPECT_EQ(terminal_block.layout_metadata_lookup_count, 3u); - EXPECT_EQ(terminal_block.layout_metadata_read_bytes, 12u); - EXPECT_EQ(terminal_block.accounted_bytes, 268u); +TEST(LlmMemoryWorkPlanTest, PrefillProductionGoldenPayloadAndAttentionMathAreExact) { + struct Case { + size_t prompt; + size_t tile; + size_t full; + size_t remainder; + size_t tiles; + size_t visits; + size_t causal_pairs; + size_t attention_pairs; + size_t fma_terms; + size_t read_bytes; + size_t write_bytes; + }; + const std::array cases = {{{5, 2, 2, 1, 3, 11, 15, 240, 1920, 2816, 1280}, + {5, 1, 5, 0, 5, 15, 15, 240, 1920, 3840, 1280}, + {5, 5, 1, 0, 1, 5, 15, 240, 1920, 1280, 1280}, + {7, 3, 2, 1, 3, 16, 28, 448, 3584, 4096, 1792}}}; + for (const auto& row : cases) { + SCOPED_TRACE(::testing::Message() << row.prompt << ":" << row.tile); + const LlmPrefillPlan plan = resolve_llm_prefill_plan(exact_prefill_request(row.prompt, row.tile)); + ASSERT_TRUE(plan.valid) << plan.reason_code; + EXPECT_EQ(plan.full_query_tile_count, row.full); + EXPECT_EQ(plan.final_query_tile_tokens, row.remainder); + EXPECT_EQ(plan.tile_count, row.tiles); + EXPECT_EQ(plan.attention_prefix_token_visits_per_sequence, row.visits); + EXPECT_EQ(plan.causal_token_pairs_per_sequence, row.causal_pairs); + EXPECT_EQ(plan.logical_attention_pairs, row.attention_pairs); + EXPECT_EQ(plan.logical_attention_fma_terms, row.fma_terms); + EXPECT_EQ(plan.kv_bytes_per_token, 128U); + EXPECT_EQ(plan.weight_read_bytes_per_work_unit, 1024U); + EXPECT_EQ(plan.kv_read_bytes_per_work_unit, row.read_bytes); + EXPECT_EQ(plan.kv_write_bytes_per_work_unit, row.write_bytes); + EXPECT_EQ(plan.kv_only_payload_bytes_per_work_unit, row.read_bytes + row.write_bytes); + EXPECT_EQ(plan.mixed_payload_bytes_per_work_unit, 1024 + row.read_bytes + row.write_bytes); + } + for (const auto& row : {std::pair{exact_prefill_request(0, 1), + LlmPrefillReason::PROMPT_TOKENS_ZERO}, + {exact_prefill_request(5, 0), LlmPrefillReason::QUERY_TILE_TOKENS_ZERO}, + {exact_prefill_request(5, 6), LlmPrefillReason::QUERY_TILE_TOKENS_EXCEEDS_PROMPT}}) { + EXPECT_EQ(resolve_llm_prefill_plan(row.first).reason_code, row.second); + } } TEST(LlmMemoryWorkPlanTest, @@ -5071,40 +4045,12 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_TRUE(validate_llm_prefill_cpu_execution_evidence(paged)); EXPECT_GT(paged.memory_budget.request.requested_block_table_mapping_bytes, 0u); - EXPECT_EQ(contiguous.geometry.prefill->prompt_tokens, 5u); - EXPECT_EQ(contiguous.geometry.prefill->attention_query_tile_tokens, 2u); - EXPECT_EQ(contiguous.geometry.prefill->tile_count, 3u); - EXPECT_EQ( - contiguous.geometry.prefill->attention_prefix_token_visits_per_sequence, - 11u); - EXPECT_EQ( - paged.geometry.prefill->paged_prefix_block_visits_per_sequence, 6u); EXPECT_NE(contiguous.plan_identity, paged.plan_identity); EXPECT_NE(contiguous.plan_identity, changed_query_tile.plan_identity); EXPECT_NE(contiguous.plan_identity, changed_prompt.plan_identity); - const LlmScenarioLimits weights = calculate_llm_scenario_limits( - paged.geometry, LlmScenario::WeightsOnly); - const LlmScenarioLimits kv = calculate_llm_scenario_limits( - paged.geometry, LlmScenario::KvOnly); - const LlmScenarioLimits mixed = calculate_llm_scenario_limits( - paged.geometry, LlmScenario::Mixed); - ASSERT_TRUE(weights.valid) << weights.reason_code; + const LlmScenarioLimits kv = calculate_llm_scenario_limits(paged.geometry, LlmScenario::KvOnly); ASSERT_TRUE(kv.valid) << kv.reason_code; - ASSERT_TRUE(mixed.valid) << mixed.reason_code; - EXPECT_EQ(weights.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(kv.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(mixed.work_unit_kind, LlmWorkUnitKind::PrefillOperation); - EXPECT_EQ(weights.kv_write_kind, LlmKvWriteKind::None); - EXPECT_EQ(kv.kv_write_kind, LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(mixed.kv_write_kind, LlmKvWriteKind::FullPromptPopulation); - EXPECT_EQ(weights.accounted_bytes_per_work_unit, 1024u); - EXPECT_EQ(kv.effective_model_payload_bytes_per_work_unit, 4096u); - EXPECT_EQ(kv.layout_metadata_lookup_count_per_work_unit, 60u); - EXPECT_EQ(kv.layout_metadata_read_bytes_per_work_unit, 240u); - EXPECT_EQ(kv.accounted_bytes_per_work_unit, 4336u); - EXPECT_EQ(mixed.effective_model_payload_bytes_per_work_unit, 5120u); - EXPECT_EQ(mixed.accounted_bytes_per_work_unit, 5360u); const std::array work_units = {2, 3, 4}; const LlmFrozenScenarioPlans frozen = @@ -5152,72 +4098,7 @@ TEST(LlmMemoryWorkPlanTest, LlmWorkPlanReason::TASK_ACCOUNTED_BYTES_CAP_EXCEEDED); } -TEST(LlmMemoryWorkPlanTest, - PrefillDescriptorAbiAndScenarioOwnershipSetsAreExact) { - EXPECT_EQ(sizeof(LlmPrefillLayerDescriptor), 48u); - EXPECT_EQ(alignof(LlmPrefillLayerDescriptor), 16u); - EXPECT_EQ(offsetof(LlmPrefillLayerDescriptor, weight_ptr), 0u); - EXPECT_EQ(offsetof(LlmPrefillLayerDescriptor, weight_bytes), 8u); - EXPECT_EQ(offsetof(LlmPrefillLayerDescriptor, first_sequence_index), 16u); - EXPECT_EQ(offsetof(LlmPrefillLayerDescriptor, sequence_count), 24u); - EXPECT_EQ(offsetof(LlmPrefillLayerDescriptor, layer_index), 32u); - EXPECT_EQ(offsetof(LlmPrefillLayerDescriptor, reserved_zero), 40u); - EXPECT_EQ(sizeof(LlmPrefillKvSequenceDescriptor), 80u); - EXPECT_EQ(alignof(LlmPrefillKvSequenceDescriptor), 16u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, k_owned_ptr), 0u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, v_owned_ptr), 8u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, first_token), 16u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, owned_token_count), 24u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, prompt_tokens), 32u); - EXPECT_EQ( - offsetof(LlmPrefillKvSequenceDescriptor, - attention_query_tile_tokens), - 40u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, record_bytes), 48u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, layer_index), 56u); - EXPECT_EQ( - offsetof(LlmPrefillKvSequenceDescriptor, batch_sequence_index), 64u); - EXPECT_EQ(offsetof(LlmPrefillKvSequenceDescriptor, reserved_zero), 72u); - EXPECT_EQ(sizeof(LlmPagedPrefillLayerDescriptor), 48u); - EXPECT_EQ(alignof(LlmPagedPrefillLayerDescriptor), 16u); - EXPECT_EQ(offsetof(LlmPagedPrefillLayerDescriptor, weight_ptr), 0u); - EXPECT_EQ(offsetof(LlmPagedPrefillLayerDescriptor, weight_bytes), 8u); - EXPECT_EQ( - offsetof(LlmPagedPrefillLayerDescriptor, first_assignment_index), 16u); - EXPECT_EQ(offsetof(LlmPagedPrefillLayerDescriptor, assignment_count), 24u); - EXPECT_EQ(offsetof(LlmPagedPrefillLayerDescriptor, layer_index), 32u); - EXPECT_EQ(offsetof(LlmPagedPrefillLayerDescriptor, reserved_zero), 40u); - EXPECT_EQ(sizeof(LlmPagedPrefillKvAssignmentDescriptor), 112u); - EXPECT_EQ(alignof(LlmPagedPrefillKvAssignmentDescriptor), 16u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, - block_table_row), 0u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, k_layer_pool), - 8u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, v_layer_pool), - 16u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, - first_logical_block), 24u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, - owned_block_count), 32u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, - blocks_per_sequence), 40u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, block_tokens), - 48u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, block_bytes), - 56u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, - last_block_valid_bytes), 64u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, prompt_tokens), - 72u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, - attention_query_tile_tokens), 80u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, record_bytes), - 88u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, layer_index), - 96u); - EXPECT_EQ(offsetof(LlmPagedPrefillKvAssignmentDescriptor, - batch_sequence_index), 104u); - +TEST(LlmMemoryWorkPlanTest, PrefillScenarioOwnershipSetsAreExact) { const LlmMemoryWorkPlan plan = build_llm_memory_work_plan( work_plan_request(integrated_prefill_geometry_request( LlmKvLayout::Contiguous, 7, 3), @@ -5441,18 +4322,17 @@ TEST(LlmMemoryWorkPlanTest, EXPECT_FALSE(cpu_execution_plan(plan).prefill.has_value()); } -TEST(LlmMemoryWorkPlanTest, - PrefillPlannerStorageAccountsExactRetainedIdentityCapacities) { - const auto expect_exact_storage = [](LlmKvLayout layout) { - const LlmMemoryWorkPlan plan = build_llm_memory_work_plan( - work_plan_request(integrated_prefill_geometry_request( - layout, 7, 3, - layout == LlmKvLayout::Paged ? 2 : 0), - 3, 3)); +TEST(LlmMemoryWorkPlanTest, PlannerStorageAccountsExactRetainedCapacitiesAndBudget) { + for (const LlmMemoryWorkPlanRequest& request : + {work_plan_request(small_geometry_request(2), 3, 3), + work_plan_request(integrated_prefill_geometry_request(LlmKvLayout::Contiguous, 7, 3, 0), 3, 3), + work_plan_request(integrated_prefill_geometry_request(LlmKvLayout::Paged, 7, 3, 2), 3, 3)}) { + LlmMemoryWorkPlanRequest budgeted = request; + budgeted.checksum_auxiliary_bytes = 7; + budgeted.orchestration_auxiliary_bytes = 9; + const LlmMemoryWorkPlan plan = build_llm_memory_work_plan(budgeted); ASSERT_TRUE(plan.valid) << plan.reason_code; const LlmCpuExecutionPlan& cpu = cpu_execution_plan(plan); - ASSERT_TRUE(cpu.prefill.has_value()); - size_t expected = plan.weight_layers.capacity() * sizeof(LlmByteRange) + cpu.workers.capacity() * sizeof(LlmWorkerWorkPlan); @@ -5476,40 +4356,33 @@ TEST(LlmMemoryWorkPlanTest, expected += value.capacity() + 1; } }; - add_external_string_capacity(cpu.prefill->identity); - for (const LlmPrefillCpuScenarioExecutionPlan& scenario : - cpu.prefill->scenarios) { - expected += scenario.ownership_scopes.capacity() * - sizeof(LlmPrefillCpuOwnershipPlan); - expected += scenario.worker_accounted_bytes_per_work_unit.capacity() * - sizeof(size_t); - add_external_string_capacity(scenario.identity); - for (const LlmPrefillCpuOwnershipPlan& scope : - scenario.ownership_scopes) { - expected += scope.assignments.capacity() * - sizeof(LlmPrefillCpuAssignment); - expected += scope.worker_weight_shard_bytes.capacity() * - sizeof(size_t); - expected += scope.worker_kv_model_payload_bytes.capacity() * - sizeof(size_t); - expected += scope.worker_layout_metadata_lookup_count.capacity() * - sizeof(size_t); - expected += scope.worker_layout_metadata_read_bytes.capacity() * - sizeof(size_t); - expected += scope.worker_scenario_accounted_bytes.capacity() * - sizeof(size_t); - add_external_string_capacity(scope.reason_code); - add_external_string_capacity(scope.identity); - EXPECT_EQ(scenario.identity.find(scope.identity), - std::string::npos); + if (cpu.prefill.has_value()) { + add_external_string_capacity(cpu.prefill->identity); + for (const LlmPrefillCpuScenarioExecutionPlan& scenario : cpu.prefill->scenarios) { + expected += scenario.ownership_scopes.capacity() * sizeof(LlmPrefillCpuOwnershipPlan); + expected += scenario.worker_accounted_bytes_per_work_unit.capacity() * sizeof(size_t); + add_external_string_capacity(scenario.identity); + for (const LlmPrefillCpuOwnershipPlan& scope : scenario.ownership_scopes) { + expected += scope.assignments.capacity() * sizeof(LlmPrefillCpuAssignment); + expected += scope.worker_weight_shard_bytes.capacity() * sizeof(size_t); + expected += scope.worker_kv_model_payload_bytes.capacity() * sizeof(size_t); + expected += scope.worker_layout_metadata_lookup_count.capacity() * sizeof(size_t); + expected += scope.worker_layout_metadata_read_bytes.capacity() * sizeof(size_t); + expected += scope.worker_scenario_accounted_bytes.capacity() * sizeof(size_t); + add_external_string_capacity(scope.reason_code); + add_external_string_capacity(scope.identity); + EXPECT_EQ(scenario.identity.find(scope.identity), std::string::npos); + } } } EXPECT_EQ(cpu.planner_storage_bytes, expected); EXPECT_EQ(plan.memory_budget.request.planner_storage_bytes, expected); - }; - - expect_exact_storage(LlmKvLayout::Contiguous); - expect_exact_storage(LlmKvLayout::Paged); + EXPECT_EQ(plan.memory_budget.request.auxiliary_bytes, cpu.descriptor_bytes + expected + 7 + 9); + EXPECT_EQ(plan.memory_budget.request.required_total_bytes, + plan.memory_budget.request.committed_data_bytes + + plan.memory_budget.request.committed_block_table_mapping_bytes + + plan.memory_budget.request.auxiliary_bytes); + } } TEST(LlmMemoryWorkPlanTest, @@ -5630,47 +4503,45 @@ TEST(LlmMemoryWorkPlanTest, TEST(LlmMemoryWorkPlanTest, PrefillSemanticTraceWritesBothBlocksBeforePerTileKThenVReads) { - const LlmPrefillPlan plan = - resolve_llm_prefill_plan(exact_prefill_request(4, 2, 2)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmPrefillSemanticTrace trace = - build_llm_prefill_semantic_trace( - plan, {LlmPrefillPartitionUnitKind::PagedBlock, 0, 2, 10}); - ASSERT_TRUE(trace.valid) << trace.reason_code; - - struct ExpectedEvent { - LlmPrefillSemanticAccess access; - LlmPrefillKvDomain domain; - size_t tile_index; - size_t tile_end; - size_t block; - size_t visit_tokens; - }; - constexpr std::array kExpected = {{ - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 4, 0, 2}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 4, 0, 2}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 4, 1, 2}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 4, 1, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 0, 2, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 0, 2, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 4, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 4, 1, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 4, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 4, 1, 2}, - }}; - ASSERT_EQ(trace.events.size(), kExpected.size()); - for (size_t index = 0; index < kExpected.size(); ++index) { - SCOPED_TRACE(index); - EXPECT_EQ(trace.events[index].access, kExpected[index].access); - EXPECT_EQ(trace.events[index].domain, kExpected[index].domain); - EXPECT_EQ(trace.events[index].tile_index, - kExpected[index].tile_index); - EXPECT_EQ(trace.events[index].tile_end_token, - kExpected[index].tile_end); - EXPECT_EQ(trace.events[index].logical_unit_index, - kExpected[index].block); - EXPECT_EQ(trace.events[index].visit_token_count, - kExpected[index].visit_tokens); + for (bool paged : {false, true}) { + SCOPED_TRACE(paged); + const size_t unit_tokens = paged ? 2 : 1; + const LlmPrefillPlan plan = + resolve_llm_prefill_plan(paged ? exact_prefill_request(4, 2, 2) : exact_prefill_request(2, 1, 0)); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const auto kind = paged ? LlmPrefillPartitionUnitKind::PagedBlock : LlmPrefillPartitionUnitKind::ContiguousToken; + const LlmPrefillSemanticTrace trace = build_llm_prefill_semantic_trace(plan, {kind, 0, 2, 10}); + ASSERT_TRUE(trace.valid) << trace.reason_code; + struct ExpectedEvent { + LlmPrefillSemanticAccess access; + LlmPrefillKvDomain domain; + size_t tile_index; + size_t tile_end; + size_t block; + size_t visit_tokens; + }; + constexpr std::array kExpected = {{ + {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 4, 0, 2}, + {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 4, 0, 2}, + {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 4, 1, 2}, + {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 4, 1, 2}, + {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 0, 2, 0, 2}, + {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 0, 2, 0, 2}, + {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 4, 0, 2}, + {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 4, 1, 2}, + {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 4, 0, 2}, + {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 4, 1, 2}, + }}; + ASSERT_EQ(trace.events.size(), kExpected.size()); + for (size_t index = 0; index < kExpected.size(); ++index) { + SCOPED_TRACE(index); + EXPECT_EQ(trace.events[index].access, kExpected[index].access); + EXPECT_EQ(trace.events[index].domain, kExpected[index].domain); + EXPECT_EQ(trace.events[index].tile_index, kExpected[index].tile_index); + EXPECT_EQ(trace.events[index].tile_end_token, kExpected[index].tile_end / 2 * unit_tokens); + EXPECT_EQ(trace.events[index].logical_unit_index, kExpected[index].block); + EXPECT_EQ(trace.events[index].visit_token_count, kExpected[index].visit_tokens / 2 * unit_tokens); + } } } diff --git a/tests/test_llm_metal_backend.cpp b/tests/test_llm_metal_backend.cpp index 0e99d23..46e5dc0 100644 --- a/tests/test_llm_metal_backend.cpp +++ b/tests/test_llm_metal_backend.cpp @@ -22,7 +22,6 @@ #include #include -#include #include #include #include @@ -1279,9 +1278,10 @@ TEST(LlmMetalBackendTest, ArgumentBufferPlanFreezesCanonicalSlotBasesCountsAndCa TEST(LlmMetalBackendTest, ContiguousPoolsUseExactCanonicalBoundarySegmentsWithoutAllocation) { const size_t capacity = Constants::LLM_METAL_SEGMENT_CAPACITY_BYTES; - const std::array lengths = {capacity - 1, capacity, capacity + 1}; - const std::array, 3> expected = {std::vector{capacity - 1}, std::vector{capacity}, - std::vector{capacity, 1}}; + const std::array lengths = {capacity - 1, capacity, capacity + 1, 3 * capacity + 17}; + const std::array, 4> expected = {std::vector{capacity - 1}, std::vector{capacity}, + std::vector{capacity, 1}, + std::vector{capacity, capacity, capacity, 17}}; for (size_t index = 0; index < lengths.size(); ++index) { const LlmGeometry geometry = contiguous_geometry(lengths[index], lengths[index]); @@ -1299,23 +1299,12 @@ TEST(LlmMetalBackendTest, ContiguousPoolsUseExactCanonicalBoundarySegmentsWithou EXPECT_EQ(plan.resources.k_segments.total_length_bytes, lengths[index]); EXPECT_EQ(plan.resources.v_segments.total_length_bytes, lengths[index]); EXPECT_FALSE(plan.resources.table_segments.has_value()); - } -} - -TEST(LlmMetalBackendTest, ContiguousPoolsRetainEveryExactLengthAcrossMultipleSegments) { - const size_t capacity = Constants::LLM_METAL_SEGMENT_CAPACITY_BYTES; - const size_t length = 3 * capacity + 17; - const LlmGeometry geometry = contiguous_geometry(length, length); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - const LlmMetalExecutionPlan plan = build_llm_metal_execution_plan(resource_request(geometry)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - - const std::vector expected = {capacity, capacity, capacity, 17}; - for (const LlmKvSegmentPlan* segments : - {&plan.resources.weight_segments, &plan.resources.k_segments, &plan.resources.v_segments}) { - EXPECT_EQ(segments->segment_lengths, expected); - EXPECT_EQ(segments->segment_count, expected.size()); - EXPECT_EQ(std::accumulate(segments->segment_lengths.begin(), segments->segment_lengths.end(), size_t{0}), length); + for (const LlmKvSegmentPlan* segments : + {&plan.resources.weight_segments, &plan.resources.k_segments, &plan.resources.v_segments}) { + EXPECT_EQ(segments->segment_count, expected[index].size()); + EXPECT_EQ(std::accumulate(segments->segment_lengths.begin(), segments->segment_lengths.end(), size_t{0}), + lengths[index]); + } } } @@ -1338,82 +1327,54 @@ TEST(LlmMetalBackendTest, MultiGibContiguousPoolRetainsExactBoundedSegmentCountW EXPECT_EQ(plan.resources.weight_segments.total_length_bytes, weight_bytes); } -TEST(LlmMetalBackendTest, PagedKvSegmentsKeepWholeBlocksAndTableSegmentsKeepWholeEntries) { - LlmMetalPlanningLimits limits; - limits.segment_capacity_bytes = 100; - limits.segment_slots_per_pool = 32; - const LlmGeometry geometry = paged_geometry(101, 49, 16, 2, 2, 3); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_EQ(geometry.kv_block_bytes, 48U); - ASSERT_EQ(geometry.total_physical_blocks, 16U); - ASSERT_EQ(geometry.block_table_entries, 8U); - - const LlmMetalExecutionPlan plan = build_llm_metal_execution_plan(resource_request(geometry, limits)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_TRUE(plan.resources.table_segments.has_value()); - EXPECT_EQ(plan.resources.k_segments.elements_per_segment, 2U); - EXPECT_EQ(plan.resources.k_segments.segment_count, 8U); - EXPECT_EQ(plan.resources.k_segments.segment_lengths, std::vector(8, 96)); - EXPECT_EQ(plan.resources.v_segments.segment_lengths, plan.resources.k_segments.segment_lengths); - EXPECT_EQ(plan.resources.k_segments.total_length_bytes, geometry.k_mapping_bytes); - EXPECT_TRUE(std::all_of(plan.resources.k_segments.segment_lengths.begin(), - plan.resources.k_segments.segment_lengths.end(), - [](size_t length) { return length % 48 == 0; })); - EXPECT_EQ(plan.resources.table_segments->element_bytes, sizeof(uint32_t)); - EXPECT_EQ(plan.resources.table_segments->segment_lengths, std::vector{32}); - EXPECT_EQ(plan.resources.table_segments->total_length_bytes, geometry.block_table_bytes); -} - -TEST(LlmMetalBackendTest, PagedPoolsHonorInjectedSegmentBoundaryMinusOneExactAndPlusOne) { - LlmMetalPlanningLimits limits; - limits.segment_capacity_bytes = 100; - limits.segment_slots_per_pool = 8; - const std::array physical_pool_bytes = {99, 100, 101}; - const std::array, 3> expected = {std::vector{99}, std::vector{100}, - std::vector{100, 1}}; - - for (size_t index = 0; index < physical_pool_bytes.size(); ++index) { - const LlmGeometry geometry = paged_geometry(1, physical_pool_bytes[index], 1); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_EQ(geometry.kv_block_bytes, 1U); - ASSERT_EQ(geometry.k_mapping_bytes, physical_pool_bytes[index]); - const LlmMetalExecutionPlan plan = build_llm_metal_execution_plan(resource_request(geometry, limits)); - SCOPED_TRACE(physical_pool_bytes[index]); +TEST(LlmMetalBackendTest, PagedTableAndPoolsUseIndependentMultiSegmentWholeElementPlans) { + struct Case { + LlmGeometry geometry; + size_t capacity; + size_t blocks_per_segment; + std::vector pool_lengths; + std::vector table_lengths; + }; + std::vector many_pool_lengths(12, 20); + many_pool_lengths.push_back(16); + const std::array cases = {{ + {paged_geometry(101, 49, 16, 2, 2, 3), 100, 2, std::vector(8, 96), {32}}, + {paged_geometry(1, 99, 1), 100, 100, {99}, {100, 100, 100, 96}}, + {paged_geometry(1, 100, 1), 100, 100, {100}, {100, 100, 100, 100}}, + {paged_geometry(1, 101, 1), 100, 100, {100, 1}, {100, 100, 100, 100, 4}}, + {paged_geometry(41, 29, 4, 2, 4, 1), 20, 5, many_pool_lengths, {20, 20, 20, 20, 20, 20, 8}}, + }}; + for (const auto& row : cases) { + SCOPED_TRACE(::testing::Message() << row.capacity << ":" << row.geometry.k_mapping_bytes); + ASSERT_TRUE(row.geometry.valid) << row.geometry.reason_code; + LlmMetalPlanningLimits limits; + limits.segment_capacity_bytes = row.capacity; + limits.segment_slots_per_pool = 32; + const auto plan = build_llm_metal_execution_plan(resource_request(row.geometry, limits)); ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.resources.k_segments.segment_lengths, expected[index]); - EXPECT_EQ(plan.resources.v_segments.segment_lengths, expected[index]); ASSERT_TRUE(plan.resources.table_segments.has_value()); - EXPECT_TRUE(std::all_of(plan.resources.table_segments->segment_lengths.begin(), - plan.resources.table_segments->segment_lengths.end(), - [](size_t length) { return length % sizeof(uint32_t) == 0 && length <= 100; })); + EXPECT_EQ(plan.resources.k_segments.elements_per_segment, row.blocks_per_segment); + for (const auto* pool : {&plan.resources.k_segments, &plan.resources.v_segments}) { + EXPECT_EQ(pool->segment_lengths, row.pool_lengths); + EXPECT_EQ(pool->segment_count, row.pool_lengths.size()); + EXPECT_EQ(pool->total_length_bytes, row.geometry.k_mapping_bytes); + for (size_t length : pool->segment_lengths) { + EXPECT_EQ(length % row.geometry.kv_block_bytes, 0U); + EXPECT_LE(length, row.capacity); + } + } + const auto& table = *plan.resources.table_segments; + EXPECT_EQ(table.element_bytes, sizeof(uint32_t)); + EXPECT_EQ(table.segment_lengths, row.table_lengths); + EXPECT_EQ(table.segment_count, row.table_lengths.size()); + EXPECT_EQ(table.total_length_bytes, row.geometry.block_table_bytes); + for (size_t length : table.segment_lengths) { + EXPECT_EQ(length % sizeof(uint32_t), 0U); + EXPECT_LE(length, row.capacity); + } } } -TEST(LlmMetalBackendTest, PagedTableAndPoolsUseIndependentMultiSegmentWholeElementPlans) { - LlmMetalPlanningLimits limits; - limits.segment_capacity_bytes = 20; - limits.segment_slots_per_pool = 32; - const LlmGeometry geometry = paged_geometry(41, 29, 4, 2, 4, 1); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_EQ(geometry.kv_block_bytes, 4U); - ASSERT_EQ(geometry.total_physical_blocks, 64U); - ASSERT_EQ(geometry.block_table_entries, 32U); - - const LlmMetalExecutionPlan plan = build_llm_metal_execution_plan(resource_request(geometry, limits)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_TRUE(plan.resources.table_segments.has_value()); - EXPECT_EQ(plan.resources.k_segments.segment_count, 13U); - EXPECT_EQ(plan.resources.k_segments.segment_lengths.back(), 16U); - EXPECT_TRUE(std::all_of(plan.resources.k_segments.segment_lengths.begin(), - plan.resources.k_segments.segment_lengths.end(), - [](size_t length) { return length % 4 == 0 && length <= 20; })); - EXPECT_EQ(plan.resources.table_segments->segment_count, 7U); - EXPECT_EQ(plan.resources.table_segments->segment_lengths, (std::vector{20, 20, 20, 20, 20, 20, 8})); - EXPECT_TRUE(std::all_of(plan.resources.table_segments->segment_lengths.begin(), - plan.resources.table_segments->segment_lengths.end(), - [](size_t length) { return length % sizeof(uint32_t) == 0 && length <= 20; })); -} - TEST(LlmMetalBackendTest, PagedIdentityBindsTailGeometryWhenPhysicalResourcesMatch) { const LlmGeometry five_token_geometry = paged_geometry(4097, 5, 4); const LlmGeometry seven_token_geometry = paged_geometry(4097, 7, 4); @@ -1444,46 +1405,35 @@ TEST(LlmMetalBackendTest, PagedIdentityBindsTailGeometryWhenPhysicalResourcesMat EXPECT_NE(five_token_plan.identity, seven_token_plan.identity); } -TEST(LlmMetalBackendTest, PagedPlannerRejectsOversizedBlocksAndMissingOrMismatchedLayouts) { +TEST(LlmMetalBackendTest, PlannerRejectsInvalidLayoutsAndResourceCaps) { LlmMetalPlanningLimits limits; limits.segment_capacity_bytes = 63; limits.segment_slots_per_pool = 8; const LlmGeometry geometry = paged_geometry(64, 17, 16, 1, 1, 4); ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_EQ(geometry.kv_block_bytes, 64U); - + struct Case { + LlmMetalResourcePlanRequest request; + const char* reason; + }; + std::vector cases; LlmMetalResourcePlanRequest request = resource_request(geometry, limits); - const LlmMetalExecutionPlan oversized = build_llm_metal_execution_plan(request); - EXPECT_FALSE(oversized.valid); - EXPECT_EQ(oversized.reason_code, LlmBackendReason::PAGED_BLOCK_EXCEEDS_SEGMENT_CAPACITY); - + cases.push_back({request, LlmBackendReason::PAGED_BLOCK_EXCEEDS_SEGMENT_CAPACITY}); request.limits.segment_capacity_bytes = 64; request.paged_layout.reset(); - const LlmMetalExecutionPlan missing = build_llm_metal_execution_plan(request); - EXPECT_FALSE(missing.valid); - EXPECT_EQ(missing.reason_code, LlmMetalPlanReason::PAGED_LAYOUT_REQUIRED); - + cases.push_back({request, LlmMetalPlanReason::PAGED_LAYOUT_REQUIRED}); request.paged_layout = paged_layout_for(paged_geometry(64, 18, 16, 1, 1, 4)); - const LlmMetalExecutionPlan mismatched = build_llm_metal_execution_plan(request); - EXPECT_FALSE(mismatched.valid); - EXPECT_EQ(mismatched.reason_code, LlmMetalPlanReason::PAGED_LAYOUT_MISMATCH); - - LlmGeometry malformed = geometry; - malformed.decode.reset(); - request.geometry = malformed; - const LlmMetalExecutionPlan invalid_geometry = build_llm_metal_execution_plan(request); - EXPECT_FALSE(invalid_geometry.valid); - EXPECT_EQ(invalid_geometry.reason_code, LlmMetalPlanReason::INVALID_GEOMETRY); -} - -TEST(LlmMetalBackendTest, ContiguousSegmentCapFailureUsesCanonicalMetalReason) { - LlmMetalPlanningLimits limits; + cases.push_back({request, LlmMetalPlanReason::PAGED_LAYOUT_MISMATCH}); + request.geometry.decode.reset(); + cases.push_back({request, LlmMetalPlanReason::INVALID_GEOMETRY}); limits.segment_capacity_bytes = 100; limits.segment_slots_per_pool = 2; - const LlmGeometry geometry = contiguous_geometry(201); - const LlmMetalExecutionPlan plan = build_llm_metal_execution_plan(resource_request(geometry, limits)); - EXPECT_FALSE(plan.valid); - EXPECT_EQ(plan.reason_code, LlmBackendReason::SEGMENT_COUNT_CAP_EXCEEDED); + cases.push_back({resource_request(contiguous_geometry(201), limits), LlmBackendReason::SEGMENT_COUNT_CAP_EXCEEDED}); + for (const auto& row : cases) { + SCOPED_TRACE(row.reason); + const auto plan = build_llm_metal_execution_plan(row.request); + EXPECT_FALSE(plan.valid); + EXPECT_EQ(plan.reason_code, row.reason); + } } TEST(LlmMetalBackendTest, RuntimeEncoderLengthAlignmentAndFirstAdmissionAreExact) { @@ -1572,27 +1522,6 @@ TEST(LlmMetalBackendTest, FirstAdmissionAppliesTheHardBudgetAtTheExactKnownOwned EXPECT_GE(admitted.resources.admitted_budget_bytes, required); } -TEST(LlmMetalBackendTest, ResourceIdentityExcludesVolatileAdmissionSample) { - const LlmGeometry geometry = contiguous_geometry(101); - LlmMetalResourcePlanRequest request = resource_request(geometry); - request.available_memory_bytes = 2 * kGiB; - const LlmMetalExecutionPlan first = build_llm_metal_execution_plan(request); - ASSERT_TRUE(first.valid) << first.reason_code; - - request.available_memory_bytes = 3 * kGiB; - const LlmMetalExecutionPlan second = build_llm_metal_execution_plan(request); - ASSERT_TRUE(second.valid) << second.reason_code; - - EXPECT_NE(first.resources.available_memory_bytes, second.resources.available_memory_bytes); - EXPECT_NE(first.resources.admitted_budget_bytes, second.resources.admitted_budget_bytes); - EXPECT_EQ(first.resources.known_owned_peak_bytes, second.resources.known_owned_peak_bytes); - EXPECT_EQ(first.resources.identity, second.resources.identity); - EXPECT_EQ(first.identity, second.identity); - EXPECT_EQ(first.resources.identity.rfind("llm-metal-resource-foundation-v1", 0), 0U); - EXPECT_EQ(first.resources.identity.find("available_memory_bytes"), std::string::npos); - EXPECT_EQ(first.resources.identity.find("admitted_budget_bytes"), std::string::npos); -} - TEST(LlmMetalBackendTest, CommittedAdmissionUsesNullableAllocatedSizeExactlyOncePerResource) { LlmMetalPlanningLimits limits; limits.segment_capacity_bytes = 100; @@ -1723,121 +1652,6 @@ TEST(LlmMetalBackendTest, GridPlanUsesInjectedWidthAndAllBoundaryCaps) { LlmMetalPlanReason::SERIAL_RANGE_VISIT_CAP_EXCEEDED); } -TEST(LlmMetalBackendTest, GridPlanReportsExactCyclicThreadgroupOwnerCostsPastGridCap) { - LlmMetalGridRequest request; - request.owner_count = 7; - request.visit_bytes = 1; - request.work_units = 1; - request.paged_semantic_lookups = 7; - request.owner_accounted_bytes = {1, 10, 100, 1000, 10000, 100000, 1000000}; - request.pipeline = {16, 64}; - request.limits.threads_per_threadgroup_cap = 64; - request.limits.maximum_threadgroups_per_grid = 3; - request.limits.maximum_owner_ordinals_per_threadgroup = 3; - request.limits.maximum_vector_iterations_per_lane_per_visit = 1; - request.limits.maximum_paged_semantic_lookups_per_task = 7; - request.limits.maximum_work_units_per_dispatch = 1; - - const LlmMetalGridPlan plan = build_llm_metal_grid_plan(request); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.actual_threadgroups, 3U); - EXPECT_EQ(plan.owner_ordinals_per_threadgroup, 3U); - EXPECT_EQ(plan.threadgroup_accounted_bytes, (std::vector{1001001, 10010, 100100})); - EXPECT_EQ(plan.minimum_threadgroup_accounted_bytes, 10010U); - EXPECT_EQ(plan.maximum_threadgroup_accounted_bytes, 1001001U); - EXPECT_EQ(plan.threadgroup_accounted_imbalance_bytes, 990991U); - EXPECT_EQ( - std::accumulate(plan.threadgroup_accounted_bytes.begin(), plan.threadgroup_accounted_bytes.end(), size_t{0}), - std::accumulate(request.owner_accounted_bytes.begin(), request.owner_accounted_bytes.end(), size_t{0})); - EXPECT_FALSE(plan.identity.empty()); - - LlmMetalGridRequest collision_left = request; - collision_left.owner_count = 3; - collision_left.paged_semantic_lookups = 3; - collision_left.owner_accounted_bytes = {1, 2, 3}; - const LlmMetalGridPlan left = build_llm_metal_grid_plan(collision_left); - ASSERT_TRUE(left.valid) << left.reason_code; - LlmMetalGridRequest collision_right = collision_left; - collision_right.owner_accounted_bytes = {2, 1, 3}; - const LlmMetalGridPlan right = build_llm_metal_grid_plan(collision_right); - ASSERT_TRUE(right.valid) << right.reason_code; - EXPECT_NE(left.threadgroup_accounted_bytes, right.threadgroup_accounted_bytes); - EXPECT_EQ(left.minimum_threadgroup_accounted_bytes, right.minimum_threadgroup_accounted_bytes); - EXPECT_EQ(left.maximum_threadgroup_accounted_bytes, right.maximum_threadgroup_accounted_bytes); - EXPECT_EQ(left.threadgroup_accounted_imbalance_bytes, right.threadgroup_accounted_imbalance_bytes); - EXPECT_NE(left.identity, right.identity); - - request.owner_accounted_bytes.pop_back(); - const LlmMetalGridPlan mismatch = build_llm_metal_grid_plan(request); - EXPECT_FALSE(mismatch.valid); - EXPECT_EQ(mismatch.reason_code, LlmMetalPlanReason::OWNER_COST_COUNT_MISMATCH); -} - -TEST(LlmMetalBackendTest, - CyclicGridPastCanonical8192CapUsesExactNonuniformPrefillBlockCosts) { - const LlmPrefillPlan prefill = - resolve_llm_prefill_plan({65, 7, 3, 2, 2, 1, 31, 31, 2}); - ASSERT_TRUE(prefill.valid) << prefill.reason_code; - ASSERT_TRUE(prefill.paged); - ASSERT_EQ(prefill.blocks_per_sequence, 4U); - - std::array block_costs{}; - for (size_t block = 0; block < block_costs.size(); ++block) { - const LlmPrefillUnitRangeCost cost = - calculate_llm_prefill_paged_block_cost(prefill, block); - ASSERT_TRUE(cost.valid) << cost.reason_code; - block_costs[block] = cost.accounted_bytes; - } - EXPECT_EQ(block_costs, - (std::array{524, 462, 392, 136})); - - constexpr size_t kOwnerCount = - Constants::LLM_METAL_MAX_THREADGROUPS_PER_GRID + 1; - LlmMetalGridRequest request; - request.owner_count = kOwnerCount; - request.visit_bytes = prefill.kv_block_tokens * - prefill.k_or_v_record_bytes_per_layer; - request.work_units = 1; - request.paged_semantic_lookups = kOwnerCount; - request.pipeline = {32, 128}; - request.owner_accounted_bytes.reserve(kOwnerCount); - for (size_t owner = 0; owner < kOwnerCount; ++owner) { - request.owner_accounted_bytes.push_back( - block_costs[owner % block_costs.size()]); - } - - const LlmMetalGridPlan plan = build_llm_metal_grid_plan(request); - ASSERT_TRUE(plan.valid) << plan.reason_code; - EXPECT_EQ(plan.owner_count, kOwnerCount); - EXPECT_EQ(plan.actual_threadgroups, - Constants::LLM_METAL_MAX_THREADGROUPS_PER_GRID); - EXPECT_EQ(plan.owner_ordinals_per_threadgroup, 2U); - ASSERT_EQ(plan.threadgroup_accounted_bytes.size(), - Constants::LLM_METAL_MAX_THREADGROUPS_PER_GRID); - - std::vector independent_costs(plan.actual_threadgroups, 0); - for (size_t owner = 0; owner < kOwnerCount; ++owner) { - independent_costs[owner % plan.actual_threadgroups] += - block_costs[owner % block_costs.size()]; - } - EXPECT_EQ(plan.threadgroup_accounted_bytes, independent_costs); - const auto [minimum, maximum] = std::minmax_element( - independent_costs.begin(), independent_costs.end()); - EXPECT_EQ(plan.minimum_threadgroup_accounted_bytes, *minimum); - EXPECT_EQ(plan.maximum_threadgroup_accounted_bytes, *maximum); - EXPECT_EQ(plan.threadgroup_accounted_imbalance_bytes, - *maximum - *minimum); - EXPECT_EQ(*minimum, 136U); - EXPECT_EQ(*maximum, 1048U); - EXPECT_EQ(std::accumulate(independent_costs.begin(), - independent_costs.end(), size_t{0}), - std::accumulate(request.owner_accounted_bytes.begin(), - request.owner_accounted_bytes.end(), size_t{0})); - EXPECT_NE(plan.identity.find( - "threadgroup_accounted_bytes_count=4:8192;"), - std::string::npos); -} - TEST(LlmMetalBackendTest, WeightGridReportsExactVectorModuloThreadgroupCostsAndTail) { constexpr size_t kWeightBytes = 181; @@ -1887,72 +1701,7 @@ TEST(LlmMetalBackendTest, std::string::npos); } -TEST(LlmMetalBackendTest, - PrefillPagedGridReportsExactCyclicNonuniformOwnerCosts) { - const LlmPrefillPlan prefill = - resolve_llm_prefill_plan({65, 7, 3, 2, 2, 1, 31, 31, 2}); - ASSERT_TRUE(prefill.valid) << prefill.reason_code; - ASSERT_TRUE(prefill.paged); - ASSERT_EQ(prefill.blocks_per_sequence, 4U); - - std::vector block_costs; - for (size_t block = 0; block < prefill.blocks_per_sequence; ++block) { - const LlmPrefillUnitRangeCost cost = - calculate_llm_prefill_paged_block_cost(prefill, block); - ASSERT_TRUE(cost.valid) << cost.reason_code; - block_costs.push_back(cost.accounted_bytes); - } - EXPECT_EQ(block_costs, (std::vector{524, 462, 392, 136})); - - constexpr size_t kWorkUnits = 2; - constexpr size_t kLayerCount = 2; - constexpr size_t kBatchSize = 2; - const std::array weight_layer_bytes = {33, 32}; - LlmMetalGridRequest request; - request.owner_count = - kLayerCount * kBatchSize * prefill.blocks_per_sequence; - request.visit_bytes = prefill.kv_block_tokens * - prefill.k_or_v_record_bytes_per_layer; - request.work_units = kWorkUnits; - request.paged_semantic_lookups = - kWorkUnits * prefill.layout_metadata_lookups_per_work_unit; - request.pipeline = {16, 64}; - request.limits.maximum_threadgroups_per_grid = 3; - request.limits.maximum_owner_ordinals_per_threadgroup = 6; - request.limits.maximum_vector_iterations_per_lane_per_visit = 1; - request.limits.maximum_work_units_per_dispatch = kWorkUnits; - request.limits.maximum_paged_semantic_lookups_per_task = - request.paged_semantic_lookups; - for (size_t owner = 0; owner < request.owner_count; ++owner) { - const size_t logical_block = owner % prefill.blocks_per_sequence; - const size_t row = owner / prefill.blocks_per_sequence; - const size_t batch = row % kBatchSize; - const size_t layer = row / kBatchSize; - size_t cost = block_costs[logical_block]; - if (batch == 0 && logical_block == 0) { - cost += weight_layer_bytes[layer]; - } - request.owner_accounted_bytes.push_back(cost * kWorkUnits); - } - - const LlmMetalGridPlan grid = build_llm_metal_grid_plan(request); - ASSERT_TRUE(grid.valid) << grid.reason_code; - EXPECT_EQ(grid.owner_count, 16U); - EXPECT_EQ(grid.actual_threadgroups, 3U); - EXPECT_EQ(grid.owner_ordinals_per_threadgroup, 6U); - EXPECT_EQ(grid.threadgroup_accounted_bytes, - (std::vector{4414, 3952, 3876})); - EXPECT_EQ(grid.minimum_threadgroup_accounted_bytes, 3876U); - EXPECT_EQ(grid.maximum_threadgroup_accounted_bytes, 4414U); - EXPECT_EQ(grid.threadgroup_accounted_imbalance_bytes, 538U); - EXPECT_EQ(std::accumulate(grid.threadgroup_accounted_bytes.begin(), - grid.threadgroup_accounted_bytes.end(), size_t{0}), - 12242U); - EXPECT_FALSE(grid.identity.empty()); -} - -TEST(LlmMetalBackendTest, - DecodePatternAndAppendHelpersHaveFrozenIndependentGoldens) { +TEST(LlmMetalBackendTest, InitializationPatternHelpersHaveFrozenIndependentGoldens) { constexpr uint64_t kSeed = UINT64_C(0x0123456789abcdef); EXPECT_EQ(llm_metal_contiguous_pattern_word(kSeed, 0), UINT32_C(0x27e347a8)); @@ -1967,24 +1716,6 @@ TEST(LlmMetalBackendTest, EXPECT_EQ(llm_metal_paged_pattern_word( kSeed, UINT32_MAX, UINT32_MAX, UINT64_MAX), UINT32_C(0x89abcdef)); - EXPECT_EQ(llm_metal_decode_append_word( - kSeed, 2, 3, 4, 5, LlmMetalResourcePool::K), - UINT32_C(0xbeae4546)); - EXPECT_EQ(llm_metal_decode_append_word( - kSeed, 2, 3, 4, 5, LlmMetalResourcePool::V), - UINT32_C(0xc9aa3846)); - EXPECT_EQ(llm_metal_prefill_write_word( - kSeed, 2, 3, 4, 5, LlmMetalResourcePool::K), - independent_prefill_write_word( - kSeed, 2, 3, 4, 5, LlmMetalResourcePool::K)); - EXPECT_EQ(llm_metal_prefill_write_word( - kSeed, 2, 3, 4, 5, LlmMetalResourcePool::V), - independent_prefill_write_word( - kSeed, 2, 3, 4, 5, LlmMetalResourcePool::V)); - EXPECT_NE(llm_metal_prefill_write_word( - kSeed, 1, 3, 4, 5, LlmMetalResourcePool::K), - llm_metal_prefill_write_word( - kSeed, 2, 3, 4, 5, LlmMetalResourcePool::K)); } TEST(LlmMetalBackendTest, @@ -2039,46 +1770,39 @@ TEST(LlmMetalBackendTest, TEST(LlmMetalBackendTest, PrefillChecksumMatchesIndependentByteOracleAcrossTilesTailsAndScenarios) { - for (size_t tail_bytes : {size_t{31}, size_t{32}, size_t{33}}) { - for (const auto [prompt_tokens, query_tile_tokens] : - {std::pair{5, 1}, {5, 5}, {5, 2}}) { - const LlmGeometry geometry = prefill_contiguous_geometry( - tail_bytes, prompt_tokens, query_tile_tokens, 2, 2, tail_bytes); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - LlmMemoryWorkPlan model = make_metal_model_plan( - geometry, - build_llm_metal_execution_plan(resource_request(geometry)), - "llm-metal-prefill-independent-byte-oracle-" + - std::to_string(tail_bytes) + "-" + - std::to_string(query_tile_tokens)); - ASSERT_TRUE(model.valid) << model.reason_code; - ASSERT_TRUE(model.prefill_plan.has_value()); - - for (LlmScenario scenario : {LlmScenario::WeightsOnly, - LlmScenario::KvOnly, - LlmScenario::Mixed}) { - const LlmScenarioWorkPlan scenario_plan = - build_llm_scenario_work_plan(model, scenario, 2, true); - ASSERT_TRUE(scenario_plan.valid) << scenario_plan.reason_code; - const LlmMetalChecksumOracle oracle = - calculate_llm_metal_prefill_contiguous_checksum(model, - scenario_plan); - ASSERT_TRUE(oracle.valid) << oracle.reason_code; - const LlmMetalDualMod32Checksum independent = - independent_prefill_checksum_byte_by_byte(model, scenario_plan); - SCOPED_TRACE(tail_bytes); - SCOPED_TRACE(query_tile_tokens); - SCOPED_TRACE(static_cast(scenario)); - EXPECT_TRUE(equal_llm_metal_checksum(oracle.checksum, independent)); - if (scenario == LlmScenario::WeightsOnly) { - EXPECT_EQ(oracle.checksum.k.a, 0U); - EXPECT_EQ(oracle.checksum.k.b, 0U); - EXPECT_EQ(oracle.checksum.v.a, 0U); - EXPECT_EQ(oracle.checksum.v.b, 0U); - } else if (scenario == LlmScenario::KvOnly) { - EXPECT_EQ(oracle.checksum.weight.a, 0U); - EXPECT_EQ(oracle.checksum.weight.b, 0U); - } + std::vector geometries; + for (size_t tail_bytes : {31U, 32U, 33U}) { + for (const auto [prompt, tile] : {std::pair{5, 1}, {5, 5}, {5, 2}}) { + geometries.push_back(prefill_contiguous_geometry(tail_bytes, prompt, tile, 2, 2, tail_bytes)); + } + } + geometries.push_back(prefill_contiguous_geometry(1, 3, 2, 2, 1, 3)); + for (const auto& geometry : geometries) { + ASSERT_TRUE(geometry.valid) << geometry.reason_code; + LlmMemoryWorkPlan model = + make_metal_model_plan(geometry, build_llm_metal_execution_plan(resource_request(geometry)), + "llm-metal-prefill-independent-byte-oracle"); + ASSERT_TRUE(model.valid) << model.reason_code; + ASSERT_TRUE(model.prefill_plan.has_value()); + + for (LlmScenario scenario : {LlmScenario::WeightsOnly, LlmScenario::KvOnly, LlmScenario::Mixed}) { + const LlmScenarioWorkPlan scenario_plan = build_llm_scenario_work_plan(model, scenario, 2, true); + ASSERT_TRUE(scenario_plan.valid) << scenario_plan.reason_code; + const LlmMetalChecksumOracle oracle = calculate_llm_metal_prefill_contiguous_checksum(model, scenario_plan); + ASSERT_TRUE(oracle.valid) << oracle.reason_code; + const LlmMetalDualMod32Checksum independent = independent_prefill_checksum_byte_by_byte(model, scenario_plan); + SCOPED_TRACE(geometry.active_weight_bytes_per_work_unit); + SCOPED_TRACE(geometry.prefill->attention_query_tile_tokens); + SCOPED_TRACE(static_cast(scenario)); + EXPECT_TRUE(equal_llm_metal_checksum(oracle.checksum, independent)); + if (scenario == LlmScenario::WeightsOnly) { + EXPECT_EQ(oracle.checksum.k.a, 0U); + EXPECT_EQ(oracle.checksum.k.b, 0U); + EXPECT_EQ(oracle.checksum.v.a, 0U); + EXPECT_EQ(oracle.checksum.v.b, 0U); + } else if (scenario == LlmScenario::KvOnly) { + EXPECT_EQ(oracle.checksum.weight.a, 0U); + EXPECT_EQ(oracle.checksum.weight.b, 0U); } } } @@ -2234,26 +1958,6 @@ TEST(LlmMetalBackendTest, LlmMetalPlanReason::VECTOR_ITERATION_CAP_EXCEEDED); } -TEST(LlmMetalBackendTest, - PrefillChecksumTreatsZeroLengthWeightLayersAsNoOps) { - const LlmGeometry geometry = - prefill_contiguous_geometry(1, 3, 2, 2, 1, 3); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - LlmMemoryWorkPlan model = make_metal_model_plan( - geometry, build_llm_metal_execution_plan(resource_request(geometry)), - "llm-metal-prefill-zero-length-weight-layer"); - ASSERT_TRUE(model.valid) << model.reason_code; - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - model, LlmScenario::Mixed, 1, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - const LlmMetalChecksumOracle oracle = - calculate_llm_metal_prefill_contiguous_checksum(model, scenario); - ASSERT_TRUE(oracle.valid) << oracle.reason_code; - EXPECT_TRUE(equal_llm_metal_checksum( - oracle.checksum, - independent_prefill_checksum_byte_by_byte(model, scenario))); -} - TEST(LlmMetalBackendTest, DecodePagedChecksumMatchesIndependentByteOracleAndDetectsEqualMultiplicitySwap) { const LlmGeometry geometry = paged_geometry(33, 6, 4, 2, 2, 3); @@ -2431,36 +2135,6 @@ TEST(LlmMetalBackendTest, } } -TEST(LlmMetalBackendTest, - DecodeChecksumFoldsTheFullScenarioSeedIntoEveryLaneDomain) { - const LlmGeometry geometry = contiguous_geometry(33, 3, 2, 2, 31); - LlmMemoryWorkPlan model = make_metal_model_plan( - geometry, build_llm_metal_execution_plan(resource_request(geometry)), - "llm-metal-full-scenario-seed-domain"); - ASSERT_TRUE(model.valid) << model.reason_code; - LlmScenarioWorkPlan low = - build_llm_scenario_work_plan(model, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(low.valid) << low.reason_code; - low.scenario_seed = UINT64_C(0x00000000a5a5a5a5); - LlmScenarioWorkPlan high = low; - high.scenario_seed = UINT64_C(0x00000001a5a5a5a5); - - const LlmMetalChecksumOracle low_oracle = - calculate_llm_metal_decode_contiguous_checksum(model, low); - const LlmMetalChecksumOracle high_oracle = - calculate_llm_metal_decode_contiguous_checksum(model, high); - ASSERT_TRUE(low_oracle.valid) << low_oracle.reason_code; - ASSERT_TRUE(high_oracle.valid) << high_oracle.reason_code; - EXPECT_FALSE(equal_llm_metal_checksum(low_oracle.checksum, - high_oracle.checksum)); - EXPECT_TRUE(equal_llm_metal_checksum( - low_oracle.checksum, - independent_decode_checksum_byte_by_byte(model, low))); - EXPECT_TRUE(equal_llm_metal_checksum( - high_oracle.checksum, - independent_decode_checksum_byte_by_byte(model, high))); -} - TEST(LlmMetalBackendTest, DecodeChecksumCrossesTheCanonicalSegmentBoundaryWithoutAllocation) { const size_t capacity = Constants::LLM_METAL_SEGMENT_CAPACITY_BYTES; @@ -2492,20 +2166,6 @@ TEST(LlmMetalBackendTest, } TEST(LlmMetalBackendTest, FoundationParameterAbiAndLayoutProbeAreExact) { - EXPECT_EQ(alignof(LlmMetalFoundationParams), 8U); - EXPECT_EQ(sizeof(LlmMetalFoundationParams), 64U); - const std::array offsets = {offsetof(LlmMetalFoundationParams, byte_count), - offsetof(LlmMetalFoundationParams, source_offset_bytes), - offsetof(LlmMetalFoundationParams, destination_offset_bytes), - offsetof(LlmMetalFoundationParams, logical_base_bytes), - offsetof(LlmMetalFoundationParams, pattern_seed), - offsetof(LlmMetalFoundationParams, block_bytes), - offsetof(LlmMetalFoundationParams, physical_blocks_per_layer), - offsetof(LlmMetalFoundationParams, pattern_kind), - offsetof(LlmMetalFoundationParams, probe_resource_kind), - offsetof(LlmMetalFoundationParams, probe_resource_slot)}; - EXPECT_EQ(offsets, (std::array{0, 8, 16, 24, 32, 40, 48, 52, 56, 60})); - LlmMetalFoundationParams parameters; parameters.byte_count = UINT64_C(0x0102030405060708); parameters.source_offset_bytes = UINT64_C(0x1112131415161718); @@ -2558,430 +2218,109 @@ TEST(LlmMetalBackendTest, FoundationParameterAbiAndLayoutProbeAreExact) { } } -TEST(LlmMetalBackendTest, DecodeContiguousParameterCpuAbiIsExact) { - EXPECT_EQ(alignof(LlmMetalDecodeContiguousParams), 8U); - EXPECT_EQ(sizeof(LlmMetalDecodeContiguousParams), 120U); - const std::array offsets = { - offsetof(LlmMetalDecodeContiguousParams, weight_bytes), - offsetof(LlmMetalDecodeContiguousParams, k_bytes), - offsetof(LlmMetalDecodeContiguousParams, v_bytes), - offsetof(LlmMetalDecodeContiguousParams, segment_capacity_bytes), - offsetof(LlmMetalDecodeContiguousParams, context_tokens), - offsetof(LlmMetalDecodeContiguousParams, layer_count), - offsetof(LlmMetalDecodeContiguousParams, batch_size), - offsetof(LlmMetalDecodeContiguousParams, record_bytes), - offsetof(LlmMetalDecodeContiguousParams, work_units), - offsetof(LlmMetalDecodeContiguousParams, weight_seed), - offsetof(LlmMetalDecodeContiguousParams, k_seed), - offsetof(LlmMetalDecodeContiguousParams, v_seed), - offsetof(LlmMetalDecodeContiguousParams, scenario_seed), - offsetof(LlmMetalDecodeContiguousParams, weight_segment_count), - offsetof(LlmMetalDecodeContiguousParams, k_segment_count), - offsetof(LlmMetalDecodeContiguousParams, v_segment_count)}; - EXPECT_EQ(offsets, - (std::array{0, 8, 16, 24, 32, 40, 48, 56, 64, - 72, 80, 88, 96, 104, 108, 112})); - EXPECT_EQ(offsetof(LlmMetalDecodeContiguousParams, reserved_zero), 116U); -} - -TEST(LlmMetalBackendTest, - DecodeContiguousParameterLayoutProbeValidatesEveryWord) { - LlmMetalDecodeContiguousParams parameters; - parameters.weight_bytes = UINT64_C(0x0102030405060708); - parameters.k_bytes = UINT64_C(0x1112131415161718); - parameters.v_bytes = UINT64_C(0x2122232425262728); - parameters.segment_capacity_bytes = UINT64_C(0x3132333435363738); - parameters.context_tokens = UINT64_C(0x4142434445464748); - parameters.layer_count = UINT64_C(0x5152535455565758); - parameters.batch_size = UINT64_C(0x6162636465666768); - parameters.record_bytes = UINT64_C(0x7172737475767778); - parameters.work_units = UINT64_C(0x8182838485868788); - parameters.weight_seed = UINT64_C(0x9192939495969798); - parameters.k_seed = UINT64_C(0xa1a2a3a4a5a6a7a8); - parameters.v_seed = UINT64_C(0xb1b2b3b4b5b6b7b8); - parameters.scenario_seed = UINT64_C(0xc1c2c3c4c5c6c7c8); - parameters.weight_segment_count = UINT32_C(0xd1d2d3d4); - parameters.k_segment_count = UINT32_C(0xe1e2e3e4); - parameters.v_segment_count = UINT32_C(0xf1f2f3f4); - parameters.reserved_zero = UINT32_C(0x01020304); - LlmMetalDecodeLayoutProbeWords words = { - 1, - 120, - 8, - 17, - 0, - 8, - 16, - 24, - 32, - 40, - 48, - 56, - 64, - 72, - 80, - 88, - 96, - 104, - 108, - 112, - 116, - parameters.weight_bytes, - parameters.k_bytes, - parameters.v_bytes, - parameters.segment_capacity_bytes, - parameters.context_tokens, - parameters.layer_count, - parameters.batch_size, - parameters.record_bytes, - parameters.work_units, - parameters.weight_seed, - parameters.k_seed, - parameters.v_seed, - parameters.scenario_seed, - parameters.weight_segment_count, - parameters.k_segment_count, - parameters.v_segment_count, - parameters.reserved_zero, - }; - ASSERT_TRUE(validate_llm_metal_decode_layout_probe(parameters, words)); - for (size_t index = 0; index < words.size(); ++index) { - LlmMetalDecodeLayoutProbeWords corrupted = words; - ++corrupted[index]; - SCOPED_TRACE(index); - EXPECT_FALSE( - validate_llm_metal_decode_layout_probe(parameters, corrupted)); - } -} - -TEST(LlmMetalBackendTest, PrefillContiguousParameterCpuAbiIsExact) { - EXPECT_STREQ(LlmMetalKernelContract::kPrefillParameterAbiRevision, - "llm-metal-prefill-contiguous-parameters-v1"); - EXPECT_EQ(alignof(LlmMetalPrefillContiguousParams), 8U); - EXPECT_EQ(sizeof(LlmMetalPrefillContiguousParams), 136U); - const std::array offsets = { - offsetof(LlmMetalPrefillContiguousParams, weight_bytes), - offsetof(LlmMetalPrefillContiguousParams, k_bytes), - offsetof(LlmMetalPrefillContiguousParams, v_bytes), - offsetof(LlmMetalPrefillContiguousParams, segment_capacity_bytes), - offsetof(LlmMetalPrefillContiguousParams, prompt_tokens), - offsetof(LlmMetalPrefillContiguousParams, - attention_query_tile_tokens), - offsetof(LlmMetalPrefillContiguousParams, tile_count), - offsetof(LlmMetalPrefillContiguousParams, layer_count), - offsetof(LlmMetalPrefillContiguousParams, batch_size), - offsetof(LlmMetalPrefillContiguousParams, record_bytes), - offsetof(LlmMetalPrefillContiguousParams, work_units), - offsetof(LlmMetalPrefillContiguousParams, weight_seed), - offsetof(LlmMetalPrefillContiguousParams, k_seed), - offsetof(LlmMetalPrefillContiguousParams, v_seed), - offsetof(LlmMetalPrefillContiguousParams, scenario_seed), - offsetof(LlmMetalPrefillContiguousParams, weight_segment_count), - offsetof(LlmMetalPrefillContiguousParams, k_segment_count), - offsetof(LlmMetalPrefillContiguousParams, v_segment_count), - offsetof(LlmMetalPrefillContiguousParams, reserved_zero), - }; - EXPECT_EQ(offsets, - (std::array{0, 8, 16, 24, 32, 40, 48, - 56, 64, 72, 80, 88, 96, 104, - 112, 120, 124, 128, 132})); -} - -TEST(LlmMetalBackendTest, - PrefillContiguousParameterLayoutProbeValidatesEveryWord) { - LlmMetalPrefillContiguousParams parameters; - parameters.weight_bytes = UINT64_C(0x0102030405060708); - parameters.k_bytes = UINT64_C(0x1112131415161718); - parameters.v_bytes = UINT64_C(0x2122232425262728); - parameters.segment_capacity_bytes = UINT64_C(0x3132333435363738); - parameters.prompt_tokens = UINT64_C(0x4142434445464748); - parameters.attention_query_tile_tokens = UINT64_C(0x5152535455565758); - parameters.tile_count = UINT64_C(0x6162636465666768); - parameters.layer_count = UINT64_C(0x7172737475767778); - parameters.batch_size = UINT64_C(0x8182838485868788); - parameters.record_bytes = UINT64_C(0x9192939495969798); - parameters.work_units = UINT64_C(0xa1a2a3a4a5a6a7a8); - parameters.weight_seed = UINT64_C(0xb1b2b3b4b5b6b7b8); - parameters.k_seed = UINT64_C(0xc1c2c3c4c5c6c7c8); - parameters.v_seed = UINT64_C(0xd1d2d3d4d5d6d7d8); - parameters.scenario_seed = UINT64_C(0xe1e2e3e4e5e6e7e8); - parameters.weight_segment_count = UINT32_C(0x11121314); - parameters.k_segment_count = UINT32_C(0x21222324); - parameters.v_segment_count = UINT32_C(0x31323334); - parameters.reserved_zero = UINT32_C(0x41424344); - constexpr std::array kOffsets = { - 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, - 80, 88, 96, 104, 112, 120, 124, 128, 132, - }; - const std::array values = { - parameters.weight_bytes, - parameters.k_bytes, - parameters.v_bytes, - parameters.segment_capacity_bytes, - parameters.prompt_tokens, - parameters.attention_query_tile_tokens, - parameters.tile_count, - parameters.layer_count, - parameters.batch_size, - parameters.record_bytes, - parameters.work_units, - parameters.weight_seed, - parameters.k_seed, - parameters.v_seed, - parameters.scenario_seed, - parameters.weight_segment_count, - parameters.k_segment_count, - parameters.v_segment_count, - parameters.reserved_zero, - }; - LlmMetalPrefillLayoutProbeWords words{}; - words[0] = 1; - words[1] = 136; - words[2] = 8; - words[3] = 19; - std::copy(kOffsets.begin(), kOffsets.end(), words.begin() + 4); - std::copy(values.begin(), values.end(), words.begin() + 23); - ASSERT_TRUE(validate_llm_metal_prefill_layout_probe(parameters, words)); - for (size_t index = 0; index < words.size(); ++index) { - LlmMetalPrefillLayoutProbeWords corrupted = words; - ++corrupted[index]; - SCOPED_TRACE(index); - EXPECT_FALSE( - validate_llm_metal_prefill_layout_probe(parameters, corrupted)); - } -} - -TEST(LlmMetalBackendTest, PrefillPagedParameterCpuAbiIsExact) { - EXPECT_STREQ(LlmMetalKernelContract::kPrefillPagedParameterAbiRevision, - "llm-metal-prefill-paged-parameters-v1"); - EXPECT_EQ(alignof(LlmMetalPrefillPagedParams), 8U); - EXPECT_EQ(sizeof(LlmMetalPrefillPagedParams), 184U); - const std::array offsets = { - offsetof(LlmMetalPrefillPagedParams, weight_bytes), - offsetof(LlmMetalPrefillPagedParams, prompt_tokens), - offsetof(LlmMetalPrefillPagedParams, attention_query_tile_tokens), - offsetof(LlmMetalPrefillPagedParams, tile_count), - offsetof(LlmMetalPrefillPagedParams, layer_count), - offsetof(LlmMetalPrefillPagedParams, batch_size), - offsetof(LlmMetalPrefillPagedParams, record_bytes), - offsetof(LlmMetalPrefillPagedParams, work_units), - offsetof(LlmMetalPrefillPagedParams, block_tokens), - offsetof(LlmMetalPrefillPagedParams, block_bytes), - offsetof(LlmMetalPrefillPagedParams, last_block_valid_bytes), - offsetof(LlmMetalPrefillPagedParams, blocks_per_sequence), - offsetof(LlmMetalPrefillPagedParams, physical_blocks_per_layer), - offsetof(LlmMetalPrefillPagedParams, blocks_per_segment), - offsetof(LlmMetalPrefillPagedParams, table_entries_per_segment), - offsetof(LlmMetalPrefillPagedParams, segment_capacity_bytes), - offsetof(LlmMetalPrefillPagedParams, weight_seed), - offsetof(LlmMetalPrefillPagedParams, k_seed), - offsetof(LlmMetalPrefillPagedParams, v_seed), - offsetof(LlmMetalPrefillPagedParams, scenario_seed), - offsetof(LlmMetalPrefillPagedParams, weight_segment_count), - offsetof(LlmMetalPrefillPagedParams, k_segment_count), - offsetof(LlmMetalPrefillPagedParams, v_segment_count), - offsetof(LlmMetalPrefillPagedParams, table_segment_count), - offsetof(LlmMetalPrefillPagedParams, reserved_zero), - offsetof(LlmMetalPrefillPagedParams, padding_zero), - }; - EXPECT_EQ(offsets, - (std::array{ - 0, 8, 16, 24, 32, 40, 48, 56, 64, - 72, 80, 88, 96, 104, 112, 120, 128, 136, - 144, 152, 160, 164, 168, 172, 176, 180})); -} +namespace { -TEST(LlmMetalBackendTest, - PrefillPagedParameterLayoutProbeValidatesEveryWord) { - LlmMetalPrefillPagedParams parameters; - parameters.weight_bytes = UINT64_C(0x0102030405060708); - parameters.prompt_tokens = UINT64_C(0x1112131415161718); - parameters.attention_query_tile_tokens = UINT64_C(0x2122232425262728); - parameters.tile_count = UINT64_C(0x3132333435363738); - parameters.layer_count = UINT64_C(0x4142434445464748); - parameters.batch_size = UINT64_C(0x5152535455565758); - parameters.record_bytes = UINT64_C(0x6162636465666768); - parameters.work_units = UINT64_C(0x7172737475767778); - parameters.block_tokens = UINT64_C(0x8182838485868788); - parameters.block_bytes = UINT64_C(0x9192939495969798); - parameters.last_block_valid_bytes = UINT64_C(0xa1a2a3a4a5a6a7a8); - parameters.blocks_per_sequence = UINT64_C(0xb1b2b3b4b5b6b7b8); - parameters.physical_blocks_per_layer = UINT64_C(0xc1c2c3c4c5c6c7c8); - parameters.blocks_per_segment = UINT64_C(0xd1d2d3d4d5d6d7d8); - parameters.table_entries_per_segment = UINT64_C(0xe1e2e3e4e5e6e7e8); - parameters.segment_capacity_bytes = UINT64_C(0xf1f2f3f4f5f6f7f8); - parameters.weight_seed = UINT64_C(0x0101010102020202); - parameters.k_seed = UINT64_C(0x0303030304040404); - parameters.v_seed = UINT64_C(0x0505050506060606); - parameters.scenario_seed = UINT64_C(0x0707070708080808); - parameters.weight_segment_count = UINT32_C(0x11121314); - parameters.k_segment_count = UINT32_C(0x21222324); - parameters.v_segment_count = UINT32_C(0x31323334); - parameters.table_segment_count = UINT32_C(0x41424344); - parameters.reserved_zero = UINT32_C(0x51525354); - parameters.padding_zero = UINT32_C(0x61626364); - - constexpr std::array kOffsets = { - 0, 8, 16, 24, 32, 40, 48, 56, 64, - 72, 80, 88, 96, 104, 112, 120, 128, 136, - 144, 152, 160, 164, 168, 172, 176, 180, - }; - const std::array values = { - parameters.weight_bytes, - parameters.prompt_tokens, - parameters.attention_query_tile_tokens, - parameters.tile_count, - parameters.layer_count, - parameters.batch_size, - parameters.record_bytes, - parameters.work_units, - parameters.block_tokens, - parameters.block_bytes, - parameters.last_block_valid_bytes, - parameters.blocks_per_sequence, - parameters.physical_blocks_per_layer, - parameters.blocks_per_segment, - parameters.table_entries_per_segment, - parameters.segment_capacity_bytes, - parameters.weight_seed, - parameters.k_seed, - parameters.v_seed, - parameters.scenario_seed, - parameters.weight_segment_count, - parameters.k_segment_count, - parameters.v_segment_count, - parameters.table_segment_count, - parameters.reserved_zero, - parameters.padding_zero, - }; - static_assert(kLlmMetalPrefillPagedLayoutProbeWordCount == 56); - LlmMetalPrefillPagedLayoutProbeWords words{}; +template +void expect_parameter_probe_words(size_t parameter_bytes, const std::array& offsets, + const std::array& wide_members, + const std::array& narrow_members, + Validator validate) { + Params parameters; + constexpr size_t field_count = WideCount + NarrowCount; + std::array words{}; words[0] = 1; - words[1] = 184; + words[1] = parameter_bytes; words[2] = 8; - words[3] = 26; - std::copy(kOffsets.begin(), kOffsets.end(), words.begin() + 4); - std::copy(values.begin(), values.end(), words.begin() + 30); - ASSERT_TRUE( - validate_llm_metal_prefill_paged_layout_probe(parameters, words)); + words[3] = field_count; + std::copy(offsets.begin(), offsets.end(), words.begin() + 4); + for (size_t index = 0; index < WideCount; ++index) { + const uint64_t value = UINT64_C(0x8102030405060708) + index * UINT64_C(0x0101010101010101); + parameters.*wide_members[index] = value; + words[4 + field_count + index] = value; + } + for (size_t index = 0; index < NarrowCount; ++index) { + const uint32_t value = UINT32_C(0x81828384) + static_cast(index) * UINT32_C(0x01010101); + parameters.*narrow_members[index] = value; + words[4 + field_count + WideCount + index] = value; + } + ASSERT_TRUE(validate(parameters, words)); for (size_t index = 0; index < words.size(); ++index) { - LlmMetalPrefillPagedLayoutProbeWords corrupted = words; + auto corrupted = words; ++corrupted[index]; SCOPED_TRACE(index); - EXPECT_FALSE( - validate_llm_metal_prefill_paged_layout_probe(parameters, corrupted)); + EXPECT_FALSE(validate(parameters, corrupted)); } } -TEST(LlmMetalBackendTest, DecodePagedParameterCpuAbiIsExact) { - EXPECT_EQ(alignof(LlmMetalDecodePagedParams), 8U); - EXPECT_EQ(sizeof(LlmMetalDecodePagedParams), 168U); - const std::array offsets = { - offsetof(LlmMetalDecodePagedParams, weight_bytes), - offsetof(LlmMetalDecodePagedParams, context_tokens), - offsetof(LlmMetalDecodePagedParams, layer_count), - offsetof(LlmMetalDecodePagedParams, batch_size), - offsetof(LlmMetalDecodePagedParams, record_bytes), - offsetof(LlmMetalDecodePagedParams, work_units), - offsetof(LlmMetalDecodePagedParams, block_bytes), - offsetof(LlmMetalDecodePagedParams, last_block_valid_bytes), - offsetof(LlmMetalDecodePagedParams, append_offset_in_last_block), - offsetof(LlmMetalDecodePagedParams, blocks_per_sequence), - offsetof(LlmMetalDecodePagedParams, physical_blocks_per_layer), - offsetof(LlmMetalDecodePagedParams, blocks_per_segment), - offsetof(LlmMetalDecodePagedParams, table_entries_per_segment), - offsetof(LlmMetalDecodePagedParams, segment_capacity_bytes), - offsetof(LlmMetalDecodePagedParams, weight_seed), - offsetof(LlmMetalDecodePagedParams, k_seed), - offsetof(LlmMetalDecodePagedParams, v_seed), - offsetof(LlmMetalDecodePagedParams, scenario_seed), - offsetof(LlmMetalDecodePagedParams, weight_segment_count), - offsetof(LlmMetalDecodePagedParams, k_segment_count), - offsetof(LlmMetalDecodePagedParams, v_segment_count), - offsetof(LlmMetalDecodePagedParams, table_segment_count), - offsetof(LlmMetalDecodePagedParams, reserved_zero), - offsetof(LlmMetalDecodePagedParams, padding_zero), - }; - EXPECT_EQ(offsets, - (std::array{0, 8, 16, 24, 32, 40, - 48, 56, 64, 72, 80, 88, - 96, 104, 112, 120, 128, 136, - 144, 148, 152, 156, 160, 164})); -} +} // namespace -TEST(LlmMetalBackendTest, - DecodePagedParameterLayoutProbeValidatesEveryWord) { - LlmMetalDecodePagedParams parameters; - parameters.weight_bytes = UINT64_C(0x0102030405060708); - parameters.context_tokens = UINT64_C(0x1112131415161718); - parameters.layer_count = UINT64_C(0x2122232425262728); - parameters.batch_size = UINT64_C(0x3132333435363738); - parameters.record_bytes = UINT64_C(0x4142434445464748); - parameters.work_units = UINT64_C(0x5152535455565758); - parameters.block_bytes = UINT64_C(0x6162636465666768); - parameters.last_block_valid_bytes = UINT64_C(0x7172737475767778); - parameters.append_offset_in_last_block = UINT64_C(0x8182838485868788); - parameters.blocks_per_sequence = UINT64_C(0x9192939495969798); - parameters.physical_blocks_per_layer = UINT64_C(0xa1a2a3a4a5a6a7a8); - parameters.blocks_per_segment = UINT64_C(0xb1b2b3b4b5b6b7b8); - parameters.table_entries_per_segment = UINT64_C(0xc1c2c3c4c5c6c7c8); - parameters.segment_capacity_bytes = UINT64_C(0xd1d2d3d4d5d6d7d8); - parameters.weight_seed = UINT64_C(0xe1e2e3e4e5e6e7e8); - parameters.k_seed = UINT64_C(0xf1f2f3f4f5f6f7f8); - parameters.v_seed = UINT64_C(0x0101010102020202); - parameters.scenario_seed = UINT64_C(0x0303030304040404); - parameters.weight_segment_count = UINT32_C(0x11121314); - parameters.k_segment_count = UINT32_C(0x21222324); - parameters.v_segment_count = UINT32_C(0x31323334); - parameters.table_segment_count = UINT32_C(0x41424344); - parameters.reserved_zero = UINT32_C(0x51525354); - parameters.padding_zero = UINT32_C(0x61626364); - - const std::array values = { - parameters.weight_bytes, - parameters.context_tokens, - parameters.layer_count, - parameters.batch_size, - parameters.record_bytes, - parameters.work_units, - parameters.block_bytes, - parameters.last_block_valid_bytes, - parameters.append_offset_in_last_block, - parameters.blocks_per_sequence, - parameters.physical_blocks_per_layer, - parameters.blocks_per_segment, - parameters.table_entries_per_segment, - parameters.segment_capacity_bytes, - parameters.weight_seed, - parameters.k_seed, - parameters.v_seed, - parameters.scenario_seed, - parameters.weight_segment_count, - parameters.k_segment_count, - parameters.v_segment_count, - parameters.table_segment_count, - parameters.reserved_zero, - parameters.padding_zero, - }; - constexpr std::array kOffsets = { - 0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, - 96, 104, 112, 120, 128, 136, 144, 148, 152, 156, 160, 164, - }; - LlmMetalDecodePagedLayoutProbeWords words{}; - words[0] = 1; - words[1] = 168; - words[2] = 8; - words[3] = 24; - std::copy(kOffsets.begin(), kOffsets.end(), words.begin() + 4); - std::copy(values.begin(), values.end(), words.begin() + 28); - ASSERT_TRUE(validate_llm_metal_decode_paged_layout_probe(parameters, - words)); - for (size_t index = 0; index < words.size(); ++index) { - LlmMetalDecodePagedLayoutProbeWords corrupted = words; - ++corrupted[index]; - SCOPED_TRACE(index); - EXPECT_FALSE(validate_llm_metal_decode_paged_layout_probe(parameters, - corrupted)); +TEST(LlmMetalBackendTest, ParameterLayoutProbesValidateEveryWord) { + { + SCOPED_TRACE("LlmMetalDecodeContiguousParams"); + using P = LlmMetalDecodeContiguousParams; + expect_parameter_probe_words( + 120, std::array{0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 108, 112, 116}, + std::array{&P::weight_bytes, &P::k_bytes, &P::v_bytes, &P::segment_capacity_bytes, &P::context_tokens, + &P::layer_count, &P::batch_size, &P::record_bytes, &P::work_units, &P::weight_seed, &P::k_seed, + &P::v_seed, &P::scenario_seed}, + std::array{&P::weight_segment_count, &P::k_segment_count, &P::v_segment_count, &P::reserved_zero}, + validate_llm_metal_decode_layout_probe); + } + { + SCOPED_TRACE("LlmMetalPrefillContiguousParams"); + using P = LlmMetalPrefillContiguousParams; + expect_parameter_probe_words( + 136, std::array{0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 124, 128, 132}, + std::array{&P::weight_bytes, &P::k_bytes, &P::v_bytes, &P::segment_capacity_bytes, &P::prompt_tokens, + &P::attention_query_tile_tokens, &P::tile_count, &P::layer_count, &P::batch_size, &P::record_bytes, + &P::work_units, &P::weight_seed, &P::k_seed, &P::v_seed, &P::scenario_seed}, + std::array{&P::weight_segment_count, &P::k_segment_count, &P::v_segment_count, &P::reserved_zero}, + validate_llm_metal_prefill_layout_probe); + } + { + SCOPED_TRACE("LlmMetalPrefillPagedParams"); + using P = LlmMetalPrefillPagedParams; + expect_parameter_probe_words( + 184, std::array{0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, + 104, 112, 120, 128, 136, 144, 152, 160, 164, 168, 172, 176, 180}, + std::array{&P::weight_bytes, + &P::prompt_tokens, + &P::attention_query_tile_tokens, + &P::tile_count, + &P::layer_count, + &P::batch_size, + &P::record_bytes, + &P::work_units, + &P::block_tokens, + &P::block_bytes, + &P::last_block_valid_bytes, + &P::blocks_per_sequence, + &P::physical_blocks_per_layer, + &P::blocks_per_segment, + &P::table_entries_per_segment, + &P::segment_capacity_bytes, + &P::weight_seed, + &P::k_seed, + &P::v_seed, + &P::scenario_seed}, + std::array{&P::weight_segment_count, &P::k_segment_count, &P::v_segment_count, &P::table_segment_count, + &P::reserved_zero, &P::padding_zero}, + validate_llm_metal_prefill_paged_layout_probe); + } + { + SCOPED_TRACE("LlmMetalDecodePagedParams"); + using P = LlmMetalDecodePagedParams; + expect_parameter_probe_words( + 168, std::array{0, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, + 96, 104, 112, 120, 128, 136, 144, 148, 152, 156, 160, 164}, + std::array{&P::weight_bytes, &P::context_tokens, &P::layer_count, &P::batch_size, &P::record_bytes, + &P::work_units, &P::block_bytes, &P::last_block_valid_bytes, &P::append_offset_in_last_block, + &P::blocks_per_sequence, &P::physical_blocks_per_layer, &P::blocks_per_segment, + &P::table_entries_per_segment, &P::segment_capacity_bytes, &P::weight_seed, &P::k_seed, &P::v_seed, + &P::scenario_seed}, + std::array{&P::weight_segment_count, &P::k_segment_count, &P::v_segment_count, &P::table_segment_count, + &P::reserved_zero, &P::padding_zero}, + validate_llm_metal_decode_paged_layout_probe); } } @@ -2991,11 +2330,6 @@ TEST(LlmMetalBackendTest, const auto require_source = [&](std::string_view token) { EXPECT_NE(source.find(token), std::string_view::npos) << token; }; - require_source("LLM_METAL_DECODE_PAGED + \\"); - require_source( - "LLM_METAL_PREFILL_CONTIGUOUS + LLM_METAL_PREFILL_PAGED) != 1"); - require_source("#if LLM_METAL_DECODE_CONTIGUOUS"); - require_source("#if LLM_METAL_DECODE_PAGED"); require_source("device const volatile uint* named_lane_table"); require_source("threadgroup_barrier(mem_flags::mem_threadgroup);"); require_source("threadgroup uint* published_physical_id [[threadgroup(1)]]"); @@ -3020,45 +2354,6 @@ TEST(LlmMetalBackendTest, std::string_view::npos); } -TEST(LlmMetalBackendTest, - PrefillContiguousSemanticTraceWritesFullPromptBeforePerTileKThenV) { - // This freezes the bounded schedule oracle. Actual MSL pipeline/layout, - // Q=1/Q=P/tail execution, and corruption rejection have real-device tests; - // the separate canonical source hash retains provenance. A trace alone - // does not attest the GPU's temporal access order. - const LlmPrefillPlan plan = resolve_llm_prefill_plan( - {64, 2, 1, 1, 1, 1, 1, 4, 0}); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmPrefillSemanticTrace trace = build_llm_prefill_semantic_trace( - plan, {LlmPrefillPartitionUnitKind::ContiguousToken, 0, 2, 10}); - const std::array expected = {{ - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 2, 0, 1}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 2, 0, 1}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 2, 1, 1}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 2, 1, 1}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 0, 1, 0, 1}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 0, 1, 0, 1}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 2, 0, 1}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 2, 1, 1}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 2, 0, 1}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 2, 1, 1}, - }}; - ASSERT_TRUE(trace.valid) << trace.reason_code; - ASSERT_EQ(trace.events.size(), expected.size()); - for (size_t index = 0; index < expected.size(); ++index) { - SCOPED_TRACE(index); - EXPECT_EQ(trace.events[index].access, expected[index].access); - EXPECT_EQ(trace.events[index].domain, expected[index].domain); - EXPECT_EQ(trace.events[index].tile_index, expected[index].tile_index); - EXPECT_EQ(trace.events[index].tile_end_token, - expected[index].tile_end_token); - EXPECT_EQ(trace.events[index].logical_unit_index, - expected[index].logical_unit_index); - EXPECT_EQ(trace.events[index].visit_token_count, - expected[index].visit_token_count); - } -} - TEST(LlmMetalBackendTest, PrefillPagedMslSourceLocksRowWritesThenPerTileKThenVContract) { const std::string_view source = LlmMetalKernelContract::kSource; @@ -3209,63 +2504,6 @@ TEST(LlmMetalBackendTest, write_loop), std::string_view::npos); - const LlmPrefillPlan plan = - resolve_llm_prefill_plan({64, 4, 2, 1, 1, 1, 1, 4, 2}); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmPrefillSemanticTrace trace = build_llm_prefill_semantic_trace( - plan, {LlmPrefillPartitionUnitKind::PagedBlock, 0, 2, 10}); - const std::array expected = {{ - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 4, 0, 2}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 4, 0, 2}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::K, 0, 4, 1, 2}, - {LlmPrefillSemanticAccess::Write, LlmPrefillKvDomain::V, 0, 4, 1, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 0, 2, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 0, 2, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 4, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::K, 1, 4, 1, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 4, 0, 2}, - {LlmPrefillSemanticAccess::Read, LlmPrefillKvDomain::V, 1, 4, 1, 2}, - }}; - ASSERT_TRUE(trace.valid) << trace.reason_code; - ASSERT_EQ(trace.events.size(), expected.size()); - for (size_t index = 0; index < expected.size(); ++index) { - SCOPED_TRACE(index); - EXPECT_EQ(trace.events[index].access, expected[index].access); - EXPECT_EQ(trace.events[index].domain, expected[index].domain); - EXPECT_EQ(trace.events[index].tile_index, expected[index].tile_index); - EXPECT_EQ(trace.events[index].tile_end_token, - expected[index].tile_end_token); - EXPECT_EQ(trace.events[index].logical_unit_index, - expected[index].logical_unit_index); - EXPECT_EQ(trace.events[index].visit_token_count, - expected[index].visit_token_count); - } -} - -TEST(LlmMetalBackendTest, CanonicalEmbeddedMslSourceHashIsFrozenLowercaseSha256) { - const std::string digest = canonical_llm_metal_kernel_source_sha256(); - EXPECT_EQ(digest, kCanonicalKernelSourceSha256); - ASSERT_EQ(digest.size(), 64U); - EXPECT_TRUE(std::all_of(digest.begin(), digest.end(), [](unsigned char value) { - return std::isdigit(value) != 0 || (value >= 'a' && value <= 'f'); - })); -} - -TEST(LlmMetalBackendTest, CheckedMetalExecutionPlanAccessorRejectsBackendOrVariantMismatch) { - LlmMemoryWorkPlan plan; - EXPECT_EQ(get_llm_metal_execution_plan(plan), nullptr); - - plan.backend = LlmMemoryBackend::Metal; - EXPECT_EQ(get_llm_metal_execution_plan(plan), nullptr); - - plan.backend_execution_plan = LlmMetalExecutionPlan{}; - EXPECT_NE(get_llm_metal_execution_plan(plan), nullptr); - const LlmMemoryWorkPlan& const_plan = plan; - EXPECT_NE(get_llm_metal_execution_plan(const_plan), nullptr); - - plan.backend = LlmMemoryBackend::Cpu; - EXPECT_EQ(get_llm_metal_execution_plan(plan), nullptr); - EXPECT_EQ(get_llm_metal_execution_plan(const_plan), nullptr); } TEST(LlmMetalBackendTest, FactoryAndDirectConstructorExposeTheMetalBackend) { @@ -3772,92 +3010,34 @@ TEST_F(LlmMetalBackendIntegrationTest, } } -TEST_F(LlmMetalBackendIntegrationTest, - DecodeContiguousExactTail31Integration) { - expect_exact_tail_all_scenarios(31); -} - -TEST_F(LlmMetalBackendIntegrationTest, - DecodeContiguousExactTail32Integration) { - expect_exact_tail_all_scenarios(32); -} - -TEST_F(LlmMetalBackendIntegrationTest, - DecodeContiguousExactTail33Integration) { - expect_exact_tail_all_scenarios(33); -} - -TEST_F(LlmMetalBackendIntegrationTest, - DecodeMixedUsesMultipleThreadgroupsWorkUnitsLayersAndBatchesIntegration) { - const size_t weight_bytes = - 2 * Constants::BYTES_PER_MB + 33; - LlmMemoryWorkPlan plan = build_device_plan( - contiguous_geometry(weight_bytes, 17, 2, 2, 8), - "llm-metal-phase9-mixed-multigroup-plan"); - resolve_and_prepare(plan); - expect_complete_scenario_task(plan, LlmScenario::Mixed, 3, 2); -} - -TEST_F(LlmMetalBackendIntegrationTest, - DecodeKvUsesMultipleThreadgroupsWorkUnitsLayersAndBatchesIntegration) { - LlmMemoryWorkPlan plan = build_device_plan( - contiguous_geometry(33, 32769, 2, 2, 33), - "llm-metal-phase9-kv-multigroup-plan"); - resolve_and_prepare(plan); - expect_complete_scenario_task(plan, LlmScenario::KvOnly, 2, 2); +TEST_F(LlmMetalBackendIntegrationTest, DecodeContiguousExactTailsIntegration) { + for (size_t tail : {31U, 32U, 33U}) { + SCOPED_TRACE(tail); + backend_ = create_llm_metal_backend(); + ASSERT_EQ(backend_->initialize(metal_config()).status, LlmBackendStatus::Ready); + expect_exact_tail_all_scenarios(tail); + } } -TEST_F(LlmMetalBackendIntegrationTest, - PrefillContiguousQOneQPromptAndRemainderTilesIntegration) { +TEST_F(LlmMetalBackendIntegrationTest, DecodeUsesMultipleThreadgroupsWorkUnitsLayersAndBatchesIntegration) { struct Case { - size_t query_tile_tokens; + size_t weight_bytes; + size_t context_tokens; size_t record_bytes; - size_t expected_prefix_visits; + LlmScenario scenario; + size_t work_units; }; - constexpr std::array kCases = {{{1, 31, 15}, - {5, 32, 5}, - {2, 33, 11}}}; - for (const Case& test_case : kCases) { + const std::array cases = { + {{2 * Constants::BYTES_PER_MB + 33, 17, 8, LlmScenario::Mixed, 3}, {33, 32769, 33, LlmScenario::KvOnly, 2}}}; + for (const Case& test_case : cases) { + SCOPED_TRACE(static_cast(test_case.scenario)); backend_ = create_llm_metal_backend(); - ASSERT_NE(backend_, nullptr); - LlmMemoryConfig config = metal_config(); - config.phase = LlmPhase::Prefill; - const LlmBackendLifecycleResult initialization = backend_->initialize(config); - ASSERT_EQ(initialization.status, LlmBackendStatus::Ready) - << initialization.reason_code << ": " - << metal_evidence().capability.error.description; - EXPECT_EQ(metal_evidence().capability.kernel_revision, - kCanonicalKernelRevision); - EXPECT_EQ(metal_evidence().capability.kernel_source_sha256, - kCanonicalKernelSourceSha256); - ASSERT_EQ(metal_evidence().capability.foundation_pipelines.size(), - kFoundationPipelineCount); - EXPECT_NE(std::find_if( - metal_evidence().capability.foundation_pipelines.begin(), - metal_evidence().capability.foundation_pipelines.end(), - [](const LlmMetalPipelineEvidence& pipeline) { - return pipeline.label == - "membenchmark.llm-metal.pipeline.prefill-contiguous-layout-probe"; - }), - metal_evidence().capability.foundation_pipelines.end()); - - const LlmGeometry geometry = prefill_contiguous_geometry( - 4097, 5, test_case.query_tile_tokens, 2, 2, - test_case.record_bytes); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_TRUE(geometry.prefill.has_value()); - EXPECT_EQ(geometry.prefill->attention_prefix_token_visits_per_sequence, - test_case.expected_prefix_visits); + ASSERT_EQ(backend_->initialize(metal_config()).status, LlmBackendStatus::Ready); LlmMemoryWorkPlan plan = build_device_plan( - geometry, "llm-metal-phase11-prefill-q-" + - std::to_string(test_case.query_tile_tokens)); + contiguous_geometry(test_case.weight_bytes, test_case.context_tokens, 2, 2, test_case.record_bytes), + "llm-metal-decode-multigroup-plan"); resolve_and_prepare(plan); - EXPECT_EQ(metal_evidence().capability.layout_probe_resource_count, 4U); - for (LlmScenario scenario : {LlmScenario::WeightsOnly, - LlmScenario::KvOnly, - LlmScenario::Mixed}) { - expect_complete_scenario_task(plan, scenario, 2); - } + expect_complete_scenario_task(plan, test_case.scenario, test_case.work_units, 2); } } @@ -3979,51 +3159,6 @@ TEST_F(LlmMetalBackendIntegrationTest, expect_complete_scenario_task(plan, LlmScenario::Mixed, 2, 2); } -TEST_F(LlmMetalBackendIntegrationTest, - PrefillPagedKvPoolsCrossCanonicalWholeBlockSegmentBoundaryIntegration) { - backend_ = create_llm_metal_backend(); - ASSERT_NE(backend_, nullptr); - LlmMemoryConfig config = metal_config(); - config.phase = LlmPhase::Prefill; - config.kv_layout = LlmKvLayout::Paged; - const LlmBackendLifecycleResult initialization = - backend_->initialize(config); - ASSERT_EQ(initialization.status, LlmBackendStatus::Ready) - << initialization.reason_code << ": " - << metal_evidence().capability.error.description; - - const size_t capacity = Constants::LLM_METAL_SEGMENT_CAPACITY_BYTES; - const size_t record_bytes = capacity / 512; - const size_t block_bytes = 2 * record_bytes; - static_assert(Constants::LLM_METAL_SEGMENT_CAPACITY_BYTES % 512 == 0); - ASSERT_EQ(capacity % block_bytes, 0U); - const LlmGeometry geometry = prefill_paged_geometry( - 33, 513, 513, 2, 1, 1, record_bytes); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_EQ(geometry.kv_blocks_per_sequence, 257U); - ASSERT_EQ(geometry.kv_block_bytes, block_bytes); - ASSERT_EQ(geometry.last_block_valid_bytes, record_bytes); - ASSERT_LT(geometry.last_block_valid_bytes, geometry.kv_block_bytes); - - LlmMemoryWorkPlan plan = build_device_plan( - geometry, "llm-metal-phase12-prefill-paged-segment-boundary", - 4 * kGiB); - const LlmMetalExecutionPlan* execution = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution, nullptr); - ASSERT_TRUE(execution->valid) << execution->reason_code; - EXPECT_EQ(execution->resources.k_segments.elements_per_segment, 256U); - EXPECT_EQ(execution->resources.k_segments.segment_lengths, - (std::vector{capacity, block_bytes})); - EXPECT_EQ(execution->resources.v_segments.segment_lengths, - execution->resources.k_segments.segment_lengths); - ASSERT_TRUE(execution->resources.table_segments.has_value()); - EXPECT_EQ(execution->resources.table_segments->segment_count, 1U); - - resolve_and_prepare(plan); - expect_complete_scenario_task(plan, LlmScenario::KvOnly, 1, 2); -} - TEST_F(LlmMetalBackendIntegrationTest, PagedPrivateTableUploadValidationAndTier2SlotsIntegration) { backend_ = create_llm_metal_backend(); ASSERT_NE(backend_, nullptr); @@ -4090,63 +3225,40 @@ TEST_F(LlmMetalBackendIntegrationTest, PagedPrivateTableUploadValidationAndTier2 } } -TEST_F(LlmMetalBackendIntegrationTest, - PagedKvPoolsCrossCanonicalSegmentBoundaryIntegration) { - backend_ = create_llm_metal_backend(); - ASSERT_NE(backend_, nullptr); - LlmMemoryConfig config = metal_config(); - config.kv_layout = LlmKvLayout::Paged; - const LlmBackendLifecycleResult paged_initialization = - backend_->initialize(config); - ASSERT_EQ(paged_initialization.status, LlmBackendStatus::Ready) - << paged_initialization.reason_code << ": " - << metal_evidence().capability.error.description; - +TEST_F(LlmMetalBackendIntegrationTest, PagedKvPoolsCrossCanonicalSegmentBoundaryIntegration) { const size_t capacity = Constants::LLM_METAL_SEGMENT_CAPACITY_BYTES; - const size_t record_bytes = capacity / 512U; - const size_t block_bytes = 2U * record_bytes; - const LlmGeometry geometry = - paged_geometry(4097, 513, 2, 1, 1, record_bytes); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - const LlmKvLayoutPlan layout = paged_layout_for(geometry); - ASSERT_TRUE(layout.valid) << layout.reason_code; - ASSERT_EQ(layout.block_bytes, block_bytes); - ASSERT_EQ(layout.blocks_per_sequence, 257U); - ASSERT_EQ(layout.last_block_valid_bytes, record_bytes); - ASSERT_EQ(layout.memory.k_physical_bytes, capacity + block_bytes); - ASSERT_EQ(layout.memory.v_physical_bytes, capacity + block_bytes); - - LlmMemoryWorkPlan plan = build_device_plan( - geometry, "llm-metal-phase10-paged-kv-multisegment-plan"); - ASSERT_TRUE(plan.valid) << plan.reason_code; - const LlmMetalExecutionPlan* execution = - get_llm_metal_execution_plan(plan); - ASSERT_NE(execution, nullptr); - ASSERT_TRUE(execution->valid) << execution->reason_code; - ASSERT_EQ(execution->resources.k_segments.segment_lengths.size(), 2U); - ASSERT_EQ(execution->resources.v_segments.segment_lengths.size(), 2U); - EXPECT_EQ(execution->resources.k_segments.segment_lengths[0], capacity); - EXPECT_EQ(execution->resources.k_segments.segment_lengths[1], block_bytes); - EXPECT_EQ(execution->resources.v_segments.segment_lengths[0], capacity); - EXPECT_EQ(execution->resources.v_segments.segment_lengths[1], block_bytes); - - resolve_and_prepare(plan); - expect_complete_scenario_task(plan, LlmScenario::KvOnly, 1, 2); - expect_complete_scenario_task(plan, LlmScenario::Mixed, 1, 2); -} - -TEST_F(LlmMetalBackendIntegrationTest, MutatedResourcePlanIsRejectedBeforeAllocationIntegration) { - LlmMemoryWorkPlan plan = build_device_plan(contiguous_geometry(4097), "llm-metal-phase8-mutated-resource-plan"); - ASSERT_TRUE(plan.valid) << plan.reason_code; - LlmMetalExecutionPlan* execution = get_llm_metal_execution_plan(plan); - ASSERT_NE(execution, nullptr); - ASSERT_FALSE(execution->resources.planned_resources.empty()); - ++execution->resources.planned_resources.front().length_bytes; - - const LlmBackendLifecycleResult resolved = backend_->resolve_execution_plan(plan); - EXPECT_EQ(resolved.status, LlmBackendStatus::Failed); - EXPECT_EQ(resolved.reason_code, LlmBackendReason::PLAN_RESOURCE_IDENTITY_MISMATCH); - EXPECT_FALSE(metal_evidence().resources.allocation_attempted); + const size_t record_bytes = capacity / 512; + const size_t block_bytes = 2 * record_bytes; + for (auto phase : {LlmPhase::Decode, LlmPhase::Prefill}) { + SCOPED_TRACE(static_cast(phase)); + backend_ = create_llm_metal_backend(); + ASSERT_NE(backend_, nullptr); + LlmMemoryConfig config = metal_config(); + config.phase = phase; + config.kv_layout = LlmKvLayout::Paged; + const auto initialization = backend_->initialize(config); + ASSERT_EQ(initialization.status, LlmBackendStatus::Ready) << initialization.reason_code; + const LlmGeometry geometry = phase == LlmPhase::Decode + ? paged_geometry(4097, 513, 2, 1, 1, record_bytes) + : prefill_paged_geometry(33, 513, 513, 2, 1, 1, record_bytes); + ASSERT_TRUE(geometry.valid) << geometry.reason_code; + EXPECT_EQ(geometry.kv_blocks_per_sequence, 257U); + EXPECT_EQ(geometry.kv_block_bytes, block_bytes); + EXPECT_EQ(geometry.last_block_valid_bytes, record_bytes); + LlmMemoryWorkPlan plan = build_device_plan(geometry, "llm-metal-paged-segment-boundary", 4 * kGiB); + ASSERT_TRUE(plan.valid) << plan.reason_code; + const auto* execution = get_llm_metal_execution_plan(plan); + ASSERT_NE(execution, nullptr); + ASSERT_TRUE(execution->valid) << execution->reason_code; + EXPECT_EQ(execution->resources.k_segments.elements_per_segment, 256U); + EXPECT_EQ(execution->resources.k_segments.segment_lengths, (std::vector{capacity, block_bytes})); + EXPECT_EQ(execution->resources.v_segments.segment_lengths, execution->resources.k_segments.segment_lengths); + ASSERT_TRUE(execution->resources.table_segments.has_value()); + EXPECT_EQ(execution->resources.table_segments->segment_count, 1U); + resolve_and_prepare(plan); + expect_complete_scenario_task(plan, LlmScenario::KvOnly, 1, 2); + if (phase == LlmPhase::Decode) expect_complete_scenario_task(plan, LlmScenario::Mixed, 1, 2); + } } TEST_F(LlmMetalBackendIntegrationTest, RuntimePageSizeRejectsASelfConsistentUnderRoundedPlanIntegration) { @@ -4247,8 +3359,7 @@ TEST_F(LlmMetalBackendIntegrationTest, ResourceLargerThanCanonicalSegmentUsesPri expect_complete_scenario_task(plan, LlmScenario::Mixed, 1); } -TEST(LlmMetalBackendFailureInjectionIntegrationTest, - DecodePagedPermutationAndPaddingHooksAreDetectedIntegration) { +TEST(LlmMetalBackendFailureInjectionIntegrationTest, PagedPermutationAndPaddingHooksAreDetectedIntegration) { struct HookCase { bool wrong_permutation; bool padding_mismatch; @@ -4259,165 +3370,66 @@ TEST(LlmMetalBackendFailureInjectionIntegrationTest, {false, true, LlmBackendReason::PADDING_CANARY_MISMATCH}, }}; - for (size_t index = 0; index < cases.size(); ++index) { - SCOPED_TRACE(index); - LlmMetalBackendTestHooks hooks; - hooks.force_wrong_paged_table_permutation = cases[index].wrong_permutation; - hooks.force_padding_canary_mismatch = cases[index].padding_mismatch; - std::unique_ptr backend = - create_llm_metal_backend_for_testing(hooks); - ASSERT_NE(backend, nullptr); - LlmMemoryConfig config = metal_config(); - config.kv_layout = LlmKvLayout::Paged; - const LlmBackendLifecycleResult initialization = - backend->initialize(config); - if (initialization.status == LlmBackendStatus::Unsupported) { - ASSERT_TRUE( - is_stable_capability_unsupported_reason(initialization.reason_code)); - GTEST_SKIP() << "LLM Metal decode-paged unsupported: " - << initialization.reason_code; - } - ASSERT_EQ(initialization.status, LlmBackendStatus::Ready) - << initialization.reason_code; - const LlmMetalBackendEvidence* initialized = - get_llm_metal_backend_evidence(backend->evidence()); - ASSERT_NE(initialized, nullptr); - - const LlmGeometry geometry = paged_geometry(4097, 17, 4, 2, 2, 8); - LlmMetalResourcePlanRequest request = resource_request(geometry); - request.argument_buffer_encoded_length = - initialized->capability.argument_buffer_encoded_length; - request.argument_buffer_alignment = - initialized->capability.argument_buffer_alignment; - request.max_buffer_length = initialized->capability.max_buffer_length; - request.available_memory_bytes = 2 * kGiB; - request.host_mapping_granularity_bytes = get_system_page_size_bytes(); - LlmMemoryWorkPlan plan = make_metal_model_plan( - geometry, build_llm_metal_execution_plan(request), - "llm-metal-phase10-paged-hook-" + std::to_string(index)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(backend->resolve_execution_plan(plan).status, - LlmBackendStatus::Ready); - const LlmBackendLifecycleResult prepared = - backend->prepare_resources(plan); - ASSERT_EQ(prepared.status, LlmBackendStatus::Ready) - << prepared.reason_code; - - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - const LlmTaskExecutionResult result = backend->execute_task( - plan, scenario, - measurement_context(LlmScenario::Mixed, - "phase-10-paged-failure-injection")); - EXPECT_EQ(result.status, LlmTaskExecutionStatus::Invalid); - EXPECT_EQ(result.reason_code, cases[index].expected_reason); - EXPECT_EQ(result.completion.completed_layout_metadata_lookup_count, - scenario.layout_metadata_lookup_count); - EXPECT_EQ(result.completion.completed_layout_metadata_read_bytes, - scenario.layout_metadata_read_bytes); - const LlmMetalTaskEvidence* task = get_llm_metal_task_evidence(result); - ASSERT_NE(task, nullptr); - EXPECT_EQ(task->checksum_valid, !cases[index].wrong_permutation); - EXPECT_TRUE(task->kv_write_validation_valid); - EXPECT_TRUE(task->padding_canary_applicable); - EXPECT_TRUE(task->padding_canary_evaluated); - EXPECT_EQ(task->padding_canary_valid, !cases[index].padding_mismatch); - EXPECT_EQ(task->post_validation_valid, !cases[index].padding_mismatch); - EXPECT_TRUE(task->cold_checks[0].valid); - EXPECT_TRUE(task->cold_checks[1].valid); - EXPECT_TRUE(task->cold_checks[2].evaluated); - EXPECT_EQ(task->cold_checks[2].valid, !cases[index].padding_mismatch); - } -} - -TEST(LlmMetalBackendFailureInjectionIntegrationTest, - PrefillPagedPermutationAndPaddingHooksAreDetectedIntegration) { - struct HookCase { - bool wrong_permutation; - bool padding_mismatch; - std::string_view expected_reason; - }; - const std::array cases = {{ - {true, false, LlmBackendReason::TIMED_CHECKSUM_MISMATCH}, - {false, true, LlmBackendReason::PADDING_CANARY_MISMATCH}, - }}; + for (LlmPhase phase : {LlmPhase::Decode, LlmPhase::Prefill}) { + SCOPED_TRACE(static_cast(phase)); + for (size_t index = 0; index < cases.size(); ++index) { + SCOPED_TRACE(index); + LlmMetalBackendTestHooks hooks; + hooks.force_wrong_paged_table_permutation = cases[index].wrong_permutation; + hooks.force_padding_canary_mismatch = cases[index].padding_mismatch; + std::unique_ptr backend = create_llm_metal_backend_for_testing(hooks); + ASSERT_NE(backend, nullptr); + LlmMemoryConfig config = metal_config(); + config.phase = phase; + config.kv_layout = LlmKvLayout::Paged; + const LlmBackendLifecycleResult initialization = backend->initialize(config); + if (initialization.status == LlmBackendStatus::Unsupported) { + ASSERT_TRUE(is_stable_capability_unsupported_reason(initialization.reason_code)); + GTEST_SKIP() << "LLM Metal paged unsupported: " << initialization.reason_code; + } + ASSERT_EQ(initialization.status, LlmBackendStatus::Ready) << initialization.reason_code; + const LlmMetalBackendEvidence* initialized = get_llm_metal_backend_evidence(backend->evidence()); + ASSERT_NE(initialized, nullptr); - for (size_t index = 0; index < cases.size(); ++index) { - SCOPED_TRACE(index); - LlmMetalBackendTestHooks hooks; - hooks.force_wrong_paged_table_permutation = - cases[index].wrong_permutation; - hooks.force_padding_canary_mismatch = cases[index].padding_mismatch; - std::unique_ptr backend = - create_llm_metal_backend_for_testing(hooks); - ASSERT_NE(backend, nullptr); - LlmMemoryConfig config = metal_config(); - config.phase = LlmPhase::Prefill; - config.kv_layout = LlmKvLayout::Paged; - const LlmBackendLifecycleResult initialization = - backend->initialize(config); - if (initialization.status == LlmBackendStatus::Unsupported) { - ASSERT_TRUE( - is_stable_capability_unsupported_reason(initialization.reason_code)); - GTEST_SKIP() << "LLM Metal prefill-paged unsupported: " - << initialization.reason_code; + const LlmGeometry geometry = phase == LlmPhase::Decode ? paged_geometry(4097, 17, 4, 2, 2, 8) + : prefill_paged_geometry(4097, 5, 2, 2, 2, 2, 33); + ASSERT_TRUE(geometry.valid) << geometry.reason_code; + ASSERT_LT(geometry.last_block_valid_bytes, geometry.kv_block_bytes); + LlmMetalResourcePlanRequest request = resource_request(geometry); + request.argument_buffer_encoded_length = initialized->capability.argument_buffer_encoded_length; + request.argument_buffer_alignment = initialized->capability.argument_buffer_alignment; + request.max_buffer_length = initialized->capability.max_buffer_length; + request.available_memory_bytes = 2 * kGiB; + request.host_mapping_granularity_bytes = get_system_page_size_bytes(); + LlmMemoryWorkPlan plan = make_metal_model_plan(geometry, build_llm_metal_execution_plan(request), + "llm-metal-phase10-paged-hook-" + std::to_string(index)); + ASSERT_TRUE(plan.valid) << plan.reason_code; + ASSERT_EQ(backend->resolve_execution_plan(plan).status, LlmBackendStatus::Ready); + const LlmBackendLifecycleResult prepared = backend->prepare_resources(plan); + ASSERT_EQ(prepared.status, LlmBackendStatus::Ready) << prepared.reason_code; + + const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan(plan, LlmScenario::Mixed, 2, true); + ASSERT_TRUE(scenario.valid) << scenario.reason_code; + const LlmTaskExecutionResult result = backend->execute_task( + plan, scenario, measurement_context(LlmScenario::Mixed, "phase-10-paged-failure-injection")); + EXPECT_EQ(result.status, LlmTaskExecutionStatus::Invalid); + EXPECT_EQ(result.reason_code, cases[index].expected_reason); + EXPECT_EQ(result.completion.completed_layout_metadata_lookup_count, scenario.layout_metadata_lookup_count); + EXPECT_EQ(result.completion.completed_layout_metadata_read_bytes, scenario.layout_metadata_read_bytes); + const LlmMetalTaskEvidence* task = get_llm_metal_task_evidence(result); + ASSERT_NE(task, nullptr); + EXPECT_EQ(task->checksum_valid, !cases[index].wrong_permutation); + EXPECT_TRUE(task->kv_write_validation_evaluated); + EXPECT_TRUE(task->kv_write_validation_valid); + EXPECT_TRUE(task->padding_canary_applicable); + EXPECT_TRUE(task->padding_canary_evaluated); + EXPECT_EQ(task->padding_canary_valid, !cases[index].padding_mismatch); + EXPECT_EQ(task->post_validation_valid, !cases[index].padding_mismatch); + EXPECT_TRUE(task->cold_checks[0].valid); + EXPECT_TRUE(task->cold_checks[1].valid); + EXPECT_TRUE(task->cold_checks[2].evaluated); + EXPECT_EQ(task->cold_checks[2].valid, !cases[index].padding_mismatch); } - ASSERT_EQ(initialization.status, LlmBackendStatus::Ready) - << initialization.reason_code; - const LlmMetalBackendEvidence* initialized = - get_llm_metal_backend_evidence(backend->evidence()); - ASSERT_NE(initialized, nullptr); - - const LlmGeometry geometry = - prefill_paged_geometry(4097, 5, 2, 2, 2, 2, 33); - ASSERT_TRUE(geometry.valid) << geometry.reason_code; - ASSERT_LT(geometry.last_block_valid_bytes, geometry.kv_block_bytes); - LlmMetalResourcePlanRequest request = resource_request(geometry); - request.argument_buffer_encoded_length = - initialized->capability.argument_buffer_encoded_length; - request.argument_buffer_alignment = - initialized->capability.argument_buffer_alignment; - request.max_buffer_length = initialized->capability.max_buffer_length; - request.available_memory_bytes = 2 * kGiB; - request.host_mapping_granularity_bytes = get_system_page_size_bytes(); - LlmMemoryWorkPlan plan = make_metal_model_plan( - geometry, build_llm_metal_execution_plan(request), - "llm-metal-phase12-prefill-paged-hook-" + std::to_string(index)); - ASSERT_TRUE(plan.valid) << plan.reason_code; - ASSERT_EQ(backend->resolve_execution_plan(plan).status, - LlmBackendStatus::Ready); - const LlmBackendLifecycleResult prepared = - backend->prepare_resources(plan); - ASSERT_EQ(prepared.status, LlmBackendStatus::Ready) - << prepared.reason_code; - - const LlmScenarioWorkPlan scenario = build_llm_scenario_work_plan( - plan, LlmScenario::Mixed, 2, true); - ASSERT_TRUE(scenario.valid) << scenario.reason_code; - const LlmTaskExecutionResult result = backend->execute_task( - plan, scenario, - measurement_context(LlmScenario::Mixed, - "phase-12-prefill-paged-failure-injection")); - EXPECT_EQ(result.status, LlmTaskExecutionStatus::Invalid); - EXPECT_EQ(result.reason_code, cases[index].expected_reason); - EXPECT_EQ(result.completion.completed_layout_metadata_lookup_count, - scenario.layout_metadata_lookup_count); - EXPECT_EQ(result.completion.completed_layout_metadata_read_bytes, - scenario.layout_metadata_read_bytes); - const LlmMetalTaskEvidence* task = get_llm_metal_task_evidence(result); - ASSERT_NE(task, nullptr); - EXPECT_EQ(task->checksum_valid, !cases[index].wrong_permutation); - EXPECT_TRUE(task->kv_write_validation_evaluated); - EXPECT_TRUE(task->kv_write_validation_valid); - EXPECT_TRUE(task->padding_canary_applicable); - EXPECT_TRUE(task->padding_canary_evaluated); - EXPECT_EQ(task->padding_canary_valid, !cases[index].padding_mismatch); - EXPECT_EQ(task->post_validation_valid, !cases[index].padding_mismatch); - EXPECT_TRUE(task->cold_checks[0].valid); - EXPECT_TRUE(task->cold_checks[1].valid); - EXPECT_TRUE(task->cold_checks[2].evaluated); - EXPECT_EQ(task->cold_checks[2].valid, !cases[index].padding_mismatch); } } @@ -4680,43 +3692,31 @@ TEST(LlmMetalBackendFailureInjectionIntegrationTest, } // namespace TEST(LlmMetalBackendTest, ColdReadbackIndependentVerdicts) { - for (bool completed : {false, true}) { - for (unsigned combination = 0; combination < 8; ++combination) { - const uint32_t flags = ((combination & 1) ? LlmMetalKernelContract::kValidationInvalidParametersBit : 0) | - ((combination & 2) ? LlmMetalKernelContract::kKvWriteValidationMismatchBit : 0) | - ((combination & 4) ? LlmMetalKernelContract::kPaddingCanaryMismatchBit : 0); - const auto checks = interpret_llm_metal_cold_checks(LlmPhase::Prefill, true, true, completed, flags); - EXPECT_EQ(checks[0].evaluated, completed); - EXPECT_EQ(checks[0].valid, completed && !(combination & 1)); - for (size_t slot = 1; slot < 3; ++slot) { - const bool mismatch = (combination & (1U << slot)) != 0; - EXPECT_EQ(checks[slot].evaluated, completed && (mismatch || !(combination & 1))); - EXPECT_EQ(checks[slot].valid, completed && !mismatch && !(combination & 1)); - } - const auto none = interpret_llm_metal_cold_checks(LlmPhase::Decode, false, true, completed, flags); - for (const auto& check : none) EXPECT_FALSE(check.applicable); - const auto contiguous = interpret_llm_metal_cold_checks(LlmPhase::Decode, true, false, completed, flags); - EXPECT_FALSE(contiguous[2].applicable); - EXPECT_EQ(contiguous[1].kind, LlmColdCheckKind::KvAppendFinal); - EXPECT_EQ(checks[1].kind, LlmColdCheckKind::KvPrefillFinalSamples); - } - } -} - -TEST(LlmMetalBackendTest, PendingColdChecksPreserveAllApplicabilityCombinations) { for (auto phase : {LlmPhase::Decode, LlmPhase::Prefill}) { for (bool writes : {false, true}) { for (bool padding : {false, true}) { - const auto checks = interpret_llm_metal_cold_checks(phase, writes, padding, false, 0); - EXPECT_EQ(checks[0].applicable, writes); - EXPECT_EQ(checks[1].applicable, writes); - EXPECT_EQ(checks[2].applicable, writes && padding); - EXPECT_EQ(checks[1].kind, phase == LlmPhase::Decode ? LlmColdCheckKind::KvAppendFinal - : LlmColdCheckKind::KvPrefillFinalSamples); - for (const auto& check : checks) { - EXPECT_FALSE(check.evaluated); - EXPECT_FALSE(check.valid); - EXPECT_EQ(check.reason_code, "not-evaluated"); + for (bool completed : {false, true}) { + for (unsigned combination = 0; combination < 8; ++combination) { + SCOPED_TRACE(::testing::Message() << static_cast(phase) << ":" << writes << ":" << padding << ":" + << completed << ":" << combination); + const uint32_t flags = ((combination & 1) ? LlmMetalKernelContract::kValidationInvalidParametersBit : 0) | + ((combination & 2) ? LlmMetalKernelContract::kKvWriteValidationMismatchBit : 0) | + ((combination & 4) ? LlmMetalKernelContract::kPaddingCanaryMismatchBit : 0); + const auto checks = interpret_llm_metal_cold_checks(phase, writes, padding, completed, flags); + EXPECT_EQ(checks[0].applicable, writes); + EXPECT_EQ(checks[1].applicable, writes); + EXPECT_EQ(checks[2].applicable, writes && padding); + EXPECT_EQ(checks[1].kind, phase == LlmPhase::Decode ? LlmColdCheckKind::KvAppendFinal + : LlmColdCheckKind::KvPrefillFinalSamples); + for (size_t slot = 0; slot < checks.size(); ++slot) { + const bool mismatch = (combination & (1U << slot)) != 0; + const bool evaluated = + checks[slot].applicable && completed && (slot == 0 || mismatch || !(combination & 1)); + EXPECT_EQ(checks[slot].evaluated, evaluated); + EXPECT_EQ(checks[slot].valid, evaluated && !mismatch && !(combination & 1)); + if (!completed) EXPECT_EQ(checks[slot].reason_code, "not-evaluated"); + } + } } } } diff --git a/tests/test_memory_manager.cpp b/tests/test_memory_manager.cpp index dddd9c5..d6f4ba8 100644 --- a/tests/test_memory_manager.cpp +++ b/tests/test_memory_manager.cpp @@ -96,21 +96,6 @@ TEST_F(MemoryManagerTest, AdviceFailureIsReportedButAllocationRemainsOwned) { Messages::warning_madvise_random_failed("advice-failure", std::strerror(EINVAL)) + "\n"); } -TEST_F(MemoryManagerTest, ZeroSizeFailsBeforeAnySystemCallWithExactMessage) { - for (const bool non_cacheable : {false, true}) { - const char* name = non_cacheable ? "zero-non-cacheable" : "zero-regular"; - testing::internal::CaptureStderr(); - MmapPtr buffer = non_cacheable ? allocate_buffer_non_cacheable(0, name) : allocate_buffer(0, name); - const std::string error = testing::internal::GetCapturedStderr(); - - EXPECT_EQ(buffer.get(), nullptr); - EXPECT_EQ(error, Messages::error_prefix() + Messages::error_buffer_size_zero(name) + "\n"); - } - EXPECT_EQ(state.map_calls, 0u); - EXPECT_EQ(state.advise_calls, 0u); - EXPECT_EQ(state.unmap_calls, 0u); -} - TEST_F(MemoryManagerTest, ReadOnlyProtectionUsesExactMappingRangeAndReportsFailure) { MmapPtr buffer = allocate_buffer(4096, "protected"); ASSERT_NE(buffer, nullptr); diff --git a/tests/test_memory_utils.cpp b/tests/test_memory_utils.cpp index 89e8cc1..ab7403a 100644 --- a/tests/test_memory_utils.cpp +++ b/tests/test_memory_utils.cpp @@ -39,7 +39,6 @@ class AlignedBuffer { class MemoryUtilsTest : public testing::Test { protected: void SetUp() override { - hooks_.page_size_bytes = page_size_bytes; hooks_.generated_seed = 0x123456789abcdef0ULL; set_memory_utils_test_hooks(&hooks_); } @@ -115,6 +114,8 @@ TEST_F(MemoryUtilsTest, SetupLatencyChainRejectsInvalidInputsWithExactReasons) { size_t buffer_size; size_t stride; std::string expected_reason; + size_t locality = 0; + LatencyChainMode mode = LatencyChainMode::Auto; }; const InvalidInputCase cases[] = { {"null buffer", true, LATENCY_STRIDE_BYTES * 2, LATENCY_STRIDE_BYTES, @@ -128,13 +129,19 @@ TEST_F(MemoryUtilsTest, SetupLatencyChainRejectsInvalidInputsWithExactReasons) { Messages::error_latency_stride_alignment(sizeof(uintptr_t) + 1, sizeof(uintptr_t))}, {"one-node buffer", false, LATENCY_STRIDE_BYTES, LATENCY_STRIDE_BYTES, Messages::error_buffer_stride_invalid_latency_chain(1, LATENCY_STRIDE_BYTES, LATENCY_STRIDE_BYTES)}, + {"locality too small", false, LATENCY_STRIDE_BYTES * 16, LATENCY_STRIDE_BYTES, + Messages::error_buffer_stride_invalid_latency_chain(1, LATENCY_STRIDE_BYTES, LATENCY_STRIDE_BYTES), + LATENCY_STRIDE_BYTES}, + {"box without locality", false, LATENCY_STRIDE_BYTES * 128, LATENCY_STRIDE_BYTES, + Messages::error_latency_chain_mode_requires_locality("diff-random-in-box"), 0, + LatencyChainMode::DiffRandomInBoxIncreasingBox}, }; for (const InvalidInputCase& test_case : cases) { SCOPED_TRACE(test_case.name); testing::internal::CaptureStderr(); - const int result = - setup_latency_chain(test_case.null_buffer ? nullptr : storage.data(), test_case.buffer_size, test_case.stride); + const int result = setup_latency_chain(test_case.null_buffer ? nullptr : storage.data(), test_case.buffer_size, + test_case.stride, test_case.locality, test_case.mode); const std::string error = testing::internal::GetCapturedStderr(); EXPECT_EQ(result, EXIT_FAILURE); @@ -142,130 +149,69 @@ TEST_F(MemoryUtilsTest, SetupLatencyChainRejectsInvalidInputsWithExactReasons) { } } -TEST_F(MemoryUtilsTest, SetupLatencyChainAcceptsMinimumTwoNodeChainsAtSupportedStrides) { - using namespace Constants; - - for (const size_t stride : {sizeof(uintptr_t), LATENCY_STRIDE_BYTES}) { - SCOPED_TRACE(stride); - const size_t buffer_size = stride * 2; - AlignedBuffer buffer(buffer_size); - ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride), EXIT_SUCCESS); - - const uintptr_t first = reinterpret_cast(buffer.data()); - const uintptr_t second = first + stride; - EXPECT_EQ(*reinterpret_cast(buffer.data()), second); - EXPECT_EQ(*reinterpret_cast(static_cast(buffer.data()) + stride), first); - } -} - // Test that setup_latency_chain creates a valid linked list TEST_F(MemoryUtilsTest, SetupLatencyChainCreatesValidChain) { using namespace Constants; - size_t buffer_size = LATENCY_STRIDE_BYTES * 4; // 4 pointers - AlignedBuffer buffer(buffer_size); - - int result = setup_latency_chain(buffer.data(), buffer_size, LATENCY_STRIDE_BYTES); - EXPECT_EQ(result, EXIT_SUCCESS); - - const size_t pointer_count = buffer_size / LATENCY_STRIDE_BYTES; - const uintptr_t buffer_start = reinterpret_cast(buffer.data()); - const uintptr_t buffer_end = buffer_start + buffer_size; - std::vector visited(pointer_count, false); - - uintptr_t current = buffer_start; - for (size_t step = 0; step < pointer_count; ++step) { - ASSERT_GE(current, buffer_start); - ASSERT_LT(current, buffer_end); - const size_t offset = static_cast(current - buffer_start); - ASSERT_EQ(offset % LATENCY_STRIDE_BYTES, 0u); - const size_t index = offset / LATENCY_STRIDE_BYTES; - ASSERT_LT(index, pointer_count); - EXPECT_FALSE(visited[index]) << "chain repeated node at step " << step; - visited[index] = true; - current = *reinterpret_cast(current); - } + for (const size_t stride : {sizeof(uintptr_t), LATENCY_STRIDE_BYTES}) { + for (const size_t pointer_count : {size_t{2}, size_t{4}}) { + SCOPED_TRACE(::testing::Message() << stride << "/" << pointer_count); + const size_t buffer_size = stride * pointer_count; + AlignedBuffer buffer(buffer_size); + + int result = setup_latency_chain(buffer.data(), buffer_size, stride); + EXPECT_EQ(result, EXIT_SUCCESS); + + const uintptr_t buffer_start = reinterpret_cast(buffer.data()); + const uintptr_t buffer_end = buffer_start + buffer_size; + std::vector visited(pointer_count, false); + + uintptr_t current = buffer_start; + for (size_t step = 0; step < pointer_count; ++step) { + ASSERT_GE(current, buffer_start); + ASSERT_LT(current, buffer_end); + const size_t offset = static_cast(current - buffer_start); + ASSERT_EQ(offset % stride, 0u); + const size_t index = offset / stride; + ASSERT_LT(index, pointer_count); + EXPECT_FALSE(visited[index]) << "chain repeated node at step " << step; + visited[index] = true; + current = *reinterpret_cast(current); + } - EXPECT_EQ(current, buffer_start); - for (size_t index = 0; index < pointer_count; ++index) { - EXPECT_TRUE(visited[index]) << "chain omitted node " << index; + EXPECT_EQ(current, buffer_start); + for (size_t index = 0; index < pointer_count; ++index) { + EXPECT_TRUE(visited[index]) << "chain omitted node " << index; + } + } } } -TEST_F(MemoryUtilsTest, SetupLatencyChainCollectsDiagnostics) { - using namespace Constants; - - const size_t page_size = page_size_bytes; - const size_t buffer_size = page_size * 4; - const size_t stride = sizeof(uintptr_t) * 8; - AlignedBuffer buffer(buffer_size); - - LatencyChainDiagnostics diagnostics; - int result = setup_latency_chain(buffer.data(), buffer_size, stride, 0, &diagnostics); - EXPECT_EQ(result, EXIT_SUCCESS); - EXPECT_EQ(diagnostics.pointer_count, buffer_size / stride); - const uintptr_t first_node = reinterpret_cast(buffer.data()); - const uintptr_t last_node = first_node + (diagnostics.pointer_count - 1) * stride; - const size_t expected_pages = last_node / page_size - first_node / page_size + 1; - EXPECT_EQ(diagnostics.unique_pages_touched, expected_pages); - EXPECT_EQ(diagnostics.page_size_bytes, page_size); - EXPECT_EQ(diagnostics.stride_bytes, stride); -} - -TEST_F(MemoryUtilsTest, SetupLatencyChainWithTooSmallTlbLocalityFails) { - using namespace Constants; - - size_t buffer_size = LATENCY_STRIDE_BYTES * 16; - AlignedBuffer buffer(buffer_size); - - testing::internal::CaptureStderr(); - int result = setup_latency_chain(buffer.data(), buffer_size, LATENCY_STRIDE_BYTES, LATENCY_STRIDE_BYTES); - std::string error_output = testing::internal::GetCapturedStderr(); - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_NE(error_output.find("Error: "), std::string::npos); -} - -TEST_F(MemoryUtilsTest, SetupLatencyChainWithBoxModeAndZeroLocalityFails) { - using namespace Constants; - - size_t buffer_size = LATENCY_STRIDE_BYTES * 128; - AlignedBuffer buffer(buffer_size); - - testing::internal::CaptureStderr(); - int result = setup_latency_chain(buffer.data(), buffer_size, LATENCY_STRIDE_BYTES, 0, nullptr, - LatencyChainMode::DiffRandomInBoxIncreasingBox); - std::string error_output = testing::internal::GetCapturedStderr(); - - EXPECT_EQ(result, EXIT_FAILURE); - EXPECT_NE(error_output.find("latency-chain-mode"), std::string::npos); -} - TEST_F(MemoryUtilsTest, ExplicitSeedsAreReproducibleAndSameRandomModeReusesBoxPermutation) { const size_t stride = 64; const size_t page_size = page_size_bytes; const size_t buffer_size = 4 * page_size; AlignedBuffer buffer(buffer_size); - ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, nullptr, - LatencyChainMode::RandomInBoxRandomBox, uint64_t{12345}), + ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, LatencyChainMode::RandomInBoxRandomBox, + uint64_t{12345}), EXIT_SUCCESS); const std::vector first = snapshot_next_indices(buffer.data(), buffer_size, stride); - ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, nullptr, - LatencyChainMode::RandomInBoxRandomBox, uint64_t{12345}), + ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, LatencyChainMode::RandomInBoxRandomBox, + uint64_t{12345}), EXIT_SUCCESS); const std::vector second = snapshot_next_indices(buffer.data(), buffer_size, stride); - ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, nullptr, - LatencyChainMode::RandomInBoxRandomBox, uint64_t{54321}), + ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, LatencyChainMode::RandomInBoxRandomBox, + uint64_t{54321}), EXIT_SUCCESS); const std::vector different = snapshot_next_indices(buffer.data(), buffer_size, stride); EXPECT_EQ(first, second); EXPECT_NE(first, different); - ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, nullptr, + ASSERT_EQ(setup_latency_chain(buffer.data(), buffer_size, stride, page_size, LatencyChainMode::SameRandomInBoxIncreasingBox, uint64_t{12345}), EXIT_SUCCESS); expect_same_permutation_per_box(snapshot_next_indices(buffer.data(), buffer_size, stride), page_size / stride); diff --git a/tests/test_messages.cpp b/tests/test_messages.cpp index 36a0463..43f9d10 100644 --- a/tests/test_messages.cpp +++ b/tests/test_messages.cpp @@ -41,64 +41,20 @@ void expect_exact_messages(const std::vector& cases) { } // namespace -TEST(MessagesFormattingTest, LinearHelpersHaveExactOutput) { - const std::vector cases = { - {"missing iterations", Messages::error_missing_value("--iterations"), "Missing value for --iterations"}, - {"missing buffer size", Messages::error_missing_value("--buffer-size"), "Missing value for --buffer-size"}, - {"missing count", Messages::error_missing_value("--count"), "Missing value for --count"}, - {"missing cache size", Messages::error_missing_value("--cache-size"), "Missing value for --cache-size"}, - {"unknown short", Messages::error_unknown_option("-unknown"), "Unknown option: -unknown"}, - {"unknown long", Messages::error_unknown_option("--invalid"), "Unknown option: --invalid"}, - {"unknown flag", Messages::error_unknown_option("-bad-flag"), "Unknown option: -bad-flag"}, - {"invalid iterations", Messages::error_invalid_value("--iterations", "abc", "must be a number"), - "Invalid value for --iterations: abc (must be a number)"}, - {"invalid cache size", Messages::error_invalid_value("--cache-size", "-1", "must be positive"), - "Invalid value for --cache-size: -1 (must be positive)"}, - {"invalid buffer size", Messages::error_invalid_value("--buffer-size", "0", "must be greater than zero"), - "Invalid value for --buffer-size: 0 (must be greater than zero)"}, - {"mmap source", Messages::error_mmap_failed("src_buffer"), "mmap failed for src_buffer"}, - {"mmap destination", Messages::error_mmap_failed("dst_buffer"), "mmap failed for dst_buffer"}, - {"mmap latency", Messages::error_mmap_failed("lat_buffer"), "mmap failed for lat_buffer"}, - {"benchmark loop 1", Messages::error_benchmark_loop(1, "memory error"), - "Error during benchmark loop 1: memory error"}, - {"benchmark loop 5", Messages::error_benchmark_loop(5, "timeout"), "Error during benchmark loop 5: timeout"}, - {"benchmark loop 10", Messages::error_benchmark_loop(10, "allocation failed"), - "Error during benchmark loop 10: allocation failed"}, - {"qos code 1", Messages::warning_qos_failed(1), "Failed to set QoS class for main thread (code: 1)"}, - {"qos code 42", Messages::warning_qos_failed(42), "Failed to set QoS class for main thread (code: 42)"}, - {"qos code 100", Messages::warning_qos_failed(100), "Failed to set QoS class for main thread (code: 100)"}, - {"qos code -1", Messages::warning_qos_failed(-1), "Failed to set QoS class for main thread (code: -1)"}, - }; - - expect_exact_messages(cases); -} - -// ============================================================================ -// Error Messages Tests -// ============================================================================ - -TEST(MessagesErrorTest, NumericValidationErrorsHaveExactOutput) { - const std::vector cases = { +TEST(MessagesErrorTest, ValidationErrorsHaveExactOutput) { + expect_exact_messages({ {"buffer calculation", Messages::error_buffer_size_calculation(1024), "Buffer size calculation error (1024 MB)."}, {"buffer too small", Messages::error_buffer_size_too_small(1024), "Final buffer size (1024 bytes) is too small."}, {"cache size", Messages::error_cache_size_invalid(16, 524288, 512), "cache-size invalid (must be between 16 KB and 524288 KB (512 MB))"}, {"iterations negative", Messages::error_iterations_invalid(-5, 1, 2147483647), "iterations invalid (must be between 1 and 2147483647, got -5)"}, - {"iterations zero", Messages::error_iterations_invalid(0, 1, 100), - "iterations invalid (must be between 1 and 100, got 0)"}, {"buffer size negative", Messages::error_buffersize_invalid(-100, 18446744073709551615UL), "buffer-size invalid (must be >= 0 and <= 18446744073709551615, got -100)"}, - {"buffer size zero", Messages::error_buffersize_invalid(0, 1000), - "buffer-size invalid (must be >= 0 and <= 1000, got 0)"}, {"count zero", Messages::error_count_invalid(0, 1, 2147483647), "count invalid (must be between 1 and 2147483647, got 0)"}, - {"count negative", Messages::error_count_invalid(-10, 1, 1000), - "count invalid (must be between 1 and 1000, got -10)"}, {"latency samples zero", Messages::error_latency_samples_invalid(0, 1, 2147483647), "latency-samples invalid (must be between 1 and 2147483647, got 0)"}, - {"latency samples negative", Messages::error_latency_samples_invalid(-1, 1, 5000), - "latency-samples invalid (must be between 1 and 5000, got -1)"}, {"latency stride range", Messages::error_latency_stride_invalid(0, 1, 9223372036854775807LL), "latency-stride-bytes invalid (must be between 1 and 9223372036854775807, got 0)"}, {"latency stride alignment", Messages::error_latency_stride_alignment(65, 8), @@ -107,93 +63,66 @@ TEST(MessagesErrorTest, NumericValidationErrorsHaveExactOutput) { "latency-tlb-locality-kb invalid (must be >= 0 and <= 1024, got -1)"}, {"latency locality page multiple", Messages::error_latency_tlb_locality_page_multiple(10, 16), "latency-tlb-locality-kb must be a multiple of system page size (16 KB), got 10 KB"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesErrorTest, JsonStdoutWriteFailureHasExactOutput) { - EXPECT_EQ(Messages::json_stdout_reason_stream_unavailable(), - "stdout stream is unavailable"); - EXPECT_EQ(Messages::json_stdout_reason_size_limit_exceeded(), - "serialized JSON exceeds stream size limits"); - EXPECT_EQ(Messages::json_stdout_reason_write_failed(), - "write operation failed"); - EXPECT_EQ(Messages::json_stdout_reason_flush_failed(), - "flush operation failed"); - EXPECT_EQ(Messages::error_json_stdout_write_failed( - Messages::json_stdout_reason_flush_failed()), - "Failed to write JSON to stdout: flush operation failed"); - EXPECT_EQ(Messages::error_json_stdout_write_failed(""), - "Failed to write JSON to stdout: unknown exception"); + {"missing iterations", Messages::error_missing_value("--iterations"), "Missing value for --iterations"}, + {"unknown short", Messages::error_unknown_option("-unknown"), "Unknown option: -unknown"}, + {"invalid iterations", Messages::error_invalid_value("--iterations", "abc", "must be a number"), + "Invalid value for --iterations: abc (must be a number)"}, + {"mmap source", Messages::error_mmap_failed("src_buffer"), "mmap failed for src_buffer"}, + {"benchmark loop 1", Messages::error_benchmark_loop(1, "memory error"), + "Error during benchmark loop 1: memory error"}, + {"qos code 1", Messages::warning_qos_failed(1), "Failed to set QoS class for main thread (code: 1)"}, + {"missing parameter", Messages::error_sweep_requires_parameter(), + "--sweep requires at least one parameter specification"}, + {"missing output", Messages::error_sweep_requires_output(), + "--sweep requires --output for the combined JSON result"}, + {"run cap", Messages::error_sweep_too_many_runs(12, 10), + "Sweep would generate 12 runs, exceeding --sweep-max-runs 10"}, + {"parameter not allowed", Messages::error_sweep_parameter_not_allowed("cache-size", "--patterns"), + "Sweep parameter 'cache-size' is not allowed with --patterns"}, + {"locality mode", Messages::error_latency_chain_mode_requires_locality("same-random-in-box-increasing-box"), + "latency-chain-mode 'same-random-in-box-increasing-box' requires --latency-tlb-locality-kb > 0"}, + {"locality minimum", Messages::error_latency_tlb_locality_too_small_for_stride(4096, 4096), + "latency-tlb-locality-kb too small for latency-stride-bytes (requires at least 8192 bytes, got 4096 bytes)"}, + {"TLB stride", Messages::error_analyze_tlb_stride_exceeds_page(32768, 16384), + "--analyze-tlb latency-stride-bytes must not exceed the system page size (16384 bytes), got 32768 bytes"}, + {"duplicate sweep", Messages::error_duplicate_sweep_parameter("latency-stride-bytes"), + "sweep parameter specified more than once: latency-stride-bytes"}, + }); } TEST(MessagesErrorTest, JsonCommandBoundaryFailuresHaveExactOutput) { - EXPECT_EQ( - Messages::error_json_output_initialization_failed("cwd unavailable"), - "JSON output initialization failed: cwd unavailable"); - EXPECT_EQ(Messages::error_json_output_initialization_failed(""), - "JSON output initialization failed: unknown exception"); - EXPECT_EQ( - Messages::error_json_payload_construction_failed("allocation failed"), - "JSON payload construction failed: allocation failed"); - EXPECT_EQ(Messages::error_json_payload_construction_failed(""), - "JSON payload construction failed: unknown exception"); - EXPECT_EQ( - Messages::error_command_execution_exception("TLB analysis", - "allocation failed"), - "TLB analysis failed with unexpected exception: allocation failed"); - EXPECT_EQ( - Messages::error_command_execution_exception("Core-to-core analysis", ""), - "Core-to-core analysis failed with unexpected exception: unknown exception"); - EXPECT_EQ( - Messages::error_sweep_nested_run_exception("allocator failed"), - "Sweep nested run failed with unexpected exception: allocator failed"); - EXPECT_EQ( - Messages::error_sweep_nested_run_exception(""), - "Sweep nested run failed with unexpected exception: unknown exception"); -} - -TEST(MessagesErrorTest, ErrorLatencyChainModeInvalid) { - std::string msg = Messages::error_latency_chain_mode_invalid(); - EXPECT_NE(msg.find("latency-chain-mode invalid"), std::string::npos); - EXPECT_NE(msg.find("random-box"), std::string::npos); -} - -TEST(MessagesErrorTest, ErrorLatencyChainModeRequiresLocality) { - const std::string mode = "same-random-in-box-increasing-box"; - EXPECT_EQ(Messages::error_latency_chain_mode_requires_locality(mode), - "latency-chain-mode 'same-random-in-box-increasing-box' requires --latency-tlb-locality-kb > 0"); -} - -TEST(MessagesErrorTest, ErrorAnalyzeTlbGlobalRandomUnsupported) { - const std::string& msg = Messages::error_analyze_tlb_global_random_unsupported(); - EXPECT_NE(msg.find("--analyze-tlb"), std::string::npos); - EXPECT_NE(msg.find("global-random"), std::string::npos); -} - -TEST(MessagesErrorTest, ErrorLatencyTlbLocalityTooSmallForStride) { - std::string msg = Messages::error_latency_tlb_locality_too_small_for_stride(4096, 4096); - EXPECT_NE(msg.find("latency-tlb-locality-kb too small for latency-stride-bytes"), std::string::npos); - EXPECT_NE(msg.find("requires at least 8192 bytes"), std::string::npos); -} - -TEST(MessagesErrorTest, ErrorAnalyzeTlbMustBeUsedAlone) { - const std::string& msg = Messages::error_analyze_tlb_must_be_used_alone(); - EXPECT_NE(msg.find("--analyze-tlb"), std::string::npos); - EXPECT_NE(msg.find("--output "), std::string::npos); - EXPECT_NE(msg.find("--latency-stride-bytes"), std::string::npos); - EXPECT_NE(msg.find("--latency-chain-mode"), std::string::npos); - EXPECT_NE(msg.find("--tlb-density"), std::string::npos); - EXPECT_NE(msg.find("--seed"), std::string::npos); - EXPECT_NE(msg.find("-S/--sweep "), std::string::npos); - EXPECT_NE(msg.find("-X/--sweep-max-runs "), std::string::npos); -} - -TEST(MessagesErrorTest, ErrorSeedRequiresEverySupportedMode) { - EXPECT_EQ(Messages::error_seed_requires_supported_mode(), - "--seed requires --benchmark, --patterns, --analyze-tlb, " - "--gpu-bandwidth, or --llm-memory"); + expect_exact_messages({ + {"error_json_output_initialization_failed", Messages::error_json_output_initialization_failed("cwd unavailable"), + "JSON output initialization failed: cwd unavailable"}, + {"error_json_output_initialization_failed empty reason", Messages::error_json_output_initialization_failed(""), + "JSON output initialization failed: unknown exception"}, + {"error_json_payload_construction_failed", Messages::error_json_payload_construction_failed("allocation failed"), + "JSON payload construction failed: allocation failed"}, + {"error_json_payload_construction_failed empty reason", Messages::error_json_payload_construction_failed(""), + "JSON payload construction failed: unknown exception"}, + {"error_command_execution_exception", + Messages::error_command_execution_exception("TLB analysis", "allocation failed"), + "TLB analysis failed with unexpected " + "exception: allocation failed"}, + {"error_command_execution_exception empty reason", + Messages::error_command_execution_exception("Core-to-core analysis", ""), + "Core-to-core analysis failed with unexpected exception: unknown exception"}, + {"error_sweep_nested_run_exception", Messages::error_sweep_nested_run_exception("allocator failed"), + "Sweep nested run failed with unexpected exception: allocator failed"}, + {"error_sweep_nested_run_exception empty reason", Messages::error_sweep_nested_run_exception(""), + "Sweep nested run failed with unexpected exception: unknown exception"}, + {"json_stdout_reason_stream_unavailable", Messages::json_stdout_reason_stream_unavailable(), + "stdout stream is unavailable"}, + {"json_stdout_reason_size_limit_exceeded", Messages::json_stdout_reason_size_limit_exceeded(), + "serialized JSON exceeds stream size limits"}, + {"json_stdout_reason_write_failed", Messages::json_stdout_reason_write_failed(), "write operation failed"}, + {"json_stdout_reason_flush_failed", Messages::json_stdout_reason_flush_failed(), "flush operation failed"}, + {"error_json_stdout_write_failed", + Messages::error_json_stdout_write_failed(Messages::json_stdout_reason_flush_failed()), + "Failed to write JSON to stdout: flush operation failed"}, + {"error_json_stdout_write_failed empty reason", Messages::error_json_stdout_write_failed(""), + "Failed to write JSON to stdout: unknown exception"}, + }); } TEST(MessagesTest, LlmMemoryCliMessagesHaveExactOutput) { @@ -276,223 +205,28 @@ TEST(MessagesTest, LlmMemoryCliMessagesHaveExactOutput) { "-t/--threads , -i/--iterations , " "-r/--count , --seed , -o/--output , and " "-h/--help (no other options allowed)"}, - {"required option", - Messages::error_llm_memory_missing_required_option("--layers"), + {"required option", Messages::error_llm_memory_missing_required_option("--layers"), "Missing required --llm-memory option: --layers"}, - {"config reason", - Messages::error_llm_memory_config_invalid( - "query-heads-not-divisible-by-kv-heads"), + {"config reason", Messages::error_llm_memory_config_invalid("query-heads-not-divisible-by-kv-heads"), "Invalid --llm-memory configuration " "(reason_code=query-heads-not-divisible-by-kv-heads)"}, - {"iteration limit", - Messages::error_llm_memory_iterations_exceed_limit(5, 4), + {"iteration limit", Messages::error_llm_memory_iterations_exceed_limit(5, 4), "LLM memory iterations exceed the exact-work guardrail " "(requested 5, maximum 4)"}, - {"runtime failure", - Messages::error_llm_memory_run_failed("checksum-mismatch"), + {"runtime failure", Messages::error_llm_memory_run_failed("checksum-mismatch"), "Synthetic LLM memory profile failed " "(reason_code=checksum-mismatch)"}, - {"paged table protection", - Messages::error_llm_paged_table_protection_failed(), + {"paged table protection", Messages::error_llm_paged_table_protection_failed(), "Failed to make the paged KV block table read-only"}, - {"positive integer", Messages::llm_memory_reason_positive_integer(), - "must be a positive integer"}, - {"backend", Messages::llm_memory_reason_backend(), - "must be exactly cpu or metal"}, - {"KV width", Messages::llm_memory_reason_kv_element_bytes(), - "must be exactly 1, 2, or 4"}, - {"phase", Messages::llm_memory_reason_phase(), - "must be exactly decode or prefill"}, - {"KV layout", Messages::llm_memory_reason_kv_layout(), - "must be exactly contiguous or paged"}, - {"platform size", Messages::llm_memory_reason_platform_size_range(), - "out of range for a platform size"}, - {"usage", Messages::llm_memory_usage_options("memory_benchmark"), - expected_usage}, - {"command name", Messages::llm_memory_command_name(), - "LLM memory profile"}, - {"report header", - Messages::report_llm_memory_header( - "cpu", "decode", "decode_step", "contiguous"), - "Synthetic LLM memory profile (backend=cpu, phase=decode, " - "work_unit=decode_step, kv_layout=contiguous, warm/cacheable)"}, - {"Metal backend", - Messages::report_llm_memory_metal_backend( - "Fake Apple GPU", std::numeric_limits::max(), true, - false, true, 4294967296ULL, 3221225472ULL), - " Metal device: name=Fake Apple GPU, " - "registry_id=18446744073709551615\n" - " Metal capabilities: apple7=true, unified=false, tier2=true\n" - " Metal limits: max_buffer_length=4294967296 bytes, " - "recommended_working_set=3221225472 bytes"}, - {"Metal resources", - Messages::report_llm_memory_metal_resources( - 2, 3, 4, 268435456, 8192, 805306368, 1073741824, - 2147483648ULL), - " Metal segments: weights=2, K=3, V=4, capacity=268435456 bytes\n" - " Metal argument buffer: encoded_length=8192 bytes\n" - " Metal memory: committed=805306368 bytes, " - "known_peak=1073741824 bytes, admitted_budget=2147483648 bytes"}, - {"Metal valid task", - Messages::report_llm_memory_metal_task( - "mixed", "llm_decode_contiguous_v1", 8, 64, true, true, - 0.00125, true, true, true, true, true, true, true, true), - " Metal task: scenario=mixed, pipeline=llm_decode_contiguous_v1, " - "threadgroups=8, threads_per_threadgroup=64\n" - " Metal timing: gpu_elapsed_seconds=0.001250000\n" - " Metal validation: checksum=valid, kv_write=valid, canary=valid"}, - {"Metal grid", - Messages::report_llm_memory_metal_grid(6, 3, 1596, 1600, 4), - " Metal owner grid: owner_count=6, " - "owner_ordinals_per_threadgroup=3, " - "cost_unit=actual-threadgroup-cost\n" - " Metal threadgroup accounted bytes: minimum=1596, maximum=1600, " - "imbalance=4"}, - {"Metal invalid task without canary", - Messages::report_llm_memory_metal_task( - "kv_only", "llm_decode_contiguous_v1", 1, 32, true, true, 0.5, - true, false, true, true, false, false, false, true), - " Metal task: scenario=kv_only, pipeline=llm_decode_contiguous_v1, " - "threadgroups=1, threads_per_threadgroup=32\n" - " Metal timing: gpu_elapsed_seconds=0.500000000\n" - " Metal validation: checksum=invalid, kv_write=invalid, " - "canary=not-applicable"}, - {"Metal weights task without KV write", - Messages::report_llm_memory_metal_task( - "weights_only", "llm_decode_contiguous_v1", 1, 32, true, true, - 0.25, false, false, false, false, true, false, false, false), - " Metal task: scenario=weights_only, " - "pipeline=llm_decode_contiguous_v1, threadgroups=1, " - "threads_per_threadgroup=32\n" - " Metal timing: gpu_elapsed_seconds=0.250000000\n" - " Metal validation: checksum=not-evaluated, " - "kv_write=not-applicable, canary=not-applicable"}, - {"Metal invalid timing has no numeric observation", - Messages::report_llm_memory_metal_task( - "mixed", "llm_decode_contiguous_v1", 1, 32, true, false, 0.0, - false, false, true, false, false, false, false, false), - " Metal task: scenario=mixed, pipeline=llm_decode_contiguous_v1, " - "threadgroups=1, threads_per_threadgroup=32\n" - " Metal timing: gpu_elapsed_seconds=invalid\n" - " Metal validation: checksum=not-evaluated, kv_write=not-evaluated, " - "canary=not-applicable"}, - {"decode unit", - Messages::report_llm_memory_work_unit_name("decode_step", false), - "decode step"}, - {"decode units", - Messages::report_llm_memory_work_unit_name("decode_step", true), - "decode steps"}, - {"prefill unit", - Messages::report_llm_memory_work_unit_name("prefill_operation", false), - "prefill operation"}, - {"prefill units", - Messages::report_llm_memory_work_unit_name("prefill_operation", true), - "prefill operations"}, - {"unknown unit", - Messages::report_llm_memory_work_unit_name("future", false), - "work unit"}, - {"unknown units", - Messages::report_llm_memory_work_unit_name("future", true), - "work units"}, - {"payload", - Messages::report_llm_memory_payload("decode step", 1024, 768, 256), - " Active weight bytes / decode step: 1024\n" - " KV read bytes / decode step: 768\n" - " KV write bytes / decode step: 256"}, - {"decode geometry", Messages::report_llm_memory_decode_geometry(8, 4.0), - " Visible context tokens: 8\n" - " Traffic crossover: 4.00 visible context tokens"}, - {"prefill geometry", - Messages::report_llm_memory_prefill_geometry( - 5, 2, 3, 11, 15, 60, 480), - " Prompt tokens (P): 5\n" - " Attention query tile tokens (Q): 2\n" - " Attention query tiles (C): 3\n" - " Prefix token visits / sequence: 11\n" - " Causal token pairs / sequence: 15\n" - " Logical attention pairs: 60\n" - " Logical attention FMA terms: 480"}, - {"paged layout", - Messages::report_llm_memory_paged_layout( - {4, 2, 2, 4, 128, 4, 128, 384, 512, 128, 384, 512, 128, - 2, 8, 4096, "permutation-v1", 99, "0123456789abcdef", - "paged-permutation-identity", 20, 80, 1104, "decode step"}), - " Paged KV block tokens (G): 4\n" - " Blocks per sequence (N): 2\n" - " Physical blocks/layer (P_b): 2\n" - " Physical block geometry: total_blocks=4, block_bytes=128\n" - " Terminal block: tokens=4, valid_bytes=128\n" - " K bytes (logical/physical/padding): 384/512/128\n" - " V bytes (logical/physical/padding): 384/512/128\n" - " Block table: 2 uint32 entries, 8 bytes, 4096 page-rounded bytes\n" - " Permutation: version=permutation-v1, seed=99, " - "sha256=0123456789abcdef\n" - " Permutation identity: paged-permutation-identity\n" - " Timed block-table metadata / KV-active decode step: 20 lookups, 80 bytes\n" - " Accounted bytes / KV-active decode step: 1104\n" - " Effective model payload excludes timed block-table metadata bytes."}, - {"weights headline", - Messages::report_llm_memory_scenario_headline( - "Weights only", "decode step", "decode steps", 1.25, 800.0, - 100.5, false), - " Weights only: 1.250 ms/decode step, " - "100.50 GB/s effective model payload"}, - {"mixed headline", - Messages::report_llm_memory_scenario_headline( - "Mixed", "decode step", "decode steps", 3.75, 266.666, 80.25, - true), - " Mixed: 3.750 ms/decode step, 266.67 synthetic decode " - "steps/s, 80.25 GB/s effective model payload"}, - {"weights label", - Messages::report_llm_memory_scenario_name("weights_only"), - "Weights only"}, - {"KV label", Messages::report_llm_memory_scenario_name("kv_only"), - "KV only"}, - {"mixed label", Messages::report_llm_memory_scenario_name("mixed"), - "Mixed"}, - {"unknown label", Messages::report_llm_memory_scenario_name("future"), - "Unknown"}, - {"interpretation", - Messages::report_llm_memory_interpretation_note( - "decode", "contiguous", "decode step"), - " Interpretation: each decode step is synthetic memory-only work, not an inference " - "token; effective model payload is logical, not physical DRAM-counter traffic.\n" - " Phase/layout: phase=decode, kv_layout=contiguous; the visible context includes the " - "current-token slot; KV uses layer/batch/token/head/dimension order.\n" - " Crossover: logical weight/KV-read payload equality is not a proven hardware " - "bottleneck transition.\n" - " Comparability: small weight or KV working sets can be cache-dominant; order imbalance, " - "high CV, non-nominal environment, QoS failures, or off-target duration reduce confidence."}, - {"prefill interpretation", - Messages::report_llm_memory_interpretation_note("prefill", "contiguous", "prefill operation"), - " Interpretation: each prefill operation is synthetic memory-only work, not an inference " - "token; effective model payload is logical, not physical DRAM-counter traffic.\n" - " Phase/layout: phase=prefill, kv_layout=contiguous; prefill performs no Transformer " - "compute and does not predict TTFT; KV uses layer/batch/token/head/dimension order.\n" - " Comparability: small weight or KV working sets can be cache-dominant; order imbalance, " - "high CV, non-nominal environment, QoS failures, or off-target duration reduce confidence."}, - {"high CV", Messages::warning_llm_memory_high_cv("kv_only", 6.25, 5.0), - "LLM KV only repeatability CV 6.25% exceeds 5.00%"}, - {"unbalanced order", Messages::warning_llm_memory_order_not_balanced(), - "LLM scenario order is not fully balanced across completed loops"}, - {"duration", - Messages::warning_llm_memory_duration_quality( - "mixed", "above-target-single-work-unit"), - "LLM Mixed duration quality is above-target-single-work-unit"}, - {"environment", Messages::warning_llm_memory_environment_not_nominal(), - "LLM result environment is not reference-eligible (thermal state or Low Power Mode)"}, - {"main QoS", - Messages::warning_llm_memory_main_thread_qos_not_applied(7), - "LLM main-thread QoS request was not applied (code: 7)"}, - {"worker QoS", Messages::warning_llm_memory_worker_qos_not_applied(), - "One or more LLM worker QoS requests were not applied"}, - {"cache", - Messages::warning_llm_memory_weight_cache_dominant(1024, 4096), - "LLM weight working set (1024 bytes) does not exceed reported L2 cache " - "(4096 bytes); the result may be cache-dominant"}, - {"KV cache", Messages::warning_llm_memory_kv_cache_dominant(2048, 4096), - "LLM KV working set (2048 bytes) does not exceed reported L2 cache " - "(4096 bytes); the result may be cache-dominant"}, + {"positive integer", Messages::llm_memory_reason_positive_integer(), "must be a positive integer"}, + {"backend", Messages::llm_memory_reason_backend(), "must be exactly cpu or metal"}, + {"KV width", Messages::llm_memory_reason_kv_element_bytes(), "must be exactly 1, 2, or 4"}, + {"phase", Messages::llm_memory_reason_phase(), "must be exactly decode or prefill"}, + {"KV layout", Messages::llm_memory_reason_kv_layout(), "must be exactly contiguous or paged"}, + {"platform size", Messages::llm_memory_reason_platform_size_range(), "out of range for a platform size"}, + {"usage", Messages::llm_memory_usage_options("memory_benchmark"), expected_usage}, + {"command name", Messages::llm_memory_command_name(), "LLM memory profile"}, + }; expect_exact_messages(cases); } @@ -525,46 +259,6 @@ TEST(MessagesTest, LlmPrefillModelContextDistinguishesUnavailableFromKnownZero) " Logical attention FMA terms: 0"); } -TEST(MessagesTest, GeneralHelpAdvertisesTheLlmBoundaryExactlyOnce) { - const std::string usage = Messages::usage_options("memory_benchmark"); - EXPECT_NE(usage.find("Platform: macOS 26 or later on Apple Silicon (ARM64)."), - std::string::npos); - EXPECT_NE(usage.find("-M, --llm-memory"), std::string::npos); - EXPECT_EQ(usage.find("-M, --llm-memory"), - usage.rfind("-M, --llm-memory")); - EXPECT_NE( - usage.find("standalone CPU/Metal synthetic LLM memory profile"), - std::string::npos); - EXPECT_NE(usage.find("Both phases support contiguous"), - std::string::npos); - EXPECT_NE( - usage.find( - "or paged KV on CPU and Metal. Metal is runtime-capability-gated"), - std::string::npos); - EXPECT_NE(usage.find("and never falls back to CPU"), - std::string::npos); - EXPECT_NE(usage.find("--weight-size-mb"), std::string::npos); - EXPECT_NE(usage.find("--query-heads"), std::string::npos); - EXPECT_NE(usage.find("--context-tokens"), std::string::npos); - EXPECT_NE(usage.find("memory-only interpretation"), std::string::npos); - EXPECT_NE(usage.find("JSON uses schema 2 methodologies "), - std::string::npos); - for (const char* selector : { - "llm-memory-v2-cpu-decode-contiguous", "llm-memory-v2-cpu-decode-paged", - "llm-memory-v2-cpu-prefill-contiguous", "llm-memory-v2-cpu-prefill-paged", - "llm-memory-v2-metal-decode-contiguous", "llm-memory-v2-metal-decode-paged", - "llm-memory-v2-metal-prefill-contiguous", "llm-memory-v2-metal-prefill-paged"}) { - SCOPED_TRACE(selector); - EXPECT_NE(usage.find(selector), std::string::npos); - } - EXPECT_NE(usage.find("Metal LLM-memory rejects --threads"), - std::string::npos); - EXPECT_NE(usage.find("LLM-memory snapshots every K=max(1,ceil(count/8)) loops"), - std::string::npos); - EXPECT_NE(usage.find("and at command terminal (abrupt loss bound: 3K completed attempts)."), - std::string::npos); -} - TEST(MessagesErrorTest, GpuMessagesHaveExactMethodologyOutput) { const std::string expected_usage = "Usage: memory_benchmark --gpu-bandwidth [options]\n" @@ -588,17 +282,12 @@ TEST(MessagesErrorTest, GpuMessagesHaveExactMethodologyOutput) { " An empty value disables JSON for this direct command.\n" " -h, --help Show this GPU-mode help and exit\n"; const std::vector cases = { - {"mode isolation", Messages::error_gpu_bandwidth_must_be_used_alone(), - "--gpu-bandwidth allows only optional -b/--buffer-size , " - "-i/--iterations , -r/--count , -o/--output , " - "--seed , and -h/--help (no other options allowed)"}, {"minimum buffer", Messages::error_gpu_buffer_size_below_minimum(32, 64), "GPU buffer-size must be at least 64 MB (got 32 MB)"}, {"iteration guard", Messages::error_gpu_iterations_exceed_limit(513, 512), "GPU iterations exceed the exact-work guardrail (requested 513, maximum 512)"}, {"run failure", Messages::error_gpu_run_failed("test-reason"), "GPU memory bandwidth benchmark failed (reason_code=test-reason)"}, - {"unknown device", Messages::gpu_unknown_device_name(), "unknown Apple GPU"}, {"usage", Messages::gpu_usage_options("memory_benchmark"), expected_usage}, {"report header", Messages::report_gpu_bandwidth_header("Apple M4", 3, true), "GPU memory bandwidth (Apple M4, private/tracked, 3 loops; headline: median)"}, @@ -606,180 +295,22 @@ TEST(MessagesErrorTest, GpuMessagesHaveExactMethodologyOutput) { " Copy: 123.46 GB/s (aggregate read + write payload)"}, {"repeatability", Messages::report_gpu_bandwidth_repeatability(1.0, 2.0, 3.0, true), " Repeatability: read CV 1.00%, write CV 2.00%, copy CV 3.00%"}, - {"interpretation", Messages::report_gpu_bandwidth_interpretation_note(), - " Note: copy is aggregate read+write throughput; DRAM residency is unverified. " - "Results can remain cache/dispatch-dominant even at the 64 MB methodology minimum."}, {"high CV", Messages::warning_gpu_high_cv("read", 5.1, 5.0), "GPU read repeatability CV 5.10% exceeds 5.00%"}, - {"unbalanced order", Messages::warning_gpu_order_not_balanced(), - "GPU operation order is not fully balanced across completed loops"}, {"duration quality", Messages::warning_gpu_duration_quality("write", "payload-cap-below-target"), "GPU write duration quality is payload-cap-below-target"}, - {"environment", Messages::warning_gpu_environment_not_nominal(), - "GPU result environment is not reference-eligible (thermal state or Low Power Mode)"}, - {"working set", Messages::warning_gpu_recommended_working_set_exceeded(), - "GPU allocation exceeds Metal's advisory recommended working-set size"}, - }; - expect_exact_messages(cases); -} - -TEST(MessagesErrorTest, ErrorMadviseFailed) { - std::string msg = Messages::error_madvise_failed("lat_buffer"); - EXPECT_EQ(msg, "madvise failed for lat_buffer"); -} - -TEST(MessagesErrorTest, ErrorBenchmarkTests) { - std::string msg = Messages::error_benchmark_tests("test failure"); - EXPECT_EQ(msg, "Error during benchmark tests: test failure"); -} - -TEST(MessagesErrorTest, ErrorOnlyFlagsRequireBenchmark) { - const std::string& msg = Messages::error_only_flags_require_benchmark(); - EXPECT_NE(msg.find("--only-bandwidth"), std::string::npos); - EXPECT_NE(msg.find("--only-latency"), std::string::npos); - EXPECT_NE(msg.find("--benchmark"), std::string::npos); -} - -TEST(MessagesErrorTest, ErrorSweepMessages) { - const std::vector cases = { - {"missing parameter", Messages::error_sweep_requires_parameter(), - "--sweep requires at least one parameter specification"}, - {"missing output", Messages::error_sweep_requires_output(), - "--sweep requires --output for the combined JSON result"}, - {"run cap", Messages::error_sweep_too_many_runs(12, 10), - "Sweep would generate 12 runs, exceeding --sweep-max-runs 10"}, - {"parameter not allowed", Messages::error_sweep_parameter_not_allowed("cache-size", "--patterns"), - "Sweep parameter 'cache-size' is not allowed with --patterns"}, }; expect_exact_messages(cases); } -// ============================================================================ -// Warning Messages Tests -// ============================================================================ - -TEST(MessagesWarningTest, WarningBufferSizeExceedsLimit) { - std::string msg = Messages::warning_buffer_size_exceeds_limit(2048, 1024); - EXPECT_NE(msg.find("2048"), std::string::npos); - EXPECT_NE(msg.find("1024"), std::string::npos); - EXPECT_NE(msg.find("Requested buffer size"), std::string::npos); -} - -TEST(MessagesWarningTest, WarningTlbMlockFailureIsBestEffort) { - const std::string msg = Messages::warning_tlb_mlock_failed(12, "Cannot allocate memory"); - EXPECT_NE(msg.find("mlock()"), std::string::npos); - EXPECT_NE(msg.find("errno 12"), std::string::npos); - EXPECT_NE(msg.find("continuing"), std::string::npos); -} - -// ============================================================================ -// Info Messages Tests (using fixture) -// ============================================================================ - -TEST(MessagesInfoTest, InfoSettingMaxFallback) { - std::string msg = Messages::info_setting_max_fallback(2048); - EXPECT_NE(msg.find("2048"), std::string::npos); - EXPECT_NE(msg.find("MB"), std::string::npos); - EXPECT_NE(msg.find("Info"), std::string::npos); -} - -TEST(MessagesInfoTest, InfoCalculatedMaxLessThanMin) { - std::string msg = Messages::info_calculated_max_less_than_min(512, 1024); - EXPECT_NE(msg.find("512"), std::string::npos); - EXPECT_NE(msg.find("1024"), std::string::npos); -} - -TEST(MessagesInfoTest, InfoCustomCacheRoundedUp) { - std::string msg = Messages::info_custom_cache_rounded_up(250, 256); - EXPECT_NE(msg.find("250"), std::string::npos); - EXPECT_NE(msg.find("256"), std::string::npos); - EXPECT_NE(msg.find("KB"), std::string::npos); -} - -// ============================================================================ -// Main Program Messages Tests -// ============================================================================ - -TEST(MessagesFormattingTest, LiteralMessagesHaveExactOutput) { - const std::vector cases = { - {"timer creation error", Messages::error_timer_creation_failed(), - "Failed to create high-resolution timer. Exiting."}, - {"memory fallback warning", Messages::warning_cannot_get_memory(), - "Cannot get available memory. Using fallback limit."}, - {"interrupted", Messages::msg_interrupted_by_user(), "\nInterrupted by user. Partial results shown."}, - {"copyright", Messages::config_copyright(), "Copyright 2025-2026 Timo Heimonen "}, - {"processor error", Messages::config_processor_name_error(), "Could not retrieve processor name."}, - {"cache header", Messages::cache_info_header(), "\nDetected Cache Sizes:"}, - {"main latency methodology", Messages::results_main_memory_latency(), - "\nMain Memory Latency Test (single-threaded, pointer chase):"}, - {"cache latency methodology", Messages::results_cache_latency(), - "\nCache Latency Tests (single-threaded, pointer chase):"}, - {"custom cache", Messages::results_custom_cache(), " Custom Cache:"}, - {"L1 cache", Messages::results_l1_cache(), " L1 Cache:"}, - {"L2 cache", Messages::results_l2_cache(), " L2 Cache:"}, - {"results separator", Messages::results_separator(), "--------------"}, - {"statistics cache read", Messages::statistics_cache_read(), " Read:"}, - {"statistics cache write", Messages::statistics_cache_write(), " Write:"}, - {"statistics cache copy", Messages::statistics_cache_copy(), " Copy:"}, - {"statistics cache latency header", Messages::statistics_cache_latency_header(), "\nCache Latency (ns):"}, - {"statistics main latency header", Messages::statistics_main_memory_latency_header(), - "\nMain Memory Latency (ns):"}, - {"statistics footer", Messages::statistics_footer(), "----------------------------------"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesFormattingTest, MsgSweepProgress) { - std::string running = Messages::msg_running_sweep(3); - EXPECT_NE(running.find("Running sweep"), std::string::npos); - EXPECT_NE(running.find("3"), std::string::npos); - - std::string progress = Messages::msg_sweep_run_progress(2, 5); - EXPECT_NE(progress.find("2/5"), std::string::npos); -} - -TEST(MessagesFormattingTest, MsgDoneTotalTime) { - const std::vector cases = { +TEST(MessagesFormattingTest, ProgressMessagesHaveExactOutput) { + expect_exact_messages({ {"ordinary duration", Messages::msg_done_total_time(123.456), "\nDone. Total execution time: 123.45600 s"}, {"short duration", Messages::msg_done_total_time(0.001), "\nDone. Total execution time: 0.00100 s"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesFormattingTest, MsgResultsSavedTo) { - std::string msg = Messages::msg_results_saved_to("results.json"); - EXPECT_NE(msg.find("Results saved to"), std::string::npos); - EXPECT_NE(msg.find("results.json"), std::string::npos); - - msg = Messages::msg_results_saved_to("/tmp/out/bench.json"); - EXPECT_NE(msg.find("/tmp/out/bench.json"), std::string::npos); -} - -TEST(MessagesFormattingTest, MsgPatternBenchmarkLoopCompleted) { - std::string msg = Messages::msg_pattern_benchmark_loop_completed(3, 10); - EXPECT_NE(msg.find("3"), std::string::npos); - EXPECT_NE(msg.find("10"), std::string::npos); - EXPECT_NE(msg.find("Pattern benchmarks"), std::string::npos); - EXPECT_NE(msg.find("completed"), std::string::npos); - - // Verify the loop separator is present - EXPECT_NE(msg.find("3/10"), std::string::npos); -} - -TEST(MessagesFormattingTest, MsgTlbAnalysisRefinementStart) { - std::string msg = Messages::msg_tlb_analysis_refinement_start(20); - EXPECT_NE(msg.find("Starting refinement sweep"), std::string::npos); - EXPECT_NE(msg.find("20"), std::string::npos); - EXPECT_NE(msg.find("points"), std::string::npos); -} - -TEST(MessagesFormattingTest, MsgTlbAnalysisValidationStart) { - const std::string msg = Messages::msg_tlb_analysis_validation_start(8); - EXPECT_NE(msg.find("validation"), std::string::npos); - EXPECT_NE(msg.find("8"), std::string::npos); + {"sweep start", Messages::msg_running_sweep(3), "\nRunning sweep with 3 runs..."}, + {"sweep progress", Messages::msg_sweep_run_progress(2, 5), "\nSweep run 2/5"}, + }); } TEST(MessagesFormattingTest, ReportTlbStatisticalConfidence) { @@ -813,27 +344,6 @@ TEST(MessagesFormattingTest, ReportTlbLatencyNormalizesNegativeZero) { EXPECT_NE(latency.find("16 KiB"), std::string::npos); } -TEST(MessagesFormattingTest, AnalyzeTlbStrideGuardMessages) { - const std::string exceeds = Messages::error_analyze_tlb_stride_exceeds_page(32768, 16384); - EXPECT_NE(exceeds.find("--analyze-tlb"), std::string::npos); - EXPECT_NE(exceeds.find("32768"), std::string::npos); - EXPECT_NE(exceeds.find("16384"), std::string::npos); -} - -TEST(MessagesFormattingTest, TlbChainSetupFailureMessage) { - const std::string msg = Messages::error_tlb_chain_setup_failed(256, "packed", "integrity-failure", "early-cycle"); - EXPECT_NE(msg.find("packed"), std::string::npos); - EXPECT_NE(msg.find("256"), std::string::npos); - EXPECT_NE(msg.find("integrity-failure"), std::string::npos); - EXPECT_NE(msg.find("early-cycle"), std::string::npos); -} - -TEST(MessagesFormattingTest, DuplicateSweepParameterMessage) { - const std::string msg = Messages::error_duplicate_sweep_parameter("latency-stride-bytes"); - EXPECT_NE(msg.find("more than once"), std::string::npos); - EXPECT_NE(msg.find("latency-stride-bytes"), std::string::npos); -} - TEST(MessagesFormattingTest, ReportTlbAnalysisStatusAndSuppressedConclusions) { const std::string status = Messages::report_tlb_analysis_status("interrupted", 29, 7, false); EXPECT_NE(status.find("interrupted"), std::string::npos); @@ -855,22 +365,24 @@ TEST(MessagesFormattingTest, ReportTlbRunAndResourceSummariesAreCompact) { EXPECT_NE(run.find("seed 570001 (user)"), std::string::npos); EXPECT_EQ(run.find('\n'), std::string::npos); - const std::string resources = - Messages::report_tlb_resource_summary(1024, true, true, true, 0, 2048, 1041 * Constants::BYTES_PER_MB); - EXPECT_NE(resources.find("1024 MiB buffer (locked)"), std::string::npos); - EXPECT_NE(resources.find("QoS applied"), std::string::npos); - EXPECT_NE(resources.find("estimated peak/budget 1041.0/2048 MiB"), std::string::npos); -} - -TEST(MessagesFormattingTest, ReportTlbResourceSummaryRetainsFailures) { - const std::string failed = - Messages::report_tlb_resource_summary(256, false, true, false, 6, 512, 300 * Constants::BYTES_PER_MB); - EXPECT_NE(failed.find("unlocked"), std::string::npos); - EXPECT_NE(failed.find("failed (code 6; best-effort)"), std::string::npos); - - const std::string not_requested = - Messages::report_tlb_resource_summary(256, false, false, false, 0, 512, 300 * Constants::BYTES_PER_MB); - EXPECT_NE(not_requested.find("QoS not requested"), std::string::npos); + struct ResourceCase { + std::string report; + std::vector fragments; + }; + const ResourceCase cases[] = { + {Messages::report_tlb_resource_summary(1024, true, true, true, 0, 2048, 1041 * Constants::BYTES_PER_MB), + {"1024 MiB buffer (locked)", "QoS applied", "estimated peak/budget 1041.0/2048 MiB"}}, + {Messages::report_tlb_resource_summary(256, false, true, false, 6, 512, 300 * Constants::BYTES_PER_MB), + {"unlocked", "failed (code 6; best-effort)"}}, + {Messages::report_tlb_resource_summary(256, false, false, false, 0, 512, 300 * Constants::BYTES_PER_MB), + {"QoS not requested"}}, + }; + for (const ResourceCase& test_case : cases) { + for (const std::string& fragment : test_case.fragments) { + SCOPED_TRACE(fragment); + EXPECT_NE(test_case.report.find(fragment), std::string::npos); + } + } } TEST(MessagesFormattingTest, ReportTlbWorkEstimateAndCompletionHaveExpectedContent) { @@ -897,18 +409,15 @@ TEST(MessagesFormattingTest, ReportTlbSweepPlanUsesIecUnits) { EXPECT_NE(disabled.find("selected 256 MiB"), std::string::npos); } -TEST(MessagesFormattingTest, ReportTlbLargeLocalityUnavailable) { - const std::string msg = Messages::report_tlb_large_locality_paired_unavailable(512, 256); - EXPECT_NE(msg.find("Large-Locality Paired Comparison"), std::string::npos); - EXPECT_NE(msg.find("N/A"), std::string::npos); - EXPECT_NE(msg.find("requires 512 MiB"), std::string::npos); - EXPECT_NE(msg.find("selected 256 MiB"), std::string::npos); -} - -TEST(MessagesFormattingTest, ReportTlbLargeLocalityInterrupted) { - const std::string msg = Messages::report_tlb_large_locality_paired_interrupted(); - EXPECT_NE(msg.find("N/A"), std::string::npos); - EXPECT_NE(msg.find("did not complete"), std::string::npos); +TEST(MessagesFormattingTest, ReportTlbLargeLocalityUnavailableReasons) { + expect_exact_messages({ + {"insufficient buffer", Messages::report_tlb_large_locality_paired_unavailable(512, 256), + "[Large-Locality Paired Comparison]\n Result: N/A (requires 512 MiB or larger analysis buffer, selected " + "256 MiB)"}, + {"interrupted", Messages::report_tlb_large_locality_paired_interrupted(), + "[Large-Locality Paired Comparison]\n Result: N/A (analysis incomplete or comparison measurement did " + "not complete)"}, + }); } TEST(MessagesFormattingTest, ReportTlbLargeLocalityPairedComparisonIsExplicit) { @@ -954,142 +463,132 @@ TEST(MessagesFormattingTest, TlbPrivateCacheMessagesHaveExactOutput) { expect_exact_messages(cases); } -// ============================================================================ -// Usage/Help Messages Tests (using formatting fixture) -// ============================================================================ - -TEST(MessagesFormattingTest, UsageHeader) { - std::string msg = Messages::usage_header("1.0.0"); - EXPECT_NE(msg.find("1.0.0"), std::string::npos); - EXPECT_NE(msg.find("Timo Heimonen"), std::string::npos); - EXPECT_NE(msg.find("GNU GPL"), std::string::npos); - EXPECT_NE(msg.find("github.com"), std::string::npos); -} - TEST(MessagesFormattingTest, UsageOptions) { - std::string msg = Messages::usage_options("memory_benchmark"); const std::string standard_schema_contract = " JSON uses standard schema " + std::to_string(Constants::BENCHMARK_JSON_SCHEMA_VERSION) + " methodology " + Constants::BENCHMARK_METHODOLOGY_VERSION + ".\n"; - EXPECT_NE(msg.find("memory_benchmark"), std::string::npos); - EXPECT_NE(msg.find("--benchmark"), std::string::npos); - EXPECT_NE(msg.find(standard_schema_contract), std::string::npos); - EXPECT_NE(msg.find("100-300 ms window"), std::string::npos); - EXPECT_NE(msg.find("--iterations"), std::string::npos); - EXPECT_NE(msg.find("--buffer-size"), std::string::npos); - EXPECT_NE(msg.find("--count"), std::string::npos); - EXPECT_NE(msg.find("--analyze-tlb"), std::string::npos); - EXPECT_NE(msg.find("schema 4"), std::string::npos); - EXPECT_NE(msg.find("exact string seeds"), std::string::npos); - EXPECT_NE(msg.find("scoped counters"), std::string::npos); - EXPECT_NE(msg.find("--tlb-density"), std::string::npos); - EXPECT_NE(msg.find("default: medium"), std::string::npos); - EXPECT_NE(msg.find("--analyze-tlb: 16"), std::string::npos); - EXPECT_NE(msg.find("calibrate toward 150 ms"), std::string::npos); - EXPECT_NE(msg.find("Reproducible workload/schedule seed for --benchmark, --patterns"), std::string::npos); - EXPECT_NE(msg.find("--gpu-bandwidth"), std::string::npos); - EXPECT_NE(msg.find(Constants::GPU_METHODOLOGY_VERSION), std::string::npos); - EXPECT_NE(msg.find("minimum buffer size is 64 MB"), std::string::npos); - EXPECT_NE(msg.find("standalone CPU/Metal synthetic LLM memory profile"), - std::string::npos); - EXPECT_NE(msg.find("prefill requires --phase prefill, --prompt-tokens"), std::string::npos); - EXPECT_NE(msg.find("--attention-query-tile-tokens"), std::string::npos); - EXPECT_NE(msg.find("Both phases support contiguous"), std::string::npos); - EXPECT_NE(msg.find("llm-memory-v2-cpu-prefill-contiguous"), std::string::npos); - EXPECT_NE(msg.find(Constants::LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION), std::string::npos); - EXPECT_NE(msg.find("--analyze-core2core"), std::string::npos); - EXPECT_NE(msg.find("acquire/release token-handoff"), std::string::npos); - EXPECT_NE(msg.find("protocol, coherence, and scheduler effects"), std::string::npos); - EXPECT_NE(msg.find("core-to-core schema 2"), std::string::npos); - EXPECT_NE(msg.find("target 250 ms"), std::string::npos); - EXPECT_NE(msg.find("Defaults to 3 loops"), std::string::npos); - EXPECT_NE(msg.find("--latency-samples"), std::string::npos); - EXPECT_NE(msg.find("--latency-stride-bytes"), std::string::npos); - EXPECT_NE(msg.find("--latency-chain-mode"), std::string::npos); - EXPECT_NE(msg.find("--latency-tlb-locality-kb"), std::string::npos); - EXPECT_NE(msg.find("Locality-using modes require"), std::string::npos); - EXPECT_NE(msg.find("explicit global-random ignores"), std::string::npos); - EXPECT_NE(msg.find("cache bandwidth uses one"), std::string::npos); - EXPECT_NE(msg.find("keeps an explicit request uncapped"), - std::string::npos); - EXPECT_NE(msg.find("latency remains"), std::string::npos); - EXPECT_NE(msg.find("single-threaded"), std::string::npos); - EXPECT_NE(msg.find("--cache-size"), std::string::npos); - EXPECT_NE(msg.find("--output "), std::string::npos); - EXPECT_NE(msg.find("Exact - writes one final JSON document to stdout"), std::string::npos); - EXPECT_NE(msg.find("one final"), std::string::npos); - EXPECT_NE(msg.find("every result-producing direct mode and CPU sweep"), - std::string::npos); - EXPECT_NE(msg.find("routes human output to stderr"), std::string::npos); - EXPECT_NE(msg.find("an empty value disables JSON for direct commands"), - std::string::npos); - EXPECT_NE(msg.find("invalid for sweeps"), std::string::npos); - EXPECT_NE(msg.find("Every other non-empty value is a file"), - std::string::npos); - EXPECT_NE(msg.find("including ./- and names such as -G"), - std::string::npos); - EXPECT_NE(msg.find("Standard, GPU, and LLM-memory files retain"), - std::string::npos); - EXPECT_NE(msg.find("LLM-memory snapshots every K=max(1,ceil(count/8)) loops"), - std::string::npos); - EXPECT_NE(msg.find("Sweep files checkpoint attempts"), std::string::npos); - EXPECT_NE(msg.find("Requires --output "), std::string::npos); - EXPECT_NE(msg.find("-h"), std::string::npos); - // Check that default values are included - EXPECT_NE(msg.find(std::to_string(Constants::DEFAULT_ITERATIONS)), std::string::npos); - EXPECT_NE(msg.find(std::to_string(Constants::DEFAULT_BUFFER_SIZE_MB)), std::string::npos); - EXPECT_NE(msg.find(std::to_string(Constants::DEFAULT_LOOP_COUNT)), std::string::npos); - EXPECT_NE(msg.find(std::to_string(Constants::DEFAULT_LATENCY_SAMPLE_COUNT)), std::string::npos); - EXPECT_NE(msg.find(std::to_string(Constants::MIN_CACHE_SIZE_KB)), std::string::npos); - EXPECT_NE(msg.find(std::to_string(Constants::MAX_CACHE_SIZE_KB)), std::string::npos); -} - -TEST(MessagesFormattingTest, UsageExample) { - std::string msg = Messages::usage_example("memory_benchmark"); - EXPECT_NE(msg.find("memory_benchmark"), std::string::npos); - EXPECT_NE(msg.find("--iterations"), std::string::npos); - EXPECT_NE(msg.find("--buffer-size"), std::string::npos); - EXPECT_NE(msg.find("--output"), std::string::npos); - EXPECT_NE(msg.find("Machine JSON:"), std::string::npos); - EXPECT_NE(msg.find("--only-bandwidth"), std::string::npos); - EXPECT_NE(msg.find("--output -"), std::string::npos); -} - -// ============================================================================ -// Configuration Output Messages Tests (using formatting fixture) -// ============================================================================ - -TEST(MessagesFormattingTest, ConfigHeader) { - std::string msg = Messages::config_header("1.0.0"); - EXPECT_NE(msg.find("1.0.0"), std::string::npos); - EXPECT_NE(msg.find("macOS-memory-benchmark"), std::string::npos); -} - -TEST(MessagesFormattingTest, ConfigLicense) { - std::string msg = Messages::config_license(); - EXPECT_NE(msg.find("free software"), std::string::npos); - EXPECT_NE(msg.find("GNU General Public License"), std::string::npos); - EXPECT_NE(msg.find("version 3"), std::string::npos); -} - -TEST(MessagesFormattingTest, ConfigBufferSize) { - std::string msg = Messages::config_buffer_size(1024.5, 1024); - EXPECT_NE(msg.find("1024.50"), std::string::npos); - EXPECT_NE(msg.find("1024"), std::string::npos); - EXPECT_NE(msg.find("MiB"), std::string::npos); - EXPECT_NE(msg.find("MB"), std::string::npos); + struct HelpCase { + std::string text; + std::vector fragments; + }; + const HelpCase cases[] = { + {Messages::usage_options("memory_benchmark"), + { + "memory_benchmark", + "--benchmark", + standard_schema_contract, + "100-300 ms window", + "--iterations", + "--buffer-size", + "--count", + "--analyze-tlb", + "schema 4", + "exact string seeds", + "scoped counters", + "--tlb-density", + "default: medium", + "--analyze-tlb: 16", + "calibrate toward 150 ms", + "Reproducible workload/schedule seed for --benchmark, --patterns", + "--gpu-bandwidth", + Constants::GPU_METHODOLOGY_VERSION, + "minimum buffer size is 64 MB", + "standalone CPU/Metal synthetic LLM memory profile", + "prefill requires --phase prefill, --prompt-tokens", + "--attention-query-tile-tokens", + "Both phases support contiguous", + "llm-memory-v2-cpu-prefill-contiguous", + Constants::LLM_CPU_PREFILL_PAGED_METHODOLOGY_VERSION, + "--analyze-core2core", + "acquire/release token-handoff", + "protocol, coherence, and scheduler effects", + "core-to-core schema 2", + "target 250 ms", + "Defaults to 3 loops", + "--latency-samples", + "--latency-stride-bytes", + "--latency-chain-mode", + "--latency-tlb-locality-kb", + "Locality-using modes require", + "explicit global-random ignores", + "cache bandwidth uses one", + "keeps an explicit request uncapped", + "latency remains", + "single-threaded", + "--cache-size", + "--output ", + "Exact - writes one final JSON document to stdout", + "one final", + "every result-producing direct mode and CPU sweep", + "routes human output to stderr", + "an empty value disables JSON for direct commands", + "invalid for sweeps", + "Every other non-empty value is a file", + "including ./- and names such as -G", + "Standard, GPU, and LLM-memory files retain", + "LLM-memory snapshots every K=max(1,ceil(count/8)) loops", + "Sweep files checkpoint attempts", + "Requires --output ", + "-h", + std::to_string(Constants::DEFAULT_ITERATIONS), + std::to_string(Constants::DEFAULT_BUFFER_SIZE_MB), + std::to_string(Constants::DEFAULT_LOOP_COUNT), + std::to_string(Constants::DEFAULT_LATENCY_SAMPLE_COUNT), + std::to_string(Constants::MIN_CACHE_SIZE_KB), + std::to_string(Constants::MAX_CACHE_SIZE_KB), + "Platform: macOS 26 or later on Apple Silicon (ARM64).", + "-M, --llm-memory", + "or paged KV on CPU and Metal. Metal is runtime-capability-gated", + "and never falls back to CPU", + "--weight-size-mb", + "--query-heads", + "--context-tokens", + "memory-only interpretation", + "JSON uses schema 2 methodologies ", + "Metal LLM-memory rejects --threads", + "and at command terminal (abrupt loss bound: 3K completed attempts).", + "llm-memory-v2-cpu-decode-contiguous", + "llm-memory-v2-cpu-decode-paged", + "llm-memory-v2-cpu-prefill-paged", + "llm-memory-v2-metal-decode-contiguous", + "llm-memory-v2-metal-decode-paged", + "llm-memory-v2-metal-prefill-contiguous", + "llm-memory-v2-metal-prefill-paged", + }}, + {Messages::usage_header("1.0.0"), {"1.0.0", "Timo Heimonen", "GNU GPL", "github.com"}}, + {Messages::usage_example("memory_benchmark"), + {"memory_benchmark", "--iterations", "--buffer-size", "--output", "Machine JSON:", "--only-bandwidth", + "--output -"}}, + }; + for (const HelpCase& test_case : cases) { + for (const std::string& fragment : test_case.fragments) { + SCOPED_TRACE(fragment); + EXPECT_NE(test_case.text.find(fragment), std::string::npos); + } + } + const std::string usage = Messages::usage_options("memory_benchmark"); + EXPECT_EQ(usage.find("-M, --llm-memory"), usage.rfind("-M, --llm-memory")); } -TEST(MessagesFormattingTest, ConfigTotalAllocation) { - std::string msg = Messages::config_total_allocation(3072.75); - EXPECT_NE(msg.find("3072.75"), std::string::npos); - EXPECT_NE(msg.find("MiB"), std::string::npos); - EXPECT_NE(msg.find("Peak Concurrent Allocation"), std::string::npos); +TEST(MessagesFormattingTest, ConfigBannerHasExactOutput) { + expect_exact_messages({ + {"header", Messages::config_header("1.0.0"), "----- macOS-memory-benchmark v1.0.0 -----"}, + {"copyright", Messages::config_copyright(), "Copyright 2025-2026 Timo Heimonen "}, + {"license", Messages::config_license(), + "This program is free software: you can redistribute it and/or modify\n" + "it under the terms of the GNU General Public License as published by\n" + "the Free Software Foundation, either version 3 of the License, or\n" + "(at your option) any later version.\n" + "This program is distributed in the hope that it will be useful,\n" + "but WITHOUT ANY WARRANTY; without even the implied warranty of\n" + "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n" + "See for more details.\n"}, + }); } TEST(MessagesFormattingTest, ScalarConfigMessagesHaveExactOutput) { - const std::vector cases = { + expect_exact_messages({ {"iterations", Messages::config_iterations(1000), "Iterations (per R/W/Copy test per loop): 1000"}, {"loop count", Messages::config_loop_count(5), "Loop Count (total benchmark repetitions): 5"}, {"latency stride", Messages::config_latency_stride(136), "Latency Stride: 136 B"}, @@ -1100,103 +599,36 @@ TEST(MessagesFormattingTest, ScalarConfigMessagesHaveExactOutput) { {"efficiency cores", Messages::config_efficiency_cores(2), " Efficiency Cores: 2"}, {"total cores", Messages::config_total_cores(10), " Total CPU Cores Detected: 10"}, {"benchmark threads", Messages::config_benchmark_threads(4), " Benchmark Threads Requested: 4"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesFormattingTest, ConfigNonCacheable) { - // Test enabled - std::string msg = Messages::config_non_cacheable(true); - EXPECT_NE(msg.find("Non-Cacheable Memory Hints"), std::string::npos); - EXPECT_NE(msg.find("Enabled"), std::string::npos); - - // Test disabled - msg = Messages::config_non_cacheable(false); - EXPECT_NE(msg.find("Non-Cacheable Memory Hints"), std::string::npos); - EXPECT_NE(msg.find("Disabled"), std::string::npos); -} - -TEST(MessagesFormattingTest, ConfigLatencyTlbLocality) { - std::string msg = Messages::config_latency_tlb_locality(0); - EXPECT_NE(msg.find("Latency TLB Locality"), std::string::npos); - EXPECT_NE(msg.find("Global random"), std::string::npos); - - msg = Messages::config_latency_tlb_locality(16 * 1024); - EXPECT_NE(msg.find("16.00"), std::string::npos); - EXPECT_NE(msg.find("KB"), std::string::npos); -} - -// ============================================================================ -// Cache Info Messages Tests (using formatting fixture) -// ============================================================================ - -TEST(MessagesFormattingTest, CacheSizeCustom) { - // Test bytes - std::string msg = Messages::cache_size_custom(512); - EXPECT_NE(msg.find("512"), std::string::npos); - EXPECT_NE(msg.find("B"), std::string::npos); - - // Test KB - msg = Messages::cache_size_custom(256 * 1024); - EXPECT_NE(msg.find("256"), std::string::npos); - EXPECT_NE(msg.find("KB"), std::string::npos); - - // Test MB - msg = Messages::cache_size_custom(2 * 1024 * 1024); - EXPECT_NE(msg.find("2"), std::string::npos); - EXPECT_NE(msg.find("MB"), std::string::npos); -} - -TEST(MessagesFormattingTest, CacheSizeCustomDisabled) { - std::string msg = Messages::cache_size_custom_disabled(); - EXPECT_NE(msg.find("Custom Cache Size"), std::string::npos); - EXPECT_NE(msg.find("Disabled"), std::string::npos); -} - -TEST(MessagesFormattingTest, CacheSizeL1) { - // Test bytes - std::string msg = Messages::cache_size_l1(128 * 1024); - EXPECT_NE(msg.find("128"), std::string::npos); - EXPECT_NE(msg.find("KB"), std::string::npos); - EXPECT_NE(msg.find("per P-core"), std::string::npos); - - // Test MB - msg = Messages::cache_size_l1(1 * 1024 * 1024); - EXPECT_NE(msg.find("1"), std::string::npos); - EXPECT_NE(msg.find("MB"), std::string::npos); -} - -TEST(MessagesFormattingTest, CacheSizeL2) { - // Test KB - std::string msg = Messages::cache_size_l2(4 * 1024 * 1024); - EXPECT_NE(msg.find("4"), std::string::npos); - EXPECT_NE(msg.find("MB"), std::string::npos); - EXPECT_NE(msg.find("per P-core cluster"), std::string::npos); -} - -// ============================================================================ -// Results Output Messages Tests (using formatting fixture) -// ============================================================================ - -TEST(MessagesFormattingTest, ResultsLoopHeader) { - const std::vector cases = { - {"first loop", Messages::results_loop_header(0), "\n--- Results (Loop 1) ---"}, - {"fifth loop", Messages::results_loop_header(4), "\n--- Results (Loop 5) ---"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesFormattingTest, ResultsMainMemoryBandwidth) { - std::string msg = Messages::results_main_memory_bandwidth(8); - EXPECT_NE(msg.find("8"), std::string::npos); - EXPECT_NE(msg.find("threads"), std::string::npos); - EXPECT_NE(msg.find("Main Memory Bandwidth"), std::string::npos); + {"buffer size", Messages::config_buffer_size(1024.5, 1024), + "Buffer Size (per buffer): 1024.50 MiB (1024 MB requested/capped)"}, + {"allocation", Messages::config_total_allocation(3072.75), "Peak Concurrent Allocation: ~3072.75 MiB"}, + {"non-cacheable enabled", Messages::config_non_cacheable(true), "Non-Cacheable Memory Hints: Enabled"}, + {"non-cacheable disabled", Messages::config_non_cacheable(false), "Non-Cacheable Memory Hints: Disabled"}, + {"global locality", Messages::config_latency_tlb_locality(0), "Latency TLB Locality: Global random"}, + {"locality window", Messages::config_latency_tlb_locality(16 * 1024), "Latency TLB Locality: 16.00 KB"}, + {"ConfigPatternAutomaticIterations", Messages::config_pattern_iterations_auto(0.150, 0.100, 0.250), + "Pattern Passes: automatic duration calibration (target 150 ms; intended window 100-250 ms)"}, + {"ConfigBenchmarkAutomaticIterations", Messages::config_benchmark_iterations_auto(0.150, 0.100, 0.250), + "Bandwidth Passes: automatic duration calibration (target 150 ms; intended window 100-250 ms)"}, + {"ConfigLatencyCalibration", Messages::config_latency_calibration(0.250, 0.100, 0.300, 16), + "Latency Headline: automatic continuous-pass calibration (target 250 ms; intended window 100-300 ms; " + "minimum 16 complete cycles)"}, + }); +} + +TEST(MessagesFormattingTest, CacheSizesHaveExactUnitsAndScope) { + expect_exact_messages({ + {"custom bytes", Messages::cache_size_custom(512), " Custom Cache Size: 512 B"}, + {"custom KB", Messages::cache_size_custom(256 * 1024), " Custom Cache Size: 256.00 KB"}, + {"custom MB", Messages::cache_size_custom(2 * 1024 * 1024), " Custom Cache Size: 2.00 MB"}, + {"L1 KB", Messages::cache_size_l1(128 * 1024), " L1 Cache Size: 128.00 KB (per P-core)"}, + {"L1 MB", Messages::cache_size_l1(1024 * 1024), " L1 Cache Size: 1.00 MB (per P-core)"}, + {"L2 MB", Messages::cache_size_l2(4 * 1024 * 1024), " L2 Cache Size: 4.00 MB (per P-core cluster)"}, + }); } TEST(MessagesFormattingTest, BandwidthResultsHaveExactOutput) { - const std::vector cases = { + expect_exact_messages({ {"main read", Messages::results_read_bandwidth(25.123, 1.456), " Read : 25.12300 GB/s (Total time: 1.45600 s)"}, {"main write", Messages::results_write_bandwidth(30.789, 2.345), " Write: 30.78900 GB/s (Total time: 2.34500 s)"}, @@ -1204,50 +636,24 @@ TEST(MessagesFormattingTest, BandwidthResultsHaveExactOutput) { {"cache read", Messages::results_cache_read_bandwidth(150.789), " Read : 150.78900 GB/s"}, {"cache write", Messages::results_cache_write_bandwidth(200.123), " Write: 200.12300 GB/s"}, {"cache copy", Messages::results_cache_copy_bandwidth(175.456), " Copy : 175.45600 GB/s"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesFormattingTest, ResultsLatencyTotalTime) { - EXPECT_EQ(Messages::results_latency_total_time(5.678), " Total time: 5.67800 s"); -} - -TEST(MessagesFormattingTest, ResultsLatencyAverage) { - std::string msg = Messages::results_latency_average(123.45, 1024 * 1024); - EXPECT_NE(msg.find("123.45"), std::string::npos); - EXPECT_NE(msg.find("ns"), std::string::npos); - EXPECT_NE(msg.find("Average latency"), std::string::npos); - EXPECT_NE(msg.find("1.00 MB locality"), std::string::npos); - - msg = Messages::results_latency_average(86.70, 0); - EXPECT_NE(msg.find("global random locality"), std::string::npos); -} - -TEST(MessagesFormattingTest, ResultsLatencyTlbHit) { - std::string msg = Messages::results_latency_tlb_hit(24.10); - EXPECT_NE(msg.find("16 KiB locality latency"), std::string::npos); - EXPECT_NE(msg.find("24.10"), std::string::npos); -} - -TEST(MessagesFormattingTest, ResultsLatencyTlbMiss) { - std::string msg = Messages::results_latency_tlb_miss(86.70); - EXPECT_NE(msg.find("Global-random latency"), std::string::npos); - EXPECT_NE(msg.find("86.70"), std::string::npos); -} - -TEST(MessagesFormattingTest, ResultsLatencyPageWalkPenalty) { - std::string msg = Messages::results_latency_page_walk_penalty(62.60); - EXPECT_NE(msg.find("Locality latency delta"), std::string::npos); - EXPECT_NE(msg.find("global - 16 KiB"), std::string::npos); - EXPECT_NE(msg.find("62.60"), std::string::npos); - EXPECT_NE(msg.find("ns"), std::string::npos); + {"first loop", Messages::results_loop_header(0), "\n--- Results (Loop 1) ---"}, + {"fifth loop", Messages::results_loop_header(4), "\n--- Results (Loop 5) ---"}, + {"cache scope", Messages::results_cache_bandwidth(1), "\nCache Bandwidth Tests (single-threaded):"}, + }); } -TEST(MessagesFormattingTest, ResultsCacheBandwidth) { - std::string msg = Messages::results_cache_bandwidth(1); - EXPECT_NE(msg.find("Cache Bandwidth"), std::string::npos); - EXPECT_NE(msg.find("single-threaded"), std::string::npos); +TEST(MessagesFormattingTest, LatencyResultsHaveExactOutput) { + expect_exact_messages({ + {"total time", Messages::results_latency_total_time(5.678), " Total time: 5.67800 s"}, + {"window latency", Messages::results_latency_average(123.45, 1024 * 1024), + " Average latency (1.00 MB locality): 123.45 ns"}, + {"global latency", Messages::results_latency_average(86.70, 0), + " Average latency (global random locality): 86.70 ns"}, + {"TLB hit", Messages::results_latency_tlb_hit(24.10), " 16 KiB locality latency: 24.10 ns"}, + {"TLB miss", Messages::results_latency_tlb_miss(86.70), " Global-random latency: 86.70 ns"}, + {"locality delta", Messages::results_latency_page_walk_penalty(62.60), + " Locality latency delta (global - 16 KiB): 62.60 ns"}, + }); } TEST(MessagesFormattingTest, BufferSizesHaveExactOutput) { @@ -1295,41 +701,6 @@ TEST(MessagesFormattingTest, NonFiniteCacheLatenciesRenderUnavailable) { expect_exact_messages(cases); } -// ============================================================================ -// Statistics Messages Tests (using formatting fixture) -// ============================================================================ - -TEST(MessagesFormattingTest, StatisticsHeader) { - std::string msg = Messages::statistics_header(5); - EXPECT_NE(msg.find("5"), std::string::npos); - EXPECT_NE(msg.find("Loops"), std::string::npos); - EXPECT_NE(msg.find("Statistics"), std::string::npos); -} - -TEST(MessagesFormattingTest, StatisticsAverage) { - std::string msg = Messages::statistics_average(25.123, 3); - EXPECT_NE(msg.find("25.123"), std::string::npos); - EXPECT_NE(msg.find("Average"), std::string::npos); - - msg = Messages::statistics_average(100.5, 1); - EXPECT_NE(msg.find("100.5"), std::string::npos); -} - -TEST(MessagesFormattingTest, StatisticsCacheBandwidthHeader) { - std::string msg = Messages::statistics_cache_bandwidth_header("L1"); - EXPECT_NE(msg.find("L1"), std::string::npos); - EXPECT_NE(msg.find("Cache Bandwidth"), std::string::npos); - EXPECT_NE(msg.find("GB/s"), std::string::npos); -} - -TEST(MessagesFormattingTest, StatisticsCacheLatencyName) { - std::string msg = Messages::statistics_cache_latency_name("L1"); - EXPECT_EQ(msg, " L1 Cache:"); - - msg = Messages::statistics_cache_latency_name("Custom"); - EXPECT_EQ(msg, " Custom Cache:"); -} - TEST(MessagesFormattingTest, WarningBenchmarkHighCv) { const std::string msg = Messages::warning_benchmark_high_cv("read bandwidth", 9.25, 7.5); EXPECT_NE(msg.find("read bandwidth"), std::string::npos); @@ -1337,74 +708,8 @@ TEST(MessagesFormattingTest, WarningBenchmarkHighCv) { EXPECT_NE(msg.find("7.5%"), std::string::npos); } -TEST(MessagesFormattingTest, WarningQosFailedBenchmarkWorker) { - const std::string msg = Messages::warning_qos_failed_benchmark_worker("read", 5); - EXPECT_NE(msg.find("read benchmark worker"), std::string::npos); - EXPECT_NE(msg.find("5"), std::string::npos); -} - -TEST(MessagesFormattingTest, BenchmarkStatusReasonsHaveExactOutput) { - const std::vector cases = { - {"before measurement", Messages::benchmark_reason_interrupted_before_measurement(), - "interrupted before measurement"}, - {"user interruption", Messages::benchmark_reason_interrupted_by_user(), "interrupted by user"}, - {"planned measurements", Messages::benchmark_reason_planned_measurements_unavailable(), - "one or more planned measurements unavailable"}, - {"locality work", Messages::benchmark_reason_invalid_locality_work(), "invalid locality-comparison work"}, - {"locality comparison", Messages::benchmark_reason_locality_comparison_unavailable(), - "paired locality comparison unavailable"}, - {"calibration interruption", Messages::benchmark_reason_interrupted_calibration_pilot(), - "interrupted during calibration pilot"}, - {"calibration duration", Messages::benchmark_reason_invalid_calibration_pilot(), - "invalid calibration pilot duration"}, - {"operation interruption", Messages::benchmark_reason_interrupted_measured_operation(), - "interrupted during measured operation"}, - {"bandwidth duration", Messages::benchmark_reason_invalid_bandwidth_duration(), - "invalid measured bandwidth duration"}, - {"bandwidth value", Messages::benchmark_reason_invalid_bandwidth_value(), "invalid measured bandwidth value"}, - {"latency pilot", Messages::benchmark_reason_interrupted_latency_pilot(), - "interrupted during latency calibration pilot"}, - {"latency interruption", Messages::benchmark_reason_interrupted_latency_measurement(), - "interrupted during latency measurement"}, - {"latency measurement", Messages::benchmark_reason_invalid_latency_measurement(), - "invalid latency duration or access count"}, - {"loops remain", Messages::benchmark_reason_loops_remain(), "benchmark loops remain"}, - {"checkpoint", Messages::benchmark_reason_checkpoint_failed(), "failed to checkpoint standard benchmark JSON"}, - {"unknown loop exception", Messages::benchmark_reason_unknown_loop_exception(), - "standard benchmark loop threw an unknown exception"}, - {"coordinator exception", Messages::benchmark_reason_coordinator_exception("boom"), - "standard benchmark coordinator exception: boom"}, - {"unknown coordinator exception", Messages::benchmark_reason_unknown_coordinator_exception(), - "standard benchmark coordinator threw an unknown exception"}, - {"latency chain", Messages::benchmark_reason_latency_chain_setup_failed("main-latency"), - "failed to construct main-latency latency chain"}, - {"preparation", Messages::benchmark_reason_prepare_failed("cache latency"), - "failed to prepare cache latency buffers"}, - {"bandwidth plan", Messages::benchmark_reason_invalid_bandwidth_plan(), "invalid bandwidth work-plan parameters"}, - {"worker partition", Messages::benchmark_reason_no_worker_partition(), "no valid aligned worker partition"}, - {"copy overflow", Messages::benchmark_reason_copy_payload_overflow(), "copy payload overflow"}, - {"payload overflow", Messages::benchmark_reason_total_payload_overflow(), "total payload overflow or pass limit"}, - {"latency plan", Messages::benchmark_reason_invalid_latency_plan(), "invalid latency work-plan parameters"}, - {"short latency chain", Messages::benchmark_reason_latency_chain_too_short(), - "latency chain requires at least two nodes"}, - {"cycle limit", Messages::benchmark_reason_minimum_cycles_exceed_limit(), - "minimum complete-cycle access count exceeds limit"}, - {"rounded access limit", Messages::benchmark_reason_rounded_accesses_exceed_limit(), - "rounded complete-cycle access count exceeds limit"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesFormattingTest, PatternStrideLabels) { - EXPECT_EQ(Messages::pattern_cache_line_64b(), "64 B stride"); - EXPECT_EQ(Messages::pattern_page_4096b(), "4096 B stride"); - EXPECT_EQ(Messages::pattern_page_16384b(), "16 KiB stride"); - EXPECT_EQ(Messages::pattern_superpage_2mb(), "2 MiB stride"); -} - TEST(MessagesFormattingTest, PatternMessagesHaveExactOutput) { - const std::vector cases = { + expect_exact_messages({ {"unavailable", Messages::pattern_measurement_unavailable("skipped", "buffer too small"), "N/A [skipped: buffer too small]"}, {"statistics header", Messages::statistics_pattern_bandwidth_header("Random"), @@ -1412,61 +717,8 @@ TEST(MessagesFormattingTest, PatternMessagesHaveExactOutput) { {"coefficient of variation", Messages::statistics_coefficient_of_variation(12.34, 1), " CV: 12.3%"}, {"noise warning", Messages::warning_pattern_measurement_noisy("Random read", 12.3, 10.0), "Noisy pattern measurement: Random read CV 12.3% exceeds 10.0%"}, - {"not completed", Messages::pattern_reason_measurement_not_completed(), "measurement not completed"}, - {"timer", Messages::pattern_reason_timer_creation_failed(), "Failed to create pattern benchmark timer."}, - {"calibration", Messages::pattern_reason_calibration_or_accounting_failed(), - "pattern calibration or byte accounting failed"}, - {"random workload", Messages::pattern_reason_no_valid_random_workload(), "no valid random access workload"}, - {"stride transition", Messages::pattern_reason_stride_transition_unavailable(), - "buffer cannot provide a valid stride transition"}, - {"copy overflow", Messages::pattern_reason_copy_accounting_overflow(), "copy payload byte accounting overflow"}, - {"strided timing", Messages::pattern_reason_invalid_strided_timing(), "invalid strided timing result"}, - {"work-plan byte overflow", Messages::pattern_reason_work_plan_byte_overflow(), - "strided work-plan byte accounting overflow"}, - {"work-plan parameters", Messages::pattern_reason_invalid_work_plan_parameters(), - "invalid strided work-plan parameters"}, - {"stride sum overflow", Messages::pattern_reason_stride_access_sum_overflow(), - "stride and access-size sum overflows"}, - {"two accesses", Messages::pattern_reason_buffer_lacks_two_strided_accesses(), - "buffer cannot provide two strided accesses"}, - {"worker partition", Messages::pattern_reason_no_valid_strided_worker_partition(), - "no valid worker partition contains a stride transition"}, - {"pass limit", Messages::pattern_reason_work_plan_pass_limit(), "strided work plan exceeds executor pass limit"}, - {"total overflow", Messages::pattern_reason_work_plan_total_overflow(), - "strided work-plan total accounting overflow"}, - {"allocation", Messages::pattern_reason_buffers_allocation_failed(), "pattern buffer allocation failed"}, - {"initialization", Messages::pattern_reason_buffers_initialization_failed(), - "pattern buffer initialization failed"}, - {"loop execution", Messages::pattern_reason_loop_execution_failed(), "pattern loop execution failed"}, - {"loop interrupted", Messages::pattern_reason_loop_interrupted(), "pattern loop interrupted by user"}, - {"loop incomplete", Messages::pattern_reason_loop_incomplete(), "pattern loop has incomplete measurements"}, - {"invalid measurement", Messages::pattern_reason_invalid_measurement(), - "pattern loop contains an invalid measurement"}, - {"loops remain", Messages::pattern_reason_loops_remain(), "pattern benchmark loops remain"}, {"loop exception", Messages::pattern_reason_loop_exception("boom"), "pattern loop threw an exception: boom"}, - {"unknown loop exception", Messages::pattern_reason_unknown_loop_exception(), - "pattern loop threw an unknown exception"}, {"coordinator exception", Messages::pattern_reason_coordinator_exception("boom"), "pattern coordinator threw an exception: boom"}, - {"unknown coordinator exception", Messages::pattern_reason_unknown_coordinator_exception(), - "pattern coordinator threw an unknown exception"}, - }; - - expect_exact_messages(cases); -} - -TEST(MessagesFormattingTest, ConfigPatternAutomaticIterations) { - EXPECT_EQ(Messages::config_pattern_iterations_auto(0.150, 0.100, 0.250), - "Pattern Passes: automatic duration calibration (target 150 ms; intended window 100-250 ms)"); -} - -TEST(MessagesFormattingTest, ConfigBenchmarkAutomaticIterations) { - EXPECT_EQ(Messages::config_benchmark_iterations_auto(0.150, 0.100, 0.250), - "Bandwidth Passes: automatic duration calibration (target 150 ms; intended window 100-250 ms)"); -} - -TEST(MessagesFormattingTest, ConfigLatencyCalibration) { - EXPECT_EQ(Messages::config_latency_calibration(0.250, 0.100, 0.300, 16), - "Latency Headline: automatic continuous-pass calibration (target 250 ms; intended window 100-300 ms; " - "minimum 16 complete cycles)"); + }); } diff --git a/tests/test_mode_selector.cpp b/tests/test_mode_selector.cpp index 986616e..589122d 100644 --- a/tests/test_mode_selector.cpp +++ b/tests/test_mode_selector.cpp @@ -43,42 +43,15 @@ PrimaryModeSelection select(const std::vector& arguments) { } // namespace -TEST(ModeSelectorTest, RecognizesEveryShortAndLongPrimaryMode) { +TEST(ModeSelectorTest, EveryPrimaryModeSpellingIsOpaqueAfterEitherOutputOption) { for (const ModeCase& mode_case : kModeCases) { - for (const char* option : {mode_case.short_option, - mode_case.long_option}) { - SCOPED_TRACE(option); - EXPECT_EQ(select({"program", option}).mode, mode_case.mode); - } - } -} - -TEST(ModeSelectorTest, - EveryShortPrimaryModeSpellingIsOpaqueAfterEitherOutputOption) { - for (const ModeCase& mode_case : kModeCases) { - for (const char* output_option : {"-o", "--output"}) { - SCOPED_TRACE(std::string(output_option) + " " + - mode_case.short_option); - const PrimaryModeSelection selection = - select({"program", output_option, mode_case.short_option}); - - EXPECT_EQ(selection.mode, PrimaryBenchmarkMode::None); - EXPECT_TRUE(selection.selected_options.empty()); - } - } -} - -TEST(ModeSelectorTest, - EveryLongPrimaryModeSpellingIsOpaqueAfterEitherOutputOption) { - for (const ModeCase& mode_case : kModeCases) { - for (const char* output_option : {"-o", "--output"}) { - SCOPED_TRACE(std::string(output_option) + " " + - mode_case.long_option); - const PrimaryModeSelection selection = - select({"program", output_option, mode_case.long_option}); - - EXPECT_EQ(selection.mode, PrimaryBenchmarkMode::None); - EXPECT_TRUE(selection.selected_options.empty()); + for (const char* mode_option : {mode_case.short_option, mode_case.long_option}) { + for (const char* output_option : {"-o", "--output"}) { + SCOPED_TRACE(std::string(output_option) + " " + mode_option); + const PrimaryModeSelection selection = select({"program", output_option, mode_option}); + EXPECT_EQ(selection.mode, PrimaryBenchmarkMode::None); + EXPECT_TRUE(selection.selected_options.empty()); + } } } } @@ -106,26 +79,6 @@ TEST(ModeSelectorTest, OutputMayAppearBeforeOrAfterTheActualSelectedMode) { } } -TEST(ModeSelectorTest, - LlmModeRemainsSelectedWhenItsOtherSpellingIsAnOpaqueOutputValue) { - for (const std::vector& arguments : { - std::vector{"program", "--output", "-M", - "--llm-memory"}, - std::vector{"program", "--llm-memory", "--output", - "-M"}, - std::vector{"program", "-o", "--llm-memory", "-M"}, - std::vector{"program", "-M", "-o", - "--llm-memory"}, - }) { - SCOPED_TRACE(::testing::PrintToString(arguments)); - const PrimaryModeSelection selection = select(arguments); - - EXPECT_EQ(selection.mode, PrimaryBenchmarkMode::LlmMemory); - ASSERT_EQ(selection.selected_options.size(), 1u); - EXPECT_EQ(selection.selected_options.front(), "--llm-memory"); - } -} - TEST(ModeSelectorTest, DistinctModesConflictIndependentOfArgvOrder) { for (size_t first_index = 0; first_index < kModeCases.size(); ++first_index) { @@ -151,25 +104,6 @@ TEST(ModeSelectorTest, DistinctModesConflictIndependentOfArgvOrder) { } } -TEST(ModeSelectorTest, - RealModeOutsideConsumedOutputValueStillConflictsInEitherOrder) { - const PrimaryModeSelection benchmark_first = - select({"program", "--output", "-P", "--benchmark", - "--gpu-bandwidth"}); - const PrimaryModeSelection gpu_first = - select({"program", "--gpu-bandwidth", "--output", "-P", - "--benchmark"}); - - EXPECT_EQ(benchmark_first.mode, PrimaryBenchmarkMode::Conflict); - EXPECT_EQ(gpu_first.mode, PrimaryBenchmarkMode::Conflict); - ASSERT_EQ(benchmark_first.selected_options.size(), 2u); - ASSERT_EQ(gpu_first.selected_options.size(), 2u); - EXPECT_EQ(benchmark_first.selected_options[0], "--benchmark"); - EXPECT_EQ(benchmark_first.selected_options[1], "--gpu-bandwidth"); - EXPECT_EQ(gpu_first.selected_options[0], "--gpu-bandwidth"); - EXPECT_EQ(gpu_first.selected_options[1], "--benchmark"); -} - TEST(ModeSelectorTest, RepeatedOneModeRemainsOwnedByItsParser) { for (const ModeCase& mode_case : kModeCases) { SCOPED_TRACE(mode_case.long_option); @@ -182,11 +116,6 @@ TEST(ModeSelectorTest, RepeatedOneModeRemainsOwnedByItsParser) { } } -TEST(ModeSelectorTest, OptionsWithoutPrimaryModeReturnNone) { - EXPECT_EQ(select({"program", "--buffer-size", "512"}).mode, - PrimaryBenchmarkMode::None); -} - TEST(ModeSelectorTest, MissingOutputValueRemainsOwnedByTheParser) { for (const char* output_option : {"-o", "--output"}) { SCOPED_TRACE(output_option); diff --git a/tests/test_numeric_utils.cpp b/tests/test_numeric_utils.cpp index 8ba48be..dde9bb8 100644 --- a/tests/test_numeric_utils.cpp +++ b/tests/test_numeric_utils.cpp @@ -124,19 +124,6 @@ TEST(NumericUtilsTest, CalibrationPolicyAndDurationScalingPreserveAllModeGoldenC EXPECT_EQ(Constants::BANDWIDTH_CALIBRATION_MAX_CORRECTIONS, 2u); EXPECT_EQ(Constants::BANDWIDTH_CALIBRATION_MIN_PILOT_BYTES, 8 * Constants::BYTES_PER_MB); EXPECT_EQ(Constants::BANDWIDTH_CALIBRATION_MAX_PASSES, 1000000000u); - - EXPECT_EQ(Constants::BENCHMARK_CALIBRATION_TARGET_SECONDS, Constants::BANDWIDTH_CALIBRATION_TARGET_SECONDS); - EXPECT_EQ(Constants::PATTERN_CALIBRATION_TARGET_SECONDS, Constants::BANDWIDTH_CALIBRATION_TARGET_SECONDS); - EXPECT_EQ(Constants::BENCHMARK_CALIBRATION_MIN_SECONDS, Constants::BANDWIDTH_CALIBRATION_MIN_SECONDS); - EXPECT_EQ(Constants::PATTERN_CALIBRATION_MIN_SECONDS, Constants::BANDWIDTH_CALIBRATION_MIN_SECONDS); - EXPECT_EQ(Constants::BENCHMARK_CALIBRATION_MAX_SECONDS, Constants::BANDWIDTH_CALIBRATION_MAX_SECONDS); - EXPECT_EQ(Constants::PATTERN_CALIBRATION_MAX_SECONDS, Constants::BANDWIDTH_CALIBRATION_MAX_SECONDS); - EXPECT_EQ(Constants::BENCHMARK_CALIBRATION_MAX_CORRECTIONS, Constants::BANDWIDTH_CALIBRATION_MAX_CORRECTIONS); - EXPECT_EQ(Constants::PATTERN_CALIBRATION_MAX_CORRECTIONS, Constants::BANDWIDTH_CALIBRATION_MAX_CORRECTIONS); - EXPECT_EQ(Constants::BENCHMARK_CALIBRATION_MIN_PILOT_BYTES, Constants::BANDWIDTH_CALIBRATION_MIN_PILOT_BYTES); - EXPECT_EQ(Constants::PATTERN_CALIBRATION_MIN_PILOT_BYTES, Constants::BANDWIDTH_CALIBRATION_MIN_PILOT_BYTES); - EXPECT_EQ(Constants::BENCHMARK_CALIBRATION_MAX_PASSES, Constants::BANDWIDTH_CALIBRATION_MAX_PASSES); - EXPECT_EQ(Constants::PATTERN_CALIBRATION_MAX_PASSES, Constants::BANDWIDTH_CALIBRATION_MAX_PASSES); } TEST(NumericUtilsTest, DurationScalingRejectsEveryInvalidInputClass) { diff --git a/tests/test_pattern_benchmark.cpp b/tests/test_pattern_benchmark.cpp index 9116c47..2b99b04 100644 --- a/tests/test_pattern_benchmark.cpp +++ b/tests/test_pattern_benchmark.cpp @@ -304,130 +304,77 @@ void expect_2mb_pattern_bandwidths_positive(const PatternResults& results) { } // namespace -TEST(PatternBenchmarkTest, PatternRunStatusStringsAreStable) { - EXPECT_STREQ(pattern_run_status_to_string(PatternRunStatus::NotStarted), "not-started"); - EXPECT_STREQ(pattern_run_status_to_string(PatternRunStatus::Complete), "complete"); - EXPECT_STREQ(pattern_run_status_to_string(PatternRunStatus::Partial), "partial"); - EXPECT_STREQ(pattern_run_status_to_string(PatternRunStatus::Interrupted), "interrupted"); - EXPECT_STREQ(pattern_run_status_to_string(PatternRunStatus::Failed), "failed"); -} - -TEST(PatternBenchmarkTest, LoopSummaryAcceptsMeasuredValuesAndSkippedMeasurements) { - const PatternResults defaults; - EXPECT_EQ(defaults.status, PatternRunStatus::NotStarted); - EXPECT_EQ(defaults.planned_measurements, kPatternMeasurementsPerLoop); - EXPECT_EQ(defaults.completed_measurements, 0u); - for (const PatternMeasurement& measurement : defaults.measurements) { - EXPECT_EQ(measurement.status, PatternMeasurementStatus::Invalid); - EXPECT_FALSE(measurement.bandwidth_gb_s.has_value()); +TEST(PatternBenchmarkTest, CollectorSumsCompletionCountersAndPreservesEarlierPartialReason) { + for (bool partial_first : {false, true}) { + SCOPED_TRACE(partial_first); + PatternStatistics statistics; + initialize_pattern_statistics(statistics, 2); + std::string partial_reason; + for (size_t index = 0; index < 2; ++index) { + const bool partial = (index == 0) == partial_first; + PatternResults loop = make_complete_pattern_loop(); + if (partial) loop.measurements.back().bandwidth_gb_s.reset(); + collect_pattern_loop_result(statistics, std::move(loop)); + EXPECT_EQ(statistics.status, PatternRunStatus::Partial); + EXPECT_EQ(statistics.planned_loops, 2u); + EXPECT_EQ(statistics.planned_measurements, 42u); + if (index == 0) { + EXPECT_EQ(statistics.completed_loops, partial ? 0u : 1u); + EXPECT_EQ(statistics.completed_measurements, partial ? 20u : 21u); + partial_reason = statistics.status_reason; + } + } + EXPECT_EQ(statistics.completed_loops, 1u); + EXPECT_EQ(statistics.completed_measurements, 41u); + ASSERT_EQ(statistics.loop_results.size(), 2u); + EXPECT_EQ(statistics.loop_results[partial_first ? 0 : 1].status, PatternRunStatus::Partial); + if (partial_first) EXPECT_EQ(statistics.status_reason, partial_reason); } - - PatternResults results = make_complete_pattern_loop(); - results.measurements[4].status = PatternMeasurementStatus::Skipped; - results.measurements[4].status_reason = "unsupported by this buffer"; - results.measurements[4].bandwidth_gb_s.reset(); - - const PatternLoopSummary summary = summarize_pattern_loop(results); - EXPECT_EQ(summary.status, PatternRunStatus::Complete); - EXPECT_TRUE(summary.status_reason.empty()); - EXPECT_EQ(summary.planned_measurements, 21u); - EXPECT_EQ(summary.completed_measurements, 21u); -} - -TEST(PatternBenchmarkTest, LoopSummaryClassifiesIncompleteInterruptedInvalidAndExecutionFailure) { - PatternResults incomplete = make_complete_pattern_loop(); - incomplete.measurements[3].bandwidth_gb_s.reset(); - EXPECT_EQ(summarize_pattern_loop(incomplete).status, PatternRunStatus::Partial); - - PatternResults interrupted = incomplete; - interrupted.measurements[3].status = PatternMeasurementStatus::Interrupted; - interrupted.measurements[3].status_reason = "stop observed"; - const PatternLoopSummary interrupted_summary = summarize_pattern_loop(interrupted); - EXPECT_EQ(interrupted_summary.status, PatternRunStatus::Interrupted); - EXPECT_EQ(interrupted_summary.status_reason, "stop observed"); - - PatternResults invalid = incomplete; - invalid.measurements[3].status = PatternMeasurementStatus::Invalid; - invalid.measurements[3].status_reason = "invalid duration"; - const PatternLoopSummary invalid_summary = summarize_pattern_loop(invalid); - EXPECT_EQ(invalid_summary.status, PatternRunStatus::Failed); - EXPECT_EQ(invalid_summary.status_reason, "invalid duration"); - - const PatternLoopSummary execution_failure = - summarize_pattern_loop(make_complete_pattern_loop(), true, true, "executor failed"); - EXPECT_EQ(execution_failure.status, PatternRunStatus::Failed); - EXPECT_EQ(execution_failure.status_reason, "executor failed"); -} - -TEST(PatternBenchmarkTest, CompleteLoopWinsLateInterruptionFlag) { - const PatternLoopSummary summary = summarize_pattern_loop(make_complete_pattern_loop(), false, true); - EXPECT_EQ(summary.status, PatternRunStatus::Complete); - EXPECT_EQ(summary.completed_measurements, 21u); } -TEST(PatternBenchmarkTest, CollectorSumsExactCompletionCounters) { - PatternStatistics statistics; - initialize_pattern_statistics(statistics, 2); - - collect_pattern_loop_result(statistics, make_complete_pattern_loop()); - EXPECT_EQ(statistics.status, PatternRunStatus::Partial); - EXPECT_EQ(statistics.planned_loops, 2u); - EXPECT_EQ(statistics.completed_loops, 1u); - EXPECT_EQ(statistics.planned_measurements, 42u); - EXPECT_EQ(statistics.completed_measurements, 21u); - - PatternResults partial = make_complete_pattern_loop(); - partial.measurements.back().bandwidth_gb_s.reset(); - collect_pattern_loop_result(statistics, std::move(partial)); - EXPECT_EQ(statistics.status, PatternRunStatus::Partial); - EXPECT_EQ(statistics.completed_loops, 1u); - EXPECT_EQ(statistics.completed_measurements, 41u); - ASSERT_EQ(statistics.loop_results.size(), 2u); - EXPECT_EQ(statistics.loop_results[1].status, PatternRunStatus::Partial); -} - -TEST(PatternBenchmarkTest, LaterCompleteLoopPreservesEarlierPartialReason) { - PatternStatistics statistics; - initialize_pattern_statistics(statistics, 2); - PatternResults partial = make_complete_pattern_loop(); - partial.measurements.back().bandwidth_gb_s.reset(); - collect_pattern_loop_result(statistics, std::move(partial)); - const std::string partial_reason = statistics.status_reason; - - collect_pattern_loop_result(statistics, make_complete_pattern_loop()); - EXPECT_EQ(statistics.status, PatternRunStatus::Partial); - EXPECT_EQ(statistics.status_reason, partial_reason); - EXPECT_EQ(statistics.completed_loops, 1u); - EXPECT_EQ(statistics.planned_loops, 2u); - EXPECT_EQ(statistics.completed_measurements, 41u); - EXPECT_EQ(statistics.planned_measurements, 42u); -} - -TEST(PatternBenchmarkTest, CoordinatorReportsBufferPreparationFailuresWithPlannedCounts) { - BenchmarkConfig config; - config.loop_count = 2; - config.buffer_size = 4096; - - PatternRunnerTestHooks allocation_failure = make_pattern_runner_hooks(); - allocation_failure.allocate_buffers = [](const BenchmarkConfig&, PatternBuffers&) { return EXIT_FAILURE; }; - PatternStatistics statistics; - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &allocation_failure), EXIT_FAILURE); - EXPECT_EQ(statistics.status, PatternRunStatus::Failed); - EXPECT_EQ(statistics.status_reason, Messages::pattern_reason_buffers_allocation_failed()); - EXPECT_EQ(statistics.planned_loops, 2u); - EXPECT_EQ(statistics.completed_loops, 0u); - EXPECT_EQ(statistics.planned_measurements, 42u); - EXPECT_EQ(statistics.completed_measurements, 0u); - EXPECT_TRUE(statistics.loop_results.empty()); - - PatternRunnerTestHooks initialization_failure = make_pattern_runner_hooks(); - initialization_failure.initialize_buffers = [](const PatternBuffers&, size_t) { return EXIT_FAILURE; }; - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &initialization_failure), EXIT_FAILURE); - EXPECT_EQ(statistics.status, PatternRunStatus::Failed); - EXPECT_EQ(statistics.status_reason, Messages::pattern_reason_buffers_initialization_failed()); - EXPECT_EQ(statistics.planned_measurements, 42u); - EXPECT_EQ(statistics.completed_measurements, 0u); - EXPECT_TRUE(statistics.loop_results.empty()); +TEST(PatternBenchmarkTest, CoordinatorContainsPreparationFailuresAndExceptionsWithPlannedCounts) { + enum class Failure { Allocation, Initialization, TypedException, UnknownException }; + for (Failure failure : + {Failure::Allocation, Failure::Initialization, Failure::TypedException, Failure::UnknownException}) { + SCOPED_TRACE(static_cast(failure)); + BenchmarkConfig config; + config.loop_count = 2; + config.buffer_size = 4096; + PatternRunnerTestHooks hooks = make_pattern_runner_hooks(); + std::string reason; + switch (failure) { + case Failure::Allocation: + hooks.allocate_buffers = [](const BenchmarkConfig&, PatternBuffers&) { return EXIT_FAILURE; }; + reason = Messages::pattern_reason_buffers_allocation_failed(); + break; + case Failure::Initialization: + hooks.initialize_buffers = [](const PatternBuffers&, size_t) { return EXIT_FAILURE; }; + reason = Messages::pattern_reason_buffers_initialization_failed(); + break; + case Failure::TypedException: + hooks.allocate_buffers = [](const BenchmarkConfig&, PatternBuffers&) -> int { + throw std::runtime_error("allocation hook exception"); + }; + reason = Messages::pattern_reason_coordinator_exception("allocation hook exception"); + break; + case Failure::UnknownException: + hooks.initialize_buffers = [](const PatternBuffers&, size_t) -> int { throw 9; }; + reason = Messages::pattern_reason_unknown_coordinator_exception(); + break; + } + PatternStatistics statistics; + testing::internal::CaptureStderr(); + const int result = run_all_pattern_benchmarks(config, statistics, &hooks); + (void)testing::internal::GetCapturedStderr(); + EXPECT_EQ(result, EXIT_FAILURE); + EXPECT_EQ(statistics.status, PatternRunStatus::Failed); + EXPECT_EQ(statistics.status_reason, reason); + EXPECT_EQ(statistics.planned_loops, 2u); + EXPECT_EQ(statistics.completed_loops, 0u); + EXPECT_EQ(statistics.planned_measurements, 42u); + EXPECT_EQ(statistics.completed_measurements, 0u); + EXPECT_TRUE(statistics.loop_results.empty()); + } } TEST(PatternBenchmarkTest, CoordinatorStopsBeforeFirstLoopWithPlannedCounts) { @@ -480,87 +427,45 @@ TEST(PatternBenchmarkTest, CoordinatorConvertsLoopExceptionToFailedEvidence) { EXPECT_EQ(statistics.status_reason, Messages::pattern_reason_unknown_loop_exception()); } -TEST(PatternBenchmarkTest, CoordinatorContainsPreparationHookExceptions) { - BenchmarkConfig config; - config.loop_count = 2; - config.buffer_size = 4096; - PatternStatistics statistics; - - PatternRunnerTestHooks hooks = make_pattern_runner_hooks(); - hooks.allocate_buffers = [](const BenchmarkConfig&, PatternBuffers&) -> int { - throw std::runtime_error("allocation hook exception"); - }; - testing::internal::CaptureStderr(); - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &hooks), EXIT_FAILURE); - (void)testing::internal::GetCapturedStderr(); - EXPECT_EQ(statistics.status, PatternRunStatus::Failed); - EXPECT_EQ(statistics.status_reason, Messages::pattern_reason_coordinator_exception("allocation hook exception")); - EXPECT_EQ(statistics.planned_measurements, 42u); - EXPECT_TRUE(statistics.loop_results.empty()); - - hooks = make_pattern_runner_hooks(); - hooks.initialize_buffers = [](const PatternBuffers&, size_t) -> int { throw 9; }; - testing::internal::CaptureStderr(); - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &hooks), EXIT_FAILURE); - (void)testing::internal::GetCapturedStderr(); - EXPECT_EQ(statistics.status_reason, Messages::pattern_reason_unknown_coordinator_exception()); - EXPECT_EQ(statistics.planned_measurements, 42u); - EXPECT_TRUE(statistics.loop_results.empty()); -} - -TEST(PatternBenchmarkTest, CoordinatorPreservesFailedAndPartialLoopEvidence) { - BenchmarkConfig config; - config.loop_count = 1; - config.buffer_size = 4096; - - PatternRunnerTestHooks failure = make_pattern_runner_hooks(); - failure.execute_loop = [](const PatternBuffers&, const BenchmarkConfig&, PatternResults& results, size_t) { - results = make_complete_pattern_loop(4.0); - return EXIT_FAILURE; - }; - PatternStatistics statistics; - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &failure), EXIT_FAILURE); - ASSERT_EQ(statistics.loop_results.size(), 1u); - EXPECT_EQ(statistics.status, PatternRunStatus::Failed); - EXPECT_EQ(statistics.loop_results[0].status, PatternRunStatus::Failed); - EXPECT_EQ(statistics.completed_loops, 0u); - EXPECT_EQ(statistics.completed_measurements, 21u); - - PatternRunnerTestHooks partial = make_pattern_runner_hooks(); - partial.execute_loop = [](const PatternBuffers&, const BenchmarkConfig&, PatternResults& results, size_t) { - results = make_complete_pattern_loop(5.0); - results.measurements.back().bandwidth_gb_s.reset(); - return EXIT_SUCCESS; - }; - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &partial), EXIT_SUCCESS); - ASSERT_EQ(statistics.loop_results.size(), 1u); - EXPECT_EQ(statistics.status, PatternRunStatus::Partial); - EXPECT_EQ(statistics.status_reason, Messages::pattern_reason_loop_incomplete()); - EXPECT_EQ(statistics.loop_results[0].status, PatternRunStatus::Partial); - EXPECT_EQ(statistics.completed_loops, 0u); - EXPECT_EQ(statistics.completed_measurements, 20u); -} - -TEST(PatternBenchmarkTest, InvalidEvidenceFailsEvenWhenExecutorReturnsSuccess) { - BenchmarkConfig config; - config.loop_count = 1; - config.buffer_size = 4096; - PatternRunnerTestHooks hooks = make_pattern_runner_hooks(); - hooks.execute_loop = [](const PatternBuffers&, const BenchmarkConfig&, PatternResults& results, size_t) { - results = make_complete_pattern_loop(); - results.measurements.back().status = PatternMeasurementStatus::Invalid; - results.measurements.back().status_reason = "invalid injected timing"; - results.measurements.back().bandwidth_gb_s.reset(); - return EXIT_SUCCESS; +TEST(PatternBenchmarkTest, CoordinatorPreservesFailedPartialInvalidAndInterruptedLoopEvidence) { + struct LoopCase { + PatternMeasurementStatus measurement; + bool missing_value; + int executor_result; + int run_result; + PatternRunStatus status; + std::string reason; }; - - PatternStatistics statistics; - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &hooks), EXIT_FAILURE); - ASSERT_EQ(statistics.loop_results.size(), 1u); - EXPECT_EQ(statistics.status, PatternRunStatus::Failed); - EXPECT_EQ(statistics.status_reason, "invalid injected timing"); - EXPECT_EQ(statistics.completed_loops, 0u); - EXPECT_EQ(statistics.completed_measurements, 20u); + for (const LoopCase& entry : + {LoopCase{PatternMeasurementStatus::Measured, false, EXIT_FAILURE, EXIT_FAILURE, PatternRunStatus::Failed, ""}, + LoopCase{PatternMeasurementStatus::Measured, true, EXIT_SUCCESS, EXIT_SUCCESS, PatternRunStatus::Partial, + Messages::pattern_reason_loop_incomplete()}, + LoopCase{PatternMeasurementStatus::Invalid, true, EXIT_SUCCESS, EXIT_FAILURE, PatternRunStatus::Failed, + "invalid injected timing"}, + LoopCase{PatternMeasurementStatus::Interrupted, true, EXIT_SUCCESS, EXIT_SUCCESS, PatternRunStatus::Interrupted, + "injected interruption"}}) { + SCOPED_TRACE(static_cast(entry.measurement)); + SCOPED_TRACE(entry.executor_result); + BenchmarkConfig config; + config.loop_count = 1; + config.buffer_size = 4096; + PatternRunnerTestHooks hooks = make_pattern_runner_hooks(); + hooks.execute_loop = [&](const PatternBuffers&, const BenchmarkConfig&, PatternResults& results, size_t) { + results = make_complete_pattern_loop(); + results.measurements.back().status = entry.measurement; + results.measurements.back().status_reason = entry.reason; + if (entry.missing_value) results.measurements.back().bandwidth_gb_s.reset(); + return entry.executor_result; + }; + PatternStatistics statistics; + EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &hooks), entry.run_result); + ASSERT_EQ(statistics.loop_results.size(), 1u); + EXPECT_EQ(statistics.status, entry.status); + EXPECT_EQ(statistics.loop_results[0].status, entry.status); + if (!entry.reason.empty()) EXPECT_EQ(statistics.status_reason, entry.reason); + EXPECT_EQ(statistics.completed_loops, 0u); + EXPECT_EQ(statistics.completed_measurements, entry.missing_value ? 20u : 21u); + } } TEST(PatternBenchmarkTest, FailedLoopEvidenceIsExcludedFromJsonAggregate) { @@ -610,27 +515,6 @@ TEST(PatternBenchmarkTest, FailedLoopEvidenceIsExcludedFromJsonAggregate) { EXPECT_DOUBLE_EQ(read["measurements"][1]["value_gb_s"].get(), 100.0); } -TEST(PatternBenchmarkTest, CoordinatorPreservesInterruptedLoopEvidence) { - BenchmarkConfig config; - config.loop_count = 1; - config.buffer_size = 4096; - PatternRunnerTestHooks hooks = make_pattern_runner_hooks(); - hooks.execute_loop = [](const PatternBuffers&, const BenchmarkConfig&, PatternResults& results, size_t) { - results = make_complete_pattern_loop(); - results.measurements.back().status = PatternMeasurementStatus::Interrupted; - results.measurements.back().status_reason = "injected interruption"; - results.measurements.back().bandwidth_gb_s.reset(); - return EXIT_SUCCESS; - }; - - PatternStatistics statistics; - EXPECT_EQ(run_all_pattern_benchmarks(config, statistics, &hooks), EXIT_SUCCESS); - ASSERT_EQ(statistics.loop_results.size(), 1u); - EXPECT_EQ(statistics.status, PatternRunStatus::Interrupted); - EXPECT_EQ(statistics.status_reason, "injected interruption"); - EXPECT_EQ(statistics.completed_measurements, 20u); -} - TEST(PatternBenchmarkTest, OneShotLateStopInterruptsOnlyWhenLoopsRemain) { auto run_with_loop_count = [](int loop_count) { BenchmarkConfig config; @@ -843,6 +727,22 @@ TEST(PatternBenchmarkTest, FinalizedPatternPlansDriveWarmupKernelsIntegration) { checksum.store(initial_checksum, std::memory_order_relaxed); warmup_read_random(source, random_workers, checksum); EXPECT_EQ(checksum.load(std::memory_order_acquire), expected_checksum); + EXPECT_EQ(random_workers[0].indices, (std::vector{0, 96})); + EXPECT_EQ(random_workers[1].indices, (std::vector{0, 96})); + + std::fill(destination, destination + buffer_size, 0xa5); + warmup_copy_random(destination, source, random_workers); + for (size_t offset = 0; offset < buffer_size; ++offset) { + const bool selected = offset < 32 || (offset >= 96 && offset < 160) || offset >= 224; + EXPECT_EQ(destination[offset], selected ? source[offset] : 0xa5) << "offset=" << offset; + } + + std::fill(destination, destination + buffer_size, 0xa5); + warmup_write_random(destination, random_workers); + for (size_t offset = 0; offset < buffer_size; ++offset) { + const bool selected = offset < 32 || (offset >= 96 && offset < 160) || offset >= 224; + EXPECT_EQ(destination[offset], selected ? 0u : 0xa5u) << "offset=" << offset; + } } TEST(PatternBenchmarkTest, CacheReadWarmupAccumulatesEveryWorkerChunkIntegration) { @@ -862,68 +762,6 @@ TEST(PatternBenchmarkTest, CacheReadWarmupAccumulatesEveryWorkerChunkIntegration EXPECT_EQ(checksum.load(std::memory_order_acquire), initial_checksum ^ expected_xor); } -TEST(PatternBenchmarkTest, RandomWarmupUsesChunkRelativeWorkerOffsetsIntegration) { - constexpr size_t buffer_size = 256; - std::vector source_storage(buffer_size + Constants::CACHE_LINE_SIZE_BYTES); - std::vector destination_storage(buffer_size + Constants::CACHE_LINE_SIZE_BYTES); - unsigned char* source = align_to_cache_line(source_storage.data()); - unsigned char* destination = align_to_cache_line(destination_storage.data()); - for (size_t offset = 0; offset < buffer_size; ++offset) { - source[offset] = static_cast((offset * 31 + 7) & 0xff); - } - - const std::vector global_indices = {0, 96, 128, 224}; - const std::vector workers = - build_random_worker_indices(buffer_size, Constants::PATTERN_ACCESS_SIZE_BYTES, 2, global_indices); - ASSERT_EQ(workers.size(), 2u); - EXPECT_EQ(workers[0].indices, (std::vector{0, 96})); - EXPECT_EQ(workers[1].indices, (std::vector{0, 96})); - - std::fill(destination, destination + buffer_size, 0xa5); - warmup_copy_random(destination, source, workers); - for (size_t offset = 0; offset < buffer_size; ++offset) { - const bool selected = offset < 32 || (offset >= 96 && offset < 160) || offset >= 224; - EXPECT_EQ(destination[offset], selected ? source[offset] : 0xa5) << "offset=" << offset; - } - - std::fill(destination, destination + buffer_size, 0xa5); - warmup_write_random(destination, workers); - for (size_t offset = 0; offset < buffer_size; ++offset) { - const bool selected = offset < 32 || (offset >= 96 && offset < 160) || offset >= 224; - EXPECT_EQ(destination[offset], selected ? 0u : 0xa5u) << "offset=" << offset; - } -} - -TEST(PatternBenchmarkTest, PhasedStridedKernelsPreserveAapcs64RegistersIntegration) { - const std::vector strides = { - Constants::PATTERN_STRIDE_CACHE_LINE, - Constants::PATTERN_STRIDE_PAGE, - Constants::PATTERN_STRIDE_PAGE_16K, - Constants::PATTERN_STRIDE_SUPERPAGE_2MB, - }; - - for (size_t stride : strides) { - SCOPED_TRACE(stride); - const size_t span = stride + Constants::PATTERN_ACCESS_SIZE_BYTES; - std::vector source(span, 0xA5); - std::vector destination(span, 0x5A); - - EXPECT_EQ( - verify_pattern_callee_saved_registers_asm(reinterpret_cast(&memory_read_strided_phased_loop_asm), - reinterpret_cast(source.data()), span, stride, 2, 0, 0), - 1u); - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(&memory_write_strided_phased_loop_asm), - reinterpret_cast(destination.data()), span, stride, 2, 0, 0), - 1u); - EXPECT_EQ( - verify_pattern_callee_saved_registers_asm(reinterpret_cast(&memory_copy_strided_phased_loop_asm), - reinterpret_cast(destination.data()), - reinterpret_cast(source.data()), span, stride, 2, 0), - 1u); - } -} - TEST(PatternBenchmarkTest, FinalizedWorkPlanDrivesInstrumentedExecutorIntegration) { constexpr size_t buffer_size = 8 * 1024 * 1024; const PatternWorkPlan pilot_plan = build_strided_pattern_work_plan( diff --git a/tests/test_pattern_validation.cpp b/tests/test_pattern_validation.cpp index c48fd39..8c8b2de 100644 --- a/tests/test_pattern_validation.cpp +++ b/tests/test_pattern_validation.cpp @@ -34,38 +34,25 @@ std::string capture_stderr(Callable&& callable) { } // namespace -TEST(PatternValidationTest, StrideAcceptsSupportedBounds) { +TEST(PatternValidationTest, StrideValidatesBufferBounds) { using namespace Constants; const struct { size_t stride; size_t buffer_size; + bool valid; } cases[] = { - {PATTERN_MIN_BUFFER_SIZE_BYTES, PATTERN_MIN_BUFFER_SIZE_BYTES}, - {PATTERN_STRIDE_CACHE_LINE, PATTERN_STRIDE_CACHE_LINE}, - {PATTERN_STRIDE_PAGE, PATTERN_STRIDE_PAGE + PATTERN_ACCESS_SIZE_BYTES}, + {PATTERN_MIN_BUFFER_SIZE_BYTES, PATTERN_MIN_BUFFER_SIZE_BYTES, true}, + {PATTERN_STRIDE_CACHE_LINE, PATTERN_STRIDE_CACHE_LINE, true}, + {PATTERN_STRIDE_PAGE, PATTERN_STRIDE_PAGE + PATTERN_ACCESS_SIZE_BYTES, true}, + {PATTERN_STRIDE_PAGE, PATTERN_STRIDE_PAGE - 1, false}, }; for (const auto& test_case : cases) { - EXPECT_TRUE(validate_stride(test_case.stride, test_case.buffer_size)); + SCOPED_TRACE(test_case.buffer_size); + EXPECT_EQ(validate_stride(test_case.stride, test_case.buffer_size), test_case.valid); } } -TEST(PatternValidationTest, StrideRejectsTooSmallValueWithCentralizedReason) { - bool result = true; - const std::string output = capture_stderr([&] { - result = validate_stride(Constants::PATTERN_MIN_BUFFER_SIZE_BYTES - 1, - Constants::PATTERN_MIN_BUFFER_SIZE_BYTES); - }); - - EXPECT_FALSE(result); - EXPECT_EQ(output, Messages::error_prefix() + Messages::error_stride_too_small() + "\n"); -} - -TEST(PatternValidationTest, StrideRejectsValueBeyondBufferWithoutBenchmarkExecution) { - EXPECT_FALSE(validate_stride(Constants::PATTERN_STRIDE_PAGE, - Constants::PATTERN_STRIDE_PAGE - 1)); -} - TEST(PatternValidationTest, RandomIndicesAcceptEveryAlignedExactlyFittingAccess) { constexpr size_t kAccess = Constants::PATTERN_ACCESS_SIZE_BYTES; EXPECT_TRUE(validate_random_indices({0, kAccess, 2 * kAccess}, 3 * kAccess)); diff --git a/tests/test_pattern_work_plan.cpp b/tests/test_pattern_work_plan.cpp index ba01303..41410f0 100644 --- a/tests/test_pattern_work_plan.cpp +++ b/tests/test_pattern_work_plan.cpp @@ -15,6 +15,7 @@ // #include +#include #include #include @@ -37,16 +38,6 @@ TEST(PatternWorkPlanTest, RejectsInvalidParameters) { EXPECT_EQ(build_plan(4096, 64, 1, 0).status, PatternMeasurementStatus::Invalid); } -TEST(PatternWorkPlanTest, SkipsBufferWithoutStrideTransition) { - const PatternWorkPlan plan = - build_plan(Constants::PATTERN_STRIDE_CACHE_LINE + Constants::PATTERN_ACCESS_SIZE_BYTES - 1, - Constants::PATTERN_STRIDE_CACHE_LINE, 1); - - EXPECT_EQ(plan.status, PatternMeasurementStatus::Skipped); - EXPECT_EQ(plan.effective_threads, 0); - EXPECT_TRUE(plan.workers.empty()); -} - TEST(PatternWorkPlanTest, IncludesLastExactlyFittingAccess) { const size_t buffer_size = Constants::PATTERN_STRIDE_PAGE + Constants::PATTERN_ACCESS_SIZE_BYTES; const PatternWorkPlan plan = build_plan(buffer_size, Constants::PATTERN_STRIDE_PAGE, 1); @@ -61,29 +52,40 @@ TEST(PatternWorkPlanTest, IncludesLastExactlyFittingAccess) { EXPECT_EQ(plan.payload_bytes_per_pass, 2u * Constants::PATTERN_ACCESS_SIZE_BYTES); } -TEST(PatternWorkPlanTest, ReducesThreadsUntilEveryWorkerHasTransition) { - const size_t buffer_size = 8 * Constants::BYTES_PER_MB; - const PatternWorkPlan plan = build_plan(buffer_size, Constants::PATTERN_STRIDE_SUPERPAGE_2MB, 10); - - ASSERT_EQ(plan.status, PatternMeasurementStatus::Measured); - EXPECT_EQ(plan.requested_threads, 10); - EXPECT_EQ(plan.effective_threads, 3); - ASSERT_EQ(plan.workers.size(), 3u); - - size_t next_offset = 0; - size_t summed_accesses = 0; - size_t summed_payload = 0; - for (const PatternWorkerRange& worker : plan.workers) { - EXPECT_EQ(worker.offset_bytes, next_offset); - EXPECT_GE(worker.accesses_per_pass, 2u); - next_offset += worker.span_bytes; - summed_accesses += worker.accesses_per_pass; - summed_payload += worker.payload_bytes_per_pass; +TEST(PatternWorkPlanTest, PartitionsUnevenBuffersAndReducesThreadsUntilEveryWorkerHasTransition) { + struct SplitCase { + size_t bytes; + size_t stride; + int requested; + int effective; + }; + for (const SplitCase& entry : + {SplitCase{8 * Constants::BYTES_PER_MB, Constants::PATTERN_STRIDE_SUPERPAGE_2MB, 10, 3}, + SplitCase{7 * Constants::BYTES_PER_MB + 123, Constants::PATTERN_STRIDE_PAGE_16K, 4, 4}}) { + SCOPED_TRACE(entry.bytes); + const size_t buffer_size = entry.bytes; + const PatternWorkPlan plan = build_plan(buffer_size, entry.stride, entry.requested); + + ASSERT_EQ(plan.status, PatternMeasurementStatus::Measured); + EXPECT_EQ(plan.requested_threads, entry.requested); + EXPECT_EQ(plan.effective_threads, entry.effective); + ASSERT_EQ(plan.workers.size(), static_cast(entry.effective)); + + size_t next_offset = 0; + size_t summed_accesses = 0; + size_t summed_payload = 0; + for (const PatternWorkerRange& worker : plan.workers) { + EXPECT_EQ(worker.offset_bytes, next_offset); + EXPECT_GE(worker.accesses_per_pass, 2u); + next_offset += worker.span_bytes; + summed_accesses += worker.accesses_per_pass; + summed_payload += worker.payload_bytes_per_pass; + } + EXPECT_EQ(next_offset, buffer_size); + EXPECT_EQ(summed_accesses, plan.accesses_per_pass); + EXPECT_EQ(summed_payload, plan.payload_bytes_per_pass); + if (entry.requested == 10) EXPECT_EQ(plan.accesses_per_pass, 6u); } - EXPECT_EQ(next_offset, buffer_size); - EXPECT_EQ(summed_accesses, plan.accesses_per_pass); - EXPECT_EQ(summed_payload, plan.payload_bytes_per_pass); - EXPECT_EQ(plan.accesses_per_pass, 6u); } TEST(PatternWorkPlanTest, UsesFinalWorkerCountsForPayloadAccounting) { @@ -102,19 +104,6 @@ TEST(PatternWorkPlanTest, UsesFinalWorkerCountsForPayloadAccounting) { EXPECT_LT(previous.total_payload_bytes, minimum_payload); } -TEST(PatternWorkPlanTest, PreservesExactCoverageForUnevenSplit) { - const size_t buffer_size = (7 * Constants::BYTES_PER_MB) + 123; - const PatternWorkPlan plan = build_plan(buffer_size, Constants::PATTERN_STRIDE_PAGE_16K, 4); - - ASSERT_EQ(plan.status, PatternMeasurementStatus::Measured); - size_t covered_bytes = 0; - for (const PatternWorkerRange& worker : plan.workers) { - EXPECT_EQ(worker.offset_bytes, covered_bytes); - covered_bytes += worker.span_bytes; - } - EXPECT_EQ(covered_bytes, buffer_size); -} - TEST(PatternWorkPlanTest, RejectsPassCountBeyondExecutorLimit) { const PatternWorkPlan plan = build_plan(Constants::PATTERN_STRIDE_CACHE_LINE + Constants::PATTERN_ACCESS_SIZE_BYTES, @@ -180,22 +169,19 @@ TEST(PatternWorkPlanTest, FullSparsePhaseCycleCoversAlignedBufferSlots) { } TEST(PatternWorkPlanTest, RandomPermutationIsSeededUniqueAlignedAndBounded) { - const size_t buffer_size = 4096; - const std::vector first = generate_random_indices(buffer_size, 100, 42); - const std::vector repeated = generate_random_indices(buffer_size, 100, 42); - const std::vector different = generate_random_indices(buffer_size, 100, 43); - - EXPECT_EQ(first, repeated); - EXPECT_NE(first, different); - EXPECT_EQ(std::unordered_set(first.begin(), first.end()).size(), first.size()); - for (size_t offset : first) { - EXPECT_EQ(offset % Constants::PATTERN_ACCESS_SIZE_BYTES, 0u); - EXPECT_LE(offset + Constants::PATTERN_ACCESS_SIZE_BYTES, buffer_size); + for (const size_t buffer_size : {size_t{4096}, size_t{96}}) { + SCOPED_TRACE(buffer_size); + const std::vector first = generate_random_indices(buffer_size, 100, 42); + const std::vector repeated = generate_random_indices(buffer_size, 100, 42); + const std::vector different = generate_random_indices(buffer_size, 100, 43); + + EXPECT_EQ(first, repeated); + if (buffer_size == 4096) EXPECT_NE(first, different); + EXPECT_EQ(first.size(), std::min(size_t{100}, buffer_size / Constants::PATTERN_ACCESS_SIZE_BYTES)); + EXPECT_EQ(std::unordered_set(first.begin(), first.end()).size(), first.size()); + for (size_t offset : first) { + EXPECT_EQ(offset % Constants::PATTERN_ACCESS_SIZE_BYTES, 0u); + EXPECT_LE(offset + Constants::PATTERN_ACCESS_SIZE_BYTES, buffer_size); + } } } - -TEST(PatternWorkPlanTest, RandomPermutationCapsAtAvailableAlignedSlots) { - const std::vector indices = generate_random_indices(96, 1000, 7); - EXPECT_EQ(indices.size(), 3u); - EXPECT_EQ(std::unordered_set(indices.begin(), indices.end()).size(), 3u); -} diff --git a/tests/test_script_examples.py b/tests/test_script_examples.py index bb61650..b5a0b7a 100644 --- a/tests/test_script_examples.py +++ b/tests/test_script_examples.py @@ -222,33 +222,24 @@ def assert_success(self, script_name: str, completed: subprocess.CompletedProces f"stderr tail:\n{completed.stderr[-4000:]}" ) - @unittest.skipUnless(shutil.which("jq"), "jq is optional and is not installed") - def test_latency_workflow_accepts_supported_result_with_jq(self): - with tempfile.TemporaryDirectory(prefix="script-example-latency-jq-") as temporary: - script_dir, completed = self.run_workflow( - Path(temporary), "latency_test_script.sh", CURRENT_CUSTOM_FIXTURE - ) - self.assert_success("latency_test_script.sh", completed) - final_output = (script_dir / "final_output.txt").read_text(encoding="utf-8") - - self.assertIn("Using jq to extract latency sample statistics", completed.stdout) - self.assertEqual(final_output.count("TLB Locality:"), 120) - self.assertIn('"average": 0.6893617567937856', final_output) - - def test_latency_workflow_accepts_supported_result_with_python(self): - with tempfile.TemporaryDirectory(prefix="script-example-latency-python-") as temporary: - script_dir, completed = self.run_workflow( - Path(temporary), - "latency_test_script.sh", - CURRENT_CUSTOM_FIXTURE, - force_python=True, - ) - self.assert_success("latency_test_script.sh", completed) - final_output = (script_dir / "final_output.txt").read_text(encoding="utf-8") - - self.assertIn("Using Python to extract latency sample statistics", completed.stdout) - self.assertEqual(final_output.count("TLB Locality:"), 120) - self.assertIn('"average": 0.6893617567937856', final_output) + def test_latency_workflow_accepts_supported_result(self): + for parser, force_python in (("Python", True), ("jq", False)): + with self.subTest(parser=parser): + if not force_python and not shutil.which("jq"): + self.skipTest("jq is optional and is not installed") + with tempfile.TemporaryDirectory(prefix="script-example-latency-") as temporary: + script_dir, completed = self.run_workflow( + Path(temporary), + "latency_test_script.sh", + CURRENT_CUSTOM_FIXTURE, + force_python=force_python, + ) + self.assert_success("latency_test_script.sh", completed) + final_output = (script_dir / "final_output.txt").read_text(encoding="utf-8") + + self.assertIn(f"Using {parser} to extract latency sample statistics", completed.stdout) + self.assertEqual(final_output.count("TLB Locality:"), 120) + self.assertIn('"average": 0.6893617567937856', final_output) def test_stride_tlb_workflow_accepts_supported_result(self): with tempfile.TemporaryDirectory(prefix="script-example-stride-tlb-") as temporary: diff --git a/tests/test_standard_kernels.cpp b/tests/test_standard_kernels.cpp index 84930c5..d8620eb 100644 --- a/tests/test_standard_kernels.cpp +++ b/tests/test_standard_kernels.cpp @@ -281,24 +281,11 @@ void verify_reverse_copy_kernel_boundaries(CopyKernel kernel) { } // namespace -TEST(StandardKernelIntegrationTest, MainReadHonorsTailsAndChecksum) { - verify_read_kernel_boundaries(memory_read_loop_asm); -} - -TEST(StandardKernelIntegrationTest, CacheReadHonorsTailsAndChecksum) { - verify_read_kernel_boundaries(memory_read_cache_loop_asm); -} - -TEST(StandardKernelIntegrationTest, CacheReadChecksumIncludesUpperVectorLane) { - alignas(64) std::array data{}; - data[8] = 0x5a; - EXPECT_EQ(memory_read_cache_loop_asm(data.data(), data.size()), 0x5aULL); -} - -TEST(PatternKernelIntegrationTest, ReverseReadChecksumIncludesUpperVectorLane) { - alignas(64) std::array data{}; - data[8] = 0x5a; - EXPECT_EQ(memory_read_reverse_loop_asm(data.data(), data.size()), 0x5aULL); +TEST(StandardKernelIntegrationTest, MainAndCacheReadsHonorTailsAndChecksum) { + for (ReadKernel kernel : {memory_read_loop_asm, memory_read_cache_loop_asm}) { + SCOPED_TRACE(kernel == memory_read_loop_asm ? "main" : "cache"); + verify_read_kernel_boundaries(kernel); + } } TEST(PatternKernelIntegrationTest, ReverseReadHonorsTailsGuardsAndChecksum) { @@ -415,92 +402,67 @@ TEST(StandardKernelIntegrationTest, MainAndCacheCopiesHonorExactBoundaries) { verify_copy_kernel_boundaries(memory_copy_cache_loop_asm); } -TEST(StandardKernelIntegrationTest, StandardKernelsPreserveCalleeSavedRegisters) { - alignas(64) std::array source{}; - alignas(64) std::array destination{}; +TEST(StandardKernelIntegrationTest, StandardAndPatternKernelsPreserveCalleeSavedRegisters) { + std::vector source(Constants::PATTERN_STRIDE_SUPERPAGE_2MB + Constants::PATTERN_ACCESS_SIZE_BYTES); + std::vector destination(source.size()); for (size_t index = 0; index < source.size(); ++index) { source[index] = static_cast(index & 0xff); } constexpr size_t kSize = 513; - for (ReadKernel kernel : {memory_read_loop_asm, - memory_read_cache_loop_asm}) { - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(kernel), - reinterpret_cast(source.data()), kSize, 0, 0, 0, - 0), - 1u); - } - for (WriteKernel kernel : {memory_write_loop_asm, - memory_write_cache_loop_asm}) { - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(kernel), - reinterpret_cast(destination.data()), kSize, 0, - 0, 0, 0), - 1u); - } - for (CopyKernel kernel : {memory_copy_loop_asm, - memory_copy_cache_loop_asm}) { - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(kernel), - reinterpret_cast(destination.data()), - reinterpret_cast(source.data()), kSize, 0, 0, 0), - 1u); - } - - ASSERT_EQ(setup_latency_chain(source.data(), source.size(), 256, 0, nullptr, - LatencyChainMode::GlobalRandom, 12345), - EXIT_SUCCESS); - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(memory_latency_chase_asm), - reinterpret_cast(source.data()), 16, 0, 0, 0, 0), - 1u); -} - -TEST(PatternKernelIntegrationTest, ReverseAndRandomKernelsPreserveCalleeSavedRegisters) { - alignas(64) std::array source{}; - alignas(64) std::array destination{}; const std::array indices = {0, 64, 128}; - for (size_t index = 0; index < source.size(); ++index) { - source[index] = static_cast((index * 17 + 3) & 0xff); + const uintptr_t src = reinterpret_cast(source.data()); + const uintptr_t dst = reinterpret_cast(destination.data()); + const uintptr_t index_data = reinterpret_cast(indices.data()); + struct KernelCase { + const char* name; + uintptr_t function; + std::array arguments; + }; + std::vector cases = { + {"read", reinterpret_cast(memory_read_loop_asm), {src, kSize, 0, 0, 0, 0}}, + {"write", reinterpret_cast(memory_write_loop_asm), {dst, kSize, 0, 0, 0, 0}}, + {"copy", reinterpret_cast(memory_copy_loop_asm), {dst, src, kSize, 0, 0, 0}}, + {"read_cache", reinterpret_cast(memory_read_cache_loop_asm), {src, kSize, 0, 0, 0, 0}}, + {"write_cache", reinterpret_cast(memory_write_cache_loop_asm), {dst, kSize, 0, 0, 0, 0}}, + {"copy_cache", reinterpret_cast(memory_copy_cache_loop_asm), {dst, src, kSize, 0, 0, 0}}, + {"read_reverse", reinterpret_cast(memory_read_reverse_loop_asm), {src, kSize, 0, 0, 0, 0}}, + {"write_reverse", reinterpret_cast(memory_write_reverse_loop_asm), {dst, kSize, 0, 0, 0, 0}}, + {"copy_reverse", reinterpret_cast(memory_copy_reverse_loop_asm), {dst, src, kSize, 0, 0, 0}}, + {"read_random", + reinterpret_cast(memory_read_random_loop_asm), + {src, index_data, indices.size(), 0, 0, 0}}, + {"write_random", + reinterpret_cast(memory_write_random_loop_asm), + {dst, index_data, indices.size(), 0, 0, 0}}, + {"copy_random", + reinterpret_cast(memory_copy_random_loop_asm), + {dst, src, index_data, indices.size(), 0, 0}}, + }; + for (size_t stride : {Constants::PATTERN_STRIDE_CACHE_LINE, Constants::PATTERN_STRIDE_PAGE, + Constants::PATTERN_STRIDE_PAGE_16K, Constants::PATTERN_STRIDE_SUPERPAGE_2MB}) { + const size_t span = stride + Constants::PATTERN_ACCESS_SIZE_BYTES; + cases.push_back({"read_strided", + reinterpret_cast(memory_read_strided_phased_loop_asm), + {src, span, stride, 2, 0, 0}}); + cases.push_back({"write_strided", + reinterpret_cast(memory_write_strided_phased_loop_asm), + {dst, span, stride, 2, 0, 0}}); + cases.push_back({"copy_strided", + reinterpret_cast(memory_copy_strided_phased_loop_asm), + {dst, src, span, stride, 2, 0}}); } - constexpr size_t kReverseSize = 513; - - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(memory_read_reverse_loop_asm), - reinterpret_cast(source.data()), kReverseSize, 0, 0, - 0, 0), - 1u); - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(memory_write_reverse_loop_asm), - reinterpret_cast(destination.data()), kReverseSize, - 0, 0, 0, 0), - 1u); - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(memory_copy_reverse_loop_asm), - reinterpret_cast(destination.data()), - reinterpret_cast(source.data()), kReverseSize, 0, 0, - 0), - 1u); - - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(memory_read_random_loop_asm), - reinterpret_cast(source.data()), - reinterpret_cast(indices.data()), indices.size(), 0, - 0, 0), - 1u); - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(memory_write_random_loop_asm), - reinterpret_cast(destination.data()), - reinterpret_cast(indices.data()), indices.size(), 0, - 0, 0), - 1u); - EXPECT_EQ(verify_pattern_callee_saved_registers_asm( - reinterpret_cast(memory_copy_random_loop_asm), - reinterpret_cast(destination.data()), - reinterpret_cast(source.data()), - reinterpret_cast(indices.data()), indices.size(), 0, - 0), + for (const KernelCase& entry : cases) { + SCOPED_TRACE(entry.name); + SCOPED_TRACE(::testing::PrintToString(entry.arguments)); + const auto& args = entry.arguments; + EXPECT_EQ( + verify_pattern_callee_saved_registers_asm(entry.function, args[0], args[1], args[2], args[3], args[4], args[5]), + 1u); + } + ASSERT_EQ(setup_latency_chain(source.data(), 1024, 256, 0, LatencyChainMode::GlobalRandom, 12345), EXIT_SUCCESS); + EXPECT_EQ(verify_pattern_callee_saved_registers_asm(reinterpret_cast(memory_latency_chase_asm), src, 16, 0, + 0, 0, 0), 1u); } diff --git a/tests/test_statistics.cpp b/tests/test_statistics.cpp index b226cdc..e3b7bcf 100644 --- a/tests/test_statistics.cpp +++ b/tests/test_statistics.cpp @@ -42,20 +42,20 @@ using test_statistics_helpers::capture_main_bandwidth; // --------------------------------------------------------------------------- // loop_count == 1 must produce no output at all. -TEST(StatisticsTest, SingleLoopProducesNoOutput) { - testing::internal::CaptureStdout(); - print_statistics(1, {100.0}, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, false, kE, kE, kE, kE, kE, kE, - kE, kE, false, false); - EXPECT_TRUE(testing::internal::GetCapturedStdout().empty()); -} - // only_latency=true with empty latency vectors suppresses output even when // loop_count > 1. -TEST(StatisticsTest, OnlyLatencyModeEmptyLatencyProducesNoOutput) { - testing::internal::CaptureStdout(); - print_statistics(3, {10.0, 20.0, 30.0}, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, false, kE, kE, kE, kE, - kE, kE, kE, kE, false, true); - EXPECT_TRUE(testing::internal::GetCapturedStdout().empty()); +TEST(StatisticsTest, SingleLoopAndEmptyLatencyModeProduceNoOutput) { + struct EmptyCase { + int loops; + bool only_latency; + }; + for (const EmptyCase& entry : {EmptyCase{1, false}, EmptyCase{3, true}}) { + SCOPED_TRACE(entry.loops); + testing::internal::CaptureStdout(); + print_statistics(entry.loops, {10.0, 20.0, 30.0}, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, kE, false, kE, + kE, kE, kE, kE, kE, kE, kE, false, entry.only_latency); + EXPECT_TRUE(testing::internal::GetCapturedStdout().empty()); + } } // --------------------------------------------------------------------------- @@ -64,29 +64,20 @@ TEST(StatisticsTest, OnlyLatencyModeEmptyLatencyProducesNoOutput) { // only_bandwidth=true: "Main Memory Latency" section must be absent even when // latency data is supplied. -TEST(StatisticsTest, OnlyBandwidthModeOmitsLatencySection) { - testing::internal::CaptureStdout(); - std::vector bw = {10.0, 20.0}; - std::vector lat = {100.0, 200.0}; - print_statistics(2, bw, bw, bw, kE, kE, kE, kE, kE, kE, kE, kE, lat, kE, kE, kE, false, kE, kE, kE, kE, kE, kE, kE, - kE, true, false); - std::string out = testing::internal::GetCapturedStdout(); - EXPECT_EQ(out.find("Main Memory Latency"), std::string::npos); - EXPECT_NE(out.find("Read Bandwidth"), std::string::npos); -} - -TEST(StatisticsTest, OnlyLatencyModeOmitsPopulatedBandwidthAndIncludesLatency) { +TEST(StatisticsTest, ModeFlagsSelectPopulatedBandwidthAndLatencySections) { const std::vector bandwidth = {10.0, 20.0}; const std::vector latency = {100.0, 200.0}; - testing::internal::CaptureStdout(); - print_statistics(2, bandwidth, bandwidth, bandwidth, kE, kE, kE, kE, kE, kE, kE, kE, latency, kE, kE, kE, false, kE, - kE, kE, kE, kE, kE, kE, kE, false, true); - const std::string output = testing::internal::GetCapturedStdout(); - - EXPECT_NE(output.find("Main Memory Latency (ns):"), std::string::npos); - EXPECT_EQ(output.find("Read Bandwidth (GB/s):"), std::string::npos); - EXPECT_EQ(output.find("Write Bandwidth (GB/s):"), std::string::npos); - EXPECT_EQ(output.find("Copy Bandwidth (GB/s):"), std::string::npos); + for (bool only_latency : {false, true}) { + SCOPED_TRACE(only_latency); + testing::internal::CaptureStdout(); + print_statistics(2, bandwidth, bandwidth, bandwidth, kE, kE, kE, kE, kE, kE, kE, kE, latency, kE, kE, kE, false, kE, + kE, kE, kE, kE, kE, kE, kE, !only_latency, only_latency); + const std::string output = testing::internal::GetCapturedStdout(); + EXPECT_EQ(output.find("Main Memory Latency (ns):") != std::string::npos, only_latency); + for (const char* label : {"Read Bandwidth (GB/s):", "Write Bandwidth (GB/s):", "Copy Bandwidth (GB/s):"}) { + EXPECT_EQ(output.find(label) != std::string::npos, !only_latency) << label; + } + } } TEST(StatisticsTest, PrintsOnlyAvailableMainBandwidthPopulations) { diff --git a/tests/test_statistics_renderer.cpp b/tests/test_statistics_renderer.cpp index bfb94b5..34d98c6 100644 --- a/tests/test_statistics_renderer.cpp +++ b/tests/test_statistics_renderer.cpp @@ -75,17 +75,11 @@ TEST(StatisticsRendererTest, RendersCanonicalOrderPrecisionAndPrefixes) { std::ostringstream undefined_output; render_statistics_summary(undefined_output, undefined_coefficient, options); - EXPECT_EQ(undefined_output.str(), - "> Average: 1.25\n" - "> Median (P50): 2.50\n" - "> P90: 7.00\n" - "> P95: 8.00\n" - "> P99: 8.80\n" - "> Stddev: 3.25\n" - "@ CV: 0.0%\n" - "@ Median absolute deviation: 0.75\n" - "> Min: -1.00\n" - "> Max: 9.00\n"); + std::string expected = output.str(); + const size_t coefficient = expected.find("@ CV: 6.5%\n"); + ASSERT_NE(coefficient, std::string::npos); + expected.replace(coefficient, std::string("@ CV: 6.5%\n").size(), "@ CV: 0.0%\n"); + EXPECT_EQ(undefined_output.str(), expected); } TEST(StatisticsRendererTest, SampleMedianOwnsIndentationAndReportsCount) { diff --git a/tests/test_sweep_runner.cpp b/tests/test_sweep_runner.cpp index f5156ee..6c6344b 100644 --- a/tests/test_sweep_runner.cpp +++ b/tests/test_sweep_runner.cpp @@ -17,31 +17,12 @@ #include "benchmark/sweep_runner.h" #include "core/config/constants.h" #include "output/console/messages/messages_api.h" -#include "output/json/json_output/json_output_session.h" namespace { using Json = nlohmann::ordered_json; -class ScopedStreamBuffers { - public: - ScopedStreamBuffers(std::streambuf* stdout_buffer, - std::streambuf* stderr_buffer) - : original_stdout_buffer_(std::cout.rdbuf(stdout_buffer)), - original_stderr_buffer_(std::cerr.rdbuf(stderr_buffer)) {} - ~ScopedStreamBuffers() { - std::cout.rdbuf(original_stdout_buffer_); - std::cerr.rdbuf(original_stderr_buffer_); - } - - ScopedStreamBuffers(const ScopedStreamBuffers&) = delete; - ScopedStreamBuffers& operator=(const ScopedStreamBuffers&) = delete; - - private: - std::streambuf* original_stdout_buffer_; - std::streambuf* original_stderr_buffer_; -}; std::vector make_parameters(size_t count) { std::vector parameters; @@ -375,309 +356,48 @@ TEST(SweepRunnerTest, CompleteSweepCheckpointsEveryRunAndValidatesConclusions) { EXPECT_EQ(announce_flags, (std::vector{false, true})); } -TEST(SweepRunnerTest, - StdoutSessionKeepsCheckpointCadenceLazyAndWritesTerminalEnvelopeOnce) { - const std::vector outcomes = { - {EXIT_SUCCESS, make_standard_result("complete", true), ""}, - {EXIT_SUCCESS, make_standard_result("complete", true), ""}, - }; - std::ostringstream stdout_capture; - std::ostringstream stderr_capture; - std::vector announce_flags; - size_t executed_runs = 0; - size_t checkpoint_calls = 0; - size_t checkpoint_builder_calls = 0; - bool stdout_empty_before_final = false; - int final_write_status = EXIT_FAILURE; - SweepExecutionResult execution; - - { - ScopedStreamBuffers capture(stdout_capture.rdbuf(), - stderr_capture.rdbuf()); - JsonOutputSession session(make_json_output_target("-")); - SweepExecutionHooks hooks; - hooks.execute_run = [&](size_t run_index) { - ++executed_runs; - return outcomes.at(run_index); - }; - hooks.stop_requested = []() { return false; }; - hooks.elapsed_seconds = []() { return 1.25; }; - hooks.utc_timestamp = []() { return "2026-01-01T00:00:00Z"; }; - hooks.write_checkpoint = [&](const Json& checkpoint, - bool announce_success) { - ++checkpoint_calls; - announce_flags.push_back(announce_success); - return session.checkpoint( - [&]() { - ++checkpoint_builder_calls; - return checkpoint; - }, - announce_success); - }; - - execution = execute_sweep_plan(SweepNestedMode::Standard, - make_parameters(2), Json::object(), hooks); - std::cout << "terminal human message\n"; - stdout_empty_before_final = stdout_capture.str().empty(); - final_write_status = session.write_final(execution.output_json); - } - - ASSERT_EQ(execution.exit_code, EXIT_SUCCESS); - EXPECT_EQ(executed_runs, 2u); - EXPECT_EQ(checkpoint_calls, 2u); - EXPECT_EQ(announce_flags, (std::vector{false, true})); - EXPECT_EQ(checkpoint_builder_calls, 0u); - EXPECT_TRUE(stdout_empty_before_final); - EXPECT_EQ(final_write_status, EXIT_SUCCESS); - EXPECT_EQ(stdout_capture.str(), execution.output_json.dump(2) + "\n"); - EXPECT_EQ(Json::parse(stdout_capture.str()), execution.output_json); - EXPECT_EQ(stderr_capture.str(), "terminal human message\n"); -} - -TEST(SweepRunnerTest, - PreRunInterruptionRetainsEmptyStdoutEnvelopeWithoutInvokingCheckpointBuilder) { - std::ostringstream stdout_capture; - std::ostringstream stderr_capture; - size_t executed_runs = 0; - size_t checkpoint_calls = 0; - size_t checkpoint_builder_calls = 0; - bool stdout_empty_before_final = false; - int final_write_status = EXIT_FAILURE; - SweepExecutionResult execution; - - { - ScopedStreamBuffers capture(stdout_capture.rdbuf(), - stderr_capture.rdbuf()); - JsonOutputSession session(make_json_output_target("-")); - SweepExecutionHooks hooks; - hooks.execute_run = [&](size_t) { - ++executed_runs; - return SweepRunOutcome{}; - }; - hooks.stop_requested = []() { return true; }; - hooks.elapsed_seconds = []() { return 0.25; }; - hooks.utc_timestamp = []() { return "2026-01-01T00:00:00Z"; }; - hooks.write_checkpoint = [&](const Json& checkpoint, - bool announce_success) { - ++checkpoint_calls; - EXPECT_TRUE(announce_success); - return session.checkpoint( - [&]() { - ++checkpoint_builder_calls; - return checkpoint; - }, - announce_success); - }; - - execution = execute_sweep_plan(SweepNestedMode::Standard, - make_parameters(2), Json::object(), hooks); - stdout_empty_before_final = stdout_capture.str().empty(); - final_write_status = session.write_final(execution.output_json); - } - - ASSERT_EQ(execution.exit_code, EXIT_SUCCESS); - EXPECT_EQ(executed_runs, 0u); - EXPECT_EQ(checkpoint_calls, 1u); - EXPECT_EQ(checkpoint_builder_calls, 0u); - EXPECT_TRUE(stdout_empty_before_final); - EXPECT_EQ(execution.output_json["status"], "interrupted"); - EXPECT_EQ(execution.output_json["status_reason"], - "interruption-requested-before-run"); - EXPECT_EQ(execution.output_json["planned_runs"], 2u); - EXPECT_EQ(execution.output_json["attempted_runs"], 0u); - EXPECT_EQ(execution.output_json["completed_runs"], 0u); - EXPECT_FALSE(execution.output_json["conclusions_valid"].get()); - EXPECT_TRUE(execution.output_json["runs"].empty()); - EXPECT_EQ(final_write_status, EXIT_SUCCESS); - EXPECT_EQ(stdout_capture.str(), execution.output_json.dump(2) + "\n"); - EXPECT_TRUE(stderr_capture.str().empty()); -} - -TEST(SweepRunnerTest, - ExecutorExceptionRetainsPriorEvidenceAndWritesOneStdoutEnvelope) { - std::ostringstream stdout_capture; - std::ostringstream stderr_capture; - size_t executed_runs = 0; - size_t checkpoint_calls = 0; - size_t checkpoint_builder_calls = 0; - bool stdout_empty_before_final = false; - int final_write_status = EXIT_FAILURE; - SweepExecutionResult execution; - - { - ScopedStreamBuffers capture(stdout_capture.rdbuf(), - stderr_capture.rdbuf()); - JsonOutputSession session(make_json_output_target("-")); - SweepExecutionHooks hooks; - hooks.execute_run = [&](size_t run_index) -> SweepRunOutcome { - ++executed_runs; - if (run_index == 0) { - return {EXIT_SUCCESS, - make_standard_result("complete", true), ""}; - } - throw std::runtime_error("injected nested failure"); - }; - hooks.stop_requested = []() { return false; }; - hooks.elapsed_seconds = []() { return 1.5; }; - hooks.utc_timestamp = []() { return "2026-01-01T00:00:00Z"; }; - hooks.write_checkpoint = [&](const Json& checkpoint, - bool announce_success) { - ++checkpoint_calls; - EXPECT_FALSE(announce_success); - return session.checkpoint( - [&]() { - ++checkpoint_builder_calls; - return checkpoint; - }, - announce_success); - }; - - execution = execute_sweep_plan(SweepNestedMode::Standard, - make_parameters(3), Json::object(), hooks); - stdout_empty_before_final = stdout_capture.str().empty(); - final_write_status = session.write_final(execution.output_json); - } - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - EXPECT_EQ(executed_runs, 2u); - EXPECT_EQ(checkpoint_calls, 2u); - EXPECT_EQ(checkpoint_builder_calls, 0u); - EXPECT_TRUE(stdout_empty_before_final); - EXPECT_EQ(execution.output_json["status"], "failed"); - EXPECT_EQ(execution.output_json["status_reason"], - "nested-run-execution-exception"); - EXPECT_EQ(execution.output_json["planned_runs"], 3u); - EXPECT_EQ(execution.output_json["attempted_runs"], 2u); - EXPECT_EQ(execution.output_json["completed_runs"], 1u); - EXPECT_FALSE(execution.output_json["conclusions_valid"].get()); - ASSERT_EQ(execution.output_json["runs"].size(), 2u); - EXPECT_EQ(execution.output_json["runs"][0]["status"], "complete"); - EXPECT_EQ(execution.output_json["runs"][1]["status"], "failed"); - EXPECT_EQ(execution.output_json["runs"][1]["status_reason"], - "nested-run-execution-exception"); - EXPECT_TRUE(execution.output_json["runs"][1]["result"].is_null()); - EXPECT_EQ(final_write_status, EXIT_SUCCESS); - EXPECT_EQ(stdout_capture.str(), execution.output_json.dump(2) + "\n"); - EXPECT_EQ(stderr_capture.str(), - Messages::error_prefix() + - Messages::error_sweep_nested_run_exception( - "injected nested failure") + - "\n"); -} - -TEST(SweepRunnerTest, UnknownExecutorExceptionUsesStableFailureReason) { - std::ostringstream stdout_capture; - std::ostringstream stderr_capture; - size_t checkpoint_calls = 0; - SweepExecutionResult execution; - - { - ScopedStreamBuffers capture(stdout_capture.rdbuf(), - stderr_capture.rdbuf()); - SweepExecutionHooks hooks; - hooks.execute_run = [](size_t) -> SweepRunOutcome { throw 7; }; - hooks.stop_requested = []() { return false; }; - hooks.elapsed_seconds = []() { return 0.5; }; - hooks.utc_timestamp = []() { return "2026-01-01T00:00:00Z"; }; - hooks.write_checkpoint = [&](const Json&, bool announce_success) { - ++checkpoint_calls; - EXPECT_FALSE(announce_success); - return EXIT_SUCCESS; - }; - - EXPECT_NO_THROW(execution = execute_sweep_plan( - SweepNestedMode::Standard, make_parameters(1), - Json::object(), hooks)); - } - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - EXPECT_EQ(checkpoint_calls, 1u); - EXPECT_EQ(execution.output_json["status_reason"], - "nested-run-execution-exception"); - ASSERT_EQ(execution.output_json["runs"].size(), 1u); - EXPECT_EQ(execution.output_json["runs"][0]["status_reason"], - "nested-run-execution-exception"); - EXPECT_TRUE(execution.output_json["runs"][0]["result"].is_null()); - EXPECT_TRUE(stdout_capture.str().empty()); - EXPECT_EQ(stderr_capture.str(), - Messages::error_prefix() + - Messages::error_sweep_nested_run_exception("") + "\n"); -} - -TEST(SweepRunnerTest, ExecutorExceptionCheckpointFailureWinsWithoutRetry) { - std::ostringstream stdout_capture; - std::ostringstream stderr_capture; - size_t checkpoint_calls = 0; - SweepExecutionHooks hooks; - hooks.execute_run = [](size_t) -> SweepRunOutcome { - throw std::runtime_error("injected nested failure"); - }; - hooks.stop_requested = []() { return false; }; - hooks.elapsed_seconds = []() { return 0.5; }; - hooks.utc_timestamp = []() { return "2026-01-01T00:00:00Z"; }; - hooks.write_checkpoint = [&](const Json&, bool) { - ++checkpoint_calls; - return EXIT_FAILURE; - }; - - SweepExecutionResult execution; - { - ScopedStreamBuffers capture(stdout_capture.rdbuf(), - stderr_capture.rdbuf()); - EXPECT_NO_THROW(execution = execute_sweep_plan( - SweepNestedMode::Standard, make_parameters(2), - Json::object(), hooks)); - } - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - EXPECT_EQ(checkpoint_calls, 1u); - EXPECT_EQ(execution.output_json["status"], "failed"); - EXPECT_EQ(execution.output_json["status_reason"], - "checkpoint-write-failed"); - EXPECT_EQ(execution.output_json["attempted_runs"], 1u); - EXPECT_EQ(execution.output_json["completed_runs"], 0u); - ASSERT_EQ(execution.output_json["runs"].size(), 1u); - EXPECT_EQ(execution.output_json["runs"][0]["status_reason"], - "nested-run-execution-exception"); - EXPECT_TRUE(execution.output_json["runs"][0]["result"].is_null()); - EXPECT_TRUE(stdout_capture.str().empty()); -} - -TEST(SweepRunnerTest, SharedEnvelopeSchemaIsAppliedForGeneralAndCoreToCoreProducers) { - struct ProducerCase { - SweepNestedMode mode; - const char* base_mode; - Json result; - }; - const std::vector producer_cases = { - {SweepNestedMode::Standard, Constants::BENCHMARK_JSON_MODE_NAME, - make_standard_result("complete", true)}, - {SweepNestedMode::CoreToCore, Constants::CORE_TO_CORE_JSON_MODE_NAME, - make_core_to_core_result("complete", true)}, - }; - - for (const ProducerCase& producer_case : producer_cases) { - const std::vector outcomes = { - {EXIT_SUCCESS, producer_case.result, ""}, - }; +TEST(SweepRunnerTest, TypedAndUnknownExecutorExceptionsRetainPriorEvidenceAndCheckpointOnce) { + for (bool typed_exception : {true, false}) { + SCOPED_TRACE(typed_exception); + const size_t prior_runs = typed_exception ? 1 : 0; + std::vector outcomes; std::vector checkpoints; std::vector announce_flags; size_t executed_runs = 0; - Json initial_output = { - {"configuration", {{"mode", Constants::SWEEP_JSON_MODE_NAME}, - {"base_mode", producer_case.base_mode}}}, + SweepExecutionHooks hooks = make_hooks(outcomes, checkpoints, announce_flags, executed_runs); + hooks.execute_run = [&](size_t run_index) -> SweepRunOutcome { + ++executed_runs; + if (run_index < prior_runs) return {EXIT_SUCCESS, make_standard_result("complete", true), ""}; + if (typed_exception) throw std::runtime_error("injected nested failure"); + throw 7; }; - - const SweepExecutionResult execution = - execute_sweep_plan(producer_case.mode, make_parameters(1), std::move(initial_output), - make_hooks(outcomes, checkpoints, announce_flags, executed_runs)); - - ASSERT_EQ(execution.exit_code, EXIT_SUCCESS); - EXPECT_EQ(execution.output_json["configuration"]["sweep_schema_version"], - Constants::SWEEP_JSON_SCHEMA_VERSION); - ASSERT_EQ(checkpoints.size(), 1u); - EXPECT_EQ(checkpoints[0]["configuration"]["sweep_schema_version"], - Constants::SWEEP_JSON_SCHEMA_VERSION); + SweepExecutionResult execution; + testing::internal::CaptureStdout(); + testing::internal::CaptureStderr(); + EXPECT_NO_THROW(execution = execute_sweep_plan(SweepNestedMode::Standard, make_parameters(typed_exception ? 3 : 1), + Json::object(), hooks)); + const std::string error = testing::internal::GetCapturedStderr(); + const std::string output = testing::internal::GetCapturedStdout(); + ASSERT_EQ(execution.exit_code, EXIT_FAILURE); + EXPECT_EQ(executed_runs, prior_runs + 1); + EXPECT_EQ(checkpoints.size(), executed_runs); + EXPECT_EQ(announce_flags, std::vector(executed_runs, false)); + EXPECT_EQ(execution.output_json["status"], "failed"); + EXPECT_EQ(execution.output_json["status_reason"], "nested-run-execution-exception"); + EXPECT_EQ(execution.output_json["planned_runs"], typed_exception ? 3u : 1u); + EXPECT_EQ(execution.output_json["attempted_runs"], executed_runs); + EXPECT_EQ(execution.output_json["completed_runs"], prior_runs); + EXPECT_FALSE(execution.output_json["conclusions_valid"].get()); + ASSERT_EQ(execution.output_json["runs"].size(), executed_runs); + if (prior_runs != 0) EXPECT_EQ(execution.output_json["runs"][0]["status"], "complete"); + const Json& failed = execution.output_json["runs"].back(); + EXPECT_EQ(failed["status"], "failed"); + EXPECT_EQ(failed["status_reason"], "nested-run-execution-exception"); + EXPECT_TRUE(failed["result"].is_null()); + EXPECT_TRUE(output.empty()); + EXPECT_EQ(error, Messages::error_prefix() + + Messages::error_sweep_nested_run_exception(typed_exception ? "injected nested failure" : "") + + "\n"); } } @@ -730,111 +450,50 @@ TEST(SweepRunnerTest, InterruptedNestedRunIsAttemptedButNotCompleted) { EXPECT_FALSE(execution.output_json["runs"][1]["result"]["tlb_analysis"].contains("status_reason")); } -TEST(SweepRunnerTest, ExplicitFailureReasonOverridesClassificationWithoutDiscardingPayload) { - const std::vector outcomes = { - {EXIT_SUCCESS, make_standard_result("complete", true), ""}, - {EXIT_FAILURE, {{"diagnostic", "runner failed after setup"}}, "simulated-execution-failure"}, - }; - std::vector checkpoints; - std::vector announce_flags; - size_t executed_runs = 0; - const SweepExecutionResult execution = - execute_sweep_plan(SweepNestedMode::Standard, make_parameters(2), Json::object(), - make_hooks(outcomes, checkpoints, announce_flags, executed_runs)); - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - EXPECT_EQ(execution.output_json["status"], "failed"); - EXPECT_EQ(execution.output_json["attempted_runs"], 2u); - EXPECT_EQ(execution.output_json["completed_runs"], 1u); - EXPECT_FALSE(execution.output_json["conclusions_valid"]); - ASSERT_EQ(execution.output_json["runs"].size(), 2u); - EXPECT_EQ(execution.output_json["runs"][0]["status"], "complete"); - EXPECT_EQ(execution.output_json["runs"][1]["status"], "failed"); - EXPECT_EQ(execution.output_json["runs"][1]["status_reason"], "simulated-execution-failure"); - EXPECT_EQ(execution.output_json["runs"][1]["result"]["diagnostic"], "runner failed after setup"); - EXPECT_EQ(checkpoints.size(), 2u); -} - -TEST(SweepRunnerTest, NonemptyFailurePayloadRetainsClassifierReasonWhenNoOverrideExists) { - const Json incomplete_result = {{"diagnostic", "runner failed after setup"}}; - const std::vector outcomes = { - {EXIT_FAILURE, incomplete_result, ""}, - }; - std::vector checkpoints; - std::vector announce_flags; - size_t executed_runs = 0; - - const SweepExecutionResult execution = - execute_sweep_plan(SweepNestedMode::Standard, make_parameters(1), Json::object(), - make_hooks(outcomes, checkpoints, announce_flags, executed_runs)); - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - ASSERT_EQ(execution.output_json["runs"].size(), 1u); - EXPECT_EQ(execution.output_json["runs"][0]["status"], "failed"); - EXPECT_EQ(execution.output_json["runs"][0]["status_reason"], - "missing-standard-schema-version"); - EXPECT_EQ(execution.output_json["runs"][0]["result"], incomplete_result); -} - -TEST(SweepRunnerTest, InitializedStandardFailureRetainsPayloadAndModeReason) { - const Json failed_result = make_standard_result("failed", false, "benchmark timer failed"); - const std::vector outcomes = { - {EXIT_FAILURE, failed_result, ""}, - }; - std::vector checkpoints; - std::vector announce_flags; - size_t executed_runs = 0; - - const SweepExecutionResult execution = - execute_sweep_plan(SweepNestedMode::Standard, make_parameters(1), Json::object(), - make_hooks(outcomes, checkpoints, announce_flags, executed_runs)); - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - EXPECT_EQ(execution.output_json["status"], "failed"); - ASSERT_EQ(execution.output_json["runs"].size(), 1u); - EXPECT_EQ(execution.output_json["runs"][0]["status"], "failed"); - EXPECT_EQ(execution.output_json["runs"][0]["status_reason"], "benchmark timer failed"); - EXPECT_EQ(execution.output_json["runs"][0]["result"], failed_result); -} - -TEST(SweepRunnerTest, TlbErrorMapsToFailedAndRetainsSchemaPayload) { - const Json error_result = make_tlb_result("error", false); - const std::vector outcomes = { - {EXIT_FAILURE, error_result, ""}, - }; - std::vector checkpoints; - std::vector announce_flags; - size_t executed_runs = 0; - - const SweepExecutionResult execution = - execute_sweep_plan(SweepNestedMode::TlbAnalysis, make_parameters(1), Json::object(), - make_hooks(outcomes, checkpoints, announce_flags, executed_runs)); - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - EXPECT_EQ(execution.output_json["status"], "failed"); - ASSERT_EQ(execution.output_json["runs"].size(), 1u); - EXPECT_EQ(execution.output_json["runs"][0]["status"], "failed"); - EXPECT_EQ(execution.output_json["runs"][0]["status_reason"], "nested-tlb-run-error"); - EXPECT_EQ(execution.output_json["runs"][0]["result"], error_result); - EXPECT_FALSE(execution.output_json["runs"][0]["result"]["tlb_analysis"].contains("status_reason")); -} - -TEST(SweepRunnerTest, EmptyFailurePayloadUsesExecutorFallbackReason) { - const std::vector outcomes = { - {EXIT_FAILURE, Json::object(), "simulated-execution-failure"}, +TEST(SweepRunnerTest, FailureClassificationRetainsPayloadAndHonorsExplicitReason) { + const Json diagnostic = {{"diagnostic", "runner failed after setup"}}; + struct FailureCase { + const char* name; + SweepNestedMode mode; + Json payload; + const char* override_reason; + const char* expected_reason; + bool prior_success; }; - std::vector checkpoints; - std::vector announce_flags; - size_t executed_runs = 0; - - const SweepExecutionResult execution = - execute_sweep_plan(SweepNestedMode::Standard, make_parameters(1), Json::object(), - make_hooks(outcomes, checkpoints, announce_flags, executed_runs)); - - ASSERT_EQ(execution.exit_code, EXIT_FAILURE); - EXPECT_EQ(execution.output_json["runs"][0]["status"], "failed"); - EXPECT_EQ(execution.output_json["runs"][0]["status_reason"], "simulated-execution-failure"); - EXPECT_TRUE(execution.output_json["runs"][0]["result"].is_null()); + for (const FailureCase& entry : + {FailureCase{"override", SweepNestedMode::Standard, diagnostic, "simulated-execution-failure", + "simulated-execution-failure", true}, + FailureCase{"classifier", SweepNestedMode::Standard, diagnostic, "", "missing-standard-schema-version", false}, + FailureCase{"mode reason", SweepNestedMode::Standard, + make_standard_result("failed", false, "benchmark timer failed"), "", "benchmark timer failed", + false}, + FailureCase{"TLB error", SweepNestedMode::TlbAnalysis, make_tlb_result("error", false), "", + "nested-tlb-run-error", false}, + FailureCase{"empty", SweepNestedMode::Standard, Json::object(), "simulated-execution-failure", + "simulated-execution-failure", false}}) { + SCOPED_TRACE(entry.name); + std::vector outcomes; + if (entry.prior_success) outcomes.push_back({EXIT_SUCCESS, make_standard_result("complete", true), ""}); + outcomes.push_back({EXIT_FAILURE, entry.payload, entry.override_reason}); + std::vector checkpoints; + std::vector announce_flags; + size_t executed_runs = 0; + const SweepExecutionResult execution = + execute_sweep_plan(entry.mode, make_parameters(outcomes.size()), Json::object(), + make_hooks(outcomes, checkpoints, announce_flags, executed_runs)); + ASSERT_EQ(execution.exit_code, EXIT_FAILURE); + EXPECT_EQ(execution.output_json["status"], "failed"); + EXPECT_EQ(execution.output_json["attempted_runs"], outcomes.size()); + EXPECT_EQ(execution.output_json["completed_runs"], entry.prior_success ? 1u : 0u); + EXPECT_FALSE(execution.output_json["conclusions_valid"]); + ASSERT_EQ(execution.output_json["runs"].size(), outcomes.size()); + const Json& failed = execution.output_json["runs"].back(); + EXPECT_EQ(failed["status"], "failed"); + EXPECT_EQ(failed["status_reason"], entry.expected_reason); + EXPECT_EQ(failed["result"], entry.payload.empty() ? Json(nullptr) : entry.payload); + if (entry.prior_success) EXPECT_EQ(execution.output_json["runs"][0]["status"], "complete"); + EXPECT_EQ(checkpoints.size(), outcomes.size()); + } } TEST(SweepRunnerTest, CheckpointWriteFailureStopsFurtherRunsAndInvalidatesSweep) { @@ -874,34 +533,34 @@ TEST(SweepRunnerTest, CheckpointWriteFailureStopsFurtherRunsAndInvalidatesSweep) EXPECT_EQ(execution.output_json["runs"].size(), 2u); } -TEST(SweepRunnerTest, InterruptionAfterCompleteRunKeepsRunCompleted) { - const std::vector outcomes = { - {EXIT_SUCCESS, make_standard_result("complete", true), ""}, - {EXIT_SUCCESS, make_standard_result("complete", true), ""}, - }; - std::vector checkpoints; - std::vector announce_flags; - size_t executed_runs = 0; - size_t stop_checks = 0; - SweepExecutionHooks hooks = make_hooks(outcomes, checkpoints, announce_flags, executed_runs); - hooks.stop_requested = [&]() { - ++stop_checks; - return stop_checks >= 2; - }; - - const SweepExecutionResult execution = - execute_sweep_plan(SweepNestedMode::Standard, make_parameters(2), Json::object(), hooks); - - ASSERT_EQ(execution.exit_code, EXIT_SUCCESS); - EXPECT_EQ(executed_runs, 1u); - EXPECT_EQ(execution.output_json["status"], "interrupted"); - EXPECT_EQ(execution.output_json["status_reason"], "interruption-requested-after-complete-run"); - EXPECT_EQ(execution.output_json["planned_runs"], 2u); - EXPECT_EQ(execution.output_json["attempted_runs"], 1u); - EXPECT_EQ(execution.output_json["completed_runs"], 1u); - EXPECT_FALSE(execution.output_json["conclusions_valid"]); - EXPECT_EQ(execution.output_json["runs"][0]["status"], "complete"); - ASSERT_EQ(checkpoints.size(), 1u); - ASSERT_EQ(announce_flags.size(), 1u); - EXPECT_TRUE(announce_flags[0]); +TEST(SweepRunnerTest, InterruptionBeforeOrAfterFirstRunPreservesCompletedEvidence) { + for (bool before_first : {true, false}) { + SCOPED_TRACE(before_first); + const std::vector outcomes = { + {EXIT_SUCCESS, make_standard_result("complete", true), ""}, + {EXIT_SUCCESS, make_standard_result("complete", true), ""}, + }; + std::vector checkpoints; + std::vector announce_flags; + size_t executed_runs = 0; + size_t stop_checks = 0; + SweepExecutionHooks hooks = make_hooks(outcomes, checkpoints, announce_flags, executed_runs); + hooks.stop_requested = [&]() { return ++stop_checks >= (before_first ? 1u : 2u); }; + const SweepExecutionResult execution = + execute_sweep_plan(SweepNestedMode::Standard, make_parameters(2), Json::object(), hooks); + ASSERT_EQ(execution.exit_code, EXIT_SUCCESS); + EXPECT_EQ(executed_runs, before_first ? 0u : 1u); + EXPECT_EQ(execution.output_json["status"], "interrupted"); + EXPECT_EQ(execution.output_json["status_reason"], + before_first ? "interruption-requested-before-run" : "interruption-requested-after-complete-run"); + EXPECT_EQ(execution.output_json["planned_runs"], 2u); + EXPECT_EQ(execution.output_json["attempted_runs"], executed_runs); + EXPECT_EQ(execution.output_json["completed_runs"], executed_runs); + EXPECT_FALSE(execution.output_json["conclusions_valid"]); + ASSERT_EQ(execution.output_json["runs"].size(), executed_runs); + if (!before_first) EXPECT_EQ(execution.output_json["runs"][0]["status"], "complete"); + ASSERT_EQ(checkpoints.size(), 1u); + ASSERT_EQ(announce_flags.size(), 1u); + EXPECT_TRUE(announce_flags[0]); + } } diff --git a/tests/test_system_info.cpp b/tests/test_system_info.cpp index 27b6340..135042c 100644 --- a/tests/test_system_info.cpp +++ b/tests/test_system_info.cpp @@ -108,44 +108,30 @@ std::string expected_sysctl_error(const std::string& operation, const std::strin } // namespace -TEST(BenchmarkQosTest, SuccessfulRequestIsReportedWithoutWarning) { - size_t setter_calls = 0; - testing::internal::CaptureStderr(); - const MainThreadQosResult result = prepare_main_thread_benchmark_qos([&setter_calls]() { - ++setter_calls; - return 0; - }); - const std::string stderr_output = testing::internal::GetCapturedStderr(); - - EXPECT_TRUE(result.requested); - EXPECT_TRUE(result.applied); - EXPECT_EQ(result.code, 0); - EXPECT_EQ(setter_calls, 1U); - EXPECT_TRUE(stderr_output.empty()); -} - -TEST(BenchmarkQosTest, FailedRequestPreservesCodeAndPrintsCentralizedWarning) { - constexpr int kFailureCode = 17; - testing::internal::CaptureStderr(); - const MainThreadQosResult result = prepare_main_thread_benchmark_qos([]() { return kFailureCode; }); - const std::string stderr_output = testing::internal::GetCapturedStderr(); - - EXPECT_TRUE(result.requested); - EXPECT_FALSE(result.applied); - EXPECT_EQ(result.code, kFailureCode); - EXPECT_EQ(stderr_output, Messages::warning_prefix() + Messages::warning_qos_failed(kFailureCode) + "\n"); -} - -TEST(BenchmarkQosTest, FailedRequestCanDeferWarningToModeSpecificReport) { - constexpr int kFailureCode = 23; - testing::internal::CaptureStderr(); - const MainThreadQosResult result = prepare_main_thread_benchmark_qos([]() { return kFailureCode; }, false); - const std::string stderr_output = testing::internal::GetCapturedStderr(); - - EXPECT_TRUE(result.requested); - EXPECT_FALSE(result.applied); - EXPECT_EQ(result.code, kFailureCode); - EXPECT_TRUE(stderr_output.empty()); +TEST(BenchmarkQosTest, RequestPreservesResultAndHonorsWarningPolicy) { + struct QosCase { + int code; + bool warn; + }; + for (const QosCase& entry : {QosCase{0, true}, QosCase{17, true}, QosCase{23, false}}) { + SCOPED_TRACE(entry.code); + size_t setter_calls = 0; + testing::internal::CaptureStderr(); + const MainThreadQosResult result = prepare_main_thread_benchmark_qos( + [&]() { + ++setter_calls; + return entry.code; + }, + entry.warn); + const std::string output = testing::internal::GetCapturedStderr(); + EXPECT_TRUE(result.requested); + EXPECT_EQ(result.applied, entry.code == 0); + EXPECT_EQ(result.code, entry.code); + EXPECT_EQ(setter_calls, 1U); + EXPECT_EQ(output, entry.code != 0 && entry.warn + ? Messages::warning_prefix() + Messages::warning_qos_failed(entry.code) + "\n" + : ""); + } } TEST(SystemInfoTest, CoreQueriesUseValidTopologyValuesAndRejectInvalidOnes) { diff --git a/tests/test_timer.cpp b/tests/test_timer.cpp index 820fb11..097b02f 100644 --- a/tests/test_timer.cpp +++ b/tests/test_timer.cpp @@ -120,17 +120,6 @@ TEST_F(HighResTimerTest, DeterministicClockProducesExactNanosecondsAndSeconds) { EXPECT_DOUBLE_EQ(timer->stop(), 100.0 / 1e9); } -TEST_F(HighResTimerTest, UnsignedTickSubtractionPreservesWraparound) { - state.numer = 1; - state.denom = 1; - set_ticks({std::numeric_limits::max() - 4, 5}); - std::optional timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - - timer->start(); - EXPECT_DOUBLE_EQ(timer->stop_ns(), 10.0); -} - TEST_F(HighResTimerTest, SnapshotRetainsOriginalBoundariesWithoutExtraClockReads) { state.numer = 125; state.denom = 3; diff --git a/tests/test_tlb_chain.cpp b/tests/test_tlb_chain.cpp index 8bda405..eb83939 100644 --- a/tests/test_tlb_chain.cpp +++ b/tests/test_tlb_chain.cpp @@ -22,9 +22,7 @@ #include #include -#include "benchmark/benchmark_tests.h" #include "benchmark/tlb_chain.h" -#include "core/timing/timer.h" namespace { @@ -211,71 +209,64 @@ TEST(TlbChainTest, ValidationReportsInvalidArgumentBeforeTraversal) { TlbChainValidationStatus::InvalidArgument); } -TEST(TlbChainTest, ValidationReportsMisalignedNode) { - constexpr size_t page_size = kTestPageSizeBytes; - PageBuffer buffer(page_size); - ASSERT_NE(buffer.get(), nullptr); - - EXPECT_EQ(validate_tlb_chain(buffer.get(), buffer.size(), static_cast(buffer.get()) + 1, - expected_chain(page_size, 1, 1)), - TlbChainValidationStatus::NodeMisaligned); -} - -TEST(TlbChainTest, ValidationReportsEarlyCycle) { - constexpr size_t page_size = kTestPageSizeBytes; - PageBuffer buffer(page_size); - ASSERT_NE(buffer.get(), nullptr); - write_link(buffer.get(), 0, address_at(buffer.get(), 0)); - - EXPECT_EQ(validate_tlb_chain(buffer.get(), buffer.size(), buffer.get(), expected_chain(page_size, 3, 1)), - TlbChainValidationStatus::EarlyCycle); -} - -TEST(TlbChainTest, ValidationReportsDuplicateNodeAtExpectedCycleLength) { - constexpr size_t page_size = kTestPageSizeBytes; - PageBuffer buffer(page_size); - ASSERT_NE(buffer.get(), nullptr); - write_link(buffer.get(), 0, address_at(buffer.get(), 64)); - write_link(buffer.get(), 64, address_at(buffer.get(), 128)); - write_link(buffer.get(), 128, address_at(buffer.get(), 64)); - - EXPECT_EQ(validate_tlb_chain(buffer.get(), buffer.size(), buffer.get(), expected_chain(page_size, 4, 1)), - TlbChainValidationStatus::DuplicateNode); -} - -TEST(TlbChainTest, ValidationReportsChainThatDoesNotReturnToHead) { - constexpr size_t page_size = kTestPageSizeBytes; - PageBuffer buffer(page_size); - ASSERT_NE(buffer.get(), nullptr); - write_link(buffer.get(), 0, address_at(buffer.get(), 64)); - write_link(buffer.get(), 64, address_at(buffer.get(), 128)); - write_link(buffer.get(), 128, address_at(buffer.get(), 192)); - - EXPECT_EQ(validate_tlb_chain(buffer.get(), buffer.size(), buffer.get(), expected_chain(page_size, 3, 1)), - TlbChainValidationStatus::DoesNotReturnToHead); -} - -TEST(TlbChainTest, ValidationReportsCacheLineReuse) { - constexpr size_t page_size = kTestPageSizeBytes; - PageBuffer buffer(page_size); - ASSERT_NE(buffer.get(), nullptr); - write_link(buffer.get(), 0, address_at(buffer.get(), sizeof(uintptr_t))); - write_link(buffer.get(), sizeof(uintptr_t), address_at(buffer.get(), 0)); - - EXPECT_EQ(validate_tlb_chain(buffer.get(), buffer.size(), buffer.get(), - expected_chain(page_size, 2, 2, TlbChainLayout::Packed)), - TlbChainValidationStatus::CacheLineReuse); -} - -TEST(TlbChainTest, ValidationReportsSpreadPageCountMismatch) { - constexpr size_t page_size = kTestPageSizeBytes; - PageBuffer buffer(2 * page_size); - ASSERT_NE(buffer.get(), nullptr); - write_link(buffer.get(), 0, address_at(buffer.get(), 64)); - write_link(buffer.get(), 64, address_at(buffer.get(), 0)); - - EXPECT_EQ(validate_tlb_chain(buffer.get(), buffer.size(), buffer.get(), expected_chain(page_size, 2, 2)), - TlbChainValidationStatus::PageCountMismatch); +TEST(TlbChainTest, ValidationReportsExactCorruptionReasons) { + struct CorruptionCase { + const char* name; + size_t allocation_pages; + size_t head_offset; + size_t nodes; + size_t pages; + TlbChainLayout layout; + std::vector> links; + TlbChainValidationStatus status; + }; + for (const CorruptionCase& entry : + {CorruptionCase{"misaligned", 1, 1, 1, 1, TlbChainLayout::Spread, {}, TlbChainValidationStatus::NodeMisaligned}, + CorruptionCase{ + "early cycle", 1, 0, 3, 1, TlbChainLayout::Spread, {{0, 0}}, TlbChainValidationStatus::EarlyCycle}, + CorruptionCase{"duplicate", + 1, + 0, + 4, + 1, + TlbChainLayout::Spread, + {{0, 64}, {64, 128}, {128, 64}}, + TlbChainValidationStatus::DuplicateNode}, + CorruptionCase{"open chain", + 1, + 0, + 3, + 1, + TlbChainLayout::Spread, + {{0, 64}, {64, 128}, {128, 192}}, + TlbChainValidationStatus::DoesNotReturnToHead}, + CorruptionCase{"cache line reuse", + 1, + 0, + 2, + 2, + TlbChainLayout::Packed, + {{0, sizeof(uintptr_t)}, {sizeof(uintptr_t), 0}}, + TlbChainValidationStatus::CacheLineReuse}, + CorruptionCase{"page count", + 2, + 0, + 2, + 2, + TlbChainLayout::Spread, + {{0, 64}, {64, 0}}, + TlbChainValidationStatus::PageCountMismatch}}) { + SCOPED_TRACE(entry.name); + PageBuffer buffer(entry.allocation_pages * kTestPageSizeBytes); + ASSERT_NE(buffer.get(), nullptr); + for (const auto& link : entry.links) { + write_link(buffer.get(), link.first, address_at(buffer.get(), link.second)); + } + EXPECT_EQ( + validate_tlb_chain(buffer.get(), buffer.size(), static_cast(buffer.get()) + entry.head_offset, + expected_chain(kTestPageSizeBytes, entry.nodes, entry.pages, entry.layout)), + entry.status); + } } TEST(TlbChainTest, RejectsInvalidStrideAndInsufficientSpreadBuffer) { @@ -386,19 +377,3 @@ TEST(TlbChainTest, ScratchReusePreservesSemanticsWithoutGrowingCapacity) { EXPECT_EQ(scratch.visited_cache_lines.bucket_count(), cache_line_buckets); EXPECT_EQ(scratch.nodes_per_page.bucket_count(), per_page_buckets); } - -TEST(TlbChainTest, SpreadAndPackedRunThroughLatencyKernelIntegration) { - const size_t page_size = static_cast(getpagesize()); - constexpr size_t kRequestedPages = 32; - PageBuffer buffer(kRequestedPages * page_size); - ASSERT_NE(buffer.get(), nullptr); - auto timer = HighResTimer::create(); - ASSERT_TRUE(timer.has_value()); - - for (TlbChainLayout layout : {TlbChainLayout::Spread, TlbChainLayout::Packed}) { - const TlbChainBuildResult result = build_tlb_chain(buffer.get(), buffer.size(), kRequestedPages, page_size, 256, - layout, TlbChainTraversalPolicy::RandomPagesRandomOffsets, 1234); - ASSERT_EQ(result.status, TlbChainBuildStatus::Success); - EXPECT_GT(run_latency_test(result.chain_head, 10000, *timer, nullptr, 0), 0.0); - } -} diff --git a/tests/test_tlb_measurement_scheduler.cpp b/tests/test_tlb_measurement_scheduler.cpp index e88408a..062ec06 100644 --- a/tests/test_tlb_measurement_scheduler.cpp +++ b/tests/test_tlb_measurement_scheduler.cpp @@ -193,75 +193,32 @@ TEST(TlbMeasurementSchedulerTest, MeasurementErrorAfterRecordsPreservesValidPref EXPECT_FALSE(result.converged); } -TEST(TlbMeasurementSchedulerTest, EmptyScheduleIsACompleteNoOp) { - size_t stop_calls = 0; - size_t measure_calls = 0; - const TlbScheduleExecutionResult result = execute_tlb_measurement_schedule( - {}, - [&stop_calls]() { - ++stop_calls; - return false; - }, - [&measure_calls](const TlbMeasurementTask&, TlbMeasurementSample&) { - ++measure_calls; - return TlbTaskMeasureStatus::Success; - }); - - EXPECT_EQ(result.status, TlbScheduleExecutionStatus::Complete); - EXPECT_EQ(stop_calls, 0u); - EXPECT_EQ(measure_calls, 0u); - EXPECT_TRUE(result.records.empty()); - EXPECT_EQ(result.rounds_completed, 0u); -} - -TEST(TlbMeasurementSchedulerTest, MissingMeasureCallbackIsAnErrorForNonEmptySchedule) { - const std::vector schedule = - build_tlb_measurement_schedule(make_points(1), 1, 7, TlbMeasurementPass::Base); - - const TlbScheduleExecutionResult result = execute_tlb_measurement_schedule(schedule, {}, {}); - - EXPECT_EQ(result.status, TlbScheduleExecutionStatus::Error); - EXPECT_TRUE(result.records.empty()); - EXPECT_EQ(result.rounds_completed, 0u); -} - -TEST(TlbMeasurementSchedulerTest, ConvergenceCallbackStopsOnlyAfterCompleteRound) { - const std::vector schedule = - build_tlb_measurement_schedule(make_points(3), 5, 7, TlbMeasurementPass::Base); - - const TlbScheduleExecutionResult result = execute_tlb_measurement_schedule( - schedule, []() { return false; }, - [](const TlbMeasurementTask&, TlbMeasurementSample& sample) { - sample.latency_ns = 1.0; - return TlbTaskMeasureStatus::Success; - }, - [](size_t completed_rounds, const std::vector& records) { - EXPECT_EQ(records.size(), completed_rounds * 3); - return completed_rounds == 2; - }); - - EXPECT_EQ(result.status, TlbScheduleExecutionStatus::Complete); - EXPECT_TRUE(result.converged); - EXPECT_EQ(result.rounds_completed, 2u); - EXPECT_EQ(result.records.size(), 6u); -} - -TEST(TlbMeasurementSchedulerTest, ReportsMaximumRoundsWithoutConvergence) { - const std::vector schedule = - build_tlb_measurement_schedule(make_points(2), 3, 7, TlbMeasurementPass::Base); - - const TlbScheduleExecutionResult result = execute_tlb_measurement_schedule( - schedule, []() { return false; }, - [](const TlbMeasurementTask&, TlbMeasurementSample& sample) { - sample.latency_ns = 1.0; - return TlbTaskMeasureStatus::Success; - }, - [](size_t, const std::vector&) { return false; }); - - EXPECT_EQ(result.status, TlbScheduleExecutionStatus::Complete); - EXPECT_FALSE(result.converged); - EXPECT_EQ(result.rounds_completed, 3u); - EXPECT_EQ(result.records.size(), 6u); +TEST(TlbMeasurementSchedulerTest, StopsAtConvergenceOrMaximumAfterCompleteRounds) { + struct ScheduleCase { + size_t points; + size_t maximum; + size_t converged_at; + size_t completed; + }; + for (const ScheduleCase& entry : {ScheduleCase{3, 5, 2, 2}, ScheduleCase{2, 3, 0, 3}}) { + SCOPED_TRACE(entry.converged_at); + const auto schedule = + build_tlb_measurement_schedule(make_points(entry.points), entry.maximum, 7, TlbMeasurementPass::Base); + const TlbScheduleExecutionResult result = execute_tlb_measurement_schedule( + schedule, []() { return false; }, + [](const TlbMeasurementTask&, TlbMeasurementSample& sample) { + sample.latency_ns = 1.0; + return TlbTaskMeasureStatus::Success; + }, + [&](size_t completed_rounds, const std::vector& records) { + EXPECT_EQ(records.size(), completed_rounds * entry.points); + return completed_rounds == entry.converged_at; + }); + EXPECT_EQ(result.status, TlbScheduleExecutionStatus::Complete); + EXPECT_EQ(result.converged, entry.converged_at != 0); + EXPECT_EQ(result.rounds_completed, entry.completed); + EXPECT_EQ(result.records.size(), entry.completed * entry.points); + } } TEST(TlbMeasurementSchedulerTest, PreservesPairedMeasurementMetadata) { diff --git a/tests/test_tlb_sweep_planner.cpp b/tests/test_tlb_sweep_planner.cpp index d2b39f6..fe010e3 100644 --- a/tests/test_tlb_sweep_planner.cpp +++ b/tests/test_tlb_sweep_planner.cpp @@ -37,12 +37,6 @@ TEST(TlbSweepPlannerTest, HighDensityBasePlanIsPageConsistent) { } } -TEST(TlbSweepPlannerTest, InvalidSizesProduceEmptyBasePlan) { - EXPECT_TRUE(build_tlb_base_sweep_plan(0, 16384, TlbSweepDensity::Low).empty()); - EXPECT_TRUE(build_tlb_base_sweep_plan(64, 0, TlbSweepDensity::Low).empty()); - EXPECT_TRUE(build_tlb_base_sweep_plan(32768, 16384, TlbSweepDensity::Low).empty()); -} - TEST(TlbSweepPlannerTest, BasePlanAcceptsAlignedStrideThatDoesNotDividePage) { const size_t page_size = 16 * Constants::BYTES_PER_KB; const size_t stride = 136; diff --git a/tests/test_utils.cpp b/tests/test_utils.cpp index 81f7777..8c6469c 100644 --- a/tests/test_utils.cpp +++ b/tests/test_utils.cpp @@ -28,18 +28,6 @@ #include "utils/seed_utils.h" #include "utils/utils.h" -TEST(SeedUtilsTest, GenerateSeedUsesInjectedProviderExactlyOnce) { - size_t provider_calls = 0; - - const uint64_t seed = SeedUtils::generate_seed([&provider_calls]() { - ++provider_calls; - return 0x123456789abcdef0ULL; - }); - - EXPECT_EQ(seed, 0x123456789abcdef0ULL); - EXPECT_EQ(provider_calls, 1U); -} - TEST(SeedUtilsTest, ZeroProviderResultFallsBackToNonZeroGeneration) { size_t provider_calls = 0; @@ -108,15 +96,3 @@ TEST(ProgressSpinnerTest, ExplicitClearMakesClearAndDestructorIdempotent) { EXPECT_EQ(output.str(), "\r| done\r \r"); } - -TEST(UtilsTest, JoinThreadsJoinsEveryWorkerAndClearsTheVector) { - std::atomic completed{0}; - std::vector threads; - threads.emplace_back([&completed]() { completed.fetch_add(1); }); - threads.emplace_back([&completed]() { completed.fetch_add(1); }); - - join_threads(threads); - - EXPECT_EQ(completed.load(), 2U); - EXPECT_TRUE(threads.empty()); -}