diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml new file mode 100644 index 0000000..fe5ab4c --- /dev/null +++ b/.github/workflows/bench.yml @@ -0,0 +1,279 @@ +name: Bench + +# Deterministic instruction-count ratchet (bench/README.md). Wall clock on a +# shared runner is noise; executed guest instructions under QEMU's TCG plugin +# are exact, so a hard two-sided gate (+-3 %) is safe here. One job per +# baseline key in bench/baselines.json: +# +# m4-softfp Cortex-M4, -mfloat-abi=soft (mps2-an386) Ooura float32 +# m4f Cortex-M4F, fpv4-sp-d16 (mps2-an386) Ooura float32 +# m33 Cortex-M33, no MVE (mps2-an505) Ooura float32 +# m55 Cortex-M55, Helium (mps3-an547) CMSIS-DSP (deployed profile) +# m55-ooura Cortex-M55, TAP_DSP_FFT_CMSIS=OFF (mps3-an547) Ooura float32 fallback +# +# Toolchain files come from the Stage 1 CI-legs PR (cmake/arm-cortex-m4-mps2.cmake +# with TAP_DSP_M4_FPU, cmake/arm-cortex-m33-mps2.cmake, cmake/arm-cortex-m55-mps3.cmake); +# the float32 test legs themselves live in ci.yml. This workflow owns only the +# ratchet and the size report. +# +# Modes, decided per key by the "Baselines state" step: +# compare the key has baselines: gate at +-3 %, two-sided. The normal case. +# seed the key is empty AND this is a push to main or a workflow_dispatch: +# run icount.py --update, print the counts, upload baselines-. +# seed-summary merges the per-key files; the seeding commit copies +# the merged file into bench/baselines.json and records that main +# run's URL and SHA plus the GCC/QEMU versions in bench/README.md. +# refuse the key is empty on a pull_request: FAIL with the seeding +# instructions. Seeding from a PR would be a gate bypass (empty the +# key, go green), so it is never allowed; and a PR that empties a key +# its base branch had seeded fails in the state step itself. +# Every run, in every mode, uploads measured- (the counts as JSON) so a +# new scenario's number is "run CI, copy, commit", never typed in. +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +# The expensive workflow: five QEMU jobs. One run per ref at a time. +concurrency: + group: bench-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + icount-ratchet: + name: icount ${{ matrix.key }} + # Pinned to 24.04, not -latest: the plugin header below is pinned to the + # QEMU this image ships (8.2.x), digest-verified. + runs-on: ubuntu-24.04 + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + # TAP_DSP_FFT_CMSIS defaults ON for every bare-metal Arm profile + # (root CMakeLists.txt), so every non-Helium key pins it OFF here + # explicitly, whatever its toolchain file also does. + # + # text_ceiling: the .text byte ceiling for the MinSizeRel + # tap_dsp_size_probe_rfft_f32_512 on this key (bench/README.md, + # "Sizes"). 0 = not yet recorded: the step prints the figure and + # does not gate. The seeding commit sets the numbers. + - key: m4-softfp + toolchain: cmake/arm-cortex-m4-mps2.cmake + flags: -DTAP_DSP_M4_FPU=OFF -DTAP_DSP_FFT_CMSIS=OFF + text_ceiling: 0 + - key: m4f + toolchain: cmake/arm-cortex-m4-mps2.cmake + flags: -DTAP_DSP_M4_FPU=ON -DTAP_DSP_FFT_CMSIS=OFF + text_ceiling: 0 + - key: m33 + toolchain: cmake/arm-cortex-m33-mps2.cmake + flags: -DTAP_DSP_FFT_CMSIS=OFF + text_ceiling: 0 + - key: m55 + toolchain: cmake/arm-cortex-m55-mps3.cmake + flags: "" + text_ceiling: 0 + - key: m55-ooura + toolchain: cmake/arm-cortex-m55-mps3.cmake + flags: -DTAP_DSP_FFT_CMSIS=OFF + text_ceiling: 0 + env: + # Commit the tag pointed at when pinned (tags are movable; commit SHAs + # are not); the header's digest is verified on download. This SHA is + # QEMU v8.2.2, matching ubuntu-24.04's qemu-system-arm. That header + # defines QEMU_PLUGIN_VERSION 1 (plugin API v1; the plugin declares + # whatever the header says). Same pin as MuTap's icount-ratchet job. + # The header is GPL-2.0-or-later, fetched here only to build a CI tool + # that is never shipped (tools/qemu_insn_plugin/insn_count.c). + QEMU_PLUGIN_HEADER_URL: https://raw.githubusercontent.com/qemu/qemu/11aa0b1ff115b86160c4d37e7c37e6a6b13b77ea/include/qemu/qemu-plugin.h + QEMU_PLUGIN_HEADER_SHA256: "c53a2af163e80e3f4bc6c60dbdfc84003db329d757e37cd8a16a77e1d82606ff" + PLUGIN: /tmp/libinsncount.so + BUILD_DIR: build-${{ matrix.key }} + SIZE_DIR: build-${{ matrix.key }}-minsizerel + KEY: ${{ matrix.key }} + steps: + - uses: actions/checkout@v4 + + - name: Install Arm toolchain and QEMU + run: > + sudo apt-get update -q && + sudo apt-get install -y -q gcc-arm-none-eabi qemu-system-arm + libglib2.0-dev pkg-config + + # These versions go into bench/README.md next to the numbers when a key + # is seeded or re-recorded; the counts are only comparable within a + # toolchain/QEMU pair. + - name: Toolchain versions + run: | + arm-none-eabi-gcc --version | head -1 + qemu-system-arm --version | head -1 + cmake --version | head -1 + + - name: Build counting plugin + run: | + curl -sfLo /tmp/qemu-plugin.h "$QEMU_PLUGIN_HEADER_URL" + actual=$(sha256sum /tmp/qemu-plugin.h | cut -d' ' -f1) + if [ "$actual" != "$QEMU_PLUGIN_HEADER_SHA256" ]; then + echo "::error::qemu-plugin.h checksum mismatch"; exit 1 + fi + grep -m1 "define QEMU_PLUGIN_VERSION" /tmp/qemu-plugin.h + gcc -shared -fPIC $(pkg-config --cflags glib-2.0) -I/tmp \ + -o "$PLUGIN" tools/qemu_insn_plugin/insn_count.c + + # Release (-O2) workloads, matching how baselines are recorded. Tests + # off: this job counts instructions; ci.yml runs the suites. + - name: Build workloads (Release) + run: > + cmake -S . -B "$BUILD_DIR" + -DCMAKE_BUILD_TYPE=Release + -DCMAKE_TOOLCHAIN_FILE=${{ matrix.toolchain }} + -DTAP_DSP_BUILD_TESTS=OFF + -DTAP_DSP_BUILD_BENCH=ON + ${{ matrix.flags }} + && cmake --build "$BUILD_DIR" -j 4 + + # The .text ceilings (docs/audit-fft-and-code-smells.md, Part 10 item 4) + # are measured on the size probe built MinSizeRel, like the test legs, + # with `size -A` (the .text row alone; Berkeley format folds .rodata and + # NOLOAD sections into "text"). Gates only once text_ceiling is nonzero. + - name: Size probe (MinSizeRel, .text) + run: | + cmake -S . -B "$SIZE_DIR" \ + -DCMAKE_BUILD_TYPE=MinSizeRel \ + -DCMAKE_TOOLCHAIN_FILE=${{ matrix.toolchain }} \ + -DTAP_DSP_BUILD_TESTS=OFF -DTAP_DSP_BUILD_BENCH=ON ${{ matrix.flags }} + cmake --build "$SIZE_DIR" -j 4 --target tap_dsp_size_probe_rfft_f32_512 + probe="$SIZE_DIR/bench/tap_dsp_size_probe_rfft_f32_512" + arm-none-eabi-size -A "$probe" | tee size.txt + text=$(awk '$1 == ".text" { print $2 }' size.txt) + ceiling=${{ matrix.text_ceiling }} + echo "$KEY: .text = $text bytes (ceiling: $ceiling; 0 = not yet recorded)" + { echo "### size $KEY (MinSizeRel, size -A)"; echo "- tap_dsp_size_probe_rfft_f32_512 .text = $text bytes (ceiling $ceiling)"; } >> "$GITHUB_STEP_SUMMARY" + if [ "$ceiling" -gt 0 ] && [ "$text" -gt "$ceiling" ]; then + echo "::error::$KEY: .text $text bytes exceeds the recorded ceiling $ceiling (bench/README.md, Sizes)" + exit 1 + fi + + # Decides the mode (see the header comment) and fails a pull request + # that emptied a key its base branch had seeded. + - name: Baselines state + id: mode + env: + EVENT: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + python3 - <<'EOF' + import json, os, subprocess, sys + key, event, base_sha = os.environ["KEY"], os.environ["EVENT"], os.environ["BASE_SHA"] + head = json.load(open("bench/baselines.json")).get(key) or {} + if event == "pull_request": + subprocess.run(["git", "fetch", "-q", "--depth=1", "origin", base_sha], check=True) + shown = subprocess.run(["git", "show", f"{base_sha}:bench/baselines.json"], + capture_output=True, text=True) + base = (json.loads(shown.stdout) if shown.returncode == 0 else {}).get(key) or {} + if base and not head: + print(f"::error::{key}: bench/baselines.json has baselines for this key on the base " + f"branch ({base_sha[:12]}) and none in this pull request. A key is never emptied; " + "re-record with icount.py --update and write the before/after and the reason in " + "bench/README.md.") + sys.exit(1) + mode = "compare" if head else ("refuse" if event == "pull_request" else "seed") + print(f"{key}: mode={mode}") + with open(os.environ["GITHUB_OUTPUT"], "a") as out: + out.write(f"mode={mode}\n") + EOF + + # One step, three modes. The counts are always measured and always + # recorded to measured-.json; the per-scenario lines go to the + # step summary; then the mode decides the verdict. + - name: Count (${{ matrix.key }}) + env: + MODE: ${{ steps.mode.outputs.mode }} + run: | + set -o pipefail + extra="" + if [ "$MODE" = seed ]; then extra="--update"; fi + rc=0 + python3 scripts/icount.py --target "$KEY" --build-dir "$BUILD_DIR" --plugin "$PLUGIN" \ + --record "measured-$KEY.json" $extra | tee icount.txt || rc=$? + { echo "### icount $KEY ($MODE)"; echo '```'; cat icount.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + case "$MODE" in + seed) + echo "::warning::$KEY had no baselines: this push-to-main run recorded them (artifact baselines-$KEY; seed-summary merges all keys). Copy the merged file into bench/baselines.json in a seeding commit that records this run's URL and SHA and the toolchain versions above in bench/README.md." + ;; + refuse) + echo "::error::$KEY has no baselines and this is a pull request. Seeding runs only on a push to main or a workflow_dispatch, never from a PR (bench/README.md, Seeding). The counts this run measured are in the measured-$KEY artifact." + exit 1 + ;; + esac + exit $rc + + - name: Upload measured counts + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: measured-${{ matrix.key }} + path: measured-${{ matrix.key }}.json + if-no-files-found: ignore + retention-days: 30 + + - name: Upload recorded baselines (seeding run) + if: steps.mode.outputs.mode == 'seed' + uses: actions/upload-artifact@v4 + with: + name: baselines-${{ matrix.key }} + path: bench/baselines.json + retention-days: 30 + + # Folds the per-key seeding artifacts (each has one key filled in) into the + # single file the seeding commit copies to bench/baselines.json. Fails when + # any icount job failed, so a partial seed from a half-red run can never be + # merged. NEVER a required check: the five `icount ` jobs are the + # required checks once seeded (bench/README.md). + seed-summary: + name: Merge seeding artifacts (never a required check) + needs: icount-ratchet + if: ${{ !cancelled() }} + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Require a green matrix + if: needs.icount-ratchet.result != 'success' + run: | + echo "::error::icount matrix result is '${{ needs.icount-ratchet.result }}'; nothing is merged from a run with a failed key" + exit 1 + + - uses: actions/checkout@v4 + + - uses: actions/download-artifact@v4 + id: download + continue-on-error: true + with: + pattern: baselines-* + path: /tmp/seeds + + - name: Merge + run: | + files=$(ls /tmp/seeds/*/baselines.json 2>/dev/null || true) + if [ -z "$files" ]; then + echo "no seeding artifacts found (every key ran in compare mode); nothing to merge" + echo "merged=false" >> "$GITHUB_ENV" + exit 0 + fi + python3 scripts/icount.py --merge $files --baselines bench/baselines.json + cat bench/baselines.json + { echo "### merged baselines.json (the seeding commit copies this; record this run's URL and SHA in bench/README.md)"; echo '```json'; cat bench/baselines.json; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + echo "merged=true" >> "$GITHUB_ENV" + + - name: Upload merged baselines + if: env.merged == 'true' + uses: actions/upload-artifact@v4 + with: + name: baselines-merged + path: bench/baselines.json + retention-days: 30 diff --git a/CMakeLists.txt b/CMakeLists.txt index ec76230..e0e0f7a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -135,6 +135,16 @@ if(TAP_DSP_BUILD_TESTS) add_subdirectory(tests) endif() +# Benchmarks: the host wall-clock microbenchmark (informational) and the +# deterministic instruction-count ratchet workloads (bench/icount), one binary +# per scenario, counted under QEMU by scripts/icount.py from +# .github/workflows/bench.yml. Builds on the host and on the cross toolchains; +# policy and workflow in bench/README.md. +option(TAP_DSP_BUILD_BENCH "Build DspTap benchmarks and instruction-count ratchet workloads" OFF) +if(TAP_DSP_BUILD_BENCH) + add_subdirectory(bench) +endif() + # C ABI shared library for FFI consumers (the notebooks drive the shipping C++ # through it via ctypes — see notebooks/dsptap_py.py). tools/capi also builds # standalone (`cmake -B build_capi -S tools/capi`), which is what the ctypes diff --git a/README.md b/README.md index b438235..ffcfbe0 100644 --- a/README.md +++ b/README.md @@ -344,6 +344,9 @@ cmake --build build-m33 ctest --test-dir build-m33 --output-on-failure ``` +Benchmarks and the per-target instruction-count ratchet build with +`-DTAP_DSP_BUILD_BENCH=ON`; policy and workflow in [`bench/README.md`](bench/README.md). + ### As a submodule ```cmake diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt new file mode 100644 index 0000000..f4b9c64 --- /dev/null +++ b/bench/CMakeLists.txt @@ -0,0 +1,59 @@ +# DspTap benchmarks (root option TAP_DSP_BUILD_BENCH; policy in README.md here). +# +# tap_dsp_bench_fft host wall-clock microbenchmark, min-of-N, informational +# only — never a gate. Host-class builds only. +# icount/ deterministic instruction-count ratchet workloads, one +# binary per scenario; built on the host as a smoke test +# (ctest runs each twice and compares) and on the QEMU +# cross legs, where .github/workflows/bench.yml counts them. +# tap_dsp_size_probe_* one profile alone, for the per-leg .text ceilings +# (built MinSizeRel by bench.yml, measured with size -A). +# +# "Host-class" is decided by CMAKE_SYSTEM_NAME STREQUAL "Generic" (bare metal) +# throughout, never by CMAKE_CROSSCOMPILING: a Linux-arm64 cross build is +# host-class and gets the double scenario; the Cortex-M legs are not. +# +# TAP_DSP_BENCH_ENGINE selects the engine everything here measures +# (bench_common.h): reference_c today; split_radix from Stage 2a. +set(TAP_DSP_BENCH_ENGINE reference_c CACHE STRING + "FFT engine the benchmark scenarios measure (reference_c; split_radix from Stage 2a)") + +if(CMAKE_SYSTEM_NAME STREQUAL "Generic") + set(_tap_dsp_bench_host_class OFF) +else() + set(_tap_dsp_bench_host_class ON) +endif() + +if(_tap_dsp_bench_host_class) + add_executable(tap_dsp_bench_fft bench_fft.cpp) + target_compile_definitions(tap_dsp_bench_fft PRIVATE + TAP_DSP_BENCH_ENGINE=${TAP_DSP_BENCH_ENGINE}) + target_include_directories(tap_dsp_bench_fft PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(tap_dsp_bench_fft PRIVATE + tap::dsp + tap_dsp_warnings) +endif() + +add_subdirectory(icount) + +# Size probes: name:precision:n, the same profiles as the icount scenarios at +# N = 512 (the ceilings are stated per profile, not per N). +set(_tap_dsp_size_probes rfft_f32_512:0:512) +if(_tap_dsp_bench_host_class) + list(APPEND _tap_dsp_size_probes rfft_f64_512:1:512) +endif() +foreach(_probe IN LISTS _tap_dsp_size_probes) + string(REPLACE ":" ";" _parts "${_probe}") + list(GET _parts 0 _name) + list(GET _parts 1 _precision) + list(GET _parts 2 _n) + add_executable(tap_dsp_size_probe_${_name} size_probe.cpp) + target_compile_definitions(tap_dsp_size_probe_${_name} PRIVATE + TAP_DSP_SC_PRECISION=${_precision} + TAP_DSP_SC_N=${_n} + TAP_DSP_BENCH_ENGINE=${TAP_DSP_BENCH_ENGINE}) + target_include_directories(tap_dsp_size_probe_${_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(tap_dsp_size_probe_${_name} PRIVATE + tap::dsp + tap_dsp_warnings) +endforeach() diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..158e081 --- /dev/null +++ b/bench/README.md @@ -0,0 +1,232 @@ +# DspTap benchmarks and the performance ratchet + +The FFT is the primitive under MuTap's, SampleRateTap's and RatioTap's own +instruction-count ratchets, so a regression here shows up downstream as a +mysterious chain-level delta someone "updates" past. The primitive needs its +own gate, per profile, per target, seeded before anything changes — which is +why this scaffold lands **before** the C++20 port (Stage 1b of +[`docs/audit-fft-and-code-smells.md`](../docs/audit-fft-and-code-smells.md), +design in its Part 11). + +Two instruments, one policy: + +| instrument | what it measures | where it runs | role | +|---|---|---|---| +| `icount/` | executed guest instructions per scenario binary, under QEMU's TCG plugin | the QEMU legs (`.github/workflows/bench.yml`) | **the gate**: ±3 %, two-sided | +| `bench_fft.cpp` | wall-clock ns per transform, min-of-N | the host, locally | informational only, never a gate | + +## Build and run (host) + +```sh +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DTAP_DSP_BUILD_BENCH=ON +cmake --build build +./build/bench/tap_dsp_bench_fft # wall clock +./build/bench/icount/tap_dsp_icount_rfft_f32_512 # prints its checksum +``` + +`bench_fft` prints ns per forward and per inverse transform for each scenario +(minimum of 25 reps of 2^20 samples per direction). Only same-machine, +same-load comparisons mean anything; record the machine, compiler, load +average and date with any number that goes into `docs/fft-design.md`. + +On the host the `tap_dsp_icount_*` binaries are a smoke test, registered with +ctest (`tap_dsp_icount__smoke`, `bench/icount/smoke.cmake`): each +binary runs twice, both runs must report `ok=1`, and the two `DONE` lines +must be byte-identical. There is no instruction count on the host; the count +comes from QEMU on the legs. + +## Scenarios + +One binary per scenario, selected by compile definitions (bare metal has no +argv), all built from `icount/icount_main.cpp`: + +| scenario | profile | N | targets | +|---|---|---|---| +| `rfft_f32_512` | float | 512 | all legs | +| `rfft_f32_2048` | float | 2048 | all legs | +| `rfft_f64_512` | double | 512 | host-class only: soft-float double is not a profile, so the bare-metal legs neither build nor baseline it | + +Stage 3b adds `rfft_q15_512`, `rfft_q31_512` and `rfft_q31_2048` (Part 11). + +Each scenario constructs one `tap::dsp::basic_real_fft`, then runs a +forward + inverse loop over a four-block xorshift corpus for 2^20 samples per +direction (2048 iterations at N = 512, 512 at N = 2048). Every output word +of every iteration — the spectrum and the scaled inverse — goes through an +integer FNV-1a-64 fold over its bit pattern (`bench_common.h`), printed at +the end: + +``` +TAP_DSP_ICOUNT_DONE ok=1 engine=reference_c backend=ooura scenario=rfft_f32_512 checksum=0x662dd085b5b88325 +``` + +The fold is exact and order-sensitive: two runs of the same binary print the +same value, and a 1-ulp change in any single output changes it (verified by +nudging one spectrum bin at one iteration with `nextafterf`: `0x662dd085b5b88325` +becomes `0x259510dc8721cba8`). A floating running sum cannot promise that — it +absorbs differences below the accumulator's ulp — which is why the checksum +is an integer hash: it is the fingerprint Stage 2b compares between the C +and the port on the QEMU legs, where the host parity TU does not run. `ok` +is a sanity check that the last iteration round-trips its input within +1e-3 in the scenario's own precision; `backend=` names what the build routed +`basic_real_fft` through (`ooura`, `cmsis` on the `m55` key, +`accelerate` on Apple), independently of the engine value. + +Why these numbers: on the host (x86-64, GCC 13 `-O2`, callgrind) the three +scenarios execute 109 M, 125 M and 108 M instructions. +Construction plus the one-time Ooura table build plus the print is under +0.2 M — well under the 1 % the design asks for. The share of the count that +is not the transform itself — the class's out-of-place copies, the 2/N +scaling loop and the fold — is 16.1 % / 14.0 % / 16.6 % +(`main` inclusive minus `rdft`/`rdft_f` inclusive); it is constant per +scenario, so the ratchet works, but a 3 % change in the FFT alone reads as +roughly 3 % × (1 − that share) at the gate, and the share is larger on a +scalar Cortex-M. The Arm counts are larger than the host's; the ratios +hold. No `` (a toolchain's libstdc++ would move the count), no +allocation in the loop, and no double anywhere in the float scenarios (the +M4 soft-float leg would otherwise measure libgcc). + +`TAP_DSP_BENCH_ENGINE` (`bench_common.h`, a CMake cache variable of the same +name) selects the engine: `reference_c` today — the class as built, which is +the vendored Ooura C, or on the `m55` key the CMSIS-DSP Helium backend behind +the same class. `split_radix` is the Stage 2a port; from then until Stage 2c +each float scenario builds twice, C and port, and the job prints both counts +and their ratio. That ratio is the Stage 2b gate. + +## Baselines and targets + +[`baselines.json`](baselines.json) is keyed by target, one dict per key: + +| key | core | QEMU machine | float32 FFT | +|---|---|---|---| +| `m4-softfp` | Cortex-M4, `-mfloat-abi=soft` | `mps2-an386` | Ooura (every float op a library call) | +| `m4f` | Cortex-M4F, `fpv4-sp-d16` | `mps2-an386` | Ooura | +| `m33` | Cortex-M33, no MVE | `mps2-an505` | Ooura | +| `m55` | Cortex-M55, Helium | `mps3-an547` | CMSIS-DSP — the deployed profile | +| `m55-ooura` | Cortex-M55, `-DTAP_DSP_FFT_CMSIS=OFF` | `mps3-an547` | Ooura — the fallback | + +JSON carries no comments, so the provenance of every recorded set lives here, +in the table below: the `main` run that recorded it, and the GCC and QEMU +versions, because the counts are only comparable within a toolchain/QEMU +pair. The plugin header is pinned to the QEMU that Ubuntu 24.04 ships +(8.2.2; that header defines `QEMU_PLUGIN_VERSION 1`, plugin API v1) and +digest-verified on download, as MuTap does. + +### Recorded baselines and updates + +Every seeding or `--update` commit adds one row per key it touched. The +shape is fixed so the record stays greppable: + +| key | scenario | before | after | delta | reason | main run (URL) | main SHA | arm-none-eabi-gcc | qemu-system-arm | +|---|---|---|---|---|---|---|---|---|---| +| all | all | — | — | — | **unseeded**: the skeleton is empty by design; numbers are measured on CI, never typed in | — | — | — | — | + +`before` is `—` for a seed. `main SHA` is the commit on `main` whose push +run measured the numbers: a pull-request head SHA stops resolving after +this repo's rebase/squash + branch-delete flow, so a seed or update is never +taken from a PR run. + +### Seeding, and how the job decides what to do + +The workflow decides per key from `baselines.json`: + +- **compare** — the key has baselines. The normal mode: ±3 %, two-sided, + and a recorded scenario with no binary fails too (`STALE BASELINE`). +- **seed** — the key is empty **and** the event is a push to `main` or a + `workflow_dispatch`. The job runs `scripts/icount.py --update`, prints + every scenario's count into the step summary and uploads + `baselines-`; the `seed-summary` job (which fails if any key failed, + and is never a required check) merges the per-key files into one + `baselines-merged` artifact. +- **refuse** — the key is empty on a `pull_request`. The job **fails** with + the seeding instructions. Seeding from a PR would be a gate bypass (empty + the key, go green), so it is never allowed; likewise a PR whose + `baselines.json` empties a key that its base branch had seeded fails in + the "Baselines state" step, before anything is counted. + +Every run, in every mode, uploads `measured-` — the counts this run +measured, as JSON — so a scenario added later (Stage 3b's `rfft_q15_512`, +`rfft_q31_*`) gets its number by "run CI, copy the artifact, commit", never +by typing one in. + +The seeding commit copies `baselines-merged` to `bench/baselines.json`, +adds the rows above with the `main` run's URL and SHA and the versions the +job printed in its "Toolchain versions" step, sets the `text_ceiling` +numbers in `bench.yml` from the same run's size step, and nothing else rides +in it. From that commit on, the key's job compares and a red ratchet is a +failing check. + +The first push to `main` after the QEMU legs (#17) and this scaffold have +merged is therefore the seeding run for all five keys (wave 2 of Part 12); +re-recording later is a `workflow_dispatch` on `main`, or the hand +procedure below. + +Re-recording by hand, in the target's environment (cross toolchain, QEMU and +the plugin on PATH): + +```sh +gcc -shared -fPIC $(pkg-config --cflags glib-2.0) -I \ + -o /tmp/libinsncount.so tools/qemu_insn_plugin/insn_count.c +cmake -S . -B build-m33 -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_TOOLCHAIN_FILE=cmake/arm-cortex-m33-mps2.cmake \ + -DTAP_DSP_BUILD_TESTS=OFF -DTAP_DSP_BUILD_BENCH=ON +cmake --build build-m33 -j +python3 scripts/icount.py --target m33 --build-dir build-m33 \ + --plugin /tmp/libinsncount.so [--update] [--record measured-m33.json] +python3 scripts/icount.py --merge a.json b.json # fold per-key files into one +``` + +## Policy + +- **Tolerance ±3 %**, the family's number. Tighter is not better: link-order + and libgcc changes move counts by fractions of a percent legitimately. +- **Two-sided.** A regression beyond tolerance fails; an *improvement* beyond + tolerance also fails, so a stale, too-high baseline can never let a later + regression hide in the slack — the winning commit re-records. +- **The ratchet runs on every pull request and on every push to `main`** + on every QEMU leg (one run per ref at a time). A red ratchet is a failing + check, not a warning. Once seeded, the five `icount ` jobs are the + required checks; the artifact-merge job never is. +- **`--update` is a written commit on its own**: the measured before/after per + key and the reason go into the table above, in its fixed shape. An expected + regression (a correctness fix that costs instructions) is written down; + there is no silent absorb. A key is never emptied to "reset" it — the + workflow fails a pull request that does. +- **Size runs in the same job, MinSizeRel.** The job builds + `tap_dsp_size_probe_rfft_f32_512` (`bench/size_probe.cpp`: one profile, + no stdio) a second time with `-DCMAKE_BUILD_TYPE=MinSizeRel` — the build + type the test legs and Part 10 item 4 use, so a ceiling here and a size in + `ci.yml` describe the same object — and reads the `.text` row of + `arm-none-eabi-size -A` (Berkeley format folds `.rodata` and NOLOAD + sections into "text"). Each key carries a `text_ceiling` in `bench.yml`; + `0` means not yet recorded and the step only prints. The ceilings are a + **promised item for wave 2**: the seeding commit sets them from the same + `main` run that seeds the counts, and they are updated the same way as the + counts, in the table above. +- **Wall clock is never a gate.** `bench_fft` is the local tool for the + desktop and Apple vDSP claims; its numbers go into `docs/fft-design.md` + with machine and date. + +## What the ratchet catches, by stage + +| stage | what a red ratchet means | +|---|---| +| 2b (routing flip to the port) | a port that vectorizes worse than the C; the C-vs-port ratio printed from 2a on is the gate | +| 3b (fixed point) | nothing yet — it seeds; every later `SMMULR`, Helium or table-layout change to the Q15/Q31 kernel then has a number to beat | +| 4 (engine as a parameter) | a backend regression on the deployed `m55` profile, or the `m55-ooura` fallback quietly getting slower | +| 6 (hygiene) | the hygiene pass slowing the hot path unnoticed | + +## Provenance + +`icount/icount_main.cpp`, `scripts/icount.py` and +`tools/qemu_insn_plugin/insn_count.c` are adapted from MuTap's (MIT, MuTap +contributors; the same pattern lives in SampleRateTap and RatioTap). Copied +rather than shared because they are small; a taphouse-style consolidation is +the eventual home (Part 11, "Sharing"). Everything under `bench/` and +`scripts/` is DspTap-authored MIT. + +The plugin is compiled against QEMU's `qemu-plugin.h`, which is +`SPDX-License-Identifier: GPL-2.0-or-later`. That header is fetched at CI +time (digest-verified), never vendored into this repo, and used only to +build a test tool that runs in CI and is not shipped; nothing in the DspTap +tree or in what consumers link is GPL, so `NOTICE.md` carries no entry for +it. diff --git a/bench/baselines.json b/bench/baselines.json new file mode 100644 index 0000000..df01ae1 --- /dev/null +++ b/bench/baselines.json @@ -0,0 +1,7 @@ +{ + "m33": {}, + "m4-softfp": {}, + "m4f": {}, + "m55": {}, + "m55-ooura": {} +} diff --git a/bench/bench_common.h b/bench/bench_common.h new file mode 100644 index 0000000..7fef2f0 --- /dev/null +++ b/bench/bench_common.h @@ -0,0 +1,128 @@ +/// @file bench_common.h +/// @brief Engine selector, deterministic input and the bit-exact fold shared by the FFT benches. +// SPDX-License-Identifier: MIT +// Copyright 2026 Timothy Place and the DspTap contributors. +// +// Shared by bench/icount/icount_main.cpp (the instruction-count ratchet +// workloads), bench/size_probe.cpp (the .text probe) and bench/bench_fft.cpp +// (the host wall-clock microbenchmark) so all measure the same engine over +// the same input (bench/README.md). + +#pragma once + +#include +#include +#include +#include +#include + +#include "tap/dsp/fft.h" + +// TAP_DSP_BENCH_ENGINE — the engine a scenario measures. A bare identifier, +// set by bench/CMakeLists.txt (cache variable of the same name; default +// reference_c) and stringized here so it is printed with every result: +// +// reference_c tap::dsp::basic_real_fft as built today: the vendored +// Ooura C (third_party/ooura/fftsg.c, fftsg_float.c). Where the +// build routes float32 through an accelerated backend behind the +// same class (TAP_DSP_FFT_CMSIS on the `m55` key, vDSP on Apple), +// that backend is what this value measures; the printed +// `backend=` field (backend_name below) says which. +// split_radix TODO(Stage 2a): the C++20 port, detail::split_radix_rdft, +// built beside the C. The port agent replaces the static_assert +// below with the second alias and makes bench/icount/CMakeLists.txt +// build every float scenario twice (docs/audit-fft-and-code-smells.md, +// Part 11: the C-vs-port ratio is the Stage 2b gate). +// +// A documented macro, on purpose — not a registry. Once Stage 4 makes the +// engine an explicit class parameter this selector becomes that parameter. +#ifndef TAP_DSP_BENCH_ENGINE +#define TAP_DSP_BENCH_ENGINE reference_c +#endif +#define TAP_DSP_BENCH_STRINGIZE_(x) #x +#define TAP_DSP_BENCH_STRINGIZE(x) TAP_DSP_BENCH_STRINGIZE_(x) + +namespace tap::dsp::bench { + + constexpr const char* k_engine_name = TAP_DSP_BENCH_STRINGIZE(TAP_DSP_BENCH_ENGINE); + static_assert(std::string_view{k_engine_name} == "reference_c", + "TAP_DSP_BENCH_ENGINE: only reference_c exists until the Stage 2a port lands split_radix"); + + /// The transform under test for the selected engine. + template + using fft_under_test = tap::dsp::basic_real_fft; + + /// Which backend basic_real_fft was built over, from the macros + /// fft.h switches on. The accelerated backends apply to float only; + /// double is always the Ooura C. + template + constexpr const char* backend_name() noexcept { + if constexpr (std::is_same_v) { +#if defined(TAP_DSP_FFT_CMSIS) + return "cmsis"; +#elif defined(TAP_DSP_FFT_ACCELERATE) + return "accelerate"; +#else + return "ooura"; +#endif + } + else { + return "ooura"; + } + } + + /// xorshift32 (Marsaglia 2003), the family's deterministic source: no + /// , so counts cannot drift with a toolchain's libstdc++, and no + /// double anywhere, so the float scenarios never touch libgcc's soft + /// double on the M4 soft-float leg. fill() lands in the CLOSED interval + /// [-0.5, +0.5]: static_cast(0xFFFFFFFF) rounds up to 2^32, so + /// the top value is reached exactly. + class xorshift32 { + public: + constexpr explicit xorshift32(std::uint32_t seed) noexcept + : m_state(seed) {} + + constexpr std::uint32_t next() noexcept { + m_state ^= m_state << 13; + m_state ^= m_state >> 17; + m_state ^= m_state << 5; + return m_state; + } + + template + void fill(Sample* out, std::size_t n) noexcept { + for (std::size_t i = 0; i < n; ++i) { + out[i] = (static_cast(next()) / Sample(4294967296.0)) - Sample(0.5); + } + } + + private: + std::uint32_t m_state; + }; + + /// FNV-1a-64 (Fowler–Noll–Vo; offset basis and prime as published) over + /// the bit pattern of every sample, one 32- or 64-bit word per step + /// rather than one octet: h ^= bits(v[i]); h *= prime. Integer only, so + /// it is exact and order-sensitive — a 1-ulp change in any single output + /// changes the printed value, which a floating running sum cannot promise + /// (it absorbs differences below the accumulator's ulp). This is what + /// makes the printed checksum a fingerprint of the engine's output, the + /// C-vs-port comparison Stage 2b needs on the QEMU legs, and not only a + /// dead-code-elimination guard. + inline constexpr std::uint64_t k_fnv1a64_offset = 0xcbf29ce484222325ull; + inline constexpr std::uint64_t k_fnv1a64_prime = 0x100000001b3ull; + + template + std::uint64_t fold(std::uint64_t h, const Sample* v, std::size_t n) noexcept { + using bits_type = std::conditional_t; + static_assert(sizeof(bits_type) == sizeof(Sample), "fold expects a 32- or 64-bit sample"); + for (std::size_t i = 0; i < n; ++i) { + bits_type bits = 0; + std::memcpy(&bits, &v[i], sizeof bits); + h ^= bits; + h *= k_fnv1a64_prime; + } + return h; + } + +} // namespace tap::dsp::bench diff --git a/bench/bench_fft.cpp b/bench/bench_fft.cpp new file mode 100644 index 0000000..89a2e6d --- /dev/null +++ b/bench/bench_fft.cpp @@ -0,0 +1,101 @@ +/// @file bench_fft.cpp +/// @brief Host wall-clock microbenchmark for the real FFT: min-of-N ns per transform. +// SPDX-License-Identifier: MIT +// Copyright 2026 Timothy Place and the DspTap contributors. +// +// Informational only (bench/README.md): wall clock on a shared runner is +// noise, so nothing gates on these numbers. This is the local tool for the +// desktop and Apple vDSP claims — record the machine, compiler, load average +// and date with any number that goes into docs/fft-design.md. The gate is the +// instruction-count ratchet in icount/. +// +// Same scenarios and engine selector as the ratchet (bench_common.h): +// rfft_f32_512, rfft_f32_2048, rfft_f64_512. Each rep runs a batch of +// out-of-place forward() calls and, separately, a batch of inverse() calls +// (the class's scaled inverse), and the minimum over reps is reported per +// transform. From Stage 2a the port and the C build side by side here via +// TAP_DSP_BENCH_ENGINE; after Stage 2c the recorded C numbers are the +// comparison. +#include +#include +#include +#include +#include + +#include "bench_common.h" + +namespace { + + constexpr std::size_t k_reps = 25; + constexpr std::size_t k_batch_samples = std::size_t{1} << 20; // per batch, per direction + constexpr std::size_t k_corpus_blocks = 4; + + struct result { + double forward_ns; + double inverse_ns; + }; + + using clock = std::chrono::steady_clock; + + double ns_since(clock::time_point t0) noexcept { + return std::chrono::duration(clock::now() - t0).count(); + } + + template + result measure(std::size_t n, Sample& sink) { + tap::dsp::bench::fft_under_test fft(n); + const std::size_t batch = k_batch_samples / n; + + std::vector corpus(k_corpus_blocks * n); + std::vector spectrum(n); + std::vector out(n); + tap::dsp::bench::xorshift32 rng(0x9E3779B9u); + rng.fill(corpus.data(), corpus.size()); + + Sample acc = Sample(0); + result best{1e300, 1e300}; + for (std::size_t rep = 0; rep < k_reps; ++rep) { + const auto t0 = clock::now(); + for (std::size_t i = 0; i < batch; ++i) { + fft.forward(corpus.data() + (i % k_corpus_blocks) * n, spectrum.data()); + acc += spectrum[i % n]; + } + const double fwd = ns_since(t0) / static_cast(batch); + + const auto t1 = clock::now(); + for (std::size_t i = 0; i < batch; ++i) { + fft.inverse(spectrum.data(), out.data()); + acc += out[i % n]; + } + const double inv = ns_since(t1) / static_cast(batch); + + best.forward_ns = fwd < best.forward_ns ? fwd : best.forward_ns; + best.inverse_ns = inv < best.inverse_ns ? inv : best.inverse_ns; + } + sink += acc; + return best; + } + + void report(const char* name, const result& r) { + std::printf("%-16s %12.1f %12.1f %14.1f\n", name, r.forward_ns, r.inverse_ns, r.forward_ns + r.inverse_ns); + } + +} // namespace + +int main() { + std::printf( + "tap_dsp_bench_fft engine=%s float backend=%s min of %zu reps, %zu samples per batch per direction\n", + tap::dsp::bench::k_engine_name, tap::dsp::bench::backend_name(), k_reps, k_batch_samples); + std::printf("%-16s %12s %12s %14s\n", "scenario", "forward ns", "inverse ns", "round-trip ns"); + + float sink32 = 0.0f; + double sink64 = 0.0; + report("rfft_f32_512", measure(512, sink32)); + report("rfft_f32_2048", measure(2048, sink32)); + report("rfft_f64_512", measure(512, sink64)); + + // Keep the transforms observable; wall clock is not gated on this value. + const bool ok = sink32 == sink32 && sink64 == sink64; + std::printf("checksum f32=%.9g f64=%.17g ok=%d\n", static_cast(sink32), sink64, ok ? 1 : 0); + return ok ? 0 : 1; +} diff --git a/bench/icount/CMakeLists.txt b/bench/icount/CMakeLists.txt new file mode 100644 index 0000000..1d57838 --- /dev/null +++ b/bench/icount/CMakeLists.txt @@ -0,0 +1,49 @@ +# Instruction-count ratchet workloads (bench/README.md). One binary per +# scenario: bare-metal targets have no argv, and per-binary instruction totals +# are what scripts/icount.py compares. The scenario name (after the +# tap_dsp_icount_ prefix) is the key in bench/baselines.json. +# +# Entries are name:precision:n — precision 0 = float, 1 = double (see +# icount_main.cpp). Adapted from MuTap's bench/icount/CMakeLists.txt. +set(_tap_dsp_icount_scenarios + rfft_f32_512:0:512 + rfft_f32_2048:0:2048) + +# The double scenario is host-class only: on the bare-metal Cortex-M legs +# double is soft-float, which is a correctness check elsewhere in the repo but +# not a profile, so it is neither built nor baselined there. +if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic") + list(APPEND _tap_dsp_icount_scenarios rfft_f64_512:1:512) +endif() + +# TODO(Stage 2a): when TAP_DSP_BENCH_ENGINE gains split_radix, build each float +# scenario twice (the C and the port, e.g. a `_port` suffix on the second +# binary's key) so the job prints both counts and their ratio. +foreach(_sc IN LISTS _tap_dsp_icount_scenarios) + string(REPLACE ":" ";" _parts "${_sc}") + list(GET _parts 0 _name) + list(GET _parts 1 _precision) + list(GET _parts 2 _n) + add_executable(tap_dsp_icount_${_name} icount_main.cpp) + target_compile_definitions(tap_dsp_icount_${_name} PRIVATE + TAP_DSP_SC_NAME="${_name}" + TAP_DSP_SC_PRECISION=${_precision} + TAP_DSP_SC_N=${_n} + TAP_DSP_BENCH_ENGINE=${TAP_DSP_BENCH_ENGINE}) + target_include_directories(tap_dsp_icount_${_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/..) + target_link_libraries(tap_dsp_icount_${_name} PRIVATE + tap::dsp + tap_dsp_warnings) + # The QEMU legs link the semihosting startup through the toolchain file's + # linker flags exactly as the test binary does; nothing extra here. + + # Host smoke test, the README's rule made executable: the binary runs + # twice, both runs report ok=1 and print the same DONE line. There is no + # instruction count on the host; that is the QEMU job's. + if(NOT CMAKE_SYSTEM_NAME STREQUAL "Generic") + add_test(NAME tap_dsp_icount_${_name}_smoke + COMMAND ${CMAKE_COMMAND} + -DBINARY=$ + -P ${CMAKE_CURRENT_SOURCE_DIR}/smoke.cmake) + endif() +endforeach() diff --git a/bench/icount/icount_main.cpp b/bench/icount/icount_main.cpp new file mode 100644 index 0000000..ec84b08 --- /dev/null +++ b/bench/icount/icount_main.cpp @@ -0,0 +1,113 @@ +/// @file icount_main.cpp +/// @brief Deterministic fixed workloads for the instruction-count ratchet. +// SPDX-License-Identifier: MIT +// Copyright 2026 Timothy Place and the DspTap contributors. +// +// Adapted from MuTap's bench/icount/icount_main.cpp (MIT, MuTap contributors) +// for the real-FFT primitive; the policy is in bench/README.md. +// +// One scenario per binary, selected at compile time because bare-metal +// targets have no argv: +// +// TAP_DSP_SC_NAME the scenario key in bench/baselines.json ("rfft_f32_512") +// TAP_DSP_SC_PRECISION 0 = float (the embedded profile), 1 = double (host-class +// targets only: soft-float double is not a profile) +// TAP_DSP_SC_N transform size (power of two) +// TAP_DSP_BENCH_ENGINE the engine under test (bench_common.h) +// +// The QEMU plugin counts the whole run including construction, so the loop +// is sized so the transforms dominate: k_total_samples samples pass through +// forward + inverse per scenario (2048 iterations at N = 512, 512 at +// N = 2048), against a one-time table build of O(N) trig calls — under 0.2 % +// of the total at either size on the host (bench/README.md has the +// callgrind figures, including the share of the count that is not the +// transform: the class's out-of-place copies, the 2/N scaling loop and the +// checksum fold, a constant dilution the ratchet's percentages sit on top +// of). The input is a small xorshift corpus generated once and cycled, so +// the loop is allocation-free. Nothing in the float scenarios is double: +// the M4 soft-float leg would otherwise measure libgcc. +// +// Every output word of every iteration goes through the integer FNV-1a-64 +// fold in bench_common.h, so the printed checksum is a bit-exact, +// order-sensitive fingerprint of the engine's output: identical between two +// runs of the same binary, and different for a single 1-ulp change in any +// output. `ok` is a sanity check that the last iteration round-trips its +// input, in the scenario's own precision. +#include +#include +#include +#include + +#include "bench_common.h" + +#ifndef TAP_DSP_SC_NAME +#define TAP_DSP_SC_NAME "rfft_f32_512" +#endif +#ifndef TAP_DSP_SC_PRECISION +#define TAP_DSP_SC_PRECISION 0 +#endif +#ifndef TAP_DSP_SC_N +#define TAP_DSP_SC_N 512 +#endif + +namespace { + +#if TAP_DSP_SC_PRECISION == 0 + using sample = float; +#else + using sample = double; +#endif + + constexpr std::size_t k_n = TAP_DSP_SC_N; + constexpr std::size_t k_total_samples = std::size_t{1} << 20; // per direction, per scenario + constexpr std::size_t k_iterations = k_total_samples / k_n; + constexpr std::size_t k_corpus_blocks = 4; + constexpr sample k_round_trip_tolerance = static_cast(1e-3); + static_assert(k_n >= 4 && (k_n & (k_n - 1)) == 0, "TAP_DSP_SC_N must be a power of two"); + static_assert(k_iterations >= 100, "the loop must dominate construction"); + + struct outcome { + std::uint64_t checksum; + bool round_trips; + }; + + outcome run() { + tap::dsp::bench::fft_under_test fft(k_n); + + std::vector corpus(k_corpus_blocks * k_n); + std::vector spectrum(k_n); + std::vector out(k_n); + tap::dsp::bench::xorshift32 rng(0x9E3779B9u); + rng.fill(corpus.data(), corpus.size()); + + std::uint64_t h = tap::dsp::bench::k_fnv1a64_offset; + const sample* in = corpus.data(); + for (std::size_t i = 0; i < k_iterations; ++i) { + in = corpus.data() + (i % k_corpus_blocks) * k_n; + fft.forward(in, spectrum.data()); + h = tap::dsp::bench::fold(h, spectrum.data(), k_n); + fft.inverse(spectrum.data(), out.data()); + h = tap::dsp::bench::fold(h, out.data(), k_n); + } + + bool round_trips = true; + for (std::size_t i = 0; i < k_n; ++i) { + const sample err = out[i] - in[i]; + if (!(err <= k_round_trip_tolerance && err >= -k_round_trip_tolerance)) { // also catches NaN + round_trips = false; + } + } + return {h, round_trips}; + } + +} // namespace + +int main() { + const outcome r = run(); + // %llx with an explicit cast rather than PRIx64: newlib's + // hides the PRI macros from C++ behind __STDC_FORMAT_MACROS. + std::printf("TAP_DSP_ICOUNT_DONE ok=%d engine=%s backend=%s scenario=%s checksum=0x%016llx\n", + r.round_trips ? 1 : 0, tap::dsp::bench::k_engine_name, tap::dsp::bench::backend_name(), + TAP_DSP_SC_NAME, static_cast(r.checksum)); + return r.round_trips ? 0 : 1; +} diff --git a/bench/icount/smoke.cmake b/bench/icount/smoke.cmake new file mode 100644 index 0000000..aa59c03 --- /dev/null +++ b/bench/icount/smoke.cmake @@ -0,0 +1,25 @@ +# Host smoke test for one icount scenario binary (bench/icount/CMakeLists.txt): +# run it twice; both runs must exit 0, report ok=1, and print byte-identical +# TAP_DSP_ICOUNT_DONE lines (the checksum is a bit-exact fingerprint). +# cmake -DBINARY= -P smoke.cmake +if(NOT BINARY) + message(FATAL_ERROR "smoke.cmake: -DBINARY= is required") +endif() +set(_lines) +foreach(_run 1 2) + execute_process(COMMAND "${BINARY}" RESULT_VARIABLE _rc OUTPUT_VARIABLE _out) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "${BINARY}: run ${_run} exited ${_rc}:\n${_out}") + endif() + string(REGEX MATCH "TAP_DSP_ICOUNT_DONE ok=1[^\n]*" _line "${_out}") + if(NOT _line) + message(FATAL_ERROR "${BINARY}: run ${_run} did not report ok=1:\n${_out}") + endif() + list(APPEND _lines "${_line}") +endforeach() +list(GET _lines 0 _first) +list(GET _lines 1 _second) +if(NOT _first STREQUAL _second) + message(FATAL_ERROR "${BINARY}: two runs differ:\n ${_first}\n ${_second}") +endif() +message(STATUS "${_first}") diff --git a/bench/size_probe.cpp b/bench/size_probe.cpp new file mode 100644 index 0000000..a8f5f05 --- /dev/null +++ b/bench/size_probe.cpp @@ -0,0 +1,44 @@ +/// @file size_probe.cpp +/// @brief One real-FFT profile alone, for the per-leg .text ceilings. +// SPDX-License-Identifier: MIT +// Copyright 2026 Timothy Place and the DspTap contributors. +// +// docs/audit-fft-and-code-smells.md, Part 10 item 4: a size probe per +// profile, built MinSizeRel like the test legs, measured with +// `arm-none-eabi-size -A` (the .text row alone; Berkeley format folds +// .rodata and NOLOAD sections into "text"). No stdio and no printf, so the +// binary is the startup, the transform and what the transform pulls in; +// the ceilings in .github/workflows/bench.yml are stated against this. +// Same TAP_DSP_SC_PRECISION / TAP_DSP_SC_N selection as icount_main.cpp. +#include + +#include "bench_common.h" + +#ifndef TAP_DSP_SC_PRECISION +#define TAP_DSP_SC_PRECISION 0 +#endif +#ifndef TAP_DSP_SC_N +#define TAP_DSP_SC_N 512 +#endif + +namespace { + +#if TAP_DSP_SC_PRECISION == 0 + using sample = float; +#else + using sample = double; +#endif + + constexpr std::size_t k_n = TAP_DSP_SC_N; + sample g_block[k_n]; + +} // namespace + +int main() { + tap::dsp::bench::fft_under_test fft(k_n); + tap::dsp::bench::xorshift32 rng(0x9E3779B9u); + rng.fill(g_block, k_n); + fft.forward_inplace(g_block); + fft.inverse_inplace(g_block); + return g_block[0] == g_block[0] ? 0 : 1; // NaN check keeps the transforms live +} diff --git a/scripts/icount.py b/scripts/icount.py new file mode 100755 index 0000000..c5f9bcb --- /dev/null +++ b/scripts/icount.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Copyright 2026 Timothy Place and the DspTap contributors. +# Adapted from MuTap's scripts/icount.py (MIT, MuTap contributors). +"""Deterministic instruction-count ratchet (see bench/README.md). + +Runs every tap_dsp_icount_* binary in a build directory under QEMU with the +instruction-counting plugin (tools/qemu_insn_plugin), then compares against +bench/baselines.json. + + icount.py --target {m4-softfp,m4f,m33,m55,m55-ooura} --build-dir DIR + --plugin LIB [--update] [--record FILE] + [--baselines bench/baselines.json] [--tolerance 0.03] + icount.py --merge FILE [FILE ...] [--baselines bench/baselines.json] + +The gate is two-sided: exit nonzero if any scenario regresses beyond +tolerance, improves beyond tolerance (the baseline must be re-recorded so +the gate stays tight), has no recorded baseline, or has a recorded baseline +but no binary (a renamed or removed workload must not linger as a dead gate +entry). --update rewrites the target's entry to exactly the measured +scenarios instead. --record FILE always writes {target: measured} as JSON, +whatever the verdict, so a run that fails on NO BASELINE still hands back +the numbers to commit (bench.yml uploads it as measured- on every run). +Seeding runs on pushes to main, never from a pull request (bench.yml); --merge +folds several per-target files (each with one target filled in) into one, +which is how the seeding commit is assembled from the job's artifacts. + +The QEMU machine per target, the binary prefix and the output markers are +DspTap's; the gate logic is MuTap's. +""" +import argparse +import glob +import json +import os +import pathlib +import re +import subprocess +import sys + +# Both M4 flavours (soft-float and M4F) run on the same board model; the +# difference is the toolchain's -mfloat-abi. m55 and m55-ooura likewise share +# the AN547: the key records which float32 backend the binaries were built +# with (CMSIS-DSP Helium, the deployed profile, or the Ooura fallback). +MACHINES = { + "m4-softfp": "mps2-an386", + "m4f": "mps2-an386", + "m33": "mps2-an505", + "m55": "mps3-an547", + "m55-ooura": "mps3-an547", +} +PREFIX = "tap_dsp_icount_" +DONE_MARKER = "TAP_DSP_ICOUNT_DONE ok=1" +COUNT_RE = re.compile(r"TAP_DSP_INSN_COUNT (\d+)") + + +def qemu_cmd(target: str, plugin: str, binary: str) -> list[str]: + # "-d plugin" routes qemu_plugin_outs() to stderr; without it the count + # line is silently dropped. + machine = MACHINES.get(target) + if machine is None: + raise SystemExit(f"unknown target {target}") + return ["qemu-system-arm", "-M", machine, "-nographic", "-semihosting", + "-d", "plugin", "-plugin", plugin, "-kernel", binary] + + +def measure(target: str, plugin: str, binary: str) -> int: + try: + proc = subprocess.run(qemu_cmd(target, plugin, binary), timeout=1200, + capture_output=True, text=True) + except subprocess.TimeoutExpired: + raise SystemExit(f"{binary}: timed out after 1200 s under QEMU") + out = proc.stdout + proc.stderr + if DONE_MARKER not in out: + print(out, file=sys.stderr) + raise SystemExit(f"{binary}: workload did not complete cleanly") + m = COUNT_RE.search(out) + if not m: + print(out, file=sys.stderr) + raise SystemExit(f"{binary}: no TAP_DSP_INSN_COUNT (plugin not loaded?)") + return int(m.group(1)) + + +def write(path: pathlib.Path, baselines: dict) -> None: + path.write_text(json.dumps(baselines, indent=2, sort_keys=True) + "\n") + + +def merge(path: pathlib.Path, files: list[str]) -> int: + baselines = json.loads(path.read_text()) if path.exists() else {} + for f in files: + for target, scenarios in json.loads(pathlib.Path(f).read_text()).items(): + if scenarios: + baselines[target] = scenarios + print(f"{target}: {len(scenarios)} scenario(s) from {f}") + write(path, baselines) + print(f"merged into {path}") + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--target", choices=sorted(MACHINES)) + ap.add_argument("--build-dir") + ap.add_argument("--plugin") + ap.add_argument("--baselines", default="bench/baselines.json") + ap.add_argument("--tolerance", type=float, default=0.03) + ap.add_argument("--update", action="store_true") + ap.add_argument("--record", metavar="FILE", + help="always write {target: measured} here, whatever the verdict") + ap.add_argument("--merge", nargs="+", metavar="FILE") + args = ap.parse_args() + + path = pathlib.Path(args.baselines) + if args.merge: + return merge(path, args.merge) + if not (args.target and args.build_dir and args.plugin): + ap.error("--target, --build-dir and --plugin are required (or --merge)") + + binaries = sorted(glob.glob(os.path.join(args.build_dir, "**", PREFIX + "*"), + recursive=True)) + binaries = [b for b in binaries if os.access(b, os.X_OK) and os.path.isfile(b)] + if not binaries: + raise SystemExit(f"no {PREFIX}* binaries under {args.build_dir}") + + baselines = json.loads(path.read_text()) if path.exists() else {} + base = baselines.get(args.target, {}) + + failures = [] + measured = {} + for binary in binaries: + scenario = os.path.basename(binary).removeprefix(PREFIX) + count = measure(args.target, args.plugin, binary) + measured[scenario] = count + recorded = base.get(scenario) + if recorded is None: + print(f"{scenario}: {count} insns (NO BASELINE — commit this value)") + if not args.update: + failures.append(scenario) + elif recorded == 0: + print(f"{scenario}: {count} insns vs baseline 0 (INVALID BASELINE)") + failures.append(scenario) + else: + delta = (count - recorded) / recorded + verdict = "ok" + if delta > args.tolerance: + verdict = "REGRESSION" + failures.append(scenario) + elif delta < -args.tolerance: + # Two-sided: a stale (too-high) baseline would let future + # regressions hide inside the slack, so improvements must be + # committed too. + verdict = ("IMPROVED beyond tolerance — run icount.py --update " + "and commit bench/baselines.json") + failures.append(scenario) + print(f"{scenario}: {count} insns vs baseline {recorded} " + f"({delta:+.2%}) {verdict}") + + # A recorded scenario with no binary is a dead gate entry (renamed or + # removed workload); compare mode fails on it, --update drops it. + for scenario in sorted(set(base) - set(measured)): + print(f"{scenario}: baseline {base[scenario]} but no binary " + "(STALE BASELINE — run icount.py --update and commit)") + if not args.update: + failures.append(scenario) + + if args.record: + pathlib.Path(args.record).write_text( + json.dumps({args.target: measured}, indent=2, sort_keys=True) + "\n") + print(f"recorded {args.record}") + + if args.update: + # Exactly the measured scenarios: stale keys for renamed/removed + # workloads must not linger as dead gate entries. + baselines[args.target] = measured + write(path, baselines) + print(f"updated {path}") + return 0 + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/qemu_insn_plugin/insn_count.c b/tools/qemu_insn_plugin/insn_count.c new file mode 100644 index 0000000..4afb617 --- /dev/null +++ b/tools/qemu_insn_plugin/insn_count.c @@ -0,0 +1,63 @@ +/* Minimal QEMU TCG plugin: counts executed guest instructions and prints + * one line at exit: + * + * TAP_DSP_INSN_COUNT + * + * Used by scripts/icount.py for the deterministic performance ratchet + * (bench/README.md). Counting uses the inline-add fast path; the single + * counter is exact for our single-vCPU deterministic workloads. + * + * Build (qemu-plugin.h fetched for the matching QEMU 8.2.x, whose header + * defines QEMU_PLUGIN_VERSION 1 — the plugin API version is the header's, and + * the pinned v8.2.2 header says 1, not 2 as MuTap's copy of this comment had it): + * gcc -shared -fPIC $(pkg-config --cflags glib-2.0) \ + * -I insn_count.c -o libinsncount.so + * + * Licensing: qemu-plugin.h is QEMU's, SPDX GPL-2.0-or-later. It is fetched at + * CI time (digest-verified), never vendored into this repo, and this file is + * compiled against it only to build a test tool that runs in CI and is not + * shipped; nothing in the DspTap tree or in what consumers link is GPL. + * This file itself is MIT. + * + * Provenance: copied from MuTap's tools/qemu_insn_plugin/insn_count.c (MIT, + * MuTap contributors) with only the output marker renamed; the same file + * pattern lives in SampleRateTap and RatioTap. A taphouse-style + * consolidation is the eventual home (docs/audit-fft-and-code-smells.md, + * Part 11, "Sharing"). + */ +// SPDX-License-Identifier: MIT +// Copyright 2026 MuTap contributors +// Copyright 2026 Timothy Place and the DspTap contributors. +#include +#include +#include +#include + +QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION; + +static uint64_t insn_count; + +static void tb_trans(qemu_plugin_id_t id, struct qemu_plugin_tb* tb) { + (void)id; + size_t n = qemu_plugin_tb_n_insns(tb); + for (size_t i = 0; i < n; i++) { + struct qemu_plugin_insn* insn = qemu_plugin_tb_get_insn(tb, i); + qemu_plugin_register_vcpu_insn_exec_inline(insn, QEMU_PLUGIN_INLINE_ADD_U64, &insn_count, 1); + } +} + +static void at_exit(qemu_plugin_id_t id, void* userdata) { + (void)id; + (void)userdata; + g_autofree gchar* msg = g_strdup_printf("TAP_DSP_INSN_COUNT %" PRIu64 "\n", insn_count); + qemu_plugin_outs(msg); +} + +QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id, const qemu_info_t* info, int argc, char** argv) { + (void)info; + (void)argc; + (void)argv; + qemu_plugin_register_vcpu_tb_trans_cb(id, tb_trans); + qemu_plugin_register_atexit_cb(id, at_exit, NULL); + return 0; +}