Stage2 cpp engine

.github/workflows/ci.yml

@@ -23,6 +23,15 @@ jobs:
         with:
           python-version: "3.12"
           cache: pip
+      # Installing the package compiles the extension, which needs an ONNX
+      # Runtime SDK matching the wheel. CMake downloads it here and reuses it on
+      # later runs.
+      - name: Cache the ONNX Runtime SDK
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/anytime-inference-planner
+          key: ort-sdk-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ort-sdk-${{ runner.os }}-
       - name: Install
         run: pip install -e ".[dev]"
       - name: ruff check
@@ -45,9 +54,28 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
           cache: pip
+      - name: Cache the ONNX Runtime SDK
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/anytime-inference-planner
+          key: ort-sdk-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ort-sdk-${{ runner.os }}-
       - name: Install
         run: pip install -e ".[bench]" pytest
+      # The extension is the runtime, so an install that silently fell back to
+      # the Python backend would hide a broken build behind passing tests.
+      - name: Assert the extension was built
+        run: |
+          python -c "
+          from anytime_serving.serving import extension_available, load_extension
+          import onnxruntime
+          assert extension_available(), 'anytime_runtime did not build or import'
+          print('anytime_runtime links ONNX Runtime', load_extension().onnxruntime_version())
+          print('onnxruntime wheel reports         ', onnxruntime.__version__)
+          "
       - name: Run tests
+        env:
+          ANYTIME_REQUIRE_BACKENDS: extension,python
         run: pytest -q
 
   test-minimal:
@@ -59,6 +87,12 @@ jobs:
         with:
           python-version: "3.12"
           cache: pip
+      - name: Cache the ONNX Runtime SDK
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/anytime-inference-planner
+          key: ort-sdk-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ort-sdk-${{ runner.os }}-
       # The serving control plane must run without the research stack. Installing
       # only the base dependencies enforces that: a module-scope torch or pandas
       # import anywhere on the serving path fails this job.
@@ -75,50 +109,47 @@ jobs:
         run: python -c "import anytime_serving.serving, anytime_serving.models.cascade"
       # The remaining suites need pandas/scipy and are covered by the `test` job.
       - name: Run serving tests and import-boundary guards
+        env:
+          ANYTIME_REQUIRE_BACKENDS: extension,python
         run: |
           pytest -q \
             tests/test_admission.py \
             tests/test_selector.py \
             tests/test_serving.py \
             tests/test_load_monitor.py \
+            tests/test_runtime_engine.py \
             tests/test_import_boundaries.py
 
-  runtime:
-    name: C++ runtime build
+  engine:
+    name: Engine parity against the reference backend
     runs-on: ubuntu-latest
-    env:
-      # Must match the onnxruntime wheel the Python side installs. The version is
-      # not cosmetic: building the worker against 1.20.1 while profiling with the
-      # 1.26.0 wheel measured DistilBERT at 98.9 ms in the worker versus 13.0 ms
-      # in the profiler, a 7.6x gap that made every admission decision wrong.
-      ONNXRUNTIME_VERSION: 1.26.0
     steps:
       - uses: actions/checkout@v4
-      - name: Fetch ONNX Runtime
-        run: |
-          curl -fsSL -o onnxruntime.tgz \
-            "https://github.com/microsoft/onnxruntime/releases/download/v${ONNXRUNTIME_VERSION}/onnxruntime-linux-x64-${ONNXRUNTIME_VERSION}.tgz"
-          mkdir -p "$RUNNER_TEMP/onnxruntime"
-          # Release tarball layouts differ between versions: some nest a
-          # versioned directory, some do not. Locate the one holding include/.
-          tar -xzf onnxruntime.tgz -C "$RUNNER_TEMP/onnxruntime" --strip-components=1
-          root="$RUNNER_TEMP/onnxruntime"
-          if [ ! -d "$root/include" ]; then
-            root=$(find "$RUNNER_TEMP/onnxruntime" -maxdepth 2 -type d -name include \
-              -exec dirname {} \; | head -1)
-          fi
-          echo "ONNXRUNTIME_ROOT_PATH=$root" >> "$GITHUB_ENV"
-      - name: Configure
-        run: |
-          cmake -S runtime_cpp -B runtime_cpp/build \
-            -DCMAKE_BUILD_TYPE=Release \
-            -DONNXRUNTIME_ROOT_PATH="$ONNXRUNTIME_ROOT_PATH"
-      - name: Build
-        run: cmake --build runtime_cpp/build -j"$(nproc)"
-      - name: Verify the binary starts and reports its usage
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+          cache: pip
+      - name: Cache the ONNX Runtime SDK
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/anytime-inference-planner
+          key: ort-sdk-${{ runner.os }}-${{ hashFiles('pyproject.toml') }}
+          restore-keys: ort-sdk-${{ runner.os }}-
+      - name: Install
+        run: pip install -e . pytest
+      # Both copies of ONNX Runtime in this process must be the same version. The
+      # extension reports what it linked; the wheel reports what it will import.
+      - name: Report the resolved ONNX Runtime versions
         run: |
-          # No --model argument is an error; the binary must exit non-zero with a
-          # message rather than hang waiting on stdin.
-          if runtime_cpp/build/anytime_runtime </dev/null; then
-            echo "expected a non-zero exit without --model"; exit 1
-          fi
+          python -c "
+          from anytime_serving.serving import load_extension
+          import onnxruntime
+          linked = load_extension().onnxruntime_version()
+          print('anytime_runtime links ONNX Runtime', linked)
+          print('onnxruntime wheel reports         ', onnxruntime.__version__)
+          assert linked == onnxruntime.__version__
+          "
+      - name: Compare the extension against the reference backend
+        env:
+          ANYTIME_REQUIRE_BACKENDS: extension,python
+        run: pytest -q tests/test_runtime_engine.py

.gitignore

@@ -59,3 +59,7 @@ logs/
 
 # C++ build artefacts
 runtime_cpp/build/
+runtime/build/
+# The compiled extension, built into the source tree by an editable install.
+anytime_runtime*.so
+anytime_runtime*.pyd

README.md

@@ -3,7 +3,7 @@
 Latency-bounded ML serving. A Python control plane watches queue depth and CPU
 load, then routes each request to one of several profiled model variants so that
 as much traffic as possible completes inside a deadline. Inference runs in a C++
-ONNX Runtime worker pool.
+ONNX Runtime engine loaded into the process as an extension module.
 
 Variant selection is a constrained optimisation: maximise expected accuracy
 subject to an M/M/c sojourn-time bound and the measured backlog.
@@ -13,22 +13,27 @@ subject to an M/M/c sojourn-time bound and the measured backlog.
 ## Results
 
 DistilBERT-SST-2 and MiniLM-L6 served from a 4-worker pool on an Apple M4 Pro,
-real SST-2 validation traffic, 38 ms deadline. Offered load is a fraction of the
-measured pool capacity of 313 rps. Goodput counts only requests that completed
-within the deadline.
+real SST-2 validation traffic, 39 ms deadline. Offered load is a fraction of the
+measured pool capacity of 310 rps. Goodput counts only requests that completed
+within the deadline; p95 is the adaptive policy's.
 
 | Offered load | Goodput, accurate-only | Goodput, adaptive | Compute cost | p95 |
 | --- | --- | --- | --- | --- |
-| ρ = 0.40 | 126 rps | 127 rps | 1.00 | 28 ms |
-| ρ = 0.80 | 99 rps | 214 rps | 0.76 | 44 ms |
-| ρ = 0.95 | 37 rps | 287 rps | 0.48 | 38 ms |
-| ρ = 1.30 | 3 rps | 403 rps | 0.41 | 18 ms |
+| ρ = 0.40 | 122 rps | 124 rps | 1.00 | 31 ms |
+| ρ = 0.80 | 144 rps | 217 rps | 0.79 | 53 ms |
+| ρ = 0.95 | 43 rps | 281 rps | 0.49 | 38 ms |
+| ρ = 1.30 | 2 rps | 394 rps | 0.41 | 19 ms |
 
 Below ρ ≈ 0.5 the planner keeps every request on the most accurate variant and
 matches the baseline exactly. As the queue builds it shifts traffic to the cheaper
-variant: at ρ = 0.95 that is **7.8x the goodput at 48% of the compute cost**, for
+variant: at ρ = 0.95 that is **6.5x the goodput at 49% of the compute cost**, for
 at most 0.92 accuracy points (91.06% to 90.14% on SST-2 validation).
 
+Measured through the serving path itself, with every variant cross-checked against
+a separate ONNX Runtime session. Repeating the sweep moved goodput by at most 6%,
+and service times carry a 1-4% run-to-run spread, so treat the third digit as
+noise.
+
 Full tables, methodology, and host details: [`docs/benchmarks.md`](docs/benchmarks.md).
 
 ## Install
@@ -48,8 +53,7 @@ python scripts/demo_serving.py
 ```
 
 Self-contained: builds a small synthetic ONNX model, drives Poisson traffic
-through both policies, and prints a JSON summary. Needs no model download and no
-C++ build.
+through both policies, and prints a JSON summary. Needs no model download.
 
 Reproduce the measured results end to end:
 
@@ -65,30 +69,35 @@ python scripts/run_load_sweep.py              # sweep load, write the figure
 pytest -q
 ```
 
-The serving tests build a tiny ONNX model on the fly, so they run without torch
-or the C++ binary. Tests needing either are marked and skip when it is absent.
+The serving tests build a tiny ONNX graph on the fly, so they run without torch.
+Tests needing the compiled extension are marked and skip when it is absent.
 
 ## Documentation
 
 | | |
 | --- | --- |
 | [Architecture](docs/architecture.md) | control plane and worker split, request lifecycle |
 | [Planner](docs/planner.md) | admission control, Erlang-C, variant selection |
-| [Runtime](docs/runtime.md) | C++ worker, build, wire protocol |
+| [Runtime](docs/runtime.md) | C++ engine, build, version matching |
 | [Quantisation](docs/quantization.md) | why the variant frontier is measured, not assumed |
 | [Benchmarks](docs/benchmarks.md) | methodology, full results, reproduction |
 | [Development](docs/development.md) | dependency groups, tests, CI, layout |
 
 ## Notes
 
-Two findings shaped the current design and are worth knowing before extending it:
+Three findings shaped the current design and are worth knowing before extending it:
 
 - **Worker count belongs in the queueing model.** Modelling a c-worker pool as a
   single server understates capacity by a factor of c. `AdaptiveServer` now
   refuses a selector that models a different pool size.
 - **Match the C++ ONNX Runtime version to the Python wheel.** A mismatch measured
   DistilBERT at 98.9 ms in the worker against 13.0 ms in the profiler, which
-  silently invalidated every admission decision.
+  silently invalidated every admission decision. The version is now derived from
+  the installed wheel at build time and checked again at import, so it cannot
+  drift.
+- **Measure through the serving path, not beside it.** That mismatch survived
+  because profiling used a separate process. Profiling now runs through the same
+  client the server dispatches to, and fails if a cross-check disagrees.
 
 ## License
 

configs/serving.yaml

@@ -1,7 +1,7 @@
-# Generated by scripts/profile_variants.py. Values are measured on the
-# host recorded in results/variant_profiles.json; re-run after changing
-# hardware, thread count, or model variants.
-deadline_ms: 38.4
+# Generated by scripts/profile_variants.py. Values are measured through the
+# serving path on the host recorded in results/variant_profiles.json;
+# re-run after changing hardware, thread count, or model variants.
+deadline_ms: 38.7
 workers: 4
 duration_s: 20.0
 arrival_rate_rps: 80.0
@@ -11,10 +11,10 @@ admission:
   safety_factor: 1.0
 variants:
   distilbert_fp32:
-    service_time_ms: 12.789
+    service_time_ms: 12.893
     accuracy: 0.9106
     compute_cost_per_request: 1.0
   minilm_fp32:
-    service_time_ms: 5.211
+    service_time_ms: 5.189
     accuracy: 0.9014
-    compute_cost_per_request: 0.4075
+    compute_cost_per_request: 0.4025

docs/architecture.md

@@ -18,12 +18,13 @@ decides, and a C++ worker pool that computes.
                 | RuntimePool                       |
                 |   N RuntimeClients, one per worker|
                 +----------------+------------------+
-                                 |  line-delimited JSON on stdin/stdout
+                                 |  in-process call, borrowed tensors
                                  v
                 +-----------------------------------+
-                | anytime_runtime (C++)             |
+                | anytime_runtime (C++ extension)   |
                 |   one ONNX Runtime session per    |
-                |   variant, 1 intra-op thread      |
+                |   variant, 1 intra-op thread,     |
+                |   GIL released during inference   |
                 +-----------------------------------+
 ```