Add experimental Praxis inference benchmark treatment

inference/README.md

@@ -42,6 +42,7 @@ repetitions whose inference backend, workload, and infrastructure must match.
 | `agentgateway-standalone` | agentgateway in the standalone EPP topology |
 | `agentgateway-gateway` | agentgateway on Kubernetes with EPP |
 | `envoy-standalone` | Envoy sidecar in the standalone EPP topology |
+| `praxis-standalone` | Experimental custom Praxis AI sidecar; see [build and qualification requirements](suites/llm-d-benchmark/PRAXIS.md) |
 
 Required variables:
 
@@ -167,7 +168,11 @@ BENCHMARK_TREATMENT=<treatment> \
 make benchmark
 ```
 
-The reference profile requires agentgateway v1.4.1. The wrapper verifies the
+The historical reference profile requires agentgateway v1.4.1.
+For v1.5.0, set `AGW_VERSION=v1.5.0` and
+`BENCHMARK_REFERENCE_PROFILE=optimized-baseline-qwen3-32b-h100-v0.9-agw-v1.5.0`.
+This separate profile preserves vLLM v0.23.0 and the same serving/EPP resources.
+For standalone mode, set `AGW_IMAGE` to the release digest to pin the binary. The wrapper verifies the
 observed vLLM, EPP, and agentgateway pod images before starting traffic.
 `BENCHMARK_ENDPOINT_PATH=internal` explicitly supplies the proxy Service
 ClusterIP to the harness; this prevents Gateway status from selecting an
@@ -433,8 +438,10 @@ not a billing quote. See the Google Cloud
 Each treatment temporarily needs about 17 GiB of local space while collecting
 raw per-request results. The three compressed per-request files total
 approximately 3 GiB, so copy long-lived evidence to durable object storage.
-`BENCHMARK_REPETITION=3` runs every treatment three times and should be
-budgeted at approximately 18 hours and USD 1,200 to 1,275.
+`BENCHMARK_REPETITION` labels one campaign invocation; it does not repeat it.
+Run the campaign separately with repetition IDs 1, 2, and 3 for three rounds,
+reversing treatment order between rounds. Budget approximately 18 hours and
+USD 1,200 to 1,275 for three rounds of the default historical matrix.
 
 ## Results
 

inference/provisioning/gke/tests/test-run-gke-campaign.sh

@@ -140,4 +140,41 @@ fi
 grep -q 'BENCHMARK_GKE_CLUSTER_LIFECYCLE must be retain or destroy' \
   "${TEMP_DIR}/invalid-lifecycle.log"
 
+: >"${TEST_COMMAND_LOG}"
+PRAXIS_IMAGE="example.invalid/praxis@sha256:$(printf '%064d' 1)" \
+BENCHMARK_TREATMENTS='agentgateway-standalone praxis-standalone' \
+BENCHMARK_CAMPAIGN_ID=test-praxis-comparison \
+  "${CAMPAIGN_SCRIPT}" >"${TEMP_DIR}/praxis.log"
+[[ "$(grep -c '^target=benchmark ' "${TEST_COMMAND_LOG}")" == 2 ]]
+grep -q '^target=benchmark treatment=praxis-standalone ' "${TEST_COMMAND_LOG}"
+grep -q '^target=benchmark-report .*comparisons=agentgateway-standalone:praxis-standalone$' "${TEST_COMMAND_LOG}"
+
+for treatments in 'service praxis-standalone' 'service typo' 'service service'; do
+  : >"${TEST_COMMAND_LOG}"
+  if PRAXIS_IMAGE=example.invalid/praxis:latest \
+    BENCHMARK_TREATMENTS="${treatments}" BENCHMARK_CAMPAIGN_ID=test-invalid-matrix \
+    "${CAMPAIGN_SCRIPT}" >"${TEMP_DIR}/invalid-matrix.log" 2>&1; then
+    echo "invalid campaign matrix unexpectedly accepted: ${treatments}" >&2
+    exit 1
+  fi
+  [[ ! -s "${TEST_COMMAND_LOG}" ]] || {
+    echo "invalid matrix reached cloud orchestration" >&2; exit 1;
+  }
+done
+
 echo "GKE campaign orchestration tests passed"
+
+: >"${TEST_COMMAND_LOG}"
+BENCHMARK_HF_TOKEN_REQUIRED=false TEST_SECRET_EXISTS=false \
+PRAXIS_IMAGE="example.invalid/praxis@sha256:$(printf '%064d' 1)" \
+BENCHMARK_TREATMENTS='service agentgateway-standalone praxis-standalone' \
+BENCHMARK_COMPARISONS='service:agentgateway-standalone agentgateway-standalone:praxis-standalone' \
+BENCHMARK_REPETITION=1 BENCHMARK_CAMPAIGN_ID=test-three-treatments \
+  "${CAMPAIGN_SCRIPT}" >"${TEMP_DIR}/three-treatments.log"
+[[ "$(grep -c '^target=benchmark ' "${TEST_COMMAND_LOG}")" == 3 ]]
+grep -q '^target=benchmark-report .*comparisons=service:agentgateway-standalone agentgateway-standalone:praxis-standalone$' "${TEST_COMMAND_LOG}"
+if grep -q '^kubectl ' "${TEST_COMMAND_LOG}"; then
+  echo 'anonymous campaign unexpectedly accessed secrets' >&2
+  exit 1
+fi
+echo "one-round paired reports and anonymous model tests passed"

inference/reporting/common/rendering.py

@@ -12,10 +12,11 @@
 
 def display_name(name: str) -> str:
     names = {
-        "service": "k8s service (RR)",
+        "service": "k8s service",
         "agentgateway-standalone": "agentgateway standalone",
         "agentgateway-gateway": "agentgateway on Kubernetes",
         "envoy-standalone": "Envoy standalone",
+        "praxis-standalone": "Praxis AI standalone",
     }
     return names.get(name, name.replace("-", " "))
 
@@ -38,6 +39,19 @@ def agentgateway_mode_note(name: str) -> str | None:
             "(https://agentgateway.dev/docs/kubernetes/latest/llm/inference/"
             "inference-routing/)."
         ),
+        "praxis-standalone": (
+            "**Praxis mode:** A custom Praxis AI build enables its experimental "
+            "`llmd-ext-proc` feature. Praxis runs beside the same EPP, uses "
+            "localhost gRPC for request scheduling, and forwards to the EPP-selected "
+            "model endpoint. The chart's generic Envoy slot is replaced by the "
+            "Praxis binary and configuration. This is a benchmark adapter, not "
+            "native Praxis Gateway API support. At the pinned source revision, the "
+            "ext_proc filter rejects response-body forwarding to the EPP and "
+            "trailer processing. This adapter sends request headers/body and "
+            "response headers only. Matching EPP configuration does not prove "
+            "equivalent request-lifecycle accounting; verify that separately "
+            "before interpreting GPU or scheduling results."
+        ),
     }
     return notes.get(name)
 
@@ -210,21 +224,31 @@ def write_markdown(
     delta_heading = "Δ% vs k8s" if baseline.name == "service" else f"Δ% vs {b_name}"
     comparison_description = (
         f"Graphs below compare {c_name} routing to a stock Kubernetes Service "
-        f"that round-robins requests across the same {identity['replicas']} "
-        f"{backend} pods (no EPP, no scoring). Tables abbreviate this baseline "
-        "as `k8s service (RR)`, where RR means round-robin."
+        f"across the same {identity['replicas']} {backend} pods (no EPP, no "
+        "scoring). Service balancing occurs at connection level; persistent "
+        "connections do not guarantee round-robin distribution of requests. "
+        "This comparison includes both gateway and scheduler effects and does "
+        "not isolate proxy overhead."
         if baseline.name == "service"
         else f"Graphs below compare {c_name} to {b_name} across the same "
         f"{identity['replicas']} {backend} pods."
     )
 
-    lines = [
-        "# Benchmark Report",
-        "",
+    topology = (
         f"The benchmark runs model {identity['model']} on {gpu_count} × {accelerator} "
         f"GPUs, distributed across {identity['replicas']} {backend} "
         f"model servers ({identity['tensor_parallelism']} GPUs per server with "
-        f"TP={identity['tensor_parallelism']}). Workload is "
+        f"TP={identity['tensor_parallelism']})."
+        if identity.get("accelerator_type") == "gpu" and backend != "sim"
+        else f"The benchmark uses {identity['replicas']} {backend} backend pods "
+        f"on {identity.get('accelerator_type', 'unspecified')} resources. "
+        "Simulator results measure routing and transport behavior, not GPU "
+        "inference throughput or model quality."
+    )
+    lines = [
+        "# Benchmark Report",
+        "",
+        f"{topology} Workload is "
         f"`{identity['workload']}` driven across the configured request-rate ladder.",
         "",
         "> [!NOTE]",
@@ -241,6 +265,15 @@ def write_markdown(
     if mode_note:
         lines.extend([mode_note, ""])
     lines.extend([comparison_description, ""])
+    if baseline.name.endswith("-standalone") and candidate.name.endswith("-standalone"):
+        lines.extend([
+            "Both treatments use the same EPP policy, backend topology, workload, "
+            "and proxy resource allocation. Differences measure the complete "
+            "gateway integration path, including EPP communication and streaming; "
+            "they do not demonstrate that one scheduler is more effective. "
+            "EPP and GPU variability remain part of end-to-end measurements.",
+            "",
+        ])
     if include_images:
         lines.extend(
             [

inference/reporting/generate.py

@@ -5,6 +5,7 @@
 
 import argparse
 import os
+import re
 import shutil
 from pathlib import Path
 
@@ -81,6 +82,42 @@ def validate_registered_repetitions(
         )
 
 
+def validate_praxis_comparison(manifest: dict, baseline: str, candidate: str) -> None:
+    """Refuse to publish a Praxis comparison with missing or unequal controls."""
+    if "praxis-standalone" not in (baseline, candidate):
+        return
+    treatments = manifest["treatments"]
+    praxis_runs = treatments["praxis-standalone"]["repetitions"]
+    builds = set()
+    for run in praxis_runs.values():
+        gateway = run.get("gateway", {})
+        image = gateway.get("image", "")
+        source = gateway.get("source_revision", "")
+        if not re.fullmatch(r"[^\s@]+@sha256:[0-9a-f]{64}", image):
+            raise ValueError("Praxis comparison requires an immutable image digest")
+        if not re.fullmatch(r"[0-9a-f]{40}", source):
+            raise ValueError("Praxis comparison requires a source revision")
+        if gateway.get("build_features") != "full,llmd-ext-proc":
+            raise ValueError("Praxis comparison requires full,llmd-ext-proc build evidence")
+        builds.add((image, source))
+    if len(builds) != 1:
+        raise ValueError("Praxis image/source must remain fixed across repetitions")
+    if not all(name.endswith("-standalone") for name in (baseline, candidate)):
+        return
+    controls = set()
+    for treatment in (baseline, candidate):
+        for run in treatments[treatment]["repetitions"].values():
+            digest = run.get("router_chart_digest", "")
+            invariant = run.get("standalone_invariant_sha256", "")
+            if not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
+                raise ValueError("standalone comparison requires the router chart digest")
+            if not re.fullmatch(r"[0-9a-f]{64}", invariant):
+                raise ValueError("standalone comparison requires the configuration invariant")
+            controls.add((run.get("router_chart_version"), digest, invariant))
+    if len(controls) != 1:
+        raise ValueError("standalone comparison has different charts, EPP policies, resources or workloads")
+
+
 def main() -> None:
     args = parser().parse_args()
     campaign = args.campaign.resolve()
@@ -112,6 +149,7 @@ def main() -> None:
             validate_registered_repetitions(
                 campaign, treatment, registered_treatments[treatment]
             )
+        validate_praxis_comparison(manifest, baseline_name, candidate_name)
         prism_treatments.update((baseline_name, candidate_name))
         baseline = read_treatment(campaign, baseline_name)
         candidate = read_treatment(campaign, candidate_name)

inference/reporting/tests/test_generate.py

@@ -176,13 +176,14 @@ def test_generates_upstream_style_artifacts(self) -> None:
             self.assertIn("agentgateway standalone", markdown)
             self.assertNotIn("Agentgateway", markdown)
             self.assertIn("<details>", markdown)
-            self.assertIn("| Rate | k8s service (RR) Output", markdown)
+            self.assertIn("| Rate | k8s service Output", markdown)
             self.assertIn(
-                "a stock Kubernetes Service that round-robins requests",
+                "a stock Kubernetes Service across the same",
                 markdown,
             )
             self.assertIn("(no EPP, no scoring)", markdown)
-            self.assertIn("where RR means round-robin", markdown)
+            self.assertIn("do not guarantee round-robin distribution of requests", markdown)
+            self.assertIn("does not isolate proxy overhead", markdown)
             self.assertIn("**agentgateway mode:**", markdown)
             self.assertIn("standalone request scheduler mode", markdown)
             self.assertIn(
@@ -317,5 +318,65 @@ def test_represents_all_failed_stage_without_fabricated_latency(self) -> None:
             )
 
 
+class PraxisComparisonTest(unittest.TestCase):
+    def create_praxis_campaign(self, root: Path) -> Path:
+        campaign = GenerateReportTest.create_campaign(self, root)
+        (campaign / "runs/service").rename(campaign / "runs/praxis-standalone")
+        manifest_path = campaign / "campaign-manifest.yaml"
+        manifest = yaml.safe_load(manifest_path.read_text())
+        manifest["identity"].update(accelerator_type="sim", backend_type="inference-sim")
+        treatments = manifest["treatments"]
+        treatments["praxis-standalone"] = treatments.pop("service")
+        for entry in treatments.values():
+            entry["repetitions"]["1"].update({
+                "router_chart_version": "v0.9.0",
+                "router_chart_digest": "sha256:" + "a" * 64,
+                "standalone_invariant_sha256": "b" * 64,
+            })
+        treatments["praxis-standalone"]["repetitions"]["1"]["gateway"] = {
+            "image": "example.invalid/praxis@sha256:" + "c" * 64,
+            "source_revision": "d" * 40,
+            "build_features": "full,llmd-ext-proc",
+        }
+        manifest_path.write_text(yaml.safe_dump(manifest))
+        return campaign
+
+    def test_praxis_report_describes_gateway_path_and_simulator_scope(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            campaign = self.create_praxis_campaign(Path(directory))
+            subprocess.run([
+                sys.executable, str(GENERATOR), "--campaign", str(campaign),
+                "--comparison", "agentgateway-standalone:praxis-standalone",
+                "--formats", "markdown,csv",
+            ], check=True, capture_output=True, text=True)
+            markdown = (campaign / "generated/agentgateway-standalone-vs-praxis-standalone/README.md").read_text()
+            self.assertIn("Praxis AI standalone", markdown)
+            self.assertIn("experimental", markdown)
+            self.assertIn("not GPU inference throughput", markdown)
+            self.assertIn("do not demonstrate that one scheduler is more effective", markdown)
+            self.assertNotIn("16 × H100 GPUs", markdown)
+
+    def test_praxis_report_rejects_incomplete_or_different_controls(self) -> None:
+        for field, value in (
+            ("standalone_invariant_sha256", "e" * 64),
+            ("standalone_invariant_sha256", ""),
+            ("router_chart_digest", "sha256:" + "e" * 64),
+            ("gateway", {"image": "praxis:latest"}),
+        ):
+            with self.subTest(field=field, value=value), tempfile.TemporaryDirectory() as directory:
+                campaign = self.create_praxis_campaign(Path(directory))
+                path = campaign / "campaign-manifest.yaml"
+                manifest = yaml.safe_load(path.read_text())
+                manifest["treatments"]["praxis-standalone"]["repetitions"]["1"][field] = value
+                path.write_text(yaml.safe_dump(manifest))
+                result = subprocess.run([
+                    sys.executable, str(GENERATOR), "--campaign", str(campaign),
+                    "--comparison", "agentgateway-standalone:praxis-standalone",
+                    "--formats", "markdown,csv",
+                ], capture_output=True, text=True)
+                self.assertNotEqual(result.returncode, 0)
+                self.assertFalse((campaign / "generated/agentgateway-standalone-vs-praxis-standalone/README.md").exists())
+
+
 if __name__ == "__main__":
     unittest.main()