feat(ci): measured local overhead harness, budgets and regression workflow (P7-CI-46)

.github/workflows/ci-bench.yml

@@ -0,0 +1,118 @@
+name: ci-bench
+
+# Local overhead of the nself ci V15 engine against the v1.4 gate (P7-CI-46).
+# Runs cibench on a hosted runner, then the cli's own `perfbench check` against the
+# per-platform budget (absolute p95) and `cibench -ratio-check` (V15 median against
+# the v1.4 median). Two controls prove the gates can fire: a result ten times over
+# every budget must fail both checks, and a result inside the relative budget must
+# pass. Budgets and baselines: hq plan/p7/evidence/CI-46-perf.md.
+
+on:
+  pull_request:
+    paths:
+      - 'free/ci/**'
+      - '.github/workflows/ci-bench.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-bench-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CGO_ENABLED: '0'
+  # nself-org/cli revision whose tools/perfbench reads the result and budget files.
+  CLI_SHA: '529e8917244bffe4ab1aa9d42b081dd3efe5ee73'
+
+jobs:
+  bench:
+    name: bench (${{ matrix.name }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: linux-amd64
+            os: ubuntu-latest
+            budget: perf-budget.json
+            services: true
+          - name: macos-arm64
+            os: macos-14
+            budget: perf-budget-darwin-arm64.json
+            services: false
+    env:
+      BUDGET: ${{ github.workspace }}/free/ci/bench/${{ matrix.budget }}
+      SLOW: ${{ github.workspace }}/free/ci/bench/testdata/slow-result.json
+      OK: ${{ github.workspace }}/free/ci/bench/testdata/ratio-ok-result.json
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          persist-credentials: false
+
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          repository: nself-org/cli
+          ref: ${{ env.CLI_SHA }}
+          path: cli
+          persist-credentials: false
+
+      # The cli module needs a newer Go than the plugin pins, and setup-go sets
+      # GOTOOLCHAIN=local, so one toolchain (the newer) builds both.
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
+        with:
+          go-version-file: cli/go.mod
+          cache: false
+
+      - name: Harness tests
+        working-directory: free/ci
+        run: go test -mod=vendor -count=1 ./bench/...
+
+      - name: Measure
+        working-directory: free/ci
+        run: go run -mod=vendor ./bench/cmd/cibench -runs 20 -warmup 2 -out "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Absolute p95 budget (cli perfbench check)
+        working-directory: cli
+        run: go run -mod=vendor ./tools/perfbench check -budget "$BUDGET" -in "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Relative budget (V15 against v1.4)
+        working-directory: free/ci
+        run: go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$RUNNER_TEMP/ci-bench.json"
+
+      - name: Control - the gates fail an injected slowdown and pass a good result
+        if: ${{ !cancelled() }}
+        run: |
+          set -u
+          rc=0
+          (cd cli && go run -mod=vendor ./tools/perfbench check -budget "$BUDGET" -in "$SLOW") || rc=$?
+          if [ "$rc" -ne 1 ]; then
+            echo "::error::perfbench check exited $rc on a result ten times over budget; want 1 (a violation)"; exit 1
+          fi
+          cd free/ci
+          rc=0
+          go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$SLOW" || rc=$?
+          if [ "$rc" -ne 1 ]; then
+            echo "::error::the ratio check exited $rc on a result ten times over budget; want 1 (a violation)"; exit 1
+          fi
+          go run -mod=vendor ./bench/cmd/cibench -ratio-check -in "$OK"
+
+      - name: Service start (warm and cold)
+        if: ${{ !cancelled() && matrix.services }}
+        run: |
+          set -eu
+          (cd cli && go build -mod=vendor -o "$RUNNER_TEMP/nself" ./cmd/nself)
+          cd free/ci
+          go run -mod=vendor ./bench/cmd/cibench -runs 5 -warmup 0 -skip-medium -services -cold -cold-runs 3 \
+            -nself "$RUNNER_TEMP/nself" -out "$RUNNER_TEMP/ci-bench-services.json"
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: ci-bench-${{ matrix.name }}
+          path: |
+            ${{ runner.temp }}/ci-bench.json
+            ${{ runner.temp }}/ci-bench-services.json
+          if-no-files-found: ignore

.github/workflows/ci-scenarios.yml

@@ -10,6 +10,9 @@
 # Constraints: actions pinned by SHA; scenario 1 cuts the network for its own processes with `sudo -n` (nft on Linux, pf on
 #   macOS) and removes the cut on every exit path. Runs on pull requests that touch these paths and on demand.
 
+# The disposable GitHub-hosted VM isolates each scenario. The source engine
+# audits the explicit operator trust override; the workflow has a read-only token
+# and receives no repository secrets.
 name: ci-scenarios
 
 on:
@@ -80,7 +83,7 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p "$RUNNER_TEMP/evidence"
-          .drill-bin/nself-ci --no-status --pipeline drills --job drill-scenario-12-external-consumer --json > "$RUNNER_TEMP/evidence/scenario-12.json" || true # exit 1 is the partial selection; the job check below decides
+          .drill-bin/nself-ci --trust owner --no-status --pipeline drills --job drill-scenario-12-external-consumer --json > "$RUNNER_TEMP/evidence/scenario-12.json" || true # exit 1 is the partial selection; the job check below decides
           # A single-job run is a partial selection, so its overall result is never pass: read the job's own check.
           jq -r '"scenario-12: " + ([.data.checks[] | select(.id == "drill-scenario-12-external-consumer") | .result][0] // "no result")' "$RUNNER_TEMP/evidence/scenario-12.json" | tee -a "$GITHUB_STEP_SUMMARY"
           jq -e '[.data.checks[] | select(.id == "drill-scenario-12-external-consumer") | .result] == ["pass"]' "$RUNNER_TEMP/evidence/scenario-12.json" >/dev/null
@@ -90,7 +93,7 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p "$RUNNER_TEMP/evidence"
-          .drill-bin/nself-ci --no-status --pipeline drills --job drill-scenario-01-local-only --json > "$RUNNER_TEMP/evidence/scenario-01.json" || true # exit 1 is the partial selection; the job check below decides
+          .drill-bin/nself-ci --trust owner --no-status --pipeline drills --job drill-scenario-01-local-only --json > "$RUNNER_TEMP/evidence/scenario-01.json" || true # exit 1 is the partial selection; the job check below decides
           # A single-job run is a partial selection, so its overall result is never pass: read the job's own check.
           jq -r '"scenario-01: " + ([.data.checks[] | select(.id == "drill-scenario-01-local-only") | .result][0] // "no result")' "$RUNNER_TEMP/evidence/scenario-01.json" | tee -a "$GITHUB_STEP_SUMMARY"
           jq -e '[.data.checks[] | select(.id == "drill-scenario-01-local-only") | .result] == ["pass"]' "$RUNNER_TEMP/evidence/scenario-01.json" >/dev/null

.github/workflows/nself-ci-source.yml

@@ -71,7 +71,10 @@ jobs:
           set -o pipefail
           mkdir -p "$RUNNER_TEMP/evidence"
           status=0
-          "$NSELF_CI_BIN" --no-status --json > "$RUNNER_TEMP/evidence/nself-ci-source.json" || status=$?
+          # This disposable GitHub-hosted VM is the outer isolation boundary.
+          # The checkout-built engine audits this explicit local-operator override;
+          # no repository secrets or writable token are available to its jobs.
+          "$NSELF_CI_BIN" --trust owner --no-status --json > "$RUNNER_TEMP/evidence/nself-ci-source.json" || status=$?
           {
             echo "### nself-ci/source (built from $GITHUB_SHA)"
             echo "exit code: $status"

free/ci/architecture/layers.yaml

@@ -172,6 +172,8 @@ layers:
     - tools/deps
     - internal/storetest
     - tools/schemagen
+    - bench
+    - bench/cmd/cibench
     - bench/sched
     - bench/cmd/schedbench
     - bench/cache

free/ci/architecture/layers_test.go

@@ -90,6 +90,9 @@ func checkImport(m layerMap, pkg, file, imported string) error {
 	}
 	local := strings.TrimPrefix(imported, module)
 	isLocal := imported != local
+	if imported == "os/exec" && pkg == "bench" && file == "spawn.go" && kind == "supporting" {
+		return nil // the timing harness starts the processes it times, outside the engine's funnel (P7-CI-46)
+	}
 	if imported == "os/exec" && pkg != "internal/exec" && pkg != "internal/legacy" {
 		// These frozen pre-fabric sites are tracked by the single-exec-site test.
 		for _, old := range []struct{ pkg, file string }{{"internal/forgejo", "forgejo.go"}, {"internal/nodes/provision", "exec.go"}, {"internal/serve", "serve_job.go"}, {"internal/serve", "serve_job_report.go"}} {

free/ci/bench/bench_test.go

@@ -0,0 +1,79 @@
+package bench
+
+// Purpose: the Go benchmarks of the in-process scenarios (`go test -bench`).
+// They call the same functions cibench times, so a benchmark number and a
+// result-JSON number measure the same code.
+
+import (
+	"context"
+	"testing"
+)
+
+func benchInproc(b *testing.B) *Inproc {
+	b.Helper()
+	dir := b.TempDir()
+	if err := Materialize(context.Background(), Tiny, dir+"/repo"); err != nil {
+		b.Fatal(err)
+	}
+	p, err := PrepareInproc(dir+"/repo", dir+"/scratch")
+	if err != nil {
+		b.Fatal(err)
+	}
+	return p
+}
+
+func BenchmarkDetect(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.Detect(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkPlan(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.Plan(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkStoreOpen(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if err := p.StoreOpen(); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkExecOverhead(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := p.ExecOverhead(context.Background()); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkRawSpawn(b *testing.B) {
+	p := benchInproc(b)
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		if _, err := p.RawSpawn(context.Background()); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkCapacityProbe(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		CapacityProbe()
+	}
+}

free/ci/bench/budget.go

@@ -0,0 +1,100 @@
+package bench
+
+// Purpose: the perf-budget.json document (perfbench-budget/v1) and its
+// derivation from a recorded baseline.
+// Inputs:  a baseline Result, a headroom factor, a metric filter.
+// Outputs: a Budget whose absolute p95 ceilings are the baseline p95 plus the
+// headroom, rounded up to a whole millisecond, never below MinCeilingMS.
+// Constraints: the field names are the ones `perfbench check` decodes with
+// DisallowUnknownFields; a metric that is not in the baseline is never given a
+// ceiling (an invented number is the one thing this Ticket must not ship).
+
+import (
+	"fmt"
+	"math"
+	"sort"
+)
+
+// BudgetSchemaV1 is the value of the "schema" field of the budget file.
+const BudgetSchemaV1 = "perfbench-budget/v1"
+
+// Headroom is the allowed growth over the baseline p95 (Ticket constraint: +25%).
+const Headroom = 1.25
+
+// MinCeilingMS is the smallest ceiling a budget gets. Below it a timing is
+// scheduler noise on a shared runner (a 0.3 ms planner run can take 3 ms), so a
+// 25 percent rule on such a baseline would fail on weather.
+const MinCeilingMS = 5.0
+
+// Budget is the budget document. P95MS maps a metric name to its p95 ceiling in ms.
+type Budget struct {
+	Note     string             `json:"_note"`
+	Schema   string             `json:"schema"`
+	Platform string             `json:"platform"`
+	P95MS    map[string]float64 `json:"p95_ms"`
+}
+
+// WorstOf folds independent baseline runs of one platform into one result whose
+// p50, p95 and max are, per metric, the highest seen in any run. One run of 20
+// samples under-states the p95 of a noisy machine; the worst of several is the
+// baseline of record. The runs must come from one goos/goarch.
+func WorstOf(runs []Result) (Result, error) {
+	if len(runs) == 0 {
+		return Result{}, fmt.Errorf("no baseline runs")
+	}
+	out := runs[0]
+	out.Metrics = append([]Metric(nil), runs[0].Metrics...)
+	for _, r := range runs[1:] {
+		if r.GOOS != out.GOOS || r.GOARCH != out.GOARCH {
+			return Result{}, fmt.Errorf("baseline runs mix %s/%s and %s/%s", out.GOOS, out.GOARCH, r.GOOS, r.GOARCH)
+		}
+		if len(r.Metrics) != len(out.Metrics) {
+			return Result{}, fmt.Errorf("baseline runs carry different metrics (%d and %d)", len(out.Metrics), len(r.Metrics))
+		}
+		for _, m := range r.Metrics {
+			base, ok := out.Find(m.Name)
+			if !ok {
+				return Result{}, fmt.Errorf("metric %s is missing from the first baseline run", m.Name)
+			}
+			for i := range out.Metrics {
+				if out.Metrics[i].Name == base.Name {
+					out.Metrics[i].P50 = math.Max(base.P50, m.P50)
+					out.Metrics[i].P95 = math.Max(base.P95, m.P95)
+					out.Metrics[i].Max = math.Max(base.Max, m.Max)
+					out.Metrics[i].N = min(base.N, m.N)
+				}
+			}
+		}
+	}
+	return out, nil
+}
+
+// BudgetFromResult derives a budget from a baseline. keep selects the metrics
+// that get a ceiling (nil keeps every metric measured in milliseconds).
+func BudgetFromResult(base Result, note string, keep func(name string) bool) (Budget, error) {
+	b := Budget{Note: note, Schema: BudgetSchemaV1,
+		Platform: fmt.Sprintf("%s/%s", base.GOOS, base.GOARCH), P95MS: map[string]float64{}}
+	for _, m := range base.Metrics {
+		if m.Unit != "ms" || (keep != nil && !keep(m.Name)) {
+			continue
+		}
+		if math.IsNaN(m.P95) || m.P95 <= 0 {
+			return b, fmt.Errorf("metric %s: baseline p95 %v is not a usable timing", m.Name, m.P95)
+		}
+		b.P95MS[m.Name] = math.Max(MinCeilingMS, math.Ceil(m.P95*Headroom))
+	}
+	if len(b.P95MS) == 0 {
+		return b, fmt.Errorf("baseline has no metric to budget")
+	}
+	return b, nil
+}
+
+// Names lists the budgeted metric names, sorted.
+func (b Budget) Names() []string {
+	out := make([]string, 0, len(b.P95MS))
+	for n := range b.P95MS {
+		out = append(out, n)
+	}
+	sort.Strings(out)
+	return out
+}

free/ci/bench/budget_test.go

@@ -0,0 +1,33 @@
+package bench
+
+import "testing"
+
+func TestWorstOf(t *testing.T) {
+	a := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P50: 1, P95: 5, Max: 6, N: 20}, {Name: "y", Unit: "ms", P95: 2, N: 20}})
+	b := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P50: 2, P95: 4, Max: 9, N: 20}, {Name: "y", Unit: "ms", P95: 3, N: 10}})
+	got, err := WorstOf([]Result{a, b})
+	if err != nil {
+		t.Fatal(err)
+	}
+	x, _ := got.Find("x")
+	y, _ := got.Find("y")
+	if x.P50 != 2 || x.P95 != 5 || x.Max != 9 || y.P95 != 3 || y.N != 10 {
+		t.Fatalf("%+v %+v", x, y)
+	}
+	if a.Metrics[0].P50 != 1 {
+		t.Fatal("WorstOf changed its input")
+	}
+	other := b
+	other.GOARCH = "riscv64"
+	if _, err := WorstOf([]Result{a, other}); err == nil {
+		t.Fatal("runs from two platforms were folded")
+	}
+	short := NewResult("", 20, []Metric{{Name: "x", Unit: "ms", P95: 1, N: 20}})
+	short.GOOS, short.GOARCH = a.GOOS, a.GOARCH
+	if _, err := WorstOf([]Result{a, short}); err == nil {
+		t.Fatal("a run with a missing metric was folded")
+	}
+	if _, err := WorstOf(nil); err == nil {
+		t.Fatal("no runs accepted")
+	}
+}