Isolate cross-process tests, fix the failing broker tests, and add CI

.github/workflows/ci.yml

@@ -0,0 +1,165 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Adapted from NVIDIA/Personal-AI-Router@b4a0cc5. PR code runs only on
+# disposable GitHub-hosted runners, without secrets or release automation.
+name: CI
+
+on:
+    pull_request:
+    push:
+        branches: [main]
+
+permissions:
+    contents: read
+
+concurrency:
+    group: ci-${{ github.workflow }}-${{ github.ref }}
+    cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+env:
+    GOTOOLCHAIN: local
+
+jobs:
+    headers:
+        name: SPDX headers
+        runs-on: ubuntu-latest
+        timeout-minutes: 5
+        steps:
+            - uses: actions/checkout@v5.0.0
+              with:
+                  persist-credentials: false
+            - uses: actions/setup-node@v5.0.0
+              with:
+                  node-version: '25.5.0'
+            - run: node scripts/spdx-headers.mjs
+
+    desktop:
+        name: Desktop checks
+        runs-on: ubuntu-latest
+        timeout-minutes: 15
+        defaults:
+            run:
+                working-directory: desktop
+        steps:
+            - uses: actions/checkout@v5.0.0
+              with:
+                  persist-credentials: false
+            - uses: actions/setup-node@v5.0.0
+              with:
+                  node-version: '25.5.0'
+                  cache: npm
+                  cache-dependency-path: desktop/package-lock.json
+            - uses: actions/setup-go@v6.0.0
+              with:
+                  go-version: '1.26.7'
+                  cache-dependency-path: services/**/go.sum
+            # The license checker is deliberately offline and enumerates every
+            # release target, including Windows-only dependencies. A native
+            # build alone would leave those dependencies out of the cache.
+            - name: Populate Go license dependency cache
+              working-directory: .
+              shell: bash
+              run: |
+                  for mod in services/*/go.mod; do
+                      (cd "$(dirname "$mod")" && go mod download)
+                  done
+                  git diff --exit-code -- services
+            - run: npm ci --prefer-offline
+            - run: npm run typecheck
+            - run: npm run lint -- --max-warnings=0
+            - run: npm run test:unit
+            - run: npm run dead-code:check
+            - run: npm run service-contracts:check
+            - run: npm run licenses:check
+
+    services:
+        name: Go modules (${{ matrix.os }})
+        # Pin the supported arm64 image: macos-latest failed runner acquisition
+        # during the macOS 26 rollout, before any test step could execute.
+        runs-on: ${{ matrix.os }}
+        timeout-minutes: 20
+        strategy:
+            fail-fast: false
+            matrix:
+                os: [macos-15, ubuntu-latest]
+        steps:
+            - uses: actions/checkout@v5.0.0
+              with:
+                  persist-credentials: false
+            - uses: actions/setup-go@v6.0.0
+              with:
+                  go-version: '1.26.7'
+                  cache-dependency-path: services/**/go.sum
+            - name: Linux test tools
+              if: runner.os == 'Linux'
+              run: sudo apt-get update -qq && sudo apt-get install -y -qq lsof iproute2
+            - name: Check Go formatting
+              shell: bash
+              run: |
+                  unformatted=$(gofmt -l services)
+                  if [ -n "$unformatted" ]; then
+                      printf '%s\n' "$unformatted"
+                      exit 1
+                  fi
+            - name: Vet and test every service module (including libraries)
+              shell: bash
+              run: |
+                  failed=0
+                  for mod in services/*/go.mod; do
+                      dir=$(dirname "$mod")
+                      # Cross-process tests run in their own job (Linux and macOS).
+                      if [ "$dir" = services/tests ]; then continue; fi
+                      echo "==> $dir"
+                      (cd "$dir" && go vet ./... && go test -count=1 -timeout=5m -v ./...) || failed=1
+                  done
+                  exit "$failed"
+
+    cross-process:
+        name: Cross-process tests (${{ matrix.os }})
+        runs-on: ${{ matrix.os }}
+        timeout-minutes: 20
+        strategy:
+            fail-fast: false
+            # Linux runners can send mDNS multicast, so discovery is exercised
+            # there. Hosted macOS VMs cannot (every send to 224.0.0.251 fails
+            # with "no route to host"); on macOS the discovery tests skip with
+            # that reason named, and everything else runs.
+            matrix:
+                os: [ubuntu-latest, macos-15]
+        steps:
+            - uses: actions/checkout@v5.0.0
+              with:
+                  persist-credentials: false
+            - uses: actions/setup-go@v6.0.0
+              with:
+                  go-version: '1.26.7'
+                  cache-dependency-path: services/**/go.sum
+            - name: Install port-owner tools
+              if: runner.os == 'Linux'
+              run: sudo apt-get update -qq && sudo apt-get install -y -qq lsof iproute2
+            # Run the source guard before executing any application binary.
+            # All runtime helpers must use per-test disposable app-data roots.
+            - name: Check process isolation
+              working-directory: services/tests
+              run: go test -count=1 -run '^Test(ProcessHelpersRequireIsolation|CommandEnvironmentIsPerTest)$' .
+            - name: Vet cross-process suite
+              working-directory: services/tests
+              run: go vet ./...
+            # Leave five minutes for setup and artifact upload. The Go deadline
+            # prints the active test and goroutines before the job is killed.
+            - name: Run isolated cross-process suite with diagnostics
+              working-directory: services/tests
+              shell: bash
+              run: |
+                  set -o pipefail
+                  go test -v -count=1 -timeout=15m -json ./... 2>&1 \
+                      | tee "$RUNNER_TEMP/cross-process.jsonl" \
+                      | jq --unbuffered -j 'select(.Output != null) | .Output'
+            - name: Upload cross-process diagnostics
+              if: always()
+              uses: actions/upload-artifact@v4.6.2
+              with:
+                  name: cross-process-${{ matrix.os }}
+                  path: ${{ runner.temp }}/cross-process.jsonl
+                  if-no-files-found: warn

services/nvpair-engine-manager/README.md

@@ -156,7 +156,11 @@ version only decides what a fresh install starts from.
 `install.creates` names `/Applications/oMLX.app`: an install that finds an app
 already there opens it but does not record it as PAIR's (and drops any earlier
 claim), so it is never offered for uninstall. A claim records the created app's
-file identity and covers that exact bundle only. A copy put there later is a
+file identity and covers that exact bundle only. On macOS this includes birth
+time (stable across the install's rename); on Linux it includes ctime so a reused
+inode cannot inherit the claim. Linux metadata changes also invalidate the claim.
+Older device/inode-only claims no longer authorize removal; reinstall through
+PAIR to establish a new claim. A copy put there later is a
 different file, and so is a bundle oMLX's own updater swaps in: after an in-app
 update PAIR no longer offers uninstall, and the app is removed from the Finder. Uninstalling an oMLX PAIR installed quits the app and removes
 `/Applications/oMLX.app` and the two shims it writes (`omlx`,

services/nvpair-engine-manager/fileid_darwin.go

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import "syscall"
+
+// Birth time survives the staging-to-destination rename and metadata changes.
+func fileGenerationTime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Birthtimespec
+}

services/nvpair-engine-manager/fileid_darwin_test.go

@@ -0,0 +1,31 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"syscall"
+	"testing"
+)
+
+// Reusing a device/inode must not transfer an install claim to a later bundle.
+func TestFileIdentityDistinguishesReusedInode(t *testing.T) {
+	st := syscall.Stat_t{Dev: 1, Ino: 2, Birthtimespec: syscall.Timespec{Sec: 3, Nsec: 4}}
+	original := fileIdentityFromStat(&st)
+	if got := fileIdentityFromStat(&st); got != original {
+		t.Fatal("unchanged bundle identity is unstable")
+	}
+	st.Ctimespec = syscall.Timespec{Sec: 5, Nsec: 6}
+	if got := fileIdentityFromStat(&st); got != original {
+		t.Fatal("metadata changes must not invalidate a Darwin bundle's identity")
+	}
+	st.Birthtimespec.Nsec++
+	if got := fileIdentityFromStat(&st); got == original {
+		t.Fatal("reused inode with a different birth nanosecond retained the old identity")
+	}
+	st.Birthtimespec.Nsec--
+	st.Birthtimespec.Sec++
+	if got := fileIdentityFromStat(&st); got == original {
+		t.Fatal("reused inode with a different birth second retained the old identity")
+	}
+}

services/nvpair-engine-manager/fileid_linux.go

@@ -0,0 +1,12 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import "syscall"
+
+// Stat_t has no birth time on Linux. ctime distinguishes reused inodes and
+// stays stable while the installed bundle is unchanged.
+func fileGenerationTime(st *syscall.Stat_t) syscall.Timespec {
+	return st.Ctim
+}

services/nvpair-engine-manager/fileid_linux_test.go

@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"syscall"
+	"testing"
+)
+
+// Linux can immediately reuse the inode of a deleted bundle directory.
+func TestFileIdentityDistinguishesReusedInode(t *testing.T) {
+	st := syscall.Stat_t{Dev: 1, Ino: 2, Ctim: syscall.Timespec{Sec: 3, Nsec: 4}}
+	original := fileIdentityFromStat(&st)
+	if got := fileIdentityFromStat(&st); got != original {
+		t.Fatal("unchanged bundle identity is unstable")
+	}
+	st.Ctim.Nsec++
+	if got := fileIdentityFromStat(&st); got == original {
+		t.Fatal("reused inode with a different change nanosecond retained the old identity")
+	}
+	st.Ctim.Nsec--
+	st.Ctim.Sec++
+	if got := fileIdentityFromStat(&st); got == original {
+		t.Fatal("reused inode with a different change second retained the old identity")
+	}
+}

services/nvpair-engine-manager/fileid_unix.go

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-//go:build !windows
+//go:build darwin || linux
 
 package main
 
@@ -11,8 +11,9 @@ import (
 	"syscall"
 )
 
-// fileIdentity is a path's device and inode, without following a final link,
-// or "" when it cannot be read.
+// fileIdentity combines device, inode, and a generation timestamp without
+// following a final link. The timestamp prevents inode reuse from transferring
+// an install claim to a replacement bundle. It returns "" on read failure.
 func fileIdentity(path string) string {
 	info, err := os.Lstat(path)
 	if err != nil {
@@ -22,5 +23,10 @@ func fileIdentity(path string) string {
 	if !ok {
 		return ""
 	}
-	return fmt.Sprintf("%d:%d", st.Dev, st.Ino)
+	return fileIdentityFromStat(st)
+}
+
+func fileIdentityFromStat(st *syscall.Stat_t) string {
+	stamp := fileGenerationTime(st)
+	return fmt.Sprintf("%d:%d:%d:%d", st.Dev, st.Ino, stamp.Sec, stamp.Nsec)
 }

services/nvpair-node-info/sampling_test.go

@@ -15,11 +15,11 @@ func TestPolicyTable(t *testing.T) {
 	epoch := time.Unix(1000000, 0)
 
 	cases := []struct {
-		name           string
-		setup          func(p *SamplingPolicy, now time.Time)
-		evalAt         time.Duration
-		wantInterval   time.Duration
-		wantMode       SamplingMode
+		name         string
+		setup        func(p *SamplingPolicy, now time.Time)
+		evalAt       time.Duration
+		wantInterval time.Duration
+		wantMode     SamplingMode
 	}{
 		{
 			name:         "clean idle: no leases, no demand",
@@ -189,9 +189,9 @@ func TestPollIntentClassification(t *testing.T) {
 		{"foreground", PollIntentForeground},
 		{"scheduling", PollIntentScheduling},
 		{"background", PollIntentBackground},
-		{"", PollIntentForeground},        // missing header behaves as foreground
-		{"unknown", PollIntentForeground},   // unknown header behaves as foreground
-		{"FOREGROUND", PollIntentForeground},// unknown/cased behaves as foreground
+		{"", PollIntentForeground},           // missing header behaves as foreground
+		{"unknown", PollIntentForeground},    // unknown header behaves as foreground
+		{"FOREGROUND", PollIntentForeground}, // unknown/cased behaves as foreground
 	}
 	for _, tc := range cases {
 		if got := parsePollIntent(tc.header); got != tc.want {

services/nvpair-ui-broker/nodeinfo_demand_test.go

@@ -36,9 +36,9 @@ func TestWriteDemandFrame(t *testing.T) {
 			}
 
 			var frame struct {
-				JSONRPC string                `json:"jsonrpc"`
-				Method  string                `json:"method"`
-				ID      json.RawMessage       `json:"id"`
+				JSONRPC string               `json:"jsonrpc"`
+				Method  string               `json:"method"`
+				ID      json.RawMessage      `json:"id"`
 				Params  noderec.DemandParams `json:"params"`
 			}
 			if err := json.Unmarshal([]byte(line), &frame); err != nil {
@@ -78,7 +78,7 @@ func TestBrokerDemandOnWorkloadTransitions(t *testing.T) {
 				continue
 			}
 			var f struct {
-				Method string                `json:"method"`
+				Method string               `json:"method"`
 				Params noderec.DemandParams `json:"params"`
 			}
 			if err := json.Unmarshal([]byte(l), &f); err == nil && f.Method == "stats/set-demand" {
@@ -205,7 +205,7 @@ func TestBrokerDemandOnNodeInfoRestart(t *testing.T) {
 		t.Fatalf("expected 1 frame on restart, got %d: %q", len(lines), buf.String())
 	}
 	var f struct {
-		Method string                `json:"method"`
+		Method string               `json:"method"`
 		Params noderec.DemandParams `json:"params"`
 	}
 	if err := json.Unmarshal([]byte(lines[0]), &f); err != nil {

services/nvpair-ui-broker/supervisor.go

@@ -79,6 +79,16 @@ func (p hangPolicy) withDefaults() hangPolicy {
 	return p
 }
 
+// startProbe transfers cancellation ownership to watch, which cancels on
+// completion, deadline, or any exit. A buffered result lets a late probe finish
+// without blocking after watch has moved on to a replacement handle.
+func (p hangPolicy) startProbe() (context.Context, context.CancelFunc, <-chan error) {
+	ctx, cancel := context.WithTimeout(context.Background(), p.deadline)
+	result := make(chan error, 1)
+	go func() { result <- p.probe(ctx) }()
+	return ctx, cancel, result
+}
+
 // restartPolicy configures the hybrid auto-restart behaviour the broker
 // applies to a crashed worker. The defaults (see defaultRestartPolicy)
 // implement: exponential backoff between attempts, a bounded budget so a
@@ -535,10 +545,8 @@ func (s *supervisor) watch(h supervisedHandle, onStable func(), onHealth func(er
 			if probeC != nil {
 				continue // Never overlap probes for the same live handle.
 			}
-			probeCtx, cancel = context.WithTimeout(context.Background(), policy.deadline)
-			result := make(chan error, 1)
-			probeC, deadlineC = result, probeCtx.Done()
-			go func(ctx context.Context) { result <- policy.probe(ctx) }(probeCtx)
+			probeCtx, cancel, probeC = policy.startProbe()
+			deadlineC = probeCtx.Done()
 		case err := <-probeC:
 			if probeCtx.Err() != nil {
 				err = probeCtx.Err() // A late success cannot beat the deadline.