From 8025ca499a231f69d2cdd22fe31a00a9c0ba8ec2 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:38:56 +0200 Subject: [PATCH 01/18] fix(clustertrust): bind peer inventory to pinned identity proofs Public identity claims must not authorize service inventory. Add bounded pinned-mTLS inventory reads, authorization bindings covering admission and certificate pins, and separate preview and manual record namespaces. Recheck the exact peer certificate after each response. Adapt the peer inventory separation from NVIDIA/Personal-AI-Router#179, with explicit provenance and proof lifetimes. Co-authored-by: Kaylee Lubick Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/shared/clustertrust/inventory.go | 67 ++++++++ .../shared/clustertrust/inventory_test.go | 155 ++++++++++++++++++ services/shared/clustertrust/membership.go | 25 ++- services/shared/clustertrust/mesh.go | 43 ++++- services/shared/noderec/noderec.go | 85 +++++++--- services/shared/noderec/provenance_test.go | 33 ++++ 6 files changed, 374 insertions(+), 34 deletions(-) create mode 100644 services/shared/clustertrust/inventory.go create mode 100644 services/shared/clustertrust/inventory_test.go create mode 100644 services/shared/noderec/provenance_test.go diff --git a/services/shared/clustertrust/inventory.go b/services/shared/clustertrust/inventory.go new file mode 100644 index 000000000..48e670e90 --- /dev/null +++ b/services/shared/clustertrust/inventory.go @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustertrust + +import ( + "context" + "encoding/json" + "io" + "net/http" +) + +// ReadPeerInventory accepts one bounded JSON response only when its TLS peer +// still matches the exact current pin and the authorization did not change +// while reading. Redirects are deliberately not followed, including HTTPS ones. +// An empty expected principal is used only to discover a manual peer; callers +// must then repeat with that exact principal before publishing inventory. +func ReadPeerInventory(ctx context.Context, mesh *Mesh, client *http.Client, principal, url, poll string, limit int64, value interface{}) (string, string, bool) { + if ctx == nil { + ctx = context.Background() + } + if client == nil || mesh == nil { + return "", "", false + } + mesh.Refresh() + local := mesh.PeerRecordID(mesh.NodeUUID()) + binding := mesh.PeerRecordID(principal) + if local == "" || (principal != "" && binding == "") { + return "", "", false + } + copyClient := *client + copyClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return "", "", false + } + if req.URL.Scheme != "https" { + return "", "", false + } + if poll != "" { + req.Header.Set("X-NVPAIR-Poll", poll) + } + resp, err := copyClient.Do(req) + if err != nil { + return "", "", false + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK || resp.TLS == nil || len(resp.TLS.PeerCertificates) == 0 { + return "", "", false + } + cert := resp.TLS.PeerCertificates[0] + if principal == "" { + principal = UUIDFromCert(cert) + } + // Read the complete bounded body: a valid JSON prefix with excessive trailing + // data is not an accepted response. + data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil || int64(len(data)) > limit || json.Unmarshal(data, value) != nil { + return "", "", false + } + mesh.Refresh() + current := mesh.PeerRecordID(principal) + if current == "" || local != mesh.PeerRecordID(mesh.NodeUUID()) || !mesh.VerifyPeerCertificate(principal, cert) || (binding != "" && binding != current) { + return "", "", false + } + return principal, current, true +} diff --git a/services/shared/clustertrust/inventory_test.go b/services/shared/clustertrust/inventory_test.go new file mode 100644 index 000000000..e93032185 --- /dev/null +++ b/services/shared/clustertrust/inventory_test.go @@ -0,0 +1,155 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clustertrust + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "nvpair-shared/clustertrusttest" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func inventoryPair(t *testing.T) (string, *Mesh, *Mesh) { + t.Helper() + selfDir, peerDir := t.TempDir(), t.TempDir() + clustertrusttest.Join(t, selfDir, "test-cluster", "self") + clustertrusttest.Join(t, peerDir, "test-cluster", "peer") + pin := func(dir, uuid, certDir string) { + cert, err := os.ReadFile(filepath.Join(certDir, "node.crt")) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(map[string]string{"nodeUuid": uuid, "certPem": string(cert)}) + if err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "trusted"), 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "trusted", uuid+".json"), data, 0600); err != nil { + t.Fatal(err) + } + } + pin(selfDir, "peer", peerDir) + pin(peerDir, "self", selfDir) + return selfDir, Open(selfDir), Open(peerDir) +} +func inventoryServer(t *testing.T, mesh *Mesh, handler http.HandlerFunc) *httptest.Server { + t.Helper() + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := mesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", 403) + return + } + handler(w, r) + })) + srv.TLS = mesh.ServerTLSConfig() + srv.StartTLS() + t.Cleanup(srv.Close) + return srv +} + +func TestReadPeerInventoryRequiresCurrentExactAuthorization(t *testing.T) { + for _, change := range []string{"none", "pin", "admission"} { + t.Run(change, func(t *testing.T) { + dir, self, peer := inventoryPair(t) + started, release := make(chan struct{}), make(chan struct{}) + srv := inventoryServer(t, peer, func(w http.ResponseWriter, _ *http.Request) { + close(started) + <-release + _, _ = w.Write([]byte(`{"value":42}`)) + }) + pool := NewPeerClientPool(self, time.Second) + defer pool.CloseIdle() + client, ok := pool.Client("peer") + if !ok { + t.Fatal("missing peer client") + } + result := make(chan bool, 1) + go func() { + var value struct { + Value int `json:"value"` + } + _, binding, ok := ReadPeerInventory(t.Context(), self, client, "peer", srv.URL, "background", 1024, &value) + result <- ok && binding != "" && value.Value == 42 + }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("request did not start") + } + if change == "pin" { + clustertrusttest.RemovePeerPin(t, dir, "peer") + } + if change == "admission" { + clustertrusttest.WriteAdmission(t, dir, "test-cluster", 2) + } + close(release) + if accepted := <-result; accepted != (change == "none") { + t.Fatalf("authorization change=%s, accepted=%v", change, accepted) + } + }) + } +} +func TestReadPeerInventoryRejectsRedirectsAndOversizedJSON(t *testing.T) { + _, self, peer := inventoryPair(t) + var publicReads atomic.Int32 + public := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + publicReads.Add(1) + _, _ = w.Write([]byte(`{"value":42}`)) + })) + defer public.Close() + srv := inventoryServer(t, peer, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/redirect" { + http.Redirect(w, r, public.URL, http.StatusFound) + return + } + _, _ = w.Write([]byte(`{"value":42}` + strings.Repeat(" ", 1024))) + }) + pool := NewPeerClientPool(self, time.Second) + defer pool.CloseIdle() + client, _ := pool.Client("peer") + for _, path := range []string{"/redirect", "/oversized"} { + var value struct{ Value int } + if _, _, ok := ReadPeerInventory(t.Context(), self, client, "peer", srv.URL+path, "", 100, &value); ok { + t.Fatalf("accepted %s", path) + } + } + if publicReads.Load() != 0 { + t.Fatal("authenticated read followed plaintext redirect") + } +} +func TestPeerRecordIDRetainsAdmissionOnReadFailureButNotClear(t *testing.T) { + dir := t.TempDir() + clustertrusttest.Join(t, dir, "test-cluster", "self") + mesh := Open(dir) + before := mesh.PeerRecordID("self") + if before == "" { + t.Fatal("missing initial proof binding") + } + path := filepath.Join(dir, "admission.json") + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + mesh.Refresh() + if mesh.PeerRecordID("self") != before { + t.Fatal("filesystem read failure discarded known admission") + } + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + mesh.Refresh() + if mesh.PeerRecordID("self") != "" { + t.Fatal("absent admission retained authorization") + } +} diff --git a/services/shared/clustertrust/membership.go b/services/shared/clustertrust/membership.go index 74c3edad6..678cc6034 100644 --- a/services/shared/clustertrust/membership.go +++ b/services/shared/clustertrust/membership.go @@ -25,25 +25,24 @@ type admissionRecord struct { Epoch uint64 `json:"epoch"` } -// hasActiveAdmission reports whether clusterDir/admission.json records a live -// admission. A missing, unreadable, malformed, or cleared file reads as false -// (not a member). This is the durable, restart-surviving membership fact, in -// contrast to keypair presence which outlives a membership by design. -// -// It is re-read on every Mesh.Refresh rather than cached: membership is exactly -// the fact that changes underneath a running service when the user creates, -// joins, or leaves a cluster. -func hasActiveAdmission(clusterDir string) bool { +// readAdmission returns the durable membership and epoch, or an empty record +// without an error when the file is absent or malformed. Other filesystem errors +// are returned so callers can retain the last known admission during read +// failures. The epoch distinguishes a rejoin using the previous certificate. +func readAdmission(clusterDir string) (admissionRecord, error) { if clusterDir == "" { - return false + return admissionRecord{}, nil } data, err := os.ReadFile(filepath.Join(clusterDir, admissionFileName)) if err != nil { - return false + if os.IsNotExist(err) { + return admissionRecord{}, nil + } + return admissionRecord{}, err } var rec admissionRecord if err := json.Unmarshal(data, &rec); err != nil { - return false + return admissionRecord{}, nil } - return rec.ClusterID != "" && rec.Epoch != 0 + return rec, nil } diff --git a/services/shared/clustertrust/mesh.go b/services/shared/clustertrust/mesh.go index 664c4da7b..5314d1cb0 100644 --- a/services/shared/clustertrust/mesh.go +++ b/services/shared/clustertrust/mesh.go @@ -5,6 +5,7 @@ package clustertrust import ( "bytes" + "crypto/sha256" "crypto/tls" "crypto/x509" "errors" @@ -72,7 +73,9 @@ type Mesh struct { // dir is the cluster subtree nvpair-cluster-manager owns (node.key/node.crt, // admission.json, trusted/). Empty means this service was given no cluster // dir and is unclustered for its whole life. - dir string + dir string + refreshMu sync.Mutex + admission admissionRecord // mu guards the identity and the derived membership answer. trust carries // its own lock and is replaced only in Open. @@ -122,6 +125,8 @@ func (m *Mesh) Refresh() { if m == nil || m.dir == "" { return } + m.refreshMu.Lock() + defer m.refreshMu.Unlock() m.trust.Reload() certPEM, certErr := os.ReadFile(filepath.Join(m.dir, "node.crt")) @@ -140,7 +145,10 @@ func (m *Mesh) Refresh() { // and durable across restarts, including a cluster of one with no peers) or // at least one pinned peer (a confirmed peer means clustered even on a // cluster dir written before admission.json existed). - m.clustered = m.id != nil && (hasActiveAdmission(m.dir) || m.trust.Count() > 0) + if admission, err := readAdmission(m.dir); err == nil { + m.admission = admission + } + m.clustered = m.id != nil && ((m.admission.ClusterID != "" && m.admission.Epoch != 0) || m.trust.Count() > 0) } // Clustered reports whether this node is CURRENTLY a cluster member, as of the @@ -236,6 +244,37 @@ func (m *Mesh) HasPin(uuid string) bool { return ok } +// PeerRecordID binds an inventory proof to the current local identity, +// admission and exact peer pin record. It is not an authenticator; only a +// successful pinned mTLS read may publish it. Call Refresh before comparing. +func (m *Mesh) PeerRecordID(uuid string) string { + if m == nil || uuid == "" { + return "" + } + m.mu.RLock() + defer m.mu.RUnlock() + if !m.clustered || m.id == nil { + return "" + } + peer := selfLeafDER(m.id) + if uuid != m.id.NodeUUID { + m.trust.mu.RLock() + peer = bytes.Clone(m.trust.raw[uuid]) + m.trust.mu.RUnlock() + if len(peer) == 0 { + return "" + } + } + binding := fmt.Sprintf("%x:%x:%s:%d", sha256.Sum256(selfLeafDER(m.id)), sha256.Sum256(peer), m.admission.ClusterID, m.admission.Epoch) + return fmt.Sprintf("peer:%s:%x", uuid, sha256.Sum256([]byte(binding))) +} + +// VerifyPeerCertificate rechecks the current exact pin after a response, since +// pooled connections and in-flight requests can outlive their authorization. +func (m *Mesh) VerifyPeerCertificate(uuid string, cert *x509.Certificate) bool { + return cert != nil && UUIDFromCert(cert) == uuid && m.matchDER(uuid, cert.Raw) +} + // matchDER is the server-side gate: accept der for uuid when it byte-for-byte // matches the pinned peer cert, or when uuid is this node and der is its own // leaf (self-trust). diff --git a/services/shared/noderec/noderec.go b/services/shared/noderec/noderec.go index b401771f4..52cad03ce 100644 --- a/services/shared/noderec/noderec.go +++ b/services/shared/noderec/noderec.go @@ -29,6 +29,7 @@ package noderec import ( + "crypto/sha256" "encoding/json" "fmt" "sort" @@ -322,18 +323,11 @@ const ( // MethodSetClusterIdentity tells nvpair-node-info the cluster principal this // node currently holds, so it can report it on /v1/node-info. // - // node-info is spawned with no cluster dir on purpose — it is the one - // inter-node surface kept plain so any LAN peer can read this host's - // inventory (see the broker's spawnNodeInfo) — which also leaves it unable to - // read membership for itself. The broker owns that fact and pushes it here on - // startup and on every change, so node-info reports one live value rather - // than deriving a second one. - // - // The value is a peer's only mDNS-independent way to learn that this node has - // left a cluster: cluster membership otherwise reaches the fleet solely as the - // cluster-uuid= TXT key, and a consumer that stops receiving this node's mDNS - // record keeps its last observed value indefinitely. + // With a cluster directory node-info reads membership from its live mesh. + // Standalone launches without one can receive it from their owning parent. MethodSetClusterIdentity = "nodeinfo:set-cluster-identity" + // MethodSetPeerMetadata supplies service inventory over the owned stdio pipe. + MethodSetPeerMetadata = "nodeinfo:set-peer-metadata" // MethodSetDemand tells nvpair-node-info whether local workload execution is active. MethodSetDemand = "stats/set-demand" @@ -419,6 +413,15 @@ type UnregisterParams struct { // MethodSetClusterIdentity. Empty means this node belongs to no cluster, which is // a meaningful value rather than an absent one — it is how a departure is // announced — so the field is always sent. +// PeerMetadataParams is broker-owned metadata authenticated with node-info. +// Public HTTP copies are display-only and cannot authorize service decisions. +type PeerMetadataParams struct { + HostUUID string `json:"hostUuid"` + Name string `json:"name"` + IPs []string `json:"ips,omitempty"` + Services map[ServiceKey]int `json:"services"` +} + type ClusterIdentityParams struct { ClusterUUID string `json:"clusterUuid"` } @@ -575,14 +578,16 @@ type DirectoryNode struct { // direct-connect link is reachable only from the machine on its far end — so // a consumer that must connect walks this list instead of treating IP as the // only answer. Empty for a node that published one address. - IPs []string `json:"ips,omitempty"` - ClusterUUID string `json:"clusterUuid,omitempty"` - Trusted bool `json:"trusted"` // we hold a cluster pin for it - Services map[ServiceKey]ServiceStatus `json:"services"` - GPUs []GPUInfo `json:"gpus,omitempty"` - CPU *CPUInfo `json:"cpu,omitempty"` - Memory *MemoryInfo `json:"memory,omitempty"` - Models []string `json:"models,omitempty"` // enriched from engine-manager's em /v1/models + IPs []string `json:"ips,omitempty"` + ClusterUUID string `json:"clusterUuid,omitempty"` + Trusted bool `json:"trusted"` // identity proven over current pinned cluster mTLS + PeerRecordID string `json:"peerRecordId,omitempty"` + IdentityVerification string `json:"identityVerification,omitempty"` // local, verified, or unverified + Services map[ServiceKey]ServiceStatus `json:"services"` + GPUs []GPUInfo `json:"gpus,omitempty"` + CPU *CPUInfo `json:"cpu,omitempty"` + Memory *MemoryInfo `json:"memory,omitempty"` + Models []string `json:"models,omitempty"` // enriched from engine-manager's em /v1/models // ModelsByEngine attributes the enriched model list to the engine that // serves each model, keyed by engine-manager engine name (e.g. "ollama", // "lmstudio"). Models is the flat de-duplicated union of these lists and is @@ -623,6 +628,47 @@ type DirectoryNode struct { LastSeen int64 `json:"lastSeen"` // Unix seconds } +// PreviewRecordID namespaces unauthenticated claims independently of host IDs. +// Addresses and ports are deliberately excluded so a changed hint updates the +// same preview rather than leaving an orphaned row. +func PreviewRecordID(name, claimedHostUUID string) string { + return fmt.Sprintf("preview:%x", sha256.Sum256([]byte(name+"\x00"+claimedHostUUID))) +} + +// ManualRecordID keeps explicit user configurations outside the peer namespace. +func ManualRecordID(id string) string { return fmt.Sprintf("manual:%x", sha256.Sum256([]byte(id))) } + +// RecordKey is the directory identity, never an unverified claimed host UUID. +func (n DirectoryNode) RecordKey() string { + if n.IdentityVerification == "local" || n.IdentityVerification == "verified" { + // Reserve preview/configuration namespaces even for a malformed peer + // principal; arbitrary certificate strings cannot collide with them. + if strings.HasPrefix(n.HostUUID, "preview:") || strings.HasPrefix(n.HostUUID, "manual:") { + return "" + } + return n.HostUUID + } + if strings.HasPrefix(n.PeerRecordID, "preview:") || strings.HasPrefix(n.PeerRecordID, "manual:") { + return n.PeerRecordID + } + return "" +} + +// RecordSelector targets an existing internal record without interpreting an +// opaque preview/configuration key as a peer host UUID. It grants no authority. +func RecordSelector(key string) DirectoryNode { + if strings.HasPrefix(key, "preview:") || strings.HasPrefix(key, "manual:") { + return DirectoryNode{PeerRecordID: key, IdentityVerification: "unverified"} + } + return DirectoryNode{HostUUID: key, IdentityVerification: "verified"} +} + +// ServicePeer distinguishes owned local inventory and authenticated peers from +// public discovery previews. A pin without an mTLS proof never qualifies. +func (n DirectoryNode) ServicePeer() bool { + return n.RecordKey() != "" && (n.IdentityVerification == "local" || (n.IdentityVerification == "verified" && n.Trusted && n.PeerRecordID != "")) +} + // Clustered reports whether the node advertises a cluster identity. func (n DirectoryNode) Clustered() bool { return n.ClusterUUID != "" } @@ -723,6 +769,7 @@ type NodeEvent struct { // scanner emits it independently of directory changes so scheduling telemetry // stays current even when discovery identity and inventory are stable. type NodeTelemetry struct { + PeerRecordID string `json:"peerRecordId,omitempty"` HostUUID string `json:"hostUuid"` GPUUtilizationPct uint32 `json:"gpuUtilizationPercent"` TelemetryValid bool `json:"telemetryValid"` diff --git a/services/shared/noderec/provenance_test.go b/services/shared/noderec/provenance_test.go new file mode 100644 index 000000000..04a83f339 --- /dev/null +++ b/services/shared/noderec/provenance_test.go @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package noderec + +import "testing" + +func TestRecordKeysSeparateClaimsPeersAndManualConfigurations(t *testing.T) { + preview := DirectoryNode{HostUUID: "peer", Name: "claim", IdentityVerification: "unverified", PeerRecordID: PreviewRecordID("claim", "peer")} + verified := DirectoryNode{HostUUID: "peer", IdentityVerification: "verified", Trusted: true, PeerRecordID: "peer:proof"} + if preview.RecordKey() == verified.RecordKey() || preview.RecordKey() == preview.HostUUID || preview.ServicePeer() || !verified.ServicePeer() { + t.Fatal("preview and peer identity namespaces overlap") + } + manual := DirectoryNode{HostUUID: "peer", IdentityVerification: "manual", PeerRecordID: ManualRecordID("peer")} + if manual.RecordKey() == verified.RecordKey() || manual.RecordKey() == preview.RecordKey() || manual.ServicePeer() { + t.Fatal("manual bypass overlaps peer identity") + } + for _, reserved := range []string{preview.RecordKey(), manual.RecordKey()} { + collision := verified + collision.HostUUID = reserved + if collision.RecordKey() != "" || collision.ServicePeer() { + t.Fatalf("peer principal claimed reserved record namespace %q", reserved) + } + } + for _, node := range []DirectoryNode{preview, verified, manual} { + if RecordSelector(node.RecordKey()).RecordKey() != node.RecordKey() { + t.Fatal("record selector changed the target identity") + } + } + if (DirectoryNode{HostUUID: "peer"}).RecordKey() != "" { + t.Fatal("unmarked claim accepted a host key") + } +} From cc7b71dd7d0daa5dbe1ef915292fbd44e4c9e5ac Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:38:56 +0200 Subject: [PATCH 02/18] fix(node-info): publish authenticated broker-owned service metadata Serve pinned cluster inventory beside the public hardware preview. Accept service metadata only from the owning broker and only for the local identity, so remote consumers can authenticate service ports without trusting discovery announcements. Adapt NVIDIA/Personal-AI-Router#179 to the shared inventory proof boundary. Co-authored-by: Kaylee Lubick Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-node-info/README.md | 12 +-- services/nvpair-node-info/main.go | 89 +++++++++------------- services/nvpair-node-info/metadata.go | 52 +++++++++++++ services/nvpair-node-info/metadata_test.go | 64 ++++++++++++++++ 4 files changed, 156 insertions(+), 61 deletions(-) create mode 100644 services/nvpair-node-info/metadata.go create mode 100644 services/nvpair-node-info/metadata_test.go diff --git a/services/nvpair-node-info/README.md b/services/nvpair-node-info/README.md index e658fbc36..ad2039b02 100644 --- a/services/nvpair-node-info/README.md +++ b/services/nvpair-node-info/README.md @@ -12,7 +12,7 @@ A Go service that exposes this machine's hardware inventory (GPUs, CPU, physical Two surfaces: - **HTTP(S)** — serves the node inventory at `/v1/node-info`. Plaintext HTTP on `:14318` by default; optional HTTPS (with optional mTLS) on `:14319` when a cert/key pair is supplied. -- **stdio JSON-RPC 2.0** — newline-delimited, used for lifecycle/control (`log/set-level`, `worker/health`, `nodeinfo:set-cluster-identity`, `stats/set-demand`) and shutdown via stdin EOF. The service is normally launched as a subprocess by the broker. +- **stdio JSON-RPC 2.0** — newline-delimited, used for lifecycle/control (`log/set-level`, `worker/health`, `nodeinfo:set-cluster-identity`, `nodeinfo:set-peer-metadata`, `stats/set-demand`) and shutdown via stdin EOF. The service is normally launched as a subprocess by the broker. | Control request | Params | Result | | --- | --- | --- | @@ -22,7 +22,7 @@ An id-bearing `worker/health` request returns `{"ok":true}` through the shared stdin control loop. This checks control-loop responsiveness, not HTTP listener or telemetry readiness. Notifications receive no response. -It does **not** advertise itself over mDNS. Discovery is centralized in the `nvpair-node-scanner` daemon: the broker registers this service's `ni` port with the daemon, which carries it on the node's one `_nvpair-node` record and fetches `/v1/node-info` over plain HTTP to enrich each node. +It does **not** advertise itself over mDNS. Discovery is centralized in the `nvpair-node-scanner` daemon: the broker registers this service's `ni` port with the daemon, which carries it on the node's one `_nvpair-node` record. Services authenticate remote inventory over pinned cluster mTLS; public HTTP copies are display-only. ## CLI Flags @@ -34,7 +34,7 @@ It does **not** advertise itself over mDNS. Discovery is centralized in the `nvp | `--key ` | _(none)_ | TLS server private key (PEM); enables HTTPS when set together with `--cert` | | `--client-ca ` | _(none)_ | PEM bundle of CAs trusted to sign client certificates; enables mTLS when set | | `--accept-http` | `false` | Keep the plaintext HTTP listener alive even when TLS is enabled (default: HTTPS-only once `--cert` is set) | -| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`); when set, the primary port carries one cluster-gated listener. While this node is a cluster member, `/v1/node-info` is served **only** over cluster-scoped pin-based mTLS, to pinned cluster peers — or this node itself via self-trust — and a plaintext caller is refused with `403`; while it is not a member the plaintext inventory is served as usual. Membership is re-read per request. Takes precedence over the `--cert` path | +| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`). The primary port serves pinned cluster mTLS inventory and a public plaintext hardware preview. TLS callers must be current pinned members; public responses confer no trust or service authority. Membership is refreshed per request and handshake. Takes precedence over `--cert` | | `--version` | | Print version and exit | `--log-level` is also accepted (registered by `nvpair-shared/applog`). @@ -89,8 +89,8 @@ Field notes: - Dynamic fields (`vram_used_bytes`, `shared_used_bytes`, `utilization_percent`, `cpu.utilization_percent`, `memory.used_bytes`) are populated by the Windows, Linux, and macOS per-tick stats collectors. - `telemetryValid` and `msSince` describe the node-wide GPU utilization snapshot. `telemetryValid` is `true` after the collector has produced a usable GPU sample; `msSince` is that sample's age in milliseconds at response time. A failed collection retains the last usable sample and lets its age increase. Before the first usable sample, and on platforms without dynamic GPU telemetry, the response reports `telemetryValid:false` and `msSince:0`; consumers must ignore the age while validity is false. - `clusterUuid` is the cluster principal this node currently holds. It has three distinct states on the wire: **absent** means unknown, **present and empty** means this node belongs to no cluster, and a value is that principal. A consumer must not read absent as unclustered — that is how a node too old to report the field answers, and also how this node answers before its parent has told it anything, so acting on it would clear a correct annotation elsewhere in the fleet. -- Under the broker, `clusterUuid` is pushed in over stdin (`nodeinfo:set-cluster-identity`) because node-info is spawned with no cluster dir and so cannot read membership itself; the field stays absent until the first push arrives. Standalone with `--cluster-dir`, it reads membership from the trust store per request instead and is therefore always known. The two sources are mutually exclusive by deployment, not a fallback chain. -- `clusterUuid` exists so a peer can learn this node's membership without its mDNS record. Membership otherwise travels only as the `cluster-uuid=` TXT key, which a consumer reads once per record *change*; a consumer that misses that change keeps the previous value indefinitely, and one still holding a departed node's principal will suppress the invite that would bring it back. +- With `--cluster-dir` (including under the broker), `clusterUuid` comes from the live trust store. Standalone without a directory can receive it over owned stdin via `nodeinfo:set-cluster-identity`; it is absent until that first update. +- `peerMetadata` contains the broker-owned `{hostUuid, name, ips?, services}` snapshot supplied by `nodeinfo:set-peer-metadata` over stdin. The service rejects metadata for a different local host UUID. Service consumers require matching pinned certificate, response identity and metadata identity; a public HTTP copy cannot establish trust or membership. - GPU field presence and per-platform memory sources are defined in the [GPU wire contract](#gpu-wire-contract) below. `DedicatedSystemMemory` is never reported as video memory; macOS system allocation is never substituted for dedicated VRAM usage. - CPU / memory fields retain their existing `omitempty` semantics; a genuinely idle CPU is still omitted. - Telemetry sampling cadence is lease-driven via the `X-NVPAIR-Poll` request header: @@ -185,7 +185,7 @@ A missing sample has no `msSince`. ## Discovery -The service no longer advertises itself over mDNS. Its parent (the broker) registers the `ni` port with the `nvpair-node-scanner` discovery daemon, which carries it on this node's single `_nvpair-node` record; a peer's daemon then fetches `/v1/node-info` over plain HTTP to enrich the node. Transport is derived from the shared per-service policy (`nvpair-shared/noderec`): node-info is served plain on the broker path. The standalone BYO-TLS / `--cluster-dir` mTLS serving above is retained for running node-info on its own, but is unused under the broker. +The service no longer advertises itself over mDNS. Its parent (the broker) registers the `ni` port with the `nvpair-node-scanner` discovery daemon, which carries it on this node's single `_nvpair-node` record; a peer's daemon authenticates `/v1/node-info` over pinned cluster mTLS before using it for service decisions. The broker passes `--cluster-dir`, enabling pinned mTLS inventory alongside public HTTP previews on the primary port. Services never authorize measurements from the public transport. The standalone BYO-TLS serving mode remains available for display clients; it does not establish pinned PAIR peer identity. ## Platform Notes diff --git a/services/nvpair-node-info/main.go b/services/nvpair-node-info/main.go index 3e842cd39..23b1042df 100644 --- a/services/nvpair-node-info/main.go +++ b/services/nvpair-node-info/main.go @@ -75,18 +75,17 @@ type MemoryInfo struct { } type NodeInfoResponse struct { - GPUs []GPUInfo `json:"GPUs"` - CPU *CPUInfo `json:"cpu,omitempty"` - Memory *MemoryInfo `json:"memory,omitempty"` - Sensors *SensorInfo `json:"sensors,omitempty"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` - SampleSeq uint64 `json:"sampleSeq,omitempty"` - // HostUUID is this node's stable per-host identity (the same value the - // node-scanner advertises as uuid= and the cluster uses as nodeUuid). It lets - // a consumer that reaches this node only over HTTP — notably a user-added - // manual node, which never sees this host's mDNS record — key it by its - // permanent identity instead of the address it was typed under. + IdentityVerification string `json:"identityVerification,omitempty"` + PeerMetadata *noderec.PeerMetadataParams `json:"peerMetadata,omitempty"` + GPUs []GPUInfo `json:"GPUs"` + CPU *CPUInfo `json:"cpu,omitempty"` + Memory *MemoryInfo `json:"memory,omitempty"` + Sensors *SensorInfo `json:"sensors,omitempty"` + TelemetryValid bool `json:"telemetryValid"` + MSSince int64 `json:"msSince"` + SampleSeq uint64 `json:"sampleSeq,omitempty"` + // HostUUID is this node's stable identity. Public copies are claims; service + // consumers require a matching pinned certificate and owned service metadata. HostUUID string `json:"hostUuid,omitempty"` // ClusterUUID is the cluster principal this node currently holds, empty when // it belongs to no cluster, and absent when this node does not (yet) know. It @@ -106,16 +105,9 @@ type NodeInfoResponse struct { ClusterUUID *string `json:"clusterUuid,omitempty"` } -// clusterIdentity is the cluster principal this node reports, kept current by -// the parent broker over stdin (noderec.MethodSetClusterIdentity). node-info -// cannot derive it under the broker: it is spawned without a cluster dir so its -// inventory stays readable by any LAN peer, which also denies it the trust store -// membership is read from. Guarded because the stdin reader and every HTTP -// handler touch it. -// -// `told` distinguishes "the broker says we hold no principal" from "the broker -// has not said anything yet", which are the same empty string but must not be -// the same answer: the first is reportable, the second has to stay absent. +// clusterIdentity supports parent-supplied membership only for standalone +// launches without --cluster-dir. Broker-managed launches read the live mesh; +// public copies of either value confer no authority. type clusterIdentity struct { mu sync.RWMutex uuid string @@ -285,17 +277,14 @@ func mergeGPUInventory(static, recovered []GPUInfo) []GPUInfo { return merged } -// nodeInfoHandler serves the marshaled node inventory from body(). While this -// node is a cluster member the endpoint is cluster-gated: the authenticated -// client must present a cert pinned for its UUID (or this node's own leaf, for a -// local self-read), or it gets a 403 — so a non-cluster peer can't read this -// host's GPU inventory in the clear, and neither can a plain-HTTP caller on the -// shared port. Refresh picks up a membership change or a peer paired after -// startup. +// nodeInfoHandler keeps plaintext hardware previews public. The TLS personality +// admits only currently pinned cluster clients. A public copy of the same JSON +// is never an identity proof: service consumers independently require pinned +// mTLS, matching identities and current authorization after the read. func nodeInfoHandler(mesh *clustertrust.Mesh, body func(*http.Request) []byte) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { mesh.Refresh() - if mesh.Clustered() { + if r.TLS != nil && mesh.Clustered() { if _, ok := mesh.VerifyClientPin(r); !ok { http.Error(w, "forbidden: not a pinned cluster peer", http.StatusForbidden) return @@ -342,7 +331,7 @@ func main() { keyPath := flag.String("key", "", "path to TLS server private key (PEM); enables HTTPS when set together with --cert") clientCAPath := flag.String("client-ca", "", "path to PEM bundle of CAs trusted to sign client certificates; enables mTLS when set") acceptHTTP := flag.Bool("accept-http", false, "keep the plaintext HTTP listener alive even when TLS is enabled (default: HTTPS-only when --cert is set)") - clusterDir := flag.String("cluster-dir", "", "cluster config dir (node.crt/node.key + trusted/); when set, /v1/node-info is served only over mTLS scoped to pinned cluster peers instead of plaintext") + clusterDir := flag.String("cluster-dir", "", "cluster config dir (node.crt/node.key + trusted/); enables pinned cluster mTLS inventory alongside public HTTP previews on the primary port") nodeID := flag.String("node-id", "", "stable per-host UUID to report on /v1/node-info; when empty it is resolved from the local identity store. The broker passes its own resolved node UUID so both agree even on a custom data root") showVersion := flag.Bool("version", false, "print version and exit") resolveLevel := applog.RegisterFlag(nil, slog.LevelInfo) @@ -361,14 +350,11 @@ func main() { // the address it was added under. hostUUID := resolveHostUUID(*nodeID, *clusterDir) - // With a --cluster-dir, /v1/node-info is served on one port that carries both - // personalities (first-byte split): while this node is a cluster member only - // pin-gated mTLS callers are admitted, closing the plaintext GPU-inventory leak - // to non-cluster peers, and while it is not a member the plaintext inventory is - // served as usual. The membership answer is live, so a node that joins or - // leaves while this process runs changes personality in place. Without a - // --cluster-dir the bring-your-own --cert path and the plaintext default are - // unchanged. + // One primary port carries public display previews and pinned cluster mTLS + // inventory. Membership changes update the authenticated personality in place; + // the public preview is not a service authority in any membership state. + // Without --cluster-dir, standalone BYO TLS and HTTP flags remain available + // for display clients but cannot establish a verified PAIR peer. mesh := clustertrust.Open(*clusterDir) clusterGated := *clusterDir != "" @@ -388,10 +374,8 @@ func main() { // haven't migrated. httpEnabled := !tlsOpts.Enabled() || *acceptHTTP if clusterGated { - // The cluster-gated listener below owns *port and serves both - // personalities on it, so there is no separate plaintext listener to - // start. A member admits only pinned peers (the local scanner reads us - // over mTLS via self-trust); a non-member serves the plaintext inventory. + // The split listener owns *port for both personalities. Public display + // reads remain available; services authenticate the TLS personality. if tlsOpts.Enabled() { log.Print("--cluster-dir set: ignoring --cert/--key (cluster mTLS takes precedence)") } @@ -425,14 +409,8 @@ func main() { collector := startStatsCollector() defer collector.Stop() - // Where the reported cluster principal comes from is decided by the - // deployment, not by a fallback chain. With --cluster-dir this process holds - // the trust store and reads membership itself, so the answer is always known - // (the handler refreshes the mesh before building the body). Under the broker - // (no cluster dir, so the inventory stays plain for every LAN peer) it cannot - // read membership at all and the broker pushes it instead — until that first - // push the answer is genuinely unknown and the field is omitted. The two - // sources are mutually exclusive by construction. + // Membership has one source per deployment: the live mesh with a cluster + // directory, or the owning parent's explicit value without one. identity := &clusterIdentity{} clusterPrincipal := func() *string { if !clusterGated { @@ -449,6 +427,7 @@ func main() { return &principal } + metadata := &peerMetadata{} mux := http.NewServeMux() mux.HandleFunc("/v1/node-info", nodeInfoHandler(mesh, func(r *http.Request) []byte { var snap statsSnapshot @@ -461,7 +440,7 @@ func main() { default: // PollIntentForeground snap = collector.Snapshot() } - return buildResponse(gpus, cpu, memTotal, snap, hostUUID, clusterPrincipal()) + return metadata.decorate(buildResponse(gpus, cpu, memTotal, snap, hostUUID, clusterPrincipal()), hostUUID, r.TLS != nil && mesh.Clustered()) })) // Listener layout (set up below depending on flags). Exactly one of the two @@ -520,9 +499,8 @@ func main() { IdleTimeout: clustertrust.PeerListenerIdleTimeout, } // The cluster leaf is resolved per handshake from the live mesh, so a - // non-member presents none and the handshake is refused; the plain - // sub-listener carries unclustered callers, whom the handler admits only - // while this node is not a member. + // non-member presents none and the handshake is refused. The plaintext + // sub-listener carries display-only previews in every membership state. serve := func(l net.Listener) { if err := tlsServer.Serve(l); err != nil && err != http.ErrServerClosed && err != net.ErrClosed { log.Fatalf("node-info server error: %v", err) @@ -601,6 +579,7 @@ func main() { go applog.StdinRPC(notifier, func(msg applog.StdinMessage) { handleClusterIdentity(msg, identity) + metadata.handle(msg, hostUUID) handleDemand(msg, collector) }, func() { log.Print("stdin closed, shutting down") diff --git a/services/nvpair-node-info/metadata.go b/services/nvpair-node-info/metadata.go new file mode 100644 index 000000000..e48031b95 --- /dev/null +++ b/services/nvpair-node-info/metadata.go @@ -0,0 +1,52 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "nvpair-shared/applog" + "nvpair-shared/noderec" + "sync" +) + +// peerMetadata is replaced atomically by our owning broker, never from the LAN. +type peerMetadata struct { + mu sync.RWMutex + value noderec.PeerMetadataParams +} + +func (m *peerMetadata) get() noderec.PeerMetadataParams { + m.mu.RLock() + defer m.mu.RUnlock() + return m.value +} +func (m *peerMetadata) decorate(data []byte, hostUUID string, verified bool) []byte { + var response NodeInfoResponse + if json.Unmarshal(data, &response) != nil { + return nil + } + response.IdentityVerification = "unverified" + if verified { + response.IdentityVerification = "verified" + } + meta := m.get() + if meta.HostUUID == hostUUID { + response.PeerMetadata = &meta + } + out, _ := json.Marshal(response) + return out +} + +func (m *peerMetadata) handle(msg applog.StdinMessage, hostUUID string) { + if msg.Method != noderec.MethodSetPeerMetadata { + return + } + var value noderec.PeerMetadataParams + if json.Unmarshal(msg.Params, &value) != nil || hostUUID == "" || value.HostUUID != hostUUID { + return + } + m.mu.Lock() + m.value = value + m.mu.Unlock() +} diff --git a/services/nvpair-node-info/metadata_test.go b/services/nvpair-node-info/metadata_test.go new file mode 100644 index 000000000..b9580d103 --- /dev/null +++ b/services/nvpair-node-info/metadata_test.go @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "nvpair-shared/applog" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" + "testing" +) + +func TestPeerMetadataAcceptsOnlyOwnedMatchingIdentity(t *testing.T) { + metadata := &peerMetadata{} + for _, body := range []string{`{`, `{"hostUuid":"other","services":{"ol":1}}`} { + metadata.handle(applog.StdinMessage{Method: noderec.MethodSetPeerMetadata, Params: json.RawMessage(body)}, "self") + } + if metadata.get().HostUUID != "" { + t.Fatal("malformed or mismatched metadata accepted") + } + data, _ := json.Marshal(noderec.PeerMetadataParams{HostUUID: "self", Name: "test-host", Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: 14318, noderec.ServiceOMLX: 1236}}) + metadata.handle(applog.StdinMessage{Method: noderec.MethodSetPeerMetadata, Params: data}, "self") + for _, verified := range []bool{false, true} { + var response NodeInfoResponse + if err := json.Unmarshal(metadata.decorate([]byte(`{"hostUuid":"self","GPUs":[]}`), "self", verified), &response); err != nil { + t.Fatal(err) + } + if response.PeerMetadata == nil || response.PeerMetadata.Services[noderec.ServiceOMLX] != 1236 { + t.Fatalf("broker metadata lost: %+v", response) + } + want := "unverified" + if verified { + want = "verified" + } + if response.IdentityVerification != want { + t.Fatalf("response provenance=%q want %q", response.IdentityVerification, want) + } + } +} + +func TestPublicHardwarePreviewRemainsAvailableAfterPairing(t *testing.T) { + dir := t.TempDir() + clustertrusttest.Join(t, dir, "test-cluster", "self", "peer") + handler := nodeInfoHandler(clustertrust.Open(dir), func(*http.Request) []byte { + return []byte(`{"GPUs":[{"name":"preview"}],"identityVerification":"unverified"}`) + }) + req := httptest.NewRequest(http.MethodGet, "http://node/v1/node-info", nil) + out := httptest.NewRecorder() + handler.ServeHTTP(out, req) + if out.Code != http.StatusOK { + t.Fatalf("public preview status=%d", out.Code) + } + var response NodeInfoResponse + if err := json.Unmarshal(out.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.IdentityVerification != "unverified" || len(response.GPUs) != 1 { + t.Fatalf("preview=%+v", response) + } +} From 554419964ec47d62f471e1e8f41a72a2d46c4792 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:38:56 +0200 Subject: [PATCH 03/18] fix(scanner): separate candidate hints from verified peers Authenticate remote hardware and service metadata before granting trust, reading model inventory, or emitting scheduler telemetry. Keep public candidates independently keyed so their updates and removal cannot displace a proven peer. Expire failed proofs using their original tokens and retain independent telemetry retry budgets. Adapt NVIDIA/Personal-AI-Router#179 with admission-bound proofs and real pinned-mTLS regression fixtures. Co-authored-by: Kaylee Lubick Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-node-scanner/README.md | 18 +- .../cluster_identity_refresh_test.go | 423 ++------- .../cluster_membership_test.go | 86 +- services/nvpair-node-scanner/daemon.go | 345 +++---- services/nvpair-node-scanner/daemon_test.go | 6 +- services/nvpair-node-scanner/directory.go | 186 +--- .../nvpair-node-scanner/directory_test.go | 20 +- .../nvpair-node-scanner/enginenative_test.go | 33 +- .../nvpair-node-scanner/ghost_node_test.go | 297 +----- services/nvpair-node-scanner/gpu_test.go | 2 +- .../inventory_lifetime_test.go | 80 ++ .../nvpair-node-scanner/peer_inventory.go | 135 +++ services/nvpair-node-scanner/refresh_test.go | 52 +- services/nvpair-node-scanner/scanner.go | 14 +- .../nvpair-node-scanner/telemetry_test.go | 873 ++++-------------- .../verified_inventory_test.go | 183 ++++ 16 files changed, 987 insertions(+), 1766 deletions(-) create mode 100644 services/nvpair-node-scanner/inventory_lifetime_test.go create mode 100644 services/nvpair-node-scanner/peer_inventory.go create mode 100644 services/nvpair-node-scanner/verified_inventory_test.go diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index 7d01a66b2..8ae598598 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -10,8 +10,8 @@ SPDX-License-Identifier: Apache-2.0 Concretely, the daemon: - **Advertises** this node's one `_nvpair-node._tcp` record, built from a registry of the local services its parent (the broker) registers — `ni`/`ol`/`lm`/`er`/`wl`/`cl`/`em`/`ec` ports plus the node identity (`uuid=`, `cluster-uuid=` when clustered, `ip=`). One record per node covers every service. -- **Browses** `_nvpair-node._tcp` for every node on the LAN (including itself) and maintains a queryable directory keyed by host UUID. -- **Enriches** each discovered node: GPU/CPU/memory inventory from its `ni` (node-info) port over plain HTTP, and the model list from its `em` (engine-manager) port (`GET /v1/models`) over cluster mTLS — a peer's model inventory is cluster data, so it is fetched only when this node holds a pin for the principal that peer advertises, while this node's own list is read over loopback in plaintext — the flat union, the per-engine breakdown (`modelsByEngine`), and the per-engine set of models loaded in memory (`loadedByEngine`), all carried onto the node's directory entry so remote cards can show loaded state. Each enrichment has a last-good cache so a transient fetch miss doesn't blank the node's card. Node-info enrichment can optionally be moved onto HTTPS (see the TLS flags below) — off by default and gated only on operator flags, never inferred per-node. +- **Browses** `_nvpair-node._tcp` for candidate endpoints. Verified/local records are keyed by authenticated host UUID; unverified previews have independent `preview:` record keys and an explicit `identityVerification: "unverified"` marker. A claimed UUID cannot overwrite or remove a verified record. +- **Authenticates before enrichment:** remote hardware and broker-owned service metadata come from `ni` over pinned cluster mTLS, with matching certificate/response/metadata identities and an exact-current-pin check after reading. Only authenticated metadata supplies service ports. Models, per-engine attribution and loaded state then come from the authenticated `em` port over pinned mTLS. Local inventory remains an owned loopback read. Public HTTP hardware is display-only and never supplies service models or scheduler telemetry. Node-info JSON decoding reads at most 1 MiB, and model inventory decoding reads at most 16 MiB. A JSON value that extends past its limit fails enrichment and leaves the last-good cache intact. @@ -22,11 +22,11 @@ Concretely, the daemon: cadence. It emits only host UUID, maximum GPU utilization, validity, and source-reported sample age; it does not mutate the public directory for each telemetry sample. -- **Annotates trust:** with `--cluster-dir`, a browsed peer whose `cluster-uuid=` this node holds a pin for is marked `trusted` (via `nvpair-shared/clustertrust`). The annotation is re-derived on `discovery:reload-trust`, which the broker sends when `nvpair-cluster-manager` reports a pin-set change, as well as when a peer's mDNS record changes. Both triggers are needed: a peer that was already advertising when this node joined never moves its record again, so the browse path alone would leave it annotated from before this node's own pins existed. A change emits one `discovery:node-updated`; a reload that finds nothing to change is silent. +- **Grants trust only after proof:** `trusted` requires a completed pinned mTLS inventory read. `peerRecordId` binds the proof to the local admission, local certificate and exact peer pin record. A pin by itself never promotes a claim. Trust changes withdraw the verified record using its original token rather than clearing the token and turning it into a preview under the host key. - **Advertises live membership:** the same reload re-derives this node's own `cluster-uuid=` from the cluster dir and re-advertises when it differs from what is being published, so joining or leaving a cluster takes effect without a restart. -- **Anti-flap:** a node has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. +- **Public-preview anti-flap:** a candidate has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. - **A scan that loses everything is treated as our own fault:** when more than one node is known and a scan returns *none* of them, no node is penalized for it — for up to six consecutive scans (~30s), after which they count normally. Independent machines do not leave the network in the same five-second window, but a process too starved to drain its multicast socket within `scanTimeout` hears silence from all of them at once; without this, one saturated machine dropped its entire cluster, idle peers included, in a single reconcile. The suppression needs two or more known nodes to engage: losing a lone peer looks the same whether it left or we stopped listening, so that case falls through to the ordinary threshold. -- **Converges without mDNS:** a 15s sweep re-reads each peer over HTTP and folds any change into the directory, independent of browse events. It covers facts that change without moving the mDNS record (a model pull/delete, a late engine start) and facts that did move it but whose announcement was never received. The second case is why a peer's cluster membership is re-read from its `ni` port (`clusterUuid` on `/v1/node-info`, which under the broker is served plain — so it answers whether or not the two nodes share a cluster; a node-info run standalone with `--cluster-dir` gates the endpoint while clustered and will not answer a non-pinned peer, which the sweep treats as no answer): membership otherwise arrives solely as the `cluster-uuid=` TXT key, read once per record change, and the anti-flap probe above deliberately keeps a still-reachable peer's entry alive — so a missed re-advertisement would leave a departed peer's principal in place for the life of the process, suppressing the invite that would bring it back. Membership is refreshed before models on each tick, since the model fetch is pinned mTLS keyed on that principal. Only a peer's explicit report is acted on: a failed fetch, or a peer too old to report the field, leaves the annotation untouched rather than inferring "unclustered" from silence. +- **Converges without mDNS:** every 15s, pending candidates retry authentication and verified peers refresh their authenticated inventory before model refresh. Candidate removal affects only its preview. A verified peer expires after 45s without authenticated inventory unless recent inference activity proves it is busy; authorization changes withdraw it immediately. Older peers lacking authenticated service metadata remain untrusted previews until upgraded, with no plaintext downgrade. ## Discovery transport lifetime @@ -60,8 +60,8 @@ Bidirectional newline-delimited JSON-RPC 2.0. Stdio by default; `--ipc ` s | Flag | Default | Description | |---|---|---| | `--ipc ` | _(stdio)_ | IPC endpoint: Unix socket or Windows named pipe | -| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`). Gates four things: a browsed peer holding a pin we trust is annotated `trusted`; this node advertises `cluster-uuid=` while it is actually a member; a peer's `em` model list is fetched over cluster mTLS; and the directory above it is the base this node's identity is resolved from | -| `--ca-bundle ` | _(none)_ | PEM CA bundle to trust for verifying node-info server certs (additive to the system trust store). Setting any TLS flag moves node-info enrichment onto HTTPS | +| `--cluster-dir ` | _(none)_ | Cluster config dir (`node.crt`/`node.key` + `trusted/`). Gates four things: a completed pinned-mTLS inventory proof grants `trusted`; this node advertises `cluster-uuid=` while it is actually a member; a peer's `em` model list is fetched over cluster mTLS; and the directory above it is the base this node's identity is resolved from | +| `--ca-bundle ` | _(none)_ | Optional CA bundle for display-only previews; never substitutes for pinned cluster authentication | | `--client-cert ` | _(none)_ | PEM client certificate to present when fetching `/v1/node-info` from TLS-enabled nodes (mTLS; requires `--client-key`) | | `--client-key ` | _(none)_ | PEM client private key matching `--client-cert` | | `--log-level ` | `info` | `error` / `warn` / `info` / `debug` | @@ -79,7 +79,7 @@ Bidirectional newline-delimited JSON-RPC 2.0. Stdio by default; `--ipc ` s | `discovery:update-txt` | `{service, port, txt?}` | `{ok:true}` — same as register with a new TXT | | `discovery:get-nodes` | `{service?}` | `{nodes:[DirectoryNode]}` — the directory, optionally filtered to nodes advertising one service | | `discovery:reload-identity` | — | `{ok:true}` — re-resolve and re-advertise this node's identity | -| `discovery:reload-trust` | — | `{ok:true}` — re-derive `trusted` on every directory entry from the current pin set | +| `discovery:reload-trust` | — | `{ok:true}` — withdraw proofs whose current authorization changed; pending candidates authenticate on the next sweep | | `discovery:set-observed-addresses` | `{addresses}` | `{ok:true}` — the local addresses peers have actually reached this node on, so address selection can rank a peer-proven address above an inferred one | | `shutdown` | — | `null` | @@ -107,7 +107,7 @@ nothing to answer and the caller must not wait: ### Notifications (daemon → caller) -Emitted as the directory changes. Each carries the affected `DirectoryNode` (identity, advertised service ports, enrichment, `trusted`). +Emitted as the directory changes. Each carries the affected `DirectoryNode`, including `peerRecordId`, `identityVerification` (`local`, `verified`, or `unverified`) and `trusted`. Verified/local records use their authenticated host UUID as the key; previews use `peerRecordId` instead. Only verified/local service metadata is eligible for service consumers. Removed verified records retain their original proof token. - `discovery:node-discovered` — a new node entered the directory - `discovery:node-updated` — a known node's record or enrichment changed. A re-observation that changes nothing emits no event, so `lastSeen` is a last-*change* stamp rather than a liveness heartbeat diff --git a/services/nvpair-node-scanner/cluster_identity_refresh_test.go b/services/nvpair-node-scanner/cluster_identity_refresh_test.go index 06b6e56ff..e4d2821bd 100644 --- a/services/nvpair-node-scanner/cluster_identity_refresh_test.go +++ b/services/nvpair-node-scanner/cluster_identity_refresh_test.go @@ -4,378 +4,67 @@ package main import ( - "encoding/json" + "net" "net/http" "net/http/httptest" - "path/filepath" - "sync" - "testing" - - "nvpair-shared/clustertrust" "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" + "strconv" + "testing" ) -// The mDNS-independent cluster-membership half of the periodic refresh loop -// (daemon.refreshClusterIdentityOnce). These tests drive it against a stub -// /v1/node-info server, with no mDNS anywhere: the whole point is that a peer's -// membership converges without a browse event ever firing. - -// nodeInfoStub is a swappable /v1/node-info handler. `body` is the raw JSON so a -// test can control exactly what the peer reports — including omitting clusterUuid -// entirely, which is how a node too old to know about the field answers. -type nodeInfoStub struct { - mu sync.Mutex - status int - body string -} - -func (s *nodeInfoStub) set(status int, body string) { - s.mu.Lock() - s.status = status - s.body = body - s.mu.Unlock() -} - -func (s *nodeInfoStub) handler() http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/node-info" { - http.NotFound(w, r) - return - } - s.mu.Lock() - status, body := s.status, s.body - s.mu.Unlock() - if status == 0 { - status = http.StatusOK - } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(status) - _, _ = w.Write([]byte(body)) - } -} - -func startNodeInfoStub(t *testing.T, body string) (*nodeInfoStub, int) { - t.Helper() - stub := &nodeInfoStub{} - stub.set(http.StatusOK, body) - srv := httptest.NewServer(stub.handler()) - t.Cleanup(srv.Close) - return stub, portFromURL(t, srv.URL) -} - -// newIdentityRefreshTestDaemon builds a daemon wired for the node-info refresh -// path only: a real directory and plain node-info client, an unclustered mesh (so -// it holds no pins, like a machine that has left), and no codec (emit is -// nil-safe). The registry self uuid is one no peer test seeds, so every seeded -// node is treated as a peer. -func newIdentityRefreshTestDaemon() *daemon { - return &daemon{ - reg: newRegistry("self-node-uuid", "", selfAddrs("127.0.0.1")), - dir: newDirectory(), - mesh: clustertrust.Open(""), - http: &http.Client{Timeout: nodeInfoFetchTimeout}, - lastInfo: make(map[string]NodeInfoResponse), - lastModels: make(map[string][]string), - lastModelsByEngine: make(map[string]map[string][]string), - lastLoadedByEngine: make(map[string]map[string][]string), - } -} - -// seedPeer puts a peer in the directory advertising node-info at port, annotated -// as belonging to clusterUUID — the state a browse event would have left behind. -func seedPeer(d *daemon, hostUUID, clusterUUID string, niPort int) { - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: hostUUID, - Name: hostUUID, - IP: "127.0.0.1", - ClusterUUID: clusterUUID, - Trusted: clusterUUID != "", - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: niPort}, - }, - }) -} - -// TestDepartedPeerConvergesWithoutMDNS is the regression guard for the reported -// failure: after every peer left the cluster, one machine went on showing them as -// belonging to another cluster with invites disabled, while the rest of the fleet -// showed them as available. -// -// Membership reaches this daemon as the cluster-uuid= key on a peer's mDNS record, -// read only when a browse *change* event fires. The peer left and re-advertised, -// but this host did not receive it — and since the peer was still reachable, the -// liveness probe kept the frozen entry alive rather than letting it be -// rediscovered. Nothing asked again, so the principal outlived the cluster. -// -// No browse event occurs anywhere in this test. The peer's own HTTP report has to -// be enough. -func TestDepartedPeerConvergesWithoutMDNS(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - // The peer has left, so it now reports no cluster principal... - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-uuid","clusterUuid":""}`) - // ...but our entry still carries the one it advertised before it left. - seedPeer(d, peerUUID, "their-old-principal", niPort) - - d.refreshClusterIdentityOnce() - - n, ok := d.dir.get(peerUUID) - if !ok { - t.Fatal("peer dropped from the directory by the refresh") - } - if n.Clustered() { - t.Errorf("peer still annotated as clustered (clusterUuid=%q) after reporting it left", n.ClusterUUID) - } - if n.Trusted { - t.Error("peer still annotated as trusted after leaving the cluster") - } - // A second pass agrees with the first and stays silent. - if d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort) { - t.Error("refresh emitted again for an unchanged peer") - } -} - -func TestClusterIdentityRefreshFallsBackToASecondPublishedAddress(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-multihomed" - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-multihomed","clusterUuid":""}`) - seedPeer(d, peerUUID, "stale-principal", niPort) - - if !d.refreshNodeClusterIdentityCandidates(peerUUID, []string{"127.0.0.2", "127.0.0.1"}, niPort) { - t.Fatal("membership did not refresh through the second published address") - } - n, ok := d.dir.get(peerUUID) - if !ok || n.Clustered() { - t.Fatalf("peer after refresh = %+v, want unclustered", n) - } -} - -// TestPeerInForeignClusterStaysSuppressed is the other half of the contract, and -// the reason the refresh trusts only an explicit report: a peer that is genuinely -// in a cluster we are not part of must keep its annotation, so the invite stays -// suppressed rather than being offered and rejected. -func TestPeerInForeignClusterStaysSuppressed(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-uuid","clusterUuid":"their-principal"}`) - seedPeer(d, peerUUID, "", niPort) - - d.refreshClusterIdentityOnce() - - n, _ := d.dir.get(peerUUID) - if !n.Clustered() { - t.Fatal("peer reporting a cluster principal was not annotated as clustered") - } - if n.ClusterUUID != "their-principal" { - t.Errorf("clusterUuid = %q, want their-principal", n.ClusterUUID) - } - // We hold no pin for it, so it is clustered but not trusted. - if n.Trusted { - t.Error("peer in a cluster we hold no pin for was annotated trusted") - } -} - -// TestPeerWithoutClusterFieldIsLeftAlone pins the version-skew rule. A node too -// old to report clusterUuid says nothing about its membership, and silence must -// not be read as "unclustered" — doing so would mark a clustered peer invitable, -// the exact failure the annotation prevents. -func TestPeerWithoutClusterFieldIsLeftAlone(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"peer-uuid"}`) - seedPeer(d, peerUUID, "their-principal", niPort) - - if d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort) { - t.Error("refresh acted on a peer that reported no clusterUuid field") - } - n, _ := d.dir.get(peerUUID) - if n.ClusterUUID != "their-principal" { - t.Errorf("clusterUuid = %q, want the prior value kept for a peer that reported nothing", n.ClusterUUID) - } -} - -// TestUnreachablePeerKeepsItsAnnotation covers the same rule for a failed fetch: -// no answer is not an answer. -func TestUnreachablePeerKeepsItsAnnotation(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - stub, niPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":""}`) - stub.set(http.StatusInternalServerError, "") - seedPeer(d, peerUUID, "their-principal", niPort) - - if d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort) { - t.Error("refresh acted on a failed node-info fetch") - } - n, _ := d.dir.get(peerUUID) - if n.ClusterUUID != "their-principal" { - t.Errorf("clusterUuid = %q, want the prior value kept across a fetch failure", n.ClusterUUID) - } -} - -// TestSelfExcludedFromIdentityRefresh keeps the sweep off this node's own entry: -// publishSelf owns the local principal from the registry, and a loopback HTTP read -// must not become a second source for it. -func TestSelfExcludedFromIdentityRefresh(t *testing.T) { - d := newIdentityRefreshTestDaemon() - self := d.reg.record().HostUUID - - // Self's node-info would report unclustered, but the registry says otherwise. - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":""}`) - seedPeer(d, self, "our-principal", niPort) - - d.refreshClusterIdentityOnce() - - n, _ := d.dir.get(self) - if n.ClusterUUID != "our-principal" { - t.Errorf("self clusterUuid = %q, want the registry-owned value untouched", n.ClusterUUID) - } -} - -// TestStrangerAtTheAddressIsNotApplied covers the case where the machine -// answering is not the one the entry describes — a wiped and reinstalled node, or -// a reused DHCP lease. node-info names who it really is, so its membership must -// not be written onto the previous occupant's entry. That entry ages out through -// the liveness probe's identity check instead, which owns the decision. -func TestStrangerAtTheAddressIsNotApplied(t *testing.T) { - d := newIdentityRefreshTestDaemon() - const peerUUID = "peer-uuid" - - // A different machine now answers here, and it belongs to no cluster. - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"hostUuid":"someone-else","clusterUuid":""}`) - seedPeer(d, peerUUID, "their-principal", niPort) - - if d.refreshNodeClusterIdentity(peerUUID, "127.0.0.1", niPort) { - t.Error("applied membership reported by a different host") - } - n, _ := d.dir.get(peerUUID) - if n.ClusterUUID != "their-principal" { - t.Errorf("clusterUuid = %q, want the entry left for the liveness probe to age out", n.ClusterUUID) - } - - // A peer that reports no hostUuid predates the field and is taken at its word. - _, oldPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":""}`) - seedPeer(d, "old-peer-uuid", "their-principal", oldPort) - if !d.refreshNodeClusterIdentity("old-peer-uuid", "127.0.0.1", oldPort) { - t.Error("skipped a peer that reported no hostUuid; it cannot be identity-checked either way") - } -} - -// TestSweepAnnotatesEveryPeerAgainstThePinSet exercises the sweep the way it -// actually runs: several peers at once, against a mesh holding a real pin. It -// covers the three answers a peer can give in one pass, and puts concurrent load -// on the fan-out so -race has something to observe. -func TestSweepAnnotatesEveryPeerAgainstThePinSet(t *testing.T) { - clusterDir := filepath.Join(t.TempDir(), "cluster") - // This node is in a cluster and holds a pin for one of the peers below. - clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", "pinned-principal") - - d := newIdentityRefreshTestDaemon() - d.mesh = clustertrust.Open(clusterDir) - - // A fellow member, a machine in someone else's cluster, and one that has left. - for _, peer := range []struct { - uuid string - reported string - }{ - {"member-uuid", "pinned-principal"}, - {"stranger-uuid", "foreign-principal"}, - {"departed-uuid", ""}, - } { - _, niPort := startNodeInfoStub(t, `{"GPUs":[],"clusterUuid":"`+peer.reported+`"}`) - // Seed each one annotated wrongly, so every case has to be corrected. - seedPeer(d, peer.uuid, "stale-principal", niPort) - } - - d.refreshClusterIdentityOnce() - - member, _ := d.dir.get("member-uuid") - if member.ClusterUUID != "pinned-principal" || !member.Trusted { - t.Errorf("member = (clusterUuid=%q, trusted=%v), want the pinned principal and trusted", - member.ClusterUUID, member.Trusted) - } - stranger, _ := d.dir.get("stranger-uuid") - if stranger.ClusterUUID != "foreign-principal" || stranger.Trusted { - t.Errorf("stranger = (clusterUuid=%q, trusted=%v), want its own principal and untrusted", - stranger.ClusterUUID, stranger.Trusted) - } - departed, _ := d.dir.get("departed-uuid") - if departed.Clustered() || departed.Trusted { - t.Errorf("departed = (clusterUuid=%q, trusted=%v), want cleared", - departed.ClusterUUID, departed.Trusted) - } -} - -// TestPinSetPassCannotWriteBackAPrincipal guards the interaction between the two -// membership passes. Both write the same two fields and they run concurrently: the -// pin-set pass on the broker's reload-trust announcement, the sweep on its own -// tick. The pin-set pass knows the pin set but not what any peer advertises, so it -// supplies no principal and trust is derived from whatever is stored when its write -// lands. Were it to pass a principal read earlier — from the snapshot it iterates — -// a sweep landing in between would be silently reverted and the invite gate would -// flip in the UI. -// -// Asserted at the write path rather than by racing the two passes, so it fails -// deterministically: a nil principal must leave the stored one alone, and the trust -// it derives must come from that stored value. -func TestPinSetPassCannotWriteBackAPrincipal(t *testing.T) { - dir := newDirectory() - dir.upsert(noderec.DirectoryNode{HostUUID: "peer", ClusterUUID: "current-principal"}) - - // Stands in for the sweep having just moved the principal: the pin-set pass - // holds a pin for the current value and none for what it used to be. - trustFor := func(clusterUUID string) bool { return clusterUUID == "current-principal" } - - n, changed := dir.applyClusterIdentity("peer", nil, trustFor) - if !changed { - t.Fatal("pin-set pass reported no change, want the trust annotation derived") - } - if n.ClusterUUID != "current-principal" { - t.Errorf("clusterUuid = %q, want the stored value untouched by a pass that supplied none", n.ClusterUUID) - } - if !n.Trusted { - t.Error("trust was not derived from the stored principal") - } - - // And a pass that does know the principal still governs it. - moved := "moved-principal" - n, _ = dir.applyClusterIdentity("peer", &moved, trustFor) - if n.ClusterUUID != "moved-principal" || n.Trusted { - t.Errorf("after an explicit move = (clusterUuid=%q, trusted=%v), want the new principal and untrusted", - n.ClusterUUID, n.Trusted) - } -} - -// TestNodeInfoResponseDistinguishesAbsentFromEmpty pins the wire contract the -// version-skew rule depends on: the pointer has to survive decoding so absent and -// present-but-empty stay distinguishable. -func TestNodeInfoResponseDistinguishesAbsentFromEmpty(t *testing.T) { - var absent NodeInfoResponse - if err := json.Unmarshal([]byte(`{"GPUs":[]}`), &absent); err != nil { - t.Fatalf("decode absent: %v", err) - } - if absent.ClusterUUID != nil { - t.Errorf("absent clusterUuid decoded to %q, want nil", *absent.ClusterUUID) - } - if absent.TelemetryValid || absent.MSSince != 0 { - t.Errorf("absent telemetry fields decoded as valid: %+v", absent) - } - - var empty NodeInfoResponse - if err := json.Unmarshal([]byte(`{"GPUs":[],"clusterUuid":"","telemetryValid":true,"msSince":137}`), &empty); err != nil { - t.Fatalf("decode empty: %v", err) - } - if empty.ClusterUUID == nil { - t.Fatal("present-but-empty clusterUuid decoded to nil, want a non-nil empty string") - } - if *empty.ClusterUUID != "" { - t.Errorf("empty clusterUuid = %q", *empty.ClusterUUID) - } - if !empty.TelemetryValid || empty.MSSince != 137 { - t.Errorf("telemetry fields = valid:%v age:%d, want true/137", empty.TelemetryValid, empty.MSSince) +// Public membership used to override cluster annotations and confer trust. It +// is now only a preview; lack of authenticated metadata cannot establish a peer. +func TestPublicIdentityRefreshCannotChangeMembership(t *testing.T) { + for _, body := range []string{`{"hostUuid":"uuid-peer","clusterUuid":"uuid-peer"}`, `{"hostUuid":"uuid-peer","clusterUuid":""}`, `{"clusterUuid":"uuid-peer"}`, `{"hostUuid":"other-peer","clusterUuid":"uuid-peer"}`} { + t.Run(body, func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(body)) })) + defer srv.Close() + host, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(p) + d := newSelfTestDaemon("uuid-self", "192.0.2.1") + old := noderec.DirectoryNode{IdentityVerification: "unverified", PeerRecordID: noderec.PreviewRecordID("candidate", "uuid-peer"), HostUUID: "uuid-peer", ClusterUUID: "uuid-peer", IP: host, Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: port}}} + d.dir.upsert(old) + if d.refreshNodeClusterIdentity(old.RecordKey(), host, port) { + t.Fatal("public response changed membership") + } + got, _ := d.dir.get(old.RecordKey()) + if got.Trusted || got.ClusterUUID != old.ClusterUUID || got.PeerRecordID != old.PeerRecordID { + t.Fatalf("public membership applied: %+v", got) + } + }) + } +} + +func TestAuthenticatedRefreshPromotesPendingPeerWithoutMDNSChange(t *testing.T) { + d, srv, mismatch := secureInventoryFixture(t, "uuid-peer") + host, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(p) + mismatch.Store(true) + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "pending", Addresses: []string{host}, TXT: []string{"v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=" + host, "ni=" + p}}}) + mismatch.Store(false) + if !d.refreshNodeClusterIdentityCandidates(noderec.PreviewRecordID("pending", "uuid-peer"), []string{"127.0.0.2", host}, port) { + t.Fatal("pending peer not authenticated via answering address") + } + got, _ := d.dir.get("uuid-peer") + if !got.ServicePeer() || got.IP != host || got.Services[noderec.ServiceOllama].Port != 11434 { + t.Fatalf("promoted peer = %+v", got) + } + clustertrusttest.RemovePeerPin(t, d.clusterDir, "uuid-peer") + d.reloadTrust() + got, _ = d.dir.get("uuid-peer") + if got.Trusted || got.ServicePeer() || len(got.GPUs) > 0 { + t.Fatalf("pin removal retained inventory: %+v", got) + } +} + +func TestAuthenticatedResponseCannotNameAnotherHost(t *testing.T) { + d, srv, mismatch := secureInventoryFixture(t, "uuid-peer") + host, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(p) + mismatch.Store(true) + node := noderec.DirectoryNode{HostUUID: "uuid-peer", ClusterUUID: "uuid-peer"} + if _, _, ok := d.authenticatedInfo(t.Context(), node, []string{host}, port); ok { + t.Fatal("authenticated certificate accepted a different response identity") } } diff --git a/services/nvpair-node-scanner/cluster_membership_test.go b/services/nvpair-node-scanner/cluster_membership_test.go index 3f287d1ec..b0fa5624c 100644 --- a/services/nvpair-node-scanner/cluster_membership_test.go +++ b/services/nvpair-node-scanner/cluster_membership_test.go @@ -57,6 +57,7 @@ func TestModelFetchTransportFollowsMembership(t *testing.T) { // Join a cluster that pins one peer. clustertrusttest.Join(t, clusterDir, "cluster-models", "principal-self", "principal-peer") d.mesh.Refresh() + d.reconcileIdentity() if _, scheme, ok := d.modelsClient("192.168.1.42", "principal-peer"); !ok || scheme != "https" { t.Fatalf("pinned peer: scheme=%q ok=%v, want https/true", scheme, ok) @@ -127,11 +128,13 @@ func TestClusterUUIDAppearsOnJoinWithoutRestart(t *testing.T) { // cluster we may or may not have joined yet. func peerEntry(hostUUID, clusterUUID string) noderec.DirectoryNode { return noderec.DirectoryNode{ - HostUUID: hostUUID, - Name: hostUUID, - IP: "192.0.2.10", - ClusterUUID: clusterUUID, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, + IdentityVerification: "unverified", + PeerRecordID: noderec.PreviewRecordID(hostUUID, hostUUID), + HostUUID: hostUUID, + Name: hostUUID, + IP: "192.0.2.10", + ClusterUUID: clusterUUID, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, } } @@ -145,36 +148,28 @@ func peerEntry(hostUUID, clusterUUID string) noderec.DirectoryNode { // process: both proxies dropped the peer as an unpinned relay peer, so the node // listed only its own models and never routed inference anywhere, while its // roster and pins on disk said it was a healthy member of a three-node cluster. -func TestReloadTrustAdoptsPeerPinnedAfterDiscovery(t *testing.T) { +func TestReloadTrustDoesNotPromoteAnUnprovenPeer(t *testing.T) { const peerUUID = "principal-peer" clusterDir := filepath.Join(t.TempDir(), "cluster") d := newMembershipTestDaemon(clusterDir) - - // Discovered before we hold any cluster identity. - d.dir.upsert(toDirectoryNodeTrusted(peerEntry("peer-host", peerUUID), d)) - if n, _ := d.dir.get("peer-host"); n.Trusted { - t.Fatal("peer annotated trusted while this node held no pins") - } - - // The join lands on disk and the cluster-manager announces it. No browse - // event accompanies it, and none ever will. - clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) - d.reloadTrust() - - if n, _ := d.dir.get("peer-host"); !n.Trusted { - t.Fatal("peer still untrusted after its pin landed; routing would drop it") - } - // The same pass converges what we advertise, so peers can pin us back. - if got := d.reg.record().ClusterUUID; got != "principal-self" { - t.Fatalf("advertised cluster-uuid=%q, want principal-self", got) + preview := peerEntry("peer-host", peerUUID) + d.dir.upsert(preview) + for _, stage := range []string{"before-join", "after-join", "after-removal"} { + if stage == "after-join" { + clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) + d.reloadTrust() + } + if stage == "after-removal" { + clustertrusttest.RemovePeerPin(t, clusterDir, peerUUID) + d.reloadTrust() + } + n, exists := d.dir.get(preview.RecordKey()) + if !exists || n.Trusted || n.RecordKey() == n.HostUUID { + t.Fatalf("%s promoted or lost preview: %+v", stage, n) + } } - - // A removal is picked up just as promptly, and for the same reason. - clustertrusttest.RemovePeerPin(t, clusterDir, peerUUID) - d.reloadTrust() - - if n, _ := d.dir.get("peer-host"); n.Trusted { - t.Fatal("peer still trusted after its pin was removed") + if d.reg.record().ClusterUUID != "principal-self" { + t.Fatal("local membership did not converge") } } @@ -183,18 +178,16 @@ func TestReloadTrustAdoptsPeerPinnedAfterDiscovery(t *testing.T) { // known node, so an unconditional write would push a directory-sized // discovery:nodes snapshot through the broker to every subscriber on each pass. func TestTrustReconcileIsSilentWhenNothingChanged(t *testing.T) { - const peerUUID = "principal-peer" clusterDir := filepath.Join(t.TempDir(), "cluster") - clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", peerUUID) + clustertrusttest.Join(t, clusterDir, "cluster-xyz", "principal-self", "principal-peer") d := newMembershipTestDaemon(clusterDir) - - d.dir.upsert(toDirectoryNodeTrusted(peerEntry("peer-host", peerUUID), d)) - d.dir.upsert(toDirectoryNodeTrusted(peerEntry("stranger", "principal-stranger"), d)) - - for _, hostUUID := range []string{"peer-host", "stranger"} { - if _, changed := d.dir.applyClusterIdentity(hostUUID, nil, d.mesh.HasPin); changed { - t.Fatalf("%s reported a trust change on a steady-state pass", hostUUID) - } + d.dir.upsert(peerEntry("peer-host", "principal-peer")) + d.dir.upsert(peerEntry("stranger", "principal-stranger")) + capture := ¬ifyCapture{} + d.codec = NewCodec(capture) + d.reconcileTrust() + if len(capture.frames) != 0 { + t.Fatal("steady-state proof reconciliation emitted notifications") } } @@ -236,7 +229,7 @@ func TestPeerLeavingItsClusterClearsInviteGate(t *testing.T) { ID: "peer", TXT: []string{"v=1", "uuid=" + peerUUID, "cluster-uuid=their-cluster", "ip=192.168.1.9"}, }}) - n, ok := d.dir.get(peerUUID) + n, ok := d.dir.get(noderec.PreviewRecordID("peer", peerUUID)) if !ok { t.Fatal("peer advertising a cluster principal was not recorded") } @@ -249,7 +242,7 @@ func TestPeerLeavingItsClusterClearsInviteGate(t *testing.T) { ID: "peer", TXT: []string{"v=1", "uuid=" + peerUUID, "ip=192.168.1.9"}, }}) - n, ok = d.dir.get(peerUUID) + n, ok = d.dir.get(noderec.PreviewRecordID("peer", peerUUID)) if !ok { t.Fatal("peer dropped from the directory when it left its cluster") } @@ -258,13 +251,6 @@ func TestPeerLeavingItsClusterClearsInviteGate(t *testing.T) { } } -// toDirectoryNodeTrusted annotates an entry the way onBrowse does at the moment -// it is folded in, so a test can reproduce "discovered at time T" faithfully. -func toDirectoryNodeTrusted(n noderec.DirectoryNode, d *daemon) noderec.DirectoryNode { - n.Trusted = d.mesh.HasPin(n.ClusterUUID) - return n -} - func mustClusterUUID(t *testing.T, d *daemon, hostUUID string) string { t.Helper() n, ok := d.dir.get(hostUUID) diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 68965d155..67139e53f 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -61,7 +61,11 @@ type daemon struct { modelsHTTP *http.Client // modelsClients is the long-lived per-peer mTLS pool for LAN /v1/models // fetches. A throwaway Transport every 15s paid a full handshake each time. - modelsClients *clustertrust.PeerClientPool + modelsClients *clustertrust.PeerClientPool + peerInfoOnce sync.Once + candidateMu sync.Mutex + candidates map[string]noderec.DirectoryNode + peerInfoClients *clustertrust.PeerClientPool // tlsHTTP, when non-nil (the operator passed --client-cert/--client-key/ // --ca-bundle), fetches node-info over HTTPS on the ni port instead of plain // HTTP. Dormant by default (nil): node-info is plain per the transport policy, @@ -473,6 +477,8 @@ func (d *daemon) run(ctx context.Context) { if d.modelsClients != nil { d.modelsClients.CloseIdle() } + d.peerInfoOnce.Do(func() { d.peerInfoClients = clustertrust.NewPeerClientPool(d.mesh, nodeInfoFetchTimeout) }) + d.peerInfoClients.CloseIdle() return case ev, ok := <-events: if !ok { @@ -483,8 +489,8 @@ func (d *daemon) run(ctx context.Context) { } } -// onBrowse folds one _nvpair-node browse event into the directory (annotating -// trust from the cluster pin set) and emits the matching discovery:node-* event. +// onBrowse treats mDNS records as independently keyed candidates. Only a +// completed pinned mTLS inventory read promotes one to a verified peer. func (d *daemon) onBrowse(ev DiscoveryEvent) { // Never let a browse event touch our own entry. Self is registry-driven // (publishSelf); folding self browse events in here is exactly what evicted @@ -494,34 +500,40 @@ func (d *daemon) onBrowse(ev DiscoveryEvent) { return } - trusted := false - if cu := clustertrust.ClusterUUIDFromTXT(ev.Node.TXT); cu != "" { - d.mesh.Refresh() - trusted = d.mesh.HasPin(cu) - } - node, ok := toDirectoryNode(ev.Node, trusted) + d.mesh.Refresh() + node, ok := toDirectoryNode(ev.Node) if !ok { // Record without a uuid= — toDirectoryNode already warned. Nothing to // store, and a removed event for it is a no-op since it was never added. return } - var method string - switch ev.Type { - case "removed": - d.dir.remove(node.HostUUID) - d.forget(node.HostUUID) - method = noderec.NotifyNodeRemoved - case "updated": - d.enrich(&node) - d.supersedingUpsert(node) - method = noderec.NotifyNodeUpdated - default: // discovered - d.enrich(&node) - d.supersedingUpsert(node) - method = noderec.NotifyNodeDiscovered + key := node.RecordKey() + if ev.Type == "removed" { + d.candidateMu.Lock() + delete(d.candidates, key) + d.candidateMu.Unlock() + if prior, exists := d.dir.get(key); exists { + d.dir.remove(key) + d.emit(noderec.NotifyNodeRemoved, prior) + } + // A candidate's disappearance says nothing about the independently + // authenticated peer. Its own mTLS refresh/expiry owns that lifetime. + return + } + d.candidateMu.Lock() + if d.candidates == nil { + d.candidates = make(map[string]noderec.DirectoryNode) } - d.emit(method, d.storedOr(node)) + d.candidates[key] = node + d.candidateMu.Unlock() + d.enrich(&node) + if node.ServicePeer() { + d.publishVerified(node, key) + return + } + d.dir.upsert(node) + d.emit(noderec.NotifyNodeUpdated, node) } // storedOr returns the record the directory holds for a node, falling back to @@ -533,99 +545,12 @@ func (d *daemon) onBrowse(ev DiscoveryEvent) { // the rebuilt one would show the client the view disappearing on every // advertisement, and reappearing at the next sweep. func (d *daemon) storedOr(node noderec.DirectoryNode) noderec.DirectoryNode { - if stored, ok := d.dir.get(node.HostUUID); ok { + if stored, ok := d.dir.get(node.RecordKey()); ok { return stored } return node } -// supersedingUpsert folds a browsed peer into the directory, evicting any record -// it is PROVEN to have replaced and emitting the removals BEFORE the caller emits -// the node's own event — so no consumer ever observes the machine under two -// identities at once. -// -// The directory can only nominate (directory.supersedeCandidates). Its signals -// are the arriving record's instance name — an unauthenticated mDNS field -// anything on the LAN can claim — plus a LastSeen gap that a healthy peer's -// frozen timestamp satisfies exactly as readily as a ghost's. Deleting on that -// alone would make mDNS a removal primitive: a record claiming a live peer's -// hostname would evict it, the browser re-reports a node only when its record -// CHANGES so it might never come back, and the loss would carry through to the -// broker's onNodeLost and to workload routing. -// -// So each candidate is confirmed the way the eviction path confirms a -// threshold-missed node: ask the machines involved who they are, and delete only -// on a definite mismatch. A candidate that cannot be confirmed stays, and is not -// left uncovered — when it stops being advertised the ordinary miss-threshold -// path (reachable) puts the same question to it. -func (d *daemon) supersedingUpsert(node noderec.DirectoryNode) { - var proven []noderec.DirectoryNode - for _, old := range d.dir.supersedeCandidates(node, d.reg.record().HostUUID) { - if d.identityReplaced(old, node) { - proven = append(proven, old) - } - } - for _, old := range d.dir.upsertEvicting(node, proven) { - slog.Warn("dropping node record proven superseded by a newer identity on the same host", - "ip", old.IP, "droppedUuid", old.HostUUID, "droppedName", old.Name, - "keptUuid", node.HostUUID, "keptName", node.Name, - "ageSeconds", node.LastSeen-old.LastSeen) - d.forget(old.HostUUID) - d.emit(noderec.NotifyNodeRemoved, old) - } -} - -// identityReplaced reports whether old is PROVEN to be a pre-wipe ghost of the -// machine now arriving as replacement — the confirmation supersedingUpsert -// requires before it deletes anything. -// -// It takes two answers, both from node-info, and both are required: -// -// - old must not still be answering for itself. Every address old published is -// asked, at old's own ni port, and any of them naming old keeps it. This is -// what tells a wipe apart from two hosts that merely share a hostname, and it -// is why a record with no ni port to ask is never evicted here: a candidate -// that cannot answer cannot be distinguished from a live namesake, and the -// miss-threshold path will reach it on its own when it stops advertising. -// - the machine at replacement's address must name someone other than old. -// That is the arriving record's own port, which is where the machine is -// answering now. -// -// Anything short of both — no ni port, no answer within the probe budget, a -// response without a hostUuid, or the 403 a clustered peer returns to a plaintext -// caller — leaves the record in place. -func (d *daemon) identityReplaced(old, replacement noderec.DirectoryNode) bool { - oldNI, ok := old.Services[noderec.ServiceNodeInfo] - if !ok || oldNI.Port == 0 { - return false - } - stillItself, _ := d.identityAlive(noderec.NodeRecord{ - HostUUID: old.HostUUID, - IP: old.IP, - Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: oldNI.Port}, - }, old.CandidateIPs()) - if stillItself { - return false - } - ni, ok := replacement.Services[noderec.ServiceNodeInfo] - if !ok || ni.Port == 0 { - return false - } - // Ask the arriving machine at every address it ranked, not just its canonical - // one. Any of them naming old keeps the record, so a wider question can only - // spare a live node; asking one address that this host happens not to reach - // yields no answer at all, and a record with only ips= has no canonical - // address to ask. - replacementIPs := replacement.CandidateIPs() - rec := noderec.NodeRecord{ - HostUUID: old.HostUUID, - IP: replacement.IP, - Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: ni.Port}, - } - alive, conclusive := d.identityAlive(rec, replacementIPs) - return conclusive && !alive -} - // reloadTrust re-derives everything this daemon caches from the cluster pin set, // in response to nvpair-cluster-manager reporting that the set moved // (discovery:reload-trust). @@ -688,16 +613,10 @@ func (d *daemon) reconcileIdentity() { // known node and emits nothing. func (d *daemon) reconcileTrust() { for _, n := range d.dir.snapshot("") { - // nil principal: this pass knows the pin set, not what each peer - // advertises, so it re-derives trust from whatever principal is stored - // when the write lands rather than from the snapshot it is iterating. - node, changed := d.dir.applyClusterIdentity(n.HostUUID, nil, d.mesh.HasPin) - if !changed { + if d.localNode(n.HostUUID) || !n.Trusted || d.currentPeer(n) { continue } - slog.Info("node trust changed", "host_uuid", n.HostUUID, - "cluster_uuid", n.ClusterUUID, "trusted", node.Trusted) - d.emit(noderec.NotifyNodeUpdated, node) + d.expireVerified(n) } } @@ -944,14 +863,15 @@ func (d *daemon) publishSelfLockedIf(shouldEmit func(noderec.DirectoryNode) bool svcs[k] = noderec.ServiceStatus{Port: port} } node := noderec.DirectoryNode{ - HostUUID: rec.HostUUID, - Name: d.instance, - IP: ip, - IPs: ips, - ClusterUUID: rec.ClusterUUID, - Trusted: trusted, - Services: svcs, - LastSeen: time.Now().Unix(), + HostUUID: rec.HostUUID, + Name: d.instance, + IP: ip, + IPs: ips, + ClusterUUID: rec.ClusterUUID, + Trusted: trusted, + IdentityVerification: "local", + Services: svcs, + LastSeen: time.Now().Unix(), } // Enrich over loopback, never the advertised LAN ip=: a first-run Windows // firewall block on inbound to our own LAN address must not blank the local @@ -1279,6 +1199,30 @@ func (d *daemon) enrichInfoAt(node *noderec.DirectoryNode, host string) { // Which address is asked, and in what order, is askRemembered's business: the one // that answered last sweep by itself, and the rest together only if it stopped. func (d *daemon) enrichInfoCandidates(node *noderec.DirectoryNode, hosts []string) { + if !d.localNode(node.HostUUID) { + node.Trusted = false + node.IdentityVerification = "unverified" + if ni, ok := node.Services[noderec.ServiceNodeInfo]; ok { + if info, host, verified := d.authenticatedInfo(context.Background(), *node, hosts, ni.Port); verified { + applyAuthenticatedInfo(node, info, host) + return + } + } + // A public preview is never cached under an authenticated identity. + if d.mesh.HasPin(node.ClusterUUID) { + return + } + for _, host := range hosts { + if ni, ok := node.Services[noderec.ServiceNodeInfo]; ok { + if info, ok := d.fetchNodeInfo(host, ni.Port); ok { + node.GPUs, node.CPU, node.Memory = info.GPUs, info.CPU, info.Memory + break + } + } + } + return + } + node.IdentityVerification = "local" ni, ok := node.Services[noderec.ServiceNodeInfo] if !ok || len(hosts) == 0 { return @@ -1463,6 +1407,12 @@ func (d *daemon) enrichModelsAt(node *noderec.DirectoryNode, host string) { // fetchModels performs the pinned mTLS handshake, so a shared address belonging // to another machine is rejected and the next published address is tried. func (d *daemon) enrichModelsCandidates(node *noderec.DirectoryNode, hosts []string) { + if !d.currentPeer(*node) { + return + } + if !d.localNode(node.HostUUID) { + hosts = node.CandidateIPs() + } em, ok := node.Services[noderec.ServiceEngineManager] if !ok || len(hosts) == 0 { return @@ -1475,6 +1425,13 @@ func (d *daemon) enrichModelsCandidates(node *noderec.DirectoryNode, hosts []str return modelInventory{models: models, byEngine: byEngine, loadedByEngine: loadedByEngine}, true } key := hostKey{hostUUID: node.HostUUID, service: noderec.ServiceEngineManager} + if !d.localNode(node.HostUUID) { + _, inventory, ok := askRemembered(&d.enrichHosts, key, hosts, ask) + if ok && d.currentPeer(*node) { + node.Models, node.ModelsByEngine, node.LoadedByEngine = inventory.models, inventory.byEngine, inventory.loadedByEngine + } + return + } if _, inventory, ok := askRemembered(&d.enrichHosts, key, hosts, ask); ok { d.infoMu.Lock() if d.lastModels == nil { @@ -1529,6 +1486,15 @@ func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, return nil, nil, nil, false } url := scheme + "://" + net.JoinHostPort(ip, strconv.Itoa(port)) + "/v1/models" + if scheme == "https" { + var body struct { + Models []string `json:"models"` + ModelsByEngine map[string][]string `json:"modelsByEngine"` + LoadedByEngine map[string][]string `json:"loadedByEngine"` + } + _, _, ok := clustertrust.ReadPeerInventory(context.Background(), d.mesh, client, clusterUUID, url, "", modelsResponseLimit, &body) + return body.Models, body.ModelsByEngine, body.LoadedByEngine, ok + } resp, err := client.Get(url) if err != nil { slog.Debug("model enrichment failed", "ip", ip, "url", url, "err", err) @@ -1555,8 +1521,8 @@ func (d *daemon) fetchModels(ip string, port int, clusterUUID string) ([]string, // false when a peer holds no pin here, which is the client-side cluster gate — // there is deliberately no plaintext fallback for a peer. func (d *daemon) modelsClient(ip, clusterUUID string) (*http.Client, string, bool) { - if isLoopbackHost(ip) { - return d.modelsHTTP, "http", true + if isLoopbackHost(ip) && (clusterUUID == "" || (d.reg != nil && clusterUUID == d.reg.record().ClusterUUID)) { + return d.modelsHTTP, "http", d.modelsHTTP != nil } d.mesh.Refresh() if d.modelsClients == nil { @@ -1603,6 +1569,17 @@ func (d *daemon) fetchEngineStats(ctx context.Context, ip string, port int, clus return nil, false } url := scheme + "://" + net.JoinHostPort(ip, strconv.Itoa(port)) + "/v1/engine-stats" + if scheme == "https" { + var stats enginenative.Stats + _, _, ok := clustertrust.ReadPeerInventory(ctx, d.mesh, client, clusterUUID, url, "", engineNativeResponseLimit, &stats) + if !ok || !stats.Supported { + return nil, false + } + if stats.ObservedAt == 0 { + stats.ObservedAt = time.Now().UnixMilli() + } + return &stats, true + } ctx, cancel := context.WithTimeout(ctx, engineNativeFetchTimeout) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) @@ -1747,6 +1724,9 @@ func (d *daemon) dispatchTelemetry( self = d.reg.record().HostUUID } for _, n := range d.dir.snapshot(noderec.ServiceNodeInfo) { + if !n.ServicePeer() { + continue + } ni, ok := n.Services[noderec.ServiceNodeInfo] if !ok { continue @@ -1811,9 +1791,23 @@ type telemetrySample struct { // a link this host cannot reach reported no telemetry at all, and the scheduler // then assigned it neutral pressure and scheduled it blind. func (d *daemon) refreshNodeTelemetryCandidates(ctx context.Context, hostUUID string, hosts []string, niPort int) bool { + var node noderec.DirectoryNode + exists := false + if d.dir != nil { + node, exists = d.dir.get(hostUUID) + } + if !d.localNode(hostUUID) && (!exists || !d.currentPeer(node)) { + return false + } ask := func(host string) (telemetrySample, bool) { startedAt := time.Now() - candidate, ok := d.fetchNodeInfoWithin(ctx, host, niPort) + var candidate NodeInfoResponse + var ok bool + if d.localNode(hostUUID) { + candidate, ok = d.fetchNodeInfoWithin(ctx, host, niPort) + } else { + candidate, _, ok = d.authenticatedInfo(ctx, node, []string{host}, niPort) + } if !ok { return telemetrySample{}, false } @@ -1830,6 +1824,12 @@ func (d *daemon) refreshNodeTelemetryCandidates(ctx context.Context, hostUUID st return false } info, startedAt := sample.info, sample.startedAt + if !d.localNode(hostUUID) { + current, exists := d.dir.get(hostUUID) + if !exists || current.PeerRecordID != info.PeerRecordID || !d.currentPeer(current) { + return false + } + } age := int64(0) if info.TelemetryValid { @@ -1852,6 +1852,7 @@ func (d *daemon) refreshNodeTelemetryCandidates(ctx context.Context, hostUUID st } d.emitTelemetry(noderec.NodeTelemetry{ HostUUID: hostUUID, + PeerRecordID: info.PeerRecordID, GPUUtilizationPct: utilization, TelemetryValid: info.TelemetryValid, MSSince: age, @@ -1888,7 +1889,14 @@ func (d *daemon) refreshClusterIdentityOnce() { d.mesh.Refresh() sem := make(chan struct{}, modelsRefreshConcurrency) var wg sync.WaitGroup - for _, n := range d.dir.snapshot(noderec.ServiceNodeInfo) { + nodes := d.dir.snapshot(noderec.ServiceNodeInfo) + nodes = append(nodes, d.candidateSnapshot()...) + seen := make(map[string]bool, len(nodes)) + for _, n := range nodes { + if seen[n.RecordKey()] { + continue + } + seen[n.RecordKey()] = true ni, ok := n.Services[noderec.ServiceNodeInfo] if !ok || n.IP == "" || n.HostUUID == self { continue @@ -1900,7 +1908,7 @@ func (d *daemon) refreshClusterIdentityOnce() { defer wg.Done() defer func() { <-sem }() d.refreshNodeClusterIdentityCandidates(hostUUID, hosts, port) - }(n.HostUUID, hosts, ni.Port) + }(n.RecordKey(), hosts, ni.Port) } wg.Wait() } @@ -1932,33 +1940,34 @@ func (d *daemon) refreshNodeClusterIdentity(hostUUID, ip string, niPort int) boo // this entry, but it is also not the end of the candidate list: shared // direct-connect addresses can legitimately resolve to a different machine. func (d *daemon) refreshNodeClusterIdentityCandidates(hostUUID string, hosts []string, niPort int) bool { - ask := func(host string) (NodeInfoResponse, bool) { - candidate, ok := d.fetchNodeInfo(host, niPort) - if !ok { - return NodeInfoResponse{}, false - } - if candidate.HostUUID != "" && candidate.HostUUID != hostUUID { - slog.Debug("skipping cluster membership reported by a different host at this address; trying next", - "expected_host_uuid", hostUUID, "reported_host_uuid", candidate.HostUUID, "ip", host) - return NodeInfoResponse{}, false - } - return candidate, true + old, exists := d.dir.get(hostUUID) + if !exists { + d.candidateMu.Lock() + old, exists = d.candidates[hostUUID] + d.candidateMu.Unlock() } - // Shares the node-info memory with the enrichment sweep, so the two node-info - // callers converge on one address per peer rather than each finding it again. - key := hostKey{hostUUID: hostUUID, service: noderec.ServiceNodeInfo} - _, info, _ := askRemembered(&d.enrichHosts, key, hosts, ask) - if info.ClusterUUID == nil { + if !exists || d.localNode(old.HostUUID) { return false } - node, changed := d.dir.applyClusterIdentity(hostUUID, info.ClusterUUID, d.mesh.HasPin) - if !changed { + info, host, verified := d.authenticatedInfo(context.Background(), old, hosts, niPort) + if !verified { + if old.ServicePeer() && time.Since(time.Unix(old.LastSeen, 0)) > 45*time.Second { + if age, active := d.activitySince(old.HostUUID); !active || age > activityFreshness { + d.expireVerified(old) + } + } return false } - slog.Info("peer cluster membership changed", "host_uuid", hostUUID, - "clustered", node.Clustered(), "trusted", node.Trusted) - d.emit(noderec.NotifyNodeUpdated, node) - return true + node := old + if node.PeerRecordID != info.PeerRecordID { + withdrawPeer(&node) + } + applyAuthenticatedInfo(&node, info, host) + candidateKey := "" + if old.IdentityVerification == "unverified" { + candidateKey = old.RecordKey() + } + return d.publishVerified(node, candidateKey) } // refreshModelsOnce snapshots every directory node advertising engine-manager @@ -1972,6 +1981,9 @@ func (d *daemon) refreshModelsOnce() { sem := make(chan struct{}, modelsRefreshConcurrency) var wg sync.WaitGroup for _, n := range d.dir.snapshot(noderec.ServiceEngineManager) { + if !n.ServicePeer() { + continue + } em, ok := n.Services[noderec.ServiceEngineManager] if !ok || n.IP == "" { continue @@ -2016,6 +2028,10 @@ func (d *daemon) refreshNodeModels(hostUUID, ip string, emPort int, clusterUUID // sweep, alone — and it shares that memory with the enrichment path, so both // converge on one address per peer instead of each rediscovering it. func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []string, emPort int, clusterUUID string) bool { + before, exists := d.dir.get(hostUUID) + if !d.localNode(hostUUID) && (!exists || !d.currentPeer(before)) { + return false + } if d.reg != nil && hostUUID == d.reg.record().HostUUID { hosts = []string{loopbackHost} } @@ -2032,6 +2048,12 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s return false } models, byEngine, loadedByEngine := inventory.models, inventory.byEngine, inventory.loadedByEngine + if !d.localNode(hostUUID) { + after, exists := d.dir.get(hostUUID) + if !exists || before.PeerRecordID != after.PeerRecordID || !d.currentPeer(after) { + return false + } + } // Refresh the enrichment cache BEFORE touching the directory: the caches // (infoMu) and the directory (dir.mu) are guarded by different locks, so if // the directory were updated first, a concurrent onBrowse->enrichModelsAt whose @@ -2113,6 +2135,9 @@ func (d *daemon) refreshEngineNativeOnce(ctx context.Context) { sem := make(chan struct{}, engineNativeRefreshConcurrency) var wg sync.WaitGroup for _, n := range d.dir.snapshot(noderec.ServiceEngineManager) { + if !n.ServicePeer() { + continue + } em, ok := n.Services[noderec.ServiceEngineManager] if !ok || n.IP == "" { continue @@ -2149,6 +2174,10 @@ func (d *daemon) refreshEngineNativeOnce(ctx context.Context) { // node. Peers are dialed at their published addresses, over the pinned mTLS // client (see fetchEngineStats). func (d *daemon) refreshNodeEngineNative(ctx context.Context, hostUUID, guardIP string, hosts []string, emPort int, clusterUUID string) bool { + before, exists := d.dir.get(hostUUID) + if !d.localNode(hostUUID) && (!exists || !d.currentPeer(before)) { + return false + } if d.reg != nil && hostUUID == d.reg.record().HostUUID { hosts = []string{loopbackHost} } @@ -2163,6 +2192,12 @@ func (d *daemon) refreshNodeEngineNative(ctx context.Context, hostUUID, guardIP if !fetched { return false } + if !d.localNode(hostUUID) { + after, exists := d.dir.get(hostUUID) + if !exists || before.PeerRecordID != after.PeerRecordID || !d.currentPeer(after) { + return false + } + } node, changed, valid := d.dir.applyEngineNative(hostUUID, guardIP, emPort, stats) if !valid { return false diff --git a/services/nvpair-node-scanner/daemon_test.go b/services/nvpair-node-scanner/daemon_test.go index 094826bfe..b96233060 100644 --- a/services/nvpair-node-scanner/daemon_test.go +++ b/services/nvpair-node-scanner/daemon_test.go @@ -70,7 +70,7 @@ func TestReloadIdentityAdoptsClusterPrincipal(t *testing.T) { // codec/responder nil: emit is nil-safe; reloadIdentity skips re-advertise. } // Simulate this node already having been browsed under its scanner-minted uuid. - d.dir.upsert(noderec.DirectoryNode{HostUUID: "scanner-minted-X", Name: "host", IP: "127.0.0.1"}) + d.dir.upsert(noderec.DirectoryNode{IdentityVerification: "local", HostUUID: "scanner-minted-X", Name: "host", IP: "127.0.0.1"}) // cluster-manager writes its principal under /cluster/identity.json. idPath := filepath.Join(base, "cluster", "identity.json") @@ -310,11 +310,11 @@ func TestPeerStillAgesOut(t *testing.T) { d := newSelfTestDaemon("self-uuid", "192.168.1.17") peerTXT := []string{"v=1", "uuid=peer-uuid", "ip=192.168.1.99"} d.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "peer", TXT: peerTXT}}) - if _, ok := d.dir.get("peer-uuid"); !ok { + if _, ok := d.dir.get(noderec.PreviewRecordID("peer", "peer-uuid")); !ok { t.Fatal("a discovered peer should be added to the directory") } d.onBrowse(DiscoveryEvent{Type: "removed", Node: RawNode{ID: "peer", TXT: peerTXT}}) - if _, ok := d.dir.get("peer-uuid"); ok { + if _, ok := d.dir.get(noderec.PreviewRecordID("peer", "peer-uuid")); ok { t.Error("a genuine remote peer must still age out when it leaves") } } diff --git a/services/nvpair-node-scanner/directory.go b/services/nvpair-node-scanner/directory.go index fc291de95..dece7dd40 100644 --- a/services/nvpair-node-scanner/directory.go +++ b/services/nvpair-node-scanner/directory.go @@ -5,6 +5,7 @@ package main import ( "log/slog" + "reflect" "sort" "sync" "time" @@ -18,7 +19,8 @@ import ( // — identity + advertised services — without enrichment or liveness (those // decorate it later in the daemon). The directory is keyed by hostUuid (the // stable identity invite/attribution correlate on); name is the mDNS instance -// (hostname). trusted is supplied by the caller from the cluster pin set. +// (hostname). Every browse record starts as an independently keyed unverified +// preview; only a completed authenticated inventory read can promote a peer. // // Addresses are carried through whole rather than reduced to one. The node ranked // its own addresses from evidence no observer has — which of its interfaces the @@ -32,7 +34,7 @@ import ( // nodeid.Resolve), so a record without it can't be keyed by a stable identity. // We skip it (and warn) rather than key it by the mutable, non-unique hostname — // node identity is the UUID everywhere, with no hostname fallback. -func toDirectoryNode(raw RawNode, trusted bool) (noderec.DirectoryNode, bool) { +func toDirectoryNode(raw RawNode) (noderec.DirectoryNode, bool) { rec := noderec.ParseTXT(raw.TXT) if rec.HostUUID == "" { slog.Warn("skipping _nvpair-node record without uuid=; cannot key it by a stable identity", @@ -51,13 +53,15 @@ func toDirectoryNode(raw RawNode, trusted bool) (noderec.DirectoryNode, bool) { svcs[k] = noderec.ServiceStatus{Port: port} } return noderec.DirectoryNode{ - HostUUID: rec.HostUUID, - Name: raw.ID, - IP: ip, - IPs: ips, - ClusterUUID: rec.ClusterUUID, - Trusted: trusted, - Services: svcs, + HostUUID: rec.HostUUID, + Name: raw.ID, + IP: ip, + IPs: ips, + ClusterUUID: rec.ClusterUUID, + Trusted: false, + IdentityVerification: "unverified", + PeerRecordID: noderec.PreviewRecordID(raw.ID, rec.HostUUID), + Services: svcs, // Models is enriched separately from engine-manager's em /v1/models // (daemon.enrichModelsAt), not carried in the record's TXT. LastSeen: time.Now().Unix(), @@ -87,8 +91,12 @@ func newDirectory() *directory { func (d *directory) upsert(n noderec.DirectoryNode) (isNew bool) { d.mu.Lock() defer d.mu.Unlock() - _, existed := d.nodes[n.HostUUID] - d.nodes[n.HostUUID] = carryEngineNative(d.nodes[n.HostUUID], n) + key := n.RecordKey() + if key == "" { + return false + } + _, existed := d.nodes[key] + d.nodes[key] = carryEngineNative(d.nodes[key], n) return !existed } @@ -99,116 +107,23 @@ func (d *directory) upsert(n noderec.DirectoryNode) (isNew bool) { // deliberately replaced by a record that carries one takes the new one: only // the absent case is carried forward. func carryEngineNative(stored, incoming noderec.DirectoryNode) noderec.DirectoryNode { - if incoming.EngineNative == nil && stored.EngineNative != nil { + if incoming.ServicePeer() && incoming.PeerRecordID == stored.PeerRecordID && incoming.EngineNative == nil && stored.EngineNative != nil { incoming.EngineNative = stored.EngineNative } return incoming } -// supersedeMinAge is how much older a record must be than the one arriving at -// its address before it is worth asking whether it has been replaced. -// -// Note what it is NOT: evidence. LastSeen is not a liveness clock — the browser -// reports a node only when its record CHANGES (shared/discovery, reconcile), so -// a healthy peer with a stable advertisement produces no events and its LastSeen -// freezes at first discovery exactly like a ghost's does. A frozen timestamp -// says nothing about whether the node is alive, and anything willing to wait out -// the window satisfies it. Nothing may be deleted on the strength of it; it -// earns its place only as a cheap filter that keeps the identity probe in -// daemon.supersedingUpsert off the common path. -const supersedeMinAge int64 = 300 // seconds, matching DirectoryNode.LastSeen - -// supersedeCandidates lists the records a newly browsed node MIGHT have replaced, -// for the caller to confirm or reject with proof. It decides nothing and deletes -// nothing. -// -// This is the peer-side counterpart to dropSelf: when a node's appdata is wiped -// it mints a fresh hostUuid, so the fleet sees the same machine arrive as a -// stranger while the pre-wipe record lingers under the old key. dropSelf handles -// that re-stamp for the local node; nothing handled it for peers, so a reset -// node showed up twice everywhere downstream. -// -// A record is nominated — and no more than nominated — when it shares the -// arriving node's Name, carries a different hostUuid, and is at least -// supersedeMinAge older: -// -// - Same Name. A wipe clears identity but not the hostname. -// - Older than supersedeMinAge (a filter, not evidence — see above). -// -// A shared address is deliberately NOT part of this. It reads as "one machine" -// only if an address belongs to one machine at a time, and it does not: a -// direct-connect link between two machines uses the same small subnet on every -// such pair, so identically-provisioned hosts genuinely advertise the same -// address on it, and matching there nominated live peers as each other's ghosts. -// Nor is it needed — a wiped machine keeps its hostname whether or not it comes -// back on the same address. -// -// Both remaining signals come from an unauthenticated mDNS record that anything -// on the LAN can claim, so together they describe the SHAPE of a wipe and equally -// the shape of a spoof — and, now that the address is gone, also the shape of two -// hosts that merely share a hostname. daemon.supersedingUpsert must confirm a -// candidate is really gone before deleting it. -// -// The arriving record still needs an address of its own, since that is where the -// confirming read is made — not because matching it against anything would mean -// something. -// -// selfUUID is never nominated, and never nominates: the local record is -// registry-driven rather than browse-driven, and its LastSeen advances only when -// this node republishes, so it must not be displaceable by a peer that happens -// to claim its name. -// -// Candidates are returned sorted by hostUuid so the resulting removals are -// emitted in a deterministic order rather than map-iteration order. -func (d *directory) supersedeCandidates(n noderec.DirectoryNode, selfUUID string) []noderec.DirectoryNode { - if n.IP == "" || n.Name == "" || n.HostUUID == selfUUID { - return nil - } - d.mu.RLock() - defer d.mu.RUnlock() - var candidates []noderec.DirectoryNode - for uuid, other := range d.nodes { - if uuid == n.HostUUID || uuid == selfUUID { - continue - } - if other.Name != n.Name { - continue - } - if n.LastSeen-other.LastSeen < supersedeMinAge { - continue - } - candidates = append(candidates, other) - } - sort.Slice(candidates, func(i, j int) bool { return candidates[i].HostUUID < candidates[j].HostUUID }) - return candidates -} - -// upsertEvicting stores a node and deletes the records the caller has PROVEN it -// replaced, returning the entries actually removed so the caller can emit -// removals and drop their caches. -// -// The proof is a network read, so it is gathered without this lock held and each -// eviction is re-validated here against the entry the caller actually judged. A -// record that was removed, re-addressed, renamed, or refreshed while the probe -// was in flight is left alone: the state the verdict was formed against is gone, -// and it must be judged again from the new one. -func (d *directory) upsertEvicting(n noderec.DirectoryNode, proven []noderec.DirectoryNode) (evicted []noderec.DirectoryNode) { +// replaceIfCurrent prevents a completed probe from overwriting a newer record. +func (d *directory) replaceIfCurrent(old, replacement noderec.DirectoryNode) bool { d.mu.Lock() defer d.mu.Unlock() - for _, old := range proven { - if old.HostUUID == n.HostUUID { - continue - } - current, present := d.nodes[old.HostUUID] - if !present || current.IP != old.IP || current.Name != old.Name || current.LastSeen != old.LastSeen { - continue - } - delete(d.nodes, old.HostUUID) - evicted = append(evicted, current) + current, ok := d.nodes[old.RecordKey()] + if !ok || !reflect.DeepEqual(current, old) { + return false } - sort.Slice(evicted, func(i, j int) bool { return evicted[i].HostUUID < evicted[j].HostUUID }) - d.nodes[n.HostUUID] = carryEngineNative(d.nodes[n.HostUUID], n) - return evicted + delete(d.nodes, old.RecordKey()) + d.nodes[replacement.RecordKey()] = replacement + return true } // remove drops a node by hostUuid and reports whether it existed. @@ -328,51 +243,6 @@ func (d *directory) applyEngineNative(hostUUID, ip string, emPort int, stats *en return n, moved, true } -// applyClusterIdentity sets one node's cluster principal and the trust annotation -// derived from it, reporting whether either changed. It is the write path for both -// membership reconciles — the pin-set pass (daemon.reconcileTrust) and the -// node-info pass (daemon.refreshClusterIdentityOnce) — and mirrors applyModels: it -// updates just those two fields in place, preserving everything the browse and -// enrichment paths own, and reports no change for an entry that already agrees so -// the caller stays silent in the steady state. -// -// The two fields move together because trust is derived from the principal: a peer -// that leaves its cluster is no longer pinned under it, so leaving one stale while -// updating the other would describe a node that cannot exist. For the same reason -// trust is computed here, from the principal this call settles on, rather than -// supplied by the caller: a caller computing it from a snapshot would race the -// other reconcile and could write back a principal that had already moved. -// -// A nil clusterUUID means "keep whatever is stored" — the pin-set pass re-derives -// trust without claiming to know the principal, since only the peer's own record -// or report can say what it is. -// -// A node that has since been evicted reports no change — there is nothing to -// annotate, and a later rediscovery evaluates both from scratch. -func (d *directory) applyClusterIdentity(hostUUID string, clusterUUID *string, trustFor func(clusterUUID string) bool) (node noderec.DirectoryNode, changed bool) { - d.mu.Lock() - defer d.mu.Unlock() - n, present := d.nodes[hostUUID] - if !present { - return n, false - } - settled := n.ClusterUUID - if clusterUUID != nil { - settled = *clusterUUID - } - trusted := false - if settled != "" { - trusted = trustFor(settled) - } - if n.ClusterUUID == settled && n.Trusted == trusted { - return n, false - } - n.ClusterUUID = settled - n.Trusted = trusted - d.nodes[hostUUID] = n - return n, true -} - // sameByEngine reports whether two per-engine model maps are semantically equal: // the same set of engine keys, each mapping to the same set of model names // (order-insensitive). A nil and an empty map compare equal (both mean "no diff --git a/services/nvpair-node-scanner/directory_test.go b/services/nvpair-node-scanner/directory_test.go index e00ef98d8..f002332fa 100644 --- a/services/nvpair-node-scanner/directory_test.go +++ b/services/nvpair-node-scanner/directory_test.go @@ -16,7 +16,7 @@ func TestToDirectoryNode(t *testing.T) { Addresses: []string{"10.221.0.9", "192.168.1.10"}, TXT: []string{"v=1", "uuid=host-uuid", "cluster-uuid=clu", "ip=192.168.1.10", "ni=14318", "ol=11434", "otel=4317"}, } - n, ok := toDirectoryNode(raw, true) + n, ok := toDirectoryNode(raw) if !ok { t.Fatal("a record with uuid= should project") } @@ -29,8 +29,8 @@ func TestToDirectoryNode(t *testing.T) { if n.IP != "192.168.1.10" { t.Errorf("IP = %q, want the ip= TXT value", n.IP) } - if !n.Trusted { - t.Error("Trusted should reflect the caller-supplied flag") + if n.Trusted || n.IdentityVerification != "unverified" || n.RecordKey() == n.HostUUID || n.PeerRecordID == "" { + t.Error("mDNS claim must have a separate unverified preview key") } if !n.HasService(noderec.ServiceNodeInfo) || !n.HasService(noderec.ServiceOllama) { t.Errorf("services wrong: %+v", n.Services) @@ -56,7 +56,7 @@ func TestToDirectoryNodeIgnoresSRVPort(t *testing.T) { Addresses: []string{"192.168.1.30"}, TXT: []string{"v=1", "uuid=host-c", "ip=192.168.1.30", "ni=14318", "ol=11434"}, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) if !ok { t.Fatal("a record with uuid= should project") } @@ -82,7 +82,7 @@ func TestToDirectoryNodeSkipsWithoutUUID(t *testing.T) { Addresses: []string{"10.221.0.9", "192.168.1.20"}, TXT: []string{"v=1", "er=14319"}, } - if _, ok := toDirectoryNode(raw, false); ok { + if _, ok := toDirectoryNode(raw); ok { t.Error("a record without uuid= must be skipped, not projected") } } @@ -100,7 +100,7 @@ func TestToDirectoryNodeIPFallback(t *testing.T) { Addresses: []string{"10.221.0.9", "192.168.1.20"}, TXT: []string{"v=1", "uuid=host-b", "er=14319"}, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) if !ok { t.Fatal("a record with uuid= should project") } @@ -128,7 +128,7 @@ func TestToDirectoryNodePreservesPublishedOrder(t *testing.T) { "ips=10.172.54.70,192.168.240.2", "ni=14318", }, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) if !ok { t.Fatal("a record with uuid= should project") } @@ -148,7 +148,7 @@ func TestToDirectoryNodeUnionsUnpublishedAddresses(t *testing.T) { Addresses: []string{"10.0.9.9"}, TXT: []string{"v=1", "uuid=host-c", "ip=10.0.0.5", "ips=10.0.0.5"}, } - n, ok := toDirectoryNode(raw, false) + n, ok := toDirectoryNode(raw) if !ok { t.Fatal("a record with uuid= should project") } @@ -160,8 +160,8 @@ func TestToDirectoryNodeUnionsUnpublishedAddresses(t *testing.T) { func TestDirectoryUpsertRemoveSnapshot(t *testing.T) { d := newDirectory() - a := noderec.DirectoryNode{HostUUID: "a", Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}} - b := noderec.DirectoryNode{HostUUID: "b", Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceErrors: {Port: 14319}}} + a := noderec.DirectoryNode{IdentityVerification: "local", HostUUID: "a", Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}} + b := noderec.DirectoryNode{IdentityVerification: "local", HostUUID: "b", Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceErrors: {Port: 14319}}} if !d.upsert(a) { t.Error("first upsert should be new") diff --git a/services/nvpair-node-scanner/enginenative_test.go b/services/nvpair-node-scanner/enginenative_test.go index dc6fcd841..cd1e63939 100644 --- a/services/nvpair-node-scanner/enginenative_test.go +++ b/services/nvpair-node-scanner/enginenative_test.go @@ -12,6 +12,7 @@ import ( "net/http" "net/http/httptest" "path/filepath" + "strconv" "sync" "testing" "time" @@ -364,25 +365,25 @@ func TestEngineNativeModelOrderIsNotAChange(t *testing.T) { // next sweep brings it back, and the client watches the field appear and // disappear. func TestEngineNativeViewSurvivesARecordReplacement(t *testing.T) { - d := newEngineNativeTestDaemon() + d, server, _ := secureInventoryFixture(t, "uuid-peer") + port := portFromURL(t, server.URL) + raw := RawNode{ID: "peer-A", Addresses: []string{"127.0.0.1"}, TXT: []string{ + "v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=127.0.0.1", "ni=" + strconv.Itoa(port), + }} + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: raw}) + if !d.refreshNodeEngineNative(t.Context(), "uuid-peer", "127.0.0.1", []string{"127.0.0.1"}, port, "uuid-peer") { + t.Fatal("the authenticated engine-native view was not applied") + } + prior, _ := d.dir.get("uuid-peer") + // Make the next genuine browse enrichment change inventory, so this test + // cannot pass by looking at an old event after a local-self early return. + prior.Models = []string{"stale-model"} + d.dir.upsert(prior) capture := ¬ifyCapture{} d.codec = NewCodec(capture) - stub := &engineStatsStub{status: http.StatusOK, body: engineNativeBody(), observedAt: 1000} - port := startEngineStatsStubServer(t, stub) - seedEMNode(d, "peer-A", "127.0.0.1", port) - - if !d.refreshNodeEngineNative(context.Background(), "peer-A", "127.0.0.1", []string{"127.0.0.1"}, port, "") { - t.Fatal("the engine-native view was not applied to the directory node") - } - - // The peer advertises again: the record is rebuilt from the mDNS TXT, - // which carries no engine-native view. - d.onBrowse(DiscoveryEvent{Type: "updated", Node: RawNode{ - ID: "peer-A", - TXT: []string{"v=1", "uuid=peer-A", "ip=127.0.0.1"}, - }}) + d.onBrowse(DiscoveryEvent{Type: "updated", Node: raw}) - stored, ok := d.dir.get("peer-A") + stored, ok := d.dir.get("uuid-peer") if !ok { t.Fatal("the advertisement dropped the node from the directory") } diff --git a/services/nvpair-node-scanner/ghost_node_test.go b/services/nvpair-node-scanner/ghost_node_test.go index 634d4907f..8c95ac394 100644 --- a/services/nvpair-node-scanner/ghost_node_test.go +++ b/services/nvpair-node-scanner/ghost_node_test.go @@ -12,22 +12,12 @@ import ( "strconv" "strings" "testing" - "time" - - "nvpair-shared/noderec" ) -// A node whose appdata is wiped mints a fresh hostUuid but keeps its hostname -// and re-binds the same fixed service ports. These tests cover the two defences -// against the duplicate ("ghost") record that leaves behind: the liveness probe -// must notice the machine is no longer the node the record describes, and a -// record arriving under that hostname must collapse the one it replaced without -// waiting for the miss threshold. -// -// Both defences delete only on the same proof — a node-info answer naming a -// different host. The directory's hostname signal comes from an unauthenticated -// mDNS record, so it nominates suspects and nothing more; the address is not a -// signal at all, since machines on a direct-connect link legitimately share one. +// These probes govern only public candidate-preview lifetime. A public identity +// response may retire a stale preview, never a verified peer. The former +// cross-record public identity eviction path has been removed; authenticated +// proof expiry and independent record keys now own verified-peer lifetime. // nodeInfoServer starts a /v1/node-info endpoint reporting hostUUID and returns // its host and port. An empty hostUUID omits the field, mimicking a peer that @@ -220,282 +210,3 @@ func TestReachableSkipsIdentityWhenNothingAnswers(t *testing.T) { t.Fatalf("node-info requests = %d, want exactly 1 once the address answered", counter.n) } } - -// testNow is captured once so every ghost() in a run shares one base: LastSeen -// is Unix SECONDS, so computing it per call would let a wall-clock rollover -// between two calls shift the gap by a second and flip a threshold assertion. -var testNow = time.Now().Unix() - -// named builds a directory entry with an explicit hostname; ghost is the common -// case where the hostname is the one a wipe would have preserved. -func named(uuid, name, ip string, ageSeconds int64) noderec.DirectoryNode { - return noderec.DirectoryNode{ - HostUUID: uuid, - Name: name, - IP: ip, - LastSeen: testNow - ageSeconds, - } -} - -func ghost(uuid, ip string, ageSeconds int64) noderec.DirectoryNode { - return named(uuid, "wiped-host", ip, ageSeconds) -} - -// withNodeInfo advertises an ni port on a directory entry — the only service the -// supersedence proof reads. -func withNodeInfo(n noderec.DirectoryNode, port int) noderec.DirectoryNode { - n.Services = map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - } - return n -} - -func candidateUUIDs(nodes []noderec.DirectoryNode) string { - ids := make([]string, 0, len(nodes)) - for _, n := range nodes { - ids = append(ids, n.HostUUID) - } - return strings.Join(ids, ",") -} - -// TestSupersedeCandidatesNominatesGhost covers the nomination: when the wiped -// machine reappears under a new uuid at the same address and hostname, the -// record it replaced is put forward for confirmation — and left in place until -// it is confirmed. -func TestSupersedeCandidatesNominatesGhost(t *testing.T) { - d := newDirectory() - d.upsert(ghost("old-uuid", "192.168.1.10", 3600)) - - candidates := d.supersedeCandidates(ghost("new-uuid", "192.168.1.10", 0), "self-uuid") - if len(candidates) != 1 || candidates[0].HostUUID != "old-uuid" { - t.Fatalf("candidates = %+v, want exactly the pre-wipe record", candidates) - } - if _, ok := d.get("old-uuid"); !ok { - t.Fatal("nominating a record must not delete it; only proof may") - } -} - -// TestSupersedeCandidatesKeepFreshNeighbours is the false-positive guard: two -// records observed at about the same time are the shape a misconfiguration -// takes, not a wipe, and are not worth a probe. -func TestSupersedeCandidatesKeepFreshNeighbours(t *testing.T) { - d := newDirectory() - d.upsert(ghost("peer-a", "192.168.1.10", supersedeMinAge-1)) - - candidates := d.supersedeCandidates(ghost("peer-b", "192.168.1.10", 0), "self-uuid") - if len(candidates) != 0 { - t.Fatalf("a gap under supersedeMinAge must not nominate: %+v", candidates) - } -} - -// TestSupersedeCandidatesRequireSameHostname keeps the nomination narrow. Two -// machines colliding on an address almost never also collide on a hostname, so a -// name mismatch means we are not looking at one machine. The wiped-AND-renamed -// case is left to the miss-threshold path, which reaches the same proof. -func TestSupersedeCandidatesRequireSameHostname(t *testing.T) { - d := newDirectory() - d.upsert(named("old-uuid", "old-name", "192.168.1.10", 3600)) - - candidates := d.supersedeCandidates(named("new-uuid", "new-name", "192.168.1.10", 0), "self-uuid") - if len(candidates) != 0 { - t.Fatalf("a different hostname on the address must not be nominated: %+v", candidates) - } -} - -// TestSupersedeCandidatesIgnoreTheAddress covers the signal that was removed. A -// wiped machine keeps its hostname whether or not it comes back on the address -// it had before, so a stale namesake elsewhere is nominated too — and, in the -// other direction, sharing an address is no longer a reason to nominate -// anything, which is what stopped two hosts on a direct-connect link from -// putting each other forward. -func TestSupersedeCandidatesIgnoreTheAddress(t *testing.T) { - d := newDirectory() - d.upsert(ghost("peer-a", "192.168.1.10", 3600)) - - candidates := d.supersedeCandidates(ghost("peer-b", "192.168.1.11", 0), "self-uuid") - if len(candidates) != 1 || candidates[0].HostUUID != "peer-a" { - t.Fatalf("candidates = %+v, want the stale namesake that moved", candidates) - } - - // The arriving record must still carry an address: that is where the - // confirming read is made, so without one there is nothing to ask. - d2 := newDirectory() - d2.upsert(ghost("peer-c", "192.168.1.10", 3600)) - if candidates := d2.supersedeCandidates(ghost("peer-d", "", 0), "self-uuid"); len(candidates) != 0 { - t.Fatalf("an unaddressable arrival can prove nothing: %+v", candidates) - } -} - -// TestSupersedeCandidatesNeverIncludeSelf protects the local card. Self is -// registry-driven, so its LastSeen advances only when this node republishes and -// can legitimately look stale — it must not be displaceable by a peer claiming -// its address. Evicting the local node is a failure mode this daemon has hit -// before (see the self-guard in onBrowse). -func TestSupersedeCandidatesNeverIncludeSelf(t *testing.T) { - d := newDirectory() - d.upsert(ghost("self-uuid", "192.168.1.10", 3600)) - - candidates := d.supersedeCandidates(ghost("peer-uuid", "192.168.1.10", 0), "self-uuid") - if len(candidates) != 0 { - t.Fatalf("self must never be nominated by a peer: %+v", candidates) - } - - // Nor may the local record nominate peers on its way in. - d2 := newDirectory() - d2.upsert(ghost("peer-uuid", "192.168.1.10", 3600)) - if candidates := d2.supersedeCandidates(ghost("self-uuid", "192.168.1.10", 0), "self-uuid"); len(candidates) != 0 { - t.Fatalf("self must not nominate peers: %+v", candidates) - } -} - -// TestSupersedeCandidatesAreDeterministic pins the order, since the caller turns -// the confirmed subset into a stream of node-removed notifications. -func TestSupersedeCandidatesAreDeterministic(t *testing.T) { - d := newDirectory() - d.upsert(ghost("uuid-c", "192.168.1.10", 3600)) - d.upsert(ghost("uuid-a", "192.168.1.10", 7200)) - d.upsert(ghost("uuid-b", "192.168.1.10", 5400)) - - got := candidateUUIDs(d.supersedeCandidates(ghost("uuid-live", "192.168.1.10", 0), "self-uuid")) - if want := "uuid-a,uuid-b,uuid-c"; got != want { - t.Fatalf("candidate order = %v, want %v (sorted by hostUuid)", got, want) - } -} - -// TestUpsertEvictingRejudgesRecordsThatMoved covers the gap the proof opens: it -// is a network read, so the directory is unlocked while it runs and the entry -// under judgement can change underneath it. A verdict formed against a state -// that no longer exists must not be applied to the state that replaced it. -func TestUpsertEvictingRejudgesRecordsThatMoved(t *testing.T) { - d := newDirectory() - judged := named("old-uuid", "wiped-host", "192.168.1.10", 3600) - d.upsert(judged) - - moved := judged - moved.IP = "192.168.1.11" - d.upsert(moved) - - arriving := named("new-uuid", "wiped-host", "192.168.1.10", 0) - if evicted := d.upsertEvicting(arriving, []noderec.DirectoryNode{judged}); len(evicted) != 0 { - t.Fatalf("a record that changed during the probe must be judged again, not evicted: %+v", evicted) - } - if _, ok := d.get("old-uuid"); !ok { - t.Fatal("the re-addressed record must survive") - } - if _, ok := d.get("new-uuid"); !ok { - t.Fatal("the arriving node must be stored either way") - } -} - -// TestSupersedingUpsertRequiresIdentityProof is the security guard. Address and -// hostname come from an unauthenticated mDNS record and LastSeen freezes for a -// healthy peer exactly as it does for a ghost, so collapsing on those alone -// would let anything on the LAN delete a live node by claiming its address — -// and the browser re-reports a node only when its record CHANGES, so the peer -// might never come back. Nothing is deleted without an answer from the machine. -func TestSupersedingUpsertRequiresIdentityProof(t *testing.T) { - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - // Nothing is listening on the record's own ni port, so it cannot speak for - // itself and the claimant's answer is the only evidence in play. - d.dir.upsert(withNodeInfo(named("live-uuid", "wiped-host", loopbackHost, 3600), closedPort(t))) - claim := named("claimant-uuid", "wiped-host", loopbackHost, 0) - - // Nothing to ask: the claimant advertises no node-info port. - d.supersedingUpsert(claim) - if _, ok := d.dir.get("live-uuid"); !ok { - t.Fatal("an unprovable claim must never evict a peer: mDNS is not a removal primitive") - } - - // Asked, but unanswerable — the same verdict. - d.supersedingUpsert(withNodeInfo(claim, closedPort(t))) - if _, ok := d.dir.get("live-uuid"); !ok { - t.Fatal("an address that cannot answer proves nothing; the peer must survive") - } - - // Answered, and the machine is still the node the record describes. - _, alivePort := nodeInfoServer(t, "live-uuid") - d.supersedingUpsert(withNodeInfo(claim, alivePort)) - if _, ok := d.dir.get("live-uuid"); !ok { - t.Fatal("node-info confirming the record's own identity must keep it") - } - - // Only a definite mismatch collapses the pair. - _, wipedPort := nodeInfoServer(t, "claimant-uuid") - d.supersedingUpsert(withNodeInfo(claim, wipedPort)) - if _, ok := d.dir.get("live-uuid"); ok { - t.Fatal("node-info naming a different host is proof; the superseded record must go") - } - if _, ok := d.dir.get("claimant-uuid"); !ok { - t.Fatal("the arriving node must be stored") - } -} - -// TestSupersedingUpsertKeepsARecordThatAnswersForItself is the case that made -// the address a liability. Two identically-provisioned machines wired back to -// back both advertise the link's address, so each looked like the other's ghost; -// the pair is now told apart by asking the older record's own endpoint who it -// is. A record that names itself is a live peer, not a leftover, whatever -// address the arriving node claims. -func TestSupersedingUpsertKeepsARecordThatAnswersForItself(t *testing.T) { - host, livePort := nodeInfoServer(t, "peer-a") - _, claimPort := nodeInfoServer(t, "peer-b") - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - d.dir.upsert(withNodeInfo(named("peer-a", "dgx-station", host, 3600), livePort)) - - d.supersedingUpsert(withNodeInfo(named("peer-b", "dgx-station", host, 0), claimPort)) - - if _, ok := d.dir.get("peer-a"); !ok { - t.Fatal("a record still answering with its own hostUuid is a live peer sharing an address, not a ghost") - } - if _, ok := d.dir.get("peer-b"); !ok { - t.Fatal("the arriving node must be stored") - } -} - -// TestSupersedingUpsertKeepsAnUnaskableRecord covers the other half of that -// defence: with no ni port to ask, a live namesake and a leftover look -// identical, so the record stays and the miss-threshold path reaches it when it -// stops advertising. -func TestSupersedingUpsertKeepsAnUnaskableRecord(t *testing.T) { - host, claimPort := nodeInfoServer(t, "claimant-uuid") - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - d.dir.upsert(named("live-uuid", "wiped-host", host, 3600)) - - d.supersedingUpsert(withNodeInfo(named("claimant-uuid", "wiped-host", host, 0), claimPort)) - - if _, ok := d.dir.get("live-uuid"); !ok { - t.Fatal("a record that cannot be asked who it is must never be evicted here") - } -} - -// TestOnBrowseEmitsSupersededRemoval covers the wiring: the daemon must tell the -// broker the ghost is gone, and drop its cached enrichment so a later -// rediscovery under that uuid re-probes instead of resurrecting stale metrics. -func TestOnBrowseEmitsSupersededRemoval(t *testing.T) { - host, port := nodeInfoServer(t, "new-uuid") - d := newSelfTestDaemon("self-uuid", "192.168.1.1") - d.dir.upsert(withNodeInfo(ghost("old-uuid", host, 3600), closedPort(t))) - d.lastInfo["old-uuid"] = NodeInfoResponse{HostUUID: "old-uuid"} - - d.onBrowse(DiscoveryEvent{ - Type: "discovered", - Node: RawNode{ - ID: "wiped-host", - Addresses: []string{host}, - TXT: []string{"v=1", "uuid=new-uuid", "ip=" + host, fmt.Sprintf("ni=%d", port)}, - }, - }) - - if _, ok := d.dir.get("old-uuid"); ok { - t.Fatal("onBrowse must collapse the record the arriving node is proven to replace") - } - if _, ok := d.dir.get("new-uuid"); !ok { - t.Fatal("the arriving node must be stored") - } - d.infoMu.Lock() - _, cached := d.lastInfo["old-uuid"] - d.infoMu.Unlock() - if cached { - t.Fatal("a superseded node's enrichment cache must be dropped") - } -} diff --git a/services/nvpair-node-scanner/gpu_test.go b/services/nvpair-node-scanner/gpu_test.go index 2a518bccb..e36130b5e 100644 --- a/services/nvpair-node-scanner/gpu_test.go +++ b/services/nvpair-node-scanner/gpu_test.go @@ -86,7 +86,7 @@ func TestTelemetryMixedMeasuredGPUReadings(t *testing.T) { t.Fatal(err) } var output bytes.Buffer - d := &daemon{codec: NewCodec(&output), http: server.Client()} + d := &daemon{codec: NewCodec(&output), http: server.Client(), reg: newRegistry("node-a", "", selfAddrs("127.0.0.1"))} if !d.refreshNodeTelemetry(context.Background(), "node-a", u.Hostname(), port) { t.Fatal("no telemetry") } diff --git a/services/nvpair-node-scanner/inventory_lifetime_test.go b/services/nvpair-node-scanner/inventory_lifetime_test.go new file mode 100644 index 000000000..0b35c61b4 --- /dev/null +++ b/services/nvpair-node-scanner/inventory_lifetime_test.go @@ -0,0 +1,80 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net" + "strconv" + "testing" + "time" + + "nvpair-shared/noderec" +) + +// Browser churn must not reset the independent verified peer's retry budget. +func TestBrowseChurnPreservesVerifiedTelemetryRetry(t *testing.T) { + d, server, _ := secureInventoryFixture(t, "uuid-peer") + host, portString, _ := net.SplitHostPort(server.Listener.Addr().String()) + port, _ := strconv.Atoi(portString) + raw := RawNode{ID: "genuine", Addresses: []string{host}, TXT: []string{ + "v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=" + host, "ni=" + portString, + }} + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: raw}) + if node, ok := d.dir.get("uuid-peer"); !ok || !node.ServicePeer() { + t.Fatal("fixture did not establish an authenticated peer") + } + now := time.Now() + target := telemetryTargetKey([]string{host}, port) + token, ok := d.telemetryRetries.claim("uuid-peer", target, now) + if !ok { + t.Fatal("initial telemetry claim denied") + } + d.telemetryRetries.finish("uuid-peer", token, false, now) + raw.TXT = append(raw.TXT, "display-hint=changed") + d.onBrowse(DiscoveryEvent{Type: "updated", Node: raw}) + if _, ok := d.telemetryRetries.claim("uuid-peer", target, now); ok { + t.Fatal("TXT refresh cleared verified telemetry backoff") + } + d.onBrowse(DiscoveryEvent{Type: "removed", Node: raw}) + if _, ok := d.telemetryRetries.claim("uuid-peer", target, now); ok { + t.Fatal("candidate removal cleared verified telemetry backoff") + } + if node, ok := d.dir.get("uuid-peer"); !ok || !node.ServicePeer() { + t.Fatal("candidate removal withdrew authenticated inventory") + } +} + +func TestFailedProofExpiresVerifiedRecordWithOriginalToken(t *testing.T) { + d, server, mismatch := secureInventoryFixture(t, "uuid-peer") + host, portString, _ := net.SplitHostPort(server.Listener.Addr().String()) + port, _ := strconv.Atoi(portString) + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "genuine", Addresses: []string{host}, TXT: []string{ + "v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=" + host, "ni=" + portString, + }}}) + node, ok := d.dir.get("uuid-peer") + if !ok || !node.ServicePeer() { + t.Fatal("fixture did not establish an authenticated peer") + } + proof := node.PeerRecordID + node.LastSeen = time.Now().Add(-time.Minute).Unix() + d.dir.upsert(node) + capture := ¬ifyCapture{} + d.codec = NewCodec(capture) + mismatch.Store(true) + d.refreshNodeClusterIdentity("uuid-peer", host, port) + if _, exists := d.dir.get("uuid-peer"); exists { + t.Fatal("failed proof retained stale verified inventory") + } + events := capture.nodes(t) + if len(events) != 1 || events[0].HostUUID != "uuid-peer" || events[0].PeerRecordID != proof || events[0].IdentityVerification != "verified" { + t.Fatalf("withdrawal lost original proof identity: %+v", events) + } + var removed bool + for _, n := range d.dir.snapshot("") { + removed = removed || n.RecordKey() == noderec.PreviewRecordID("genuine", "uuid-peer") + } + if removed { + t.Fatal("withdrawal incorrectly relabelled old proof as a public preview") + } +} diff --git a/services/nvpair-node-scanner/peer_inventory.go b/services/nvpair-node-scanner/peer_inventory.go new file mode 100644 index 000000000..d810728b4 --- /dev/null +++ b/services/nvpair-node-scanner/peer_inventory.go @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "net" + "nvpair-shared/clustertrust" + "nvpair-shared/noderec" + "slices" + "strconv" + "time" +) + +func (d *daemon) localNode(hostUUID string) bool { + return d.reg != nil && hostUUID == d.reg.record().HostUUID +} + +func (d *daemon) currentPeer(node noderec.DirectoryNode) bool { + if d.localNode(node.HostUUID) { + return true + } + d.mesh.Refresh() + return node.ServicePeer() && node.PeerRecordID == d.mesh.PeerRecordID(node.ClusterUUID) +} + +func (d *daemon) authenticatedInfo(ctx context.Context, node noderec.DirectoryNode, hosts []string, port int) (NodeInfoResponse, string, bool) { + if node.HostUUID == "" || node.ClusterUUID == "" || node.HostUUID != node.ClusterUUID { + return NodeInfoResponse{}, "", false + } + d.mesh.Refresh() + d.peerInfoOnce.Do(func() { d.peerInfoClients = clustertrust.NewPeerClientPool(d.mesh, nodeInfoFetchTimeout) }) + client, ok := d.peerInfoClients.Client(node.ClusterUUID) + if !ok { + return NodeInfoResponse{}, "", false + } + for _, host := range hosts { + var info NodeInfoResponse + url := "https://" + net.JoinHostPort(host, strconv.Itoa(port)) + "/v1/node-info" + principal, binding, ok := clustertrust.ReadPeerInventory(ctx, d.mesh, client, node.ClusterUUID, url, d.pollIntent(), nodeInfoResponseLimit, &info) + if !ok || info.HostUUID != principal || info.ClusterUUID == nil || *info.ClusterUUID != principal || info.PeerMetadata == nil || info.PeerMetadata.HostUUID != principal || info.PeerMetadata.Name == "" || info.PeerMetadata.Services[noderec.ServiceNodeInfo] != port { + continue + } + info.PeerRecordID = binding + return info, host, true + } + return NodeInfoResponse{}, "", false +} + +func applyAuthenticatedInfo(node *noderec.DirectoryNode, info NodeInfoResponse, host string) { + meta := info.PeerMetadata + node.Trusted = true + node.IdentityVerification = "verified" + node.PeerRecordID = info.PeerRecordID + node.Name = meta.Name + // Only the endpoint which actually proved its identity is promoted. Other + // mDNS addresses remain hints for a later authentication, never routing data. + node.IP = host + node.IPs = []string{host} + node.Services = make(map[noderec.ServiceKey]noderec.ServiceStatus, len(meta.Services)) + for service, port := range meta.Services { + if port > 0 && port <= 65535 { + node.Services[service] = noderec.ServiceStatus{Port: port} + } + } + node.GPUs, node.CPU, node.Memory = info.GPUs, info.CPU, info.Memory + node.LastSeen = time.Now().Unix() +} + +func (d *daemon) candidateSnapshot() []noderec.DirectoryNode { + d.candidateMu.Lock() + defer d.candidateMu.Unlock() + out := make([]noderec.DirectoryNode, 0, len(d.candidates)) + for _, node := range d.candidates { + out = append(out, node) + } + return out +} + +func (d *daemon) publishVerified(node noderec.DirectoryNode, candidateKey string) bool { + if !d.currentPeer(node) { + return false + } + if candidateKey != "" { + d.candidateMu.Lock() + hint, exists := d.candidates[candidateKey] + d.candidateMu.Unlock() + if !exists || hint.HostUUID != node.HostUUID || !slices.Contains(hint.CandidateIPs(), node.IP) { + return false + } + } + if prior, exists := d.dir.get(node.HostUUID); exists && prior.PeerRecordID == node.PeerRecordID { + if node.Models == nil { + node.Models, node.ModelsByEngine, node.LoadedByEngine = prior.Models, prior.ModelsByEngine, prior.LoadedByEngine + } + if node.EngineNative == nil { + node.EngineNative = prior.EngineNative + } + if sameIgnoringLastSeen(prior, node) { + d.dir.replaceIfCurrent(prior, node) + return false + } + } else if exists { + d.forget(node.HostUUID) + } + if candidateKey != "" { + if preview, exists := d.dir.get(candidateKey); exists { + d.dir.remove(candidateKey) + d.emit(noderec.NotifyNodeRemoved, preview) + } + } + d.dir.upsert(node) + d.emit(noderec.NotifyNodeUpdated, node) + return true +} + +func (d *daemon) expireVerified(node noderec.DirectoryNode) { + current, exists := d.dir.get(node.HostUUID) + if !exists || current.PeerRecordID != node.PeerRecordID || current.LastSeen != node.LastSeen || current.IP != node.IP { + return + } + d.dir.remove(node.HostUUID) + d.forget(node.HostUUID) + d.emit(noderec.NotifyNodeRemoved, node) +} + +func withdrawPeer(node *noderec.DirectoryNode) { + node.Trusted = false + node.PeerRecordID = "" + node.IdentityVerification = "unverified" + node.GPUs, node.CPU, node.Memory = nil, nil, nil + node.Models, node.ModelsByEngine, node.LoadedByEngine = nil, nil, nil + node.EngineNative = nil +} diff --git a/services/nvpair-node-scanner/refresh_test.go b/services/nvpair-node-scanner/refresh_test.go index e61678b16..0ae19c00e 100644 --- a/services/nvpair-node-scanner/refresh_test.go +++ b/services/nvpair-node-scanner/refresh_test.go @@ -5,6 +5,7 @@ package main import ( "encoding/json" + "net" "net/http" "net/http/httptest" "net/url" @@ -105,10 +106,14 @@ func newRefreshTestDaemon() *daemon { // seedEMNode adds a directory entry advertising engine-manager (em) at ip:port, // with no model inventory yet (the state before any enrichment). func seedEMNode(d *daemon, hostUUID, ip string, emPort int) { + // These tests exercise owned loopback inventory/cache mechanics. Remote + // authentication is covered by the real mTLS inventory fixture. + d.reg = newRegistry(hostUUID, "", selfAddrs(ip)) d.dir.upsert(noderec.DirectoryNode{ - HostUUID: hostUUID, - Name: hostUUID, - IP: ip, + IdentityVerification: "local", + HostUUID: hostUUID, + Name: hostUUID, + IP: ip, Services: map[noderec.ServiceKey]noderec.ServiceStatus{ noderec.ServiceEngineManager: {Port: emPort}, }, @@ -140,27 +145,17 @@ func TestRefreshPopulatesInitialEmptyInventory(t *testing.T) { } func TestRefreshModelsFallsBackToASecondPublishedAddress(t *testing.T) { - d := newRefreshTestDaemon() - _, port := startModelsStub(t, http.StatusOK, map[string]any{ - "models": []string{"reachable-model"}, - }) - const peerUUID = "peer-multihomed-models" - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: peerUUID, - Name: peerUUID, - IP: "127.0.0.2", - IPs: []string{"127.0.0.2", "127.0.0.1"}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceEngineManager: {Port: port}, - }, - }) - - if !d.refreshNodeModelsCandidates(peerUUID, "127.0.0.2", []string{"127.0.0.2", "127.0.0.1"}, port, "") { - t.Fatal("model inventory did not refresh through the second published address") - } - got, _ := d.dir.get(peerUUID) - if want := []string{"reachable-model"}; !reflect.DeepEqual(got.Models, want) { - t.Fatalf("Models = %v, want %v", got.Models, want) + d, srv, _ := secureInventoryFixture(t, "uuid-peer") + host, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(p) + n := noderec.DirectoryNode{HostUUID: "uuid-peer", ClusterUUID: "uuid-peer", IP: host, Trusted: true, IdentityVerification: "verified", PeerRecordID: d.mesh.PeerRecordID("uuid-peer"), Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: port}}} + d.dir.upsert(n) + if !d.refreshNodeModelsCandidates("uuid-peer", host, []string{"127.0.0.2", host}, port, "uuid-peer") { + t.Fatal("model inventory did not refresh through authenticated second address") + } + got, _ := d.dir.get("uuid-peer") + if !reflect.DeepEqual(got.Models, []string{"authenticated-model"}) { + t.Fatalf("Models=%v", got.Models) } } @@ -454,10 +449,11 @@ func TestApplyModelsGuard(t *testing.T) { // Re-addressed mid-sweep: node present but IP changed -> not overwritten. d.upsert(noderec.DirectoryNode{ - HostUUID: "peer-E", - IP: "10.0.0.9", - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14322}}, - Models: []string{"old"}, + IdentityVerification: "local", + HostUUID: "peer-E", + IP: "10.0.0.9", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14322}}, + Models: []string{"old"}, }) if _, changed, ok := d.applyModels("peer-E", "127.0.0.1", 14322, []string{"new"}, nil, nil); ok || changed { t.Errorf("applyModels with a stale IP = (changed %v, ok %v), want (false, false)", changed, ok) diff --git a/services/nvpair-node-scanner/scanner.go b/services/nvpair-node-scanner/scanner.go index 6cddfb29e..cb49770db 100644 --- a/services/nvpair-node-scanner/scanner.go +++ b/services/nvpair-node-scanner/scanner.go @@ -35,12 +35,14 @@ type ( // NodeInfoResponse is the /v1/node-info body the daemon fetches to enrich a // node (see daemon.fetchNodeInfo). type NodeInfoResponse struct { - GPUs []GPUInfo `json:"GPUs"` - CPU *CPUInfo `json:"cpu,omitempty"` - Memory *MemoryInfo `json:"memory,omitempty"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` - SampleSeq uint64 `json:"sampleSeq,omitempty"` + PeerMetadata *noderec.PeerMetadataParams `json:"peerMetadata,omitempty"` + PeerRecordID string `json:"-"` + GPUs []GPUInfo `json:"GPUs"` + CPU *CPUInfo `json:"cpu,omitempty"` + Memory *MemoryInfo `json:"memory,omitempty"` + TelemetryValid bool `json:"telemetryValid"` + MSSince int64 `json:"msSince"` + SampleSeq uint64 `json:"sampleSeq,omitempty"` // HostUUID is the identity the node itself reports (nvpair-node-info's // hostUuid). It is not enrichment data — it is what lets the liveness probe // tell "this node still answers" from "some other node now answers at this diff --git a/services/nvpair-node-scanner/telemetry_test.go b/services/nvpair-node-scanner/telemetry_test.go index db10d8b04..257402305 100644 --- a/services/nvpair-node-scanner/telemetry_test.go +++ b/services/nvpair-node-scanner/telemetry_test.go @@ -4,766 +4,299 @@ package main import ( - "bytes" "context" "encoding/json" + "fmt" + "net" "net/http" "net/http/httptest" - "net/url" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" + "path/filepath" "strconv" "sync" "sync/atomic" "testing" "time" - - "nvpair-shared/noderec" ) -func TestRefreshNodeTelemetryEmitsFreshnessAndUtilization(t *testing.T) { - response := NodeInfoResponse{ - GPUs: []GPUInfo{ - {Name: "GPU 0", UtilizationPercent: gpuReading(uint32(0))}, - {Name: "GPU 1", UtilizationPercent: gpuReading(uint32(84))}, - }, - TelemetryValid: true, - MSSince: 137, - SampleSeq: 42, - HostUUID: "node-a", - } - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { +// Scheduling tests use actual pinned TLS, not public inventory fixtures. The +// initial directory record represents an already authenticated discovery; each +// telemetry request must independently prove its certificate and response ID. +type telemetryHarness struct { + t *testing.T + d *daemon + selfDir string +} + +func newTelemetryHarness(t *testing.T) *telemetryHarness { + t.Helper() + dir := t.TempDir() + clustertrusttest.Join(t, dir, "telemetry-cluster", "self") + d := newSelfTestDaemon("self", "192.0.2.1") + d.mesh = clustertrust.Open(dir) + return &telemetryHarness{t: t, d: d, selfDir: dir} +} +func (h *telemetryHarness) peer(uuid string, body func(*http.Request) NodeInfoResponse) (string, int) { + t := h.t + t.Helper() + dir := t.TempDir() + clustertrusttest.Join(t, dir, "telemetry-cluster", uuid) + writePinFromCert(t, h.selfDir, uuid, filepath.Join(dir, "node.crt")) + writePinFromCert(t, dir, "self", filepath.Join(h.selfDir, "node.crt")) + mesh := clustertrust.Open(dir) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := mesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", 403) + return + } + response := body(r) + _, p, _ := net.SplitHostPort(r.Host) + port, _ := strconv.Atoi(p) + response.ClusterUUID = &uuid + response.PeerMetadata = &noderec.PeerMetadataParams{HostUUID: uuid, Name: uuid, Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port}} _ = json.NewEncoder(w).Encode(response) })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - if err != nil { - t.Fatalf("parse server URL: %v", err) - } - port, err := strconv.Atoi(serverURL.Port()) - if err != nil { - t.Fatalf("parse server port: %v", err) - } - - var output bytes.Buffer - d := &daemon{codec: NewCodec(&output), http: server.Client()} - if !d.refreshNodeTelemetry(context.Background(), "node-a", serverURL.Hostname(), port) { - t.Fatal("valid node-info response did not emit telemetry") - } + srv.TLS = mesh.ServerTLSConfig() + srv.StartTLS() + t.Cleanup(srv.Close) + host, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(p) + h.d.mesh.Refresh() + h.d.dir.upsert(noderec.DirectoryNode{HostUUID: uuid, ClusterUUID: uuid, Name: uuid, IP: host, Trusted: true, IdentityVerification: "verified", PeerRecordID: h.d.mesh.PeerRecordID(uuid), Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: port}}}) + return host, port +} +func TestRefreshNodeTelemetryEmitsFreshnessAndUtilization(t *testing.T) { + h := newTelemetryHarness(t) + host, port := h.peer("node-a", func(*http.Request) NodeInfoResponse { + return NodeInfoResponse{HostUUID: "node-a", TelemetryValid: true, MSSince: 137, SampleSeq: 42, GPUs: []GPUInfo{{Name: "GPU 0", UtilizationPercent: gpuReading(uint32(0))}, {Name: "GPU 1", UtilizationPercent: gpuReading(uint32(84))}}} + }) + capture := ¬ifyCapture{} + h.d.codec = NewCodec(capture) + if !h.d.refreshNodeTelemetry(t.Context(), "node-a", host, port) { + t.Fatal("pinned telemetry rejected") + } + capture.mu.Lock() + data := append([]byte(nil), capture.frames...) + capture.mu.Unlock() var message Message - if err := json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message); err != nil { - t.Fatalf("decode notification: %v", err) - } - if message.Method != noderec.NotifyNodeTelemetry { - t.Fatalf("notification method = %q, want %q", message.Method, noderec.NotifyNodeTelemetry) + if err := json.Unmarshal(data, &message); err != nil { + t.Fatal(err) } var got noderec.NodeTelemetry if err := json.Unmarshal(message.Params, &got); err != nil { - t.Fatalf("decode telemetry: %v", err) - } - if got.HostUUID != "node-a" || !got.TelemetryValid { - t.Fatalf("telemetry identity/validity = %+v", got) + t.Fatal(err) } - if got.GPUUtilizationPct != 84 { - t.Fatalf("GPU utilization = %d, want max 84", got.GPUUtilizationPct) - } - if got.MSSince < 137 || got.MSSince >= 3_500 { - t.Fatalf("age = %d, want node age plus bounded fetch time", got.MSSince) - } - if got.SampleSeq != 42 { - t.Fatalf("sampleSeq = %d, want 42", got.SampleSeq) + if message.Method != noderec.NotifyNodeTelemetry || got.HostUUID != "node-a" || !got.TelemetryValid || got.GPUUtilizationPct != 84 || got.SampleSeq != 42 || got.PeerRecordID == "" || got.MSSince < 137 || got.MSSince >= 3500 { + t.Fatalf("telemetry = %+v", got) } } - func TestRefreshNodeTelemetryRejectsMismatchedIdentity(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(NodeInfoResponse{ - HostUUID: "node-b", - TelemetryValid: true, - }) - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - if err != nil { - t.Fatalf("parse server URL: %v", err) - } - port, err := strconv.Atoi(serverURL.Port()) - if err != nil { - t.Fatalf("parse server port: %v", err) - } - - var output bytes.Buffer - d := &daemon{codec: NewCodec(&output), http: server.Client()} - if d.refreshNodeTelemetry(context.Background(), "node-a", serverURL.Hostname(), port) { - t.Fatal("mismatched host emitted telemetry") - } - if output.Len() != 0 { - t.Fatalf("mismatched host wrote notification: %s", output.String()) + h := newTelemetryHarness(t) + host, port := h.peer("node-a", func(*http.Request) NodeInfoResponse { + return NodeInfoResponse{HostUUID: "node-b", TelemetryValid: true} + }) + if h.d.refreshNodeTelemetry(t.Context(), "node-a", host, port) { + t.Fatal("mismatched response accepted") } } - -// TestRefreshTelemetryFailsOverToAnAnsweringAddress: this sweep is the only source -// of GPU pressure the scheduler gets, and it dialed the canonical address alone. -// A node whose canonical address is a link this host cannot reach therefore -// reported nothing at all, and the scheduler assigned it neutral pressure and -// scheduled it blind — while an address that answers sat unused in the same -// published list. func TestRefreshTelemetryFailsOverToAnAnsweringAddress(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - _ = json.NewEncoder(w).Encode(NodeInfoResponse{ - HostUUID: "node-a", - TelemetryValid: true, - GPUs: []GPUInfo{{Name: "GPU 0", UtilizationPercent: gpuReading(uint32(61))}}, - }) - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - if err != nil { - t.Fatalf("parse server URL: %v", err) - } - port, err := strconv.Atoi(serverURL.Port()) - if err != nil { - t.Fatalf("parse server port: %v", err) - } - - var output bytes.Buffer - d := &daemon{ - codec: NewCodec(&output), - dir: newDirectory(), - // A short client timeout stands in for the daemon's own dial budget, so - // the address that never answers is written off promptly. - http: &http.Client{Timeout: 500 * time.Millisecond}, - } - // TEST-NET-1 (RFC 5737) is routed nowhere, so only the second published - // address can answer. - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-a", - IP: "192.0.2.1", - IPs: []string{"192.0.2.1", serverURL.Hostname()}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, + h := newTelemetryHarness(t) + host, port := h.peer("node-a", func(*http.Request) NodeInfoResponse { + return NodeInfoResponse{HostUUID: "node-a", TelemetryValid: true} }) - - d.refreshTelemetryOnce(context.Background()) - - var message Message - if err := json.Unmarshal(bytes.TrimSpace(output.Bytes()), &message); err != nil { - t.Fatalf("decode notification: %v (output %q)", err, output.String()) - } - var got noderec.NodeTelemetry - if err := json.Unmarshal(message.Params, &got); err != nil { - t.Fatalf("decode telemetry: %v", err) - } - if got.HostUUID != "node-a" || got.GPUUtilizationPct != 61 || !got.TelemetryValid { - t.Fatalf("telemetry = %+v, want node-a at 61%% from its second address", got) - } - - // And the address that answered is remembered — in the memory the enrichment - // sweeps share — so the sweeps behind this one ask it alone rather than paying - // the unreachable address on every due telemetry attempt. - key := hostKey{hostUUID: "node-a", service: noderec.ServiceNodeInfo} - if remembered := d.enrichHosts.get(key); remembered != serverURL.Hostname() { - t.Fatalf("remembered address = %q, want %q", remembered, serverURL.Hostname()) + if !h.d.refreshNodeTelemetryCandidates(t.Context(), "node-a", []string{"127.0.0.2", host}, port) { + t.Fatal("answering second address ignored") } } - func TestTelemetryIntervalForNodeIsStableAndBounded(t *testing.T) { - if got := telemetryIntervalForNode(""); got != telemetryRefreshInterval { - t.Fatalf("empty identity interval = %v, want %v", got, telemetryRefreshInterval) + if telemetryIntervalForNode("") != telemetryRefreshInterval { + t.Fatal("default interval changed") } first := telemetryIntervalForNode("node-a") - if again := telemetryIntervalForNode("node-a"); again != first { - t.Fatalf("node jitter changed from %v to %v", first, again) - } - minimum := telemetryRefreshInterval - telemetryRefreshJitter - maximum := telemetryRefreshInterval + telemetryRefreshJitter - if first < minimum || first > maximum { - t.Fatalf("node-a interval = %v, want within [%v,%v]", first, minimum, maximum) - } - if second := telemetryIntervalForNode("node-b"); second == first { - t.Fatalf("distinct identities received identical test intervals: %v", first) + if first != telemetryIntervalForNode("node-a") || first < telemetryRefreshInterval-telemetryRefreshJitter || first > telemetryRefreshInterval+telemetryRefreshJitter || first == telemetryIntervalForNode("node-b") { + t.Fatal("jitter is unstable or out of bounds") } } - func TestTelemetryRetryDelay(t *testing.T) { - cases := []struct { + for _, tc := range []struct { failures int want time.Duration - }{ - {0, 2 * time.Second}, - {1, 4 * time.Second}, - {2, 8 * time.Second}, - {3, 16 * time.Second}, - {4, 30 * time.Second}, - {100, 30 * time.Second}, - } - for _, test := range cases { - if got := telemetryRetryDelay(test.failures); got != test.want { - t.Errorf("telemetryRetryDelay(%d) = %v, want %v", test.failures, got, test.want) + }{{0, 2 * time.Second}, {1, 4 * time.Second}, {2, 8 * time.Second}, {3, 16 * time.Second}, {4, 30 * time.Second}, {100, 30 * time.Second}} { + if got := telemetryRetryDelay(tc.failures); got != tc.want { + t.Errorf("delay(%d)=%v, want %v", tc.failures, got, tc.want) } } } - func TestTelemetryRetryGateBacksOffAndResets(t *testing.T) { var gate telemetryRetryGate - startedAt := time.Unix(1_000, 0) - targetKey := telemetryTargetKey([]string{"192.0.2.1"}, 14318) - - first, ok := gate.claim("node-a", targetKey, startedAt) + now := time.Unix(1000, 0) + key := telemetryTargetKey([]string{"192.0.2.1"}, 14318) + first, ok := gate.claim("node-a", key, now) if !ok || first == 0 { - t.Fatal("first telemetry attempt was not due") + t.Fatal("first claim denied") } - if _, ok := gate.claim("node-a", targetKey, startedAt); ok { - t.Fatal("a second attempt started while the first was in flight") + if _, ok := gate.claim("node-a", key, now); ok { + t.Fatal("overlapping claim") } - gate.finish("node-a", first, false, startedAt) - - if _, ok := gate.claim("node-a", targetKey, startedAt.Add(4*time.Second-time.Nanosecond)); ok { - t.Fatal("failed telemetry retried before its first backoff elapsed") + gate.finish("node-a", first, false, now) + if _, ok := gate.claim("node-a", key, now.Add(4*time.Second-time.Nanosecond)); ok { + t.Fatal("early retry") } - second, ok := gate.claim("node-a", targetKey, startedAt.Add(4*time.Second)) + second, ok := gate.claim("node-a", key, now.Add(4*time.Second)) if !ok || second == first { - t.Fatal("failed telemetry was not due at its retry deadline") + t.Fatal("retry not due") } - gate.finish("node-a", second, true, startedAt.Add(4*time.Second)) - - if _, ok := gate.claim("node-a", targetKey, startedAt.Add(4*time.Second)); !ok { - t.Fatal("successful telemetry did not reset the retry gate") + gate.finish("node-a", second, true, now.Add(4*time.Second)) + if _, ok := gate.claim("node-a", key, now.Add(4*time.Second)); !ok { + t.Fatal("success did not reset retry") } } - func TestTelemetryRetryGateChangedTargetWaitsForActiveAttempt(t *testing.T) { - oldTarget := telemetryTargetKey([]string{"192.0.2.1"}, 14318) - changedTargets := map[string]string{ - "hosts": telemetryTargetKey([]string{"10.0.0.1"}, 14318), - "port": telemetryTargetKey([]string{"192.0.2.1"}, 14319), - } - for name, changedTarget := range changedTargets { - t.Run(name, func(t *testing.T) { - var gate telemetryRetryGate - now := time.Unix(1_500, 0) - oldToken, ok := gate.claim("node-a", oldTarget, now) - if !ok { - t.Fatal("old endpoint telemetry attempt was not due") - } - if _, ok := gate.claim("node-a", changedTarget, now); ok { - t.Fatal("changed endpoint started while the old attempt was in flight") - } - - gate.finish("node-a", oldToken, false, now) - if _, ok := gate.claim("node-a", changedTarget, now); !ok { - t.Fatal("old endpoint failure backed off the changed endpoint") - } - }) + for _, target := range []string{telemetryTargetKey([]string{"10.0.0.1"}, 14318), telemetryTargetKey([]string{"192.0.2.1"}, 14319)} { + var gate telemetryRetryGate + now := time.Unix(1500, 0) + old := telemetryTargetKey([]string{"192.0.2.1"}, 14318) + token, ok := gate.claim("node-a", old, now) + if !ok { + t.Fatal("initial claim denied") + } + if _, ok := gate.claim("node-a", target, now); ok { + t.Fatal("replacement overlapped active request") + } + gate.finish("node-a", token, false, now) + if _, ok := gate.claim("node-a", target, now); !ok { + t.Fatal("old failure backed off replacement") + } } } - func TestTelemetryRetryGateRemoveRediscoverPreservesActiveClaim(t *testing.T) { var gate telemetryRetryGate - now := time.Unix(2_000, 0) - targetKey := telemetryTargetKey([]string{"192.0.2.1"}, 14318) - token, ok := gate.claim("node-a", targetKey, now) + now := time.Unix(2000, 0) + key := telemetryTargetKey([]string{"192.0.2.1"}, 14318) + token, ok := gate.claim("node-a", key, now) if !ok { - t.Fatal("first telemetry attempt was not due") + t.Fatal("initial claim denied") } - gate.forget("node-a") - if _, ok := gate.claim("node-a", targetKey, now); ok { - t.Fatal("re-discovered node started telemetry before its removed attempt finished") + if _, ok := gate.claim("node-a", key, now); ok { + t.Fatal("rediscovery overlapped old request") } gate.finish("node-a", token, false, now) - if _, ok := gate.claim("node-a", targetKey, now); !ok { - t.Fatal("removed attempt's late failure backed off the re-discovered node") + if _, ok := gate.claim("node-a", key, now); !ok { + t.Fatal("old failure backed off rediscovery") } } - func TestRefreshTelemetrySkipsBackedOffPeerWithoutDelayingHealthyPeer(t *testing.T) { - var backedOffCalls atomic.Int32 - backedOff := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - backedOffCalls.Add(1) - _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-backed-off"}) - })) - defer backedOff.Close() - backedOffURL, err := url.Parse(backedOff.URL) - if err != nil { - t.Fatalf("parse backed-off server URL: %v", err) - } - backedOffPort, err := strconv.Atoi(backedOffURL.Port()) - if err != nil { - t.Fatalf("parse backed-off server port: %v", err) - } - - var healthyCalls atomic.Int32 - healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - healthyCalls.Add(1) - _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-healthy"}) - })) - defer healthy.Close() - healthyURL, err := url.Parse(healthy.URL) - if err != nil { - t.Fatalf("parse healthy server URL: %v", err) - } - healthyPort, err := strconv.Atoi(healthyURL.Port()) - if err != nil { - t.Fatalf("parse healthy server port: %v", err) - } - - var output bytes.Buffer - d := &daemon{ - codec: NewCodec(&output), - dir: newDirectory(), - http: &http.Client{Timeout: time.Second}, - } - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-backed-off", - IP: backedOffURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: backedOffPort}, - }, - }) - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-healthy", - IP: healthyURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: healthyPort}, - }, - }) + h := newTelemetryHarness(t) + var backed, healthy atomic.Int32 + host, port := h.peer("backed", func(*http.Request) NodeInfoResponse { backed.Add(1); return NodeInfoResponse{HostUUID: "backed"} }) + h.peer("healthy", func(*http.Request) NodeInfoResponse { healthy.Add(1); return NodeInfoResponse{HostUUID: "healthy"} }) now := time.Now() - targetKey := telemetryTargetKey([]string{backedOffURL.Hostname()}, backedOffPort) - token, ok := d.telemetryRetries.claim("node-backed-off", targetKey, now) - if !ok { - t.Fatal("could not seed backed-off peer") - } - d.telemetryRetries.finish("node-backed-off", token, false, now) - - d.refreshTelemetryOnce(context.Background()) - - if got := backedOffCalls.Load(); got != 0 { - t.Fatalf("backed-off peer received %d requests, want 0", got) - } - if got := healthyCalls.Load(); got != 1 { - t.Fatalf("healthy peer received %d requests, want 1", got) + token, _ := h.d.telemetryRetries.claim("backed", telemetryTargetKey([]string{host}, port), now) + h.d.telemetryRetries.finish("backed", token, false, now) + h.d.refreshTelemetryOnce(t.Context()) + if backed.Load() != 0 || healthy.Load() != 1 { + t.Fatalf("requests: backed=%d healthy=%d", backed.Load(), healthy.Load()) } } - -func TestBrowseTXTUpdatePreservesTelemetryRetry(t *testing.T) { - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - calls.Add(1) - _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "peer-uuid"}) - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - if err != nil { - t.Fatalf("parse server URL: %v", err) - } - port, err := strconv.Atoi(serverURL.Port()) - if err != nil { - t.Fatalf("parse server port: %v", err) - } - - d := newSelfTestDaemon("self-uuid", "127.0.0.1") - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "peer-uuid", - IP: serverURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, - }) - now := time.Now() - targetKey := telemetryTargetKey([]string{serverURL.Hostname()}, port) - token, ok := d.telemetryRetries.claim("peer-uuid", targetKey, now) - if !ok { - t.Fatal("could not seed peer retry state") - } - d.telemetryRetries.finish("peer-uuid", token, false, now) - - d.onBrowse(DiscoveryEvent{ - Type: "updated", - Node: RawNode{ - ID: "peer", - Addresses: []string{serverURL.Hostname()}, - TXT: []string{ - "v=1", - "uuid=peer-uuid", - "ip=" + serverURL.Hostname(), - "ni=" + strconv.Itoa(port), - "peer-controlled-field=changed", - }, - }, +func TestTelemetryLoopKeepsHealthyNodeOnCadenceWhilePeerIsBlocked(t *testing.T) { + h := newTelemetryHarness(t) + blocked := make(chan struct{}) + release := make(chan struct{}) + var once sync.Once + h.peer("slow", func(*http.Request) NodeInfoResponse { + once.Do(func() { close(blocked) }) + <-release + return NodeInfoResponse{HostUUID: "slow"} }) - - if got := calls.Load(); got != 1 { - t.Fatalf("browse enrichment made %d node-info requests, want 1", got) - } - if _, ok := d.telemetryRetries.claim("peer-uuid", targetKey, now); ok { - t.Fatal("TXT-only update cleared telemetry backoff for an unchanged endpoint") - } -} - -func TestBrowseRemovalClearsTelemetryRetry(t *testing.T) { - d := newSelfTestDaemon("self-uuid", "127.0.0.1") - now := time.Now() - host := "192.0.2.81" - port := 14318 - targetKey := telemetryTargetKey([]string{host}, port) - token, ok := d.telemetryRetries.claim("peer-uuid", targetKey, now) - if !ok { - t.Fatal("could not seed peer retry state") - } - d.telemetryRetries.finish("peer-uuid", token, false, now) - - d.onBrowse(DiscoveryEvent{ - Type: "removed", - Node: RawNode{ - ID: "peer", - Addresses: []string{host}, - TXT: []string{ - "v=1", - "uuid=peer-uuid", - "ip=" + host, - "ni=" + strconv.Itoa(port), - }, - }, + healthy := make(chan struct{}, 20) + h.peer("healthy", func(*http.Request) NodeInfoResponse { + healthy <- struct{}{} + return NodeInfoResponse{HostUUID: "healthy"} }) - - if _, ok := d.telemetryRetries.claim("peer-uuid", targetKey, now); !ok { - t.Fatal("removed peer endpoint remained backed off") - } -} - -func TestBrowseEndpointUpdateWaitsForOldTelemetryBeforeClaimingReplacement(t *testing.T) { - newEntered := make(chan struct{}, 1) - releaseNew := make(chan struct{}, 1) - newServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - newEntered <- struct{}{} - <-releaseNew - _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "peer-uuid"}) - })) - oldEntered := make(chan struct{}, 1) - releaseOld := make(chan struct{}, 1) - oldServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - oldEntered <- struct{}{} - <-releaseOld - w.WriteHeader(http.StatusServiceUnavailable) - })) - release := func(ch chan<- struct{}) { - select { - case ch <- struct{}{}: - default: - } + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan struct{}) + go func() { h.d.runTelemetryLoop(ctx, 20*time.Millisecond); close(done) }() + defer func() { cancel(); close(release); <-done }() + select { + case <-blocked: + case <-time.After(2 * time.Second): + t.Fatal("slow request did not start") } - defer func() { - release(releaseNew) - release(releaseOld) - newServer.Close() - oldServer.Close() - }() - waitFor := func(ch <-chan struct{}, what string) { + for i := 0; i < 2; i++ { select { - case <-ch: - case <-time.After(time.Second): - t.Fatalf("timed out waiting for %s", what) + case <-healthy: + case <-time.After(2 * time.Second): + t.Fatal("blocked peer stalled healthy cadence") } } - - newURL, err := url.Parse(newServer.URL) - if err != nil { - t.Fatalf("parse new server URL: %v", err) - } - newPort, err := strconv.Atoi(newURL.Port()) - if err != nil { - t.Fatalf("parse new server port: %v", err) - } - oldURL, err := url.Parse(oldServer.URL) - if err != nil { - t.Fatalf("parse old server URL: %v", err) - } - oldPort, err := strconv.Atoi(oldURL.Port()) - if err != nil { - t.Fatalf("parse old server port: %v", err) - } - - d := newSelfTestDaemon("self-uuid", "127.0.0.1") - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "peer-uuid", - IP: oldURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: oldPort}, - }, - }) - - browseDone := make(chan struct{}) - go func() { - d.onBrowse(DiscoveryEvent{ - Type: "updated", - Node: RawNode{ - ID: "peer", - Addresses: []string{newURL.Hostname()}, - TXT: []string{ - "v=1", - "uuid=peer-uuid", - "ip=" + newURL.Hostname(), - "ni=" + strconv.Itoa(newPort), - }, - }, - }) - close(browseDone) - }() - waitFor(newEntered, "replacement endpoint enrichment") - - telemetryDone := make(chan struct{}) - go func() { - d.refreshTelemetryOnce(context.Background()) - close(telemetryDone) - }() - waitFor(oldEntered, "old endpoint telemetry") - - release(releaseNew) - waitFor(browseDone, "directory update") - replacementTarget := telemetryTargetKey([]string{newURL.Hostname()}, newPort) - if _, ok := d.telemetryRetries.claim("peer-uuid", replacementTarget, time.Now()); ok { - t.Fatal("replacement endpoint started while old endpoint telemetry was in flight") - } - release(releaseOld) - waitFor(telemetryDone, "old endpoint result") - - if _, ok := d.telemetryRetries.claim("peer-uuid", replacementTarget, time.Now()); !ok { - t.Fatal("old endpoint failure backed off the replacement endpoint") - } } - -func TestTelemetryLoopKeepsHealthyNodeOnCadenceWhilePeerIsBlocked(t *testing.T) { - slowStarted := make(chan struct{}, 1) - slow := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { - select { - case slowStarted <- struct{}{}: - default: - } - <-r.Context().Done() - })) - defer slow.Close() - slowURL, err := url.Parse(slow.URL) - if err != nil { - t.Fatalf("parse slow server URL: %v", err) - } - slowPort, err := strconv.Atoi(slowURL.Port()) - if err != nil { - t.Fatalf("parse slow server port: %v", err) - } - - healthyThird := make(chan struct{}, 1) - var healthyCalls atomic.Int32 - healthy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - if healthyCalls.Add(1) >= 3 { - select { - case healthyThird <- struct{}{}: - default: - } +func TestTelemetryLoopDoesNotOverlapNodePolls(t *testing.T) { + h := newTelemetryHarness(t) + var active, maxActive atomic.Int32 + started := make(chan struct{}, 10) + release := make(chan struct{}) + h.peer("node-a", func(*http.Request) NodeInfoResponse { + v := active.Add(1) + if v > maxActive.Load() { + maxActive.Store(v) } - _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-healthy"}) - })) - defer healthy.Close() - healthyURL, err := url.Parse(healthy.URL) - if err != nil { - t.Fatalf("parse healthy server URL: %v", err) - } - healthyPort, err := strconv.Atoi(healthyURL.Port()) - if err != nil { - t.Fatalf("parse healthy server port: %v", err) - } - - d := &daemon{ - dir: newDirectory(), - http: &http.Client{Timeout: time.Second}, - } - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-slow", - IP: slowURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: slowPort}, - }, - }) - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-healthy", - IP: healthyURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: healthyPort}, - }, + started <- struct{}{} + <-release + active.Add(-1) + return NodeInfoResponse{HostUUID: "node-a"} }) - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithCancel(t.Context()) done := make(chan struct{}) - go func() { - d.runTelemetryLoop(ctx, 5*time.Millisecond) - close(done) - }() - + go func() { h.d.runTelemetryLoop(ctx, 10*time.Millisecond); close(done) }() select { - case <-slowStarted: - case <-time.After(time.Second): - t.Fatal("blocked peer telemetry did not start") + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("poll did not start") } select { - case <-healthyThird: - case <-time.After(500 * time.Millisecond): - t.Fatal("blocked peer delayed repeated healthy telemetry polls") + case <-started: + t.Error("overlapping request") + case <-time.After(60 * time.Millisecond): } cancel() - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("telemetry loop did not drain blocked work after cancellation") - } -} - -func TestTelemetryLoopDoesNotOverlapNodePolls(t *testing.T) { - var active atomic.Int32 - var maxActive atomic.Int32 - var calls atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - current := active.Add(1) - defer active.Add(-1) - calls.Add(1) - for { - seen := maxActive.Load() - if current <= seen || maxActive.CompareAndSwap(seen, current) { - break - } - } - select { - case <-time.After(20 * time.Millisecond): - _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "node-a"}) - case <-r.Context().Done(): - } - })) - defer server.Close() - serverURL, err := url.Parse(server.URL) - if err != nil { - t.Fatalf("parse server URL: %v", err) - } - port, err := strconv.Atoi(serverURL.Port()) - if err != nil { - t.Fatalf("parse server port: %v", err) - } - - d := &daemon{ - dir: newDirectory(), - http: server.Client(), - } - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: "node-a", - IP: serverURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, - }) - ctx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond) - defer cancel() - d.runTelemetryLoop(ctx, 5*time.Millisecond) - - if calls.Load() < 2 { - t.Fatalf("telemetry loop made %d calls, want at least two", calls.Load()) - } + close(release) + <-done if maxActive.Load() != 1 { - t.Fatalf("maximum concurrent requests for one node = %d, want 1", maxActive.Load()) + t.Fatalf("concurrent polls=%d", maxActive.Load()) } } - func TestTelemetryLoopBoundsConcurrentWorkAcrossTicks(t *testing.T) { - const nodeCount = telemetryRefreshConcurrency + 1 - entered := make(chan struct{}, nodeCount) - releaseWork := make(chan struct{}) - var releaseOnce sync.Once - var active atomic.Int32 - var maxActive atomic.Int32 - d := &daemon{ - dir: newDirectory(), - http: &http.Client{Timeout: time.Second}, - } - - for i := range nodeCount { - nodeID := "node-" + strconv.Itoa(i) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - current := active.Add(1) - defer active.Add(-1) - for { - seen := maxActive.Load() - if current <= seen || maxActive.CompareAndSwap(seen, current) { - break - } - } - entered <- struct{}{} - <-releaseWork - w.WriteHeader(http.StatusServiceUnavailable) - })) - t.Cleanup(server.Close) - serverURL, err := url.Parse(server.URL) - if err != nil { - t.Fatalf("parse server URL: %v", err) - } - port, err := strconv.Atoi(serverURL.Port()) - if err != nil { - t.Fatalf("parse server port: %v", err) - } - d.dir.upsert(noderec.DirectoryNode{ - HostUUID: nodeID, - IP: serverURL.Hostname(), - Services: map[noderec.ServiceKey]noderec.ServiceStatus{ - noderec.ServiceNodeInfo: {Port: port}, - }, + h := newTelemetryHarness(t) + started := make(chan struct{}, 50) + release := make(chan struct{}) + for i := 0; i < telemetryRefreshConcurrency+2; i++ { + uuid := fmt.Sprintf("node-%d", i) + h.peer(uuid, func(*http.Request) NodeInfoResponse { + started <- struct{}{} + <-release + return NodeInfoResponse{HostUUID: uuid} }) } - t.Cleanup(func() { - releaseOnce.Do(func() { close(releaseWork) }) - }) - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - d.runTelemetryLoop(ctx, 5*time.Millisecond) - close(done) - }() - - for range telemetryRefreshConcurrency { + ctx, cancel := context.WithCancel(t.Context()) + sem := make(chan struct{}, telemetryRefreshConcurrency) + var workers sync.WaitGroup + h.d.dispatchTelemetry(ctx, time.Now(), sem, &workers) + defer func() { cancel(); close(release); workers.Wait() }() + for i := 0; i < telemetryRefreshConcurrency; i++ { select { - case <-entered: - case <-time.After(2 * time.Second): - releaseOnce.Do(func() { close(releaseWork) }) - cancel() - <-done - t.Fatal("telemetry loop did not fill its concurrency allowance") + case <-started: + case <-time.After(3 * time.Second): + t.Fatal("concurrency budget not filled") } } - overflowed := false + h.d.dispatchTelemetry(ctx, time.Now(), sem, &workers) select { - case <-entered: - overflowed = true + case <-started: + t.Fatal("concurrency cap exceeded") case <-time.After(50 * time.Millisecond): } - - releaseOnce.Do(func() { close(releaseWork) }) - if !overflowed { - select { - case <-entered: - case <-time.After(time.Second): - cancel() - <-done - t.Fatal("queued telemetry did not start when capacity became available") - } - } - cancel() - select { - case <-done: - case <-time.After(time.Second): - t.Fatal("telemetry loop did not drain bounded workers after cancellation") - } - - if overflowed { - t.Fatalf("more than %d telemetry requests ran concurrently", telemetryRefreshConcurrency) - } - if got := maxActive.Load(); got != telemetryRefreshConcurrency { - t.Fatalf("maximum concurrent telemetry work = %d, want %d", got, telemetryRefreshConcurrency) - } } diff --git a/services/nvpair-node-scanner/verified_inventory_test.go b/services/nvpair-node-scanner/verified_inventory_test.go new file mode 100644 index 000000000..9e5fbda37 --- /dev/null +++ b/services/nvpair-node-scanner/verified_inventory_test.go @@ -0,0 +1,183 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "context" + "encoding/json" + "net" + "net/http" + "net/http/httptest" + "nvpair-shared/clustertrust" + "nvpair-shared/noderec" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + + "nvpair-shared/clustertrusttest" +) + +// A public inventory and an mDNS principal are claims, even when that principal +// has a pin. Neither proves that the machine serving them possesses its key. +func TestClaimedPinnedPrincipalCannotPublishServiceInventory(t *testing.T) { + const peer = "principal-peer" + dir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, dir, "cluster-test", "principal-self", peer) + d := newMembershipTestDaemon(dir) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/models" { + _ = json.NewEncoder(w).Encode(map[string][]string{"models": {"forged-model"}}) + return + } + _, portString, _ := net.SplitHostPort(r.Host) + port, _ := strconv.Atoi(portString) + principal := peer + _ = json.NewEncoder(w).Encode(NodeInfoResponse{ + HostUUID: peer, ClusterUUID: &principal, TelemetryValid: true, + GPUs: []GPUInfo{{Name: "forged-gpu", UtilizationPercent: gpuReading(uint32(99))}}, + PeerMetadata: &noderec.PeerMetadataParams{HostUUID: peer, Name: "forged-peer", Services: map[noderec.ServiceKey]int{ + noderec.ServiceNodeInfo: port, noderec.ServiceEngineManager: port, noderec.ServiceOllama: 11434, + }}, + }) + })) + defer srv.Close() + d.http = srv.Client() + d.modelsHTTP = srv.Client() + host, port, err := net.SplitHostPort(srv.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: RawNode{ID: "forged-peer", Addresses: []string{host}, TXT: []string{"v=1", "uuid=" + peer, "cluster-uuid=" + peer, "ip=" + host, "ni=" + port, "em=" + port, "ol=11434"}}}) + if n, ok := d.dir.get(peer); ok && (n.Trusted || len(n.Models) > 0 || len(n.ModelsByEngine) > 0) { + t.Errorf("unauthenticated claim reached trusted/routing inventory: %+v", n) + } + p, _ := strconv.Atoi(port) + if d.refreshNodeTelemetryCandidates(t.Context(), peer, []string{host}, p) { + t.Error("plaintext inventory accepted as scheduler telemetry") + } +} + +// secureInventoryFixture uses real mutually pinned cluster leaves and listeners; +// no synthetic TLS state or mock transport can accidentally confer provenance. +func secureInventoryFixture(t *testing.T, principal string) (*daemon, *httptest.Server, *atomic.Bool) { + t.Helper() + selfDir, peerDir := t.TempDir(), t.TempDir() + clustertrusttest.WriteKeypair(t, selfDir, "uuid-self") + clustertrusttest.WriteKeypair(t, peerDir, principal) + clustertrusttest.WriteAdmission(t, selfDir, "cluster-test", 1) + clustertrusttest.WriteAdmission(t, peerDir, "cluster-test", 1) + writePinFromCert(t, selfDir, principal, filepath.Join(peerDir, "node.crt")) + writePinFromCert(t, peerDir, "uuid-self", filepath.Join(selfDir, "node.crt")) + mesh := clustertrust.Open(peerDir) + d := newSelfTestDaemon("uuid-self", "192.0.2.1") + d.mesh = clustertrust.Open(selfDir) + d.clusterDir = selfDir + d.modelsClients = clustertrust.NewPeerClientPool(d.mesh, modelsFetchTimeout) + mismatch := &atomic.Bool{} + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := mesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", 403) + return + } + if r.URL.Path == "/v1/models" { + _ = json.NewEncoder(w).Encode(map[string][]string{"models": {"authenticated-model"}}) + return + } + if r.URL.Path == "/v1/engine-stats" { + body := engineNativeBody() + body["observedAt"] = 1000 + _ = json.NewEncoder(w).Encode(body) + return + } + _, p, _ := net.SplitHostPort(r.Host) + port, _ := strconv.Atoi(p) + host := principal + if mismatch.Load() { + host = "other-peer" + } + info := NodeInfoResponse{HostUUID: host, ClusterUUID: &principal, TelemetryValid: true, SampleSeq: 12, GPUs: []GPUInfo{{Name: "authenticated-gpu", UtilizationPercent: gpuReading(uint32(43))}}, PeerMetadata: &noderec.PeerMetadataParams{HostUUID: host, Name: "authenticated-name", Services: map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port, noderec.ServiceEngineManager: port, noderec.ServiceOllama: 11434}}} + _ = json.NewEncoder(w).Encode(info) + })) + srv.TLS = mesh.ServerTLSConfig() + srv.StartTLS() + t.Cleanup(srv.Close) + t.Cleanup(func() { + d.modelsClients.CloseIdle() + if d.peerInfoClients != nil { + d.peerInfoClients.CloseIdle() + } + }) + return d, srv, mismatch +} + +func TestPublicPreviewHasIndependentKeyAndCannotRemoveVerifiedPeer(t *testing.T) { + d, srv, _ := secureInventoryFixture(t, "uuid-peer") + host, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + genuine := RawNode{ID: "genuine", Addresses: []string{host}, TXT: []string{"v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=" + host, "ni=" + p}} + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: genuine}) + before, exists := d.dir.get("uuid-peer") + if !exists || !before.ServicePeer() { + t.Fatal("verified fixture not promoted") + } + public := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "uuid-peer", GPUs: []GPUInfo{{Name: "public-preview"}}}) + })) + defer public.Close() + ph, pp, _ := net.SplitHostPort(public.Listener.Addr().String()) + hint := RawNode{ID: "forged-preview", Addresses: []string{ph}, TXT: []string{"v=1", "uuid=uuid-peer", "ip=" + ph, "ni=" + pp, "ol=1"}} + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: hint}) + key := noderec.PreviewRecordID(hint.ID, "uuid-peer") + preview, exists := d.dir.get(key) + if !exists || preview.RecordKey() != key || preview.IdentityVerification != "unverified" || preview.Trusted { + t.Fatalf("preview not independently keyed: %+v", preview) + } + if after, _ := d.dir.get("uuid-peer"); after.PeerRecordID != before.PeerRecordID || !after.ServicePeer() { + t.Fatal("preview displaced verified identity") + } + d.onBrowse(DiscoveryEvent{Type: "removed", Node: hint}) + if _, exists := d.dir.get(key); exists { + t.Fatal("preview removal left its own row") + } + if after, _ := d.dir.get("uuid-peer"); after.PeerRecordID != before.PeerRecordID || !after.ServicePeer() { + t.Fatal("preview removal withdrew verified inventory") + } +} + +func TestPinnedInventoryPromotesOnlyAuthenticatedMetadata(t *testing.T) { + d, srv, mismatch := secureInventoryFixture(t, "uuid-peer") + host, p, _ := net.SplitHostPort(srv.Listener.Addr().String()) + port, _ := strconv.Atoi(p) + raw := RawNode{ID: "forged-name", Addresses: []string{host}, TXT: []string{"v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=" + host, "ni=" + p, "em=1", "ol=1"}} + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: raw}) + n, ok := d.dir.get("uuid-peer") + if !ok || !n.ServicePeer() || n.PeerRecordID == "" || n.Name != "authenticated-name" || n.Services[noderec.ServiceOllama].Port != 11434 || len(n.Models) != 1 || n.Models[0] != "authenticated-model" { + t.Fatalf("authenticated projection = %+v", n) + } + var out bytes.Buffer + d.codec = NewCodec(&out) + if !d.refreshNodeTelemetryCandidates(context.Background(), n.HostUUID, []string{host}, port) { + t.Fatal("pinned telemetry rejected") + } + var message Message + _ = json.Unmarshal(out.Bytes(), &message) + var telemetry noderec.NodeTelemetry + _ = json.Unmarshal(message.Params, &telemetry) + if !telemetry.TelemetryValid || telemetry.PeerRecordID != n.PeerRecordID || telemetry.GPUUtilizationPct != 43 { + t.Fatalf("authenticated telemetry = %+v", telemetry) + } + mismatch.Store(true) + if d.refreshNodeTelemetryCandidates(context.Background(), n.HostUUID, []string{host}, port) { + t.Fatal("mismatched authenticated response identity accepted") + } + mismatch.Store(false) + // Same certificate, different local admission: old proofs are not reusable. + clustertrusttest.WriteAdmission(t, d.clusterDir, "cluster-test", 2) + d.reloadTrust() + n, _ = d.dir.get("uuid-peer") + if n.Trusted || n.ServicePeer() || len(n.Models) > 0 || len(n.GPUs) > 0 { + t.Fatalf("admission change retained inventory: %+v", n) + } +} From 3cbf85ee8fc8c76ab9c1ac1cb21f967109d1885f Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:38:56 +0200 Subject: [PATCH 04/18] fix(manual-nodes): require pinned proof for PAIR peer inventory Read PAIR identity, models, engines and telemetry only over pinned cluster mTLS. Keep explicitly configured bare engines as a separately labelled manual bypass, and prevent a recognized PAIR configuration from downgrading during its lifetime in the worker. Preserve independent manual configuration identities. Adapt NVIDIA/Personal-AI-Router#179; manual-entry kind persistence across worker restarts remains follow-up work. Co-authored-by: Kaylee Lubick Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-manual-nodes/README.md | 10 +- .../nvpair-manual-nodes/enginenative_test.go | 27 +- services/nvpair-manual-nodes/gpu_test.go | 15 +- .../inventory_fixture_test.go | 154 +++++++ services/nvpair-manual-nodes/manager.go | 425 +++++------------- services/nvpair-manual-nodes/manager_test.go | 386 ++++++++-------- .../nvpair-manual-nodes/peer_inventory.go | 98 ++++ .../response_bounds_test.go | 2 +- .../verified_inventory_test.go | 67 +++ 9 files changed, 657 insertions(+), 527 deletions(-) create mode 100644 services/nvpair-manual-nodes/inventory_fixture_test.go create mode 100644 services/nvpair-manual-nodes/peer_inventory.go create mode 100644 services/nvpair-manual-nodes/verified_inventory_test.go diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index 9e5cdbf40..76d11709d 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -30,7 +30,7 @@ Uses bidirectional newline-delimited JSON-RPC 2.0. By default, communication is | `--client-cert ` | _(none)_ | PEM client certificate to present when probing TLS-enabled manual nodes (requires `--client-key`) | | `--client-key ` | _(none)_ | PEM client private key matching `--client-cert` | | `--ca-bundle ` | _(none)_ | PEM bundle of CAs to trust when verifying server certificates (additive to the system trust store) | -| `--cluster-dir ` | _(none)_ | Cluster config dir; when set, a TLS manual node's node-info is probed over cluster mTLS, presenting this node's leaf and accepting any currently-pinned server cert | +| `--cluster-dir ` | _(none)_ | Cluster config dir for pinned mTLS peer inventory. A manual peer is discovered through a pinned certificate, then re-read against that exact principal before publishing its proof | | `--log-level ` | `info` | `error` / `warn` / `info` / `debug`; falls back to `$NVPAIR_LOG_LEVEL`. Also changeable at runtime via the `log/set-level` method | | `--version` | | Print version and exit | @@ -82,7 +82,9 @@ Emitted when a manually added node has been probed and its initial status determ } ``` -Each node is probed for inference engines: Ollama on its default `:11434` (`GET /` + `/api/tags`), LM Studio on its default `:1234` (`GET /v1/models`), and oMLX on its default `:1236` (OpenAI-compatible `/v1/models` or loopback-only 403 proxy identification). Nodes running `nvpair-engine-manager` on `:14322` are also probed for authoritative multi-engine inventory. `lmstudio_*` and `omlx_*` mirror the `ollama_*` fields and let a supervising broker bridge the node into `lmstudio-proxy` and `omlx-proxy` the same way it bridges Ollama into `ollama-proxy`. A node can run any engine combination. +PAIR peers must first prove identity through pinned cluster mTLS node-info on `:14318` (or the configured `tls_port`). Matching certificate, response and broker-owned `peerMetadata` identities are required. Only authenticated metadata supplies service ports; models and engine state are then read over pinned mTLS with no HTTP or operator-CA downgrade. Completed reads recheck the exact current pin and admission-bound `peerRecordId`. + +Explicitly user-added bare engines retain independent manual routes: Ollama on `:11434`, LM Studio on `:1234`, and oMLX on `:1236`. They remain labelled `identityVerification: "manual"`, never acquire peer trust, and provide no hardware scheduling telemetry. Recognized PAIR configurations that cannot authenticate are labelled `unverified`, not silently downgraded into this bypass. The broker namespaces manual configurations separately from authenticated host UUIDs. Recognition lasts only for the configuration's lifetime in this worker: kind is not yet persisted in `manual-nodes.json`. After a worker restart, an address previously added for a PAIR peer can be routed as an unauthenticated plaintext bare engine if no PAIR endpoint answers there (including if another machine answers instead). Persisting PAIR-vs-bare-engine intent is follow-up work. `engineNative` is the remote engine's own report of itself — the counters, the rates and the active models the node's engine-manager publishes at `GET /v1/engine-stats`. It is read over the same transport as the model list beside it, because it is the same kind of data: a peer's engine counters are cluster data, so a clustered peer is read over cluster mTLS pinned to the principal the node advertises, and there is no weaker fallback for a peer. The field is absent when the node publishes no view — an engine with no admin API, an engine that could not be reached, or a node this one cannot read. A node whose view could not be re-read keeps the view it had (the last-good rule the node-scanner follows for a discovered node). It is read on a slower cadence than the probe (once a minute), because the engine's counters move slowly and every reading is a fresh observation. @@ -142,7 +144,7 @@ A hostname is preferred over an IP literal: probe clients disable keep-alives sp |---|---|---| | `address` | Yes | IP address or hostname of the node, with no port | | `name` | No | Friendly name (used as node ID; defaults to `manual:
`) | -| `tls_port` | No | Probe node-info over HTTPS on this port instead of plain HTTP on `14318`. Echoed back as `tls_enabled` | +| `tls_port` | No | Probe pinned cluster mTLS node-info on this port instead of the primary `14318`. Echoed back as `tls_enabled`; no plaintext downgrade | | `mtls` | No | Stored and echoed back as `mtls_required`. The probe transport itself is chosen by `tls_port` and live cluster membership, so this field records intent rather than driving it | Response: the initial node status object. @@ -200,7 +202,7 @@ Each manual node is probed every 10 seconds, with a 3-second timeout per leg, fo - **Ollama** on port 11434: health check (`GET /`) and model list (`GET /api/tags`) - **LM Studio** on port 1234: `GET /v1/models`, which doubles as the liveness check and the model list -- **Node Info** on port 14318, or `tls_port` over HTTPS: hardware inventory and identity (`GET /v1/node-info`) +- **Node Info** on port 14318, or `tls_port`: pinned cluster mTLS hardware inventory, matching identity and authenticated service metadata (`GET /v1/node-info`). Public responses cannot establish identity or scheduling measurements. A node can have any combination of these, or none if the target is unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. diff --git a/services/nvpair-manual-nodes/enginenative_test.go b/services/nvpair-manual-nodes/enginenative_test.go index 4f4d5d3d6..0ced71075 100644 --- a/services/nvpair-manual-nodes/enginenative_test.go +++ b/services/nvpair-manual-nodes/enginenative_test.go @@ -18,6 +18,7 @@ import ( "nvpair-shared/clustertrust" "nvpair-shared/clustertrusttest" "nvpair-shared/enginenative" + "nvpair-shared/noderec" ) // The engine-native half of a manual node: the engine's own report of itself, @@ -147,8 +148,9 @@ func startPeerEngineManager(t *testing.T, selfDir string, body map[string]any) ( }) ts := httptest.NewUnstartedServer(mux) ts.TLS = peerMesh.ServerTLSConfig() - ts.StartTLS() - t.Cleanup(ts.Close) + ts.Config.ReadTimeout = 2 * time.Second + ts.Config.WriteTimeout = 2 * time.Second + t.Cleanup(func() { ts.CloseClientConnections(); ts.Close() }) _, portStr, err := net.SplitHostPort(ts.Listener.Addr().String()) if err != nil { @@ -158,6 +160,20 @@ func startPeerEngineManager(t *testing.T, selfDir string, body map[string]any) ( if err != nil { t.Fatalf("parse listener port: %v", err) } + mux.HandleFunc("/v1/node-info", func(w http.ResponseWriter, r *http.Request) { + if _, ok := peerMesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + principal := "uuid-peer" + _ = json.NewEncoder(w).Encode(NodeInfoResponse{ + HostUUID: principal, ClusterUUID: &principal, + PeerMetadata: &noderec.PeerMetadataParams{HostUUID: principal, Services: map[noderec.ServiceKey]int{ + noderec.ServiceNodeInfo: port, noderec.ServiceEngineManager: port, noderec.ServiceOMLX: 1236, + }}, + }) + }) + ts.StartTLS() // "localhost" is not a loopback IP literal, so the read is a peer's read // and not this node's own engine-manager. return "localhost", port, stub @@ -173,6 +189,7 @@ func newClusteredTestManager(t *testing.T, selfDir string, emPort int) *Manager if err != nil { t.Fatalf("NewManager: %v", err) } + bindTestManager(t, m) m.emPort = emPort // The plain-HTTP and operator-TLS fallbacks must not answer: a peer is // read over cluster mTLS, and a weaker transport is not a view. @@ -237,7 +254,7 @@ func TestManualNodeCarriesThePeerEngineNativeView(t *testing.T) { host, port, stub := startPeerEngineManager(t, selfDir, engineNativeBody()) m := newClusteredTestManager(t, selfDir, port) - entry := ManualEntry{Address: host, Name: "peer"} + entry := ManualEntry{Address: host, Name: "peer", TLSPort: port, MTLS: true} m.addNode(entry) status := waitForProbedNode(t, m, nodeID(entry)) @@ -281,7 +298,7 @@ func TestManualNodeWithNoEngineViewCarriesNone(t *testing.T) { host, port, _ := startPeerEngineManager(t, selfDir, map[string]any{"engine": "omlx", "supported": false}) m := newClusteredTestManager(t, selfDir, port) - entry := ManualEntry{Address: host, Name: "peer"} + entry := ManualEntry{Address: host, Name: "peer", TLSPort: port, MTLS: true} m.addNode(entry) status := waitForProbedNode(t, m, nodeID(entry)) @@ -298,7 +315,7 @@ func TestManualNodeKeepsTheLastGoodEngineView(t *testing.T) { host, port, stub := startPeerEngineManager(t, selfDir, engineNativeBody()) m := newClusteredTestManager(t, selfDir, port) - entry := ManualEntry{Address: host, Name: "peer"} + entry := ManualEntry{Address: host, Name: "peer", TLSPort: port, MTLS: true} m.addNode(entry) status := waitForProbedNode(t, m, nodeID(entry)) diff --git a/services/nvpair-manual-nodes/gpu_test.go b/services/nvpair-manual-nodes/gpu_test.go index a66831f80..054e4539c 100644 --- a/services/nvpair-manual-nodes/gpu_test.go +++ b/services/nvpair-manual-nodes/gpu_test.go @@ -4,7 +4,6 @@ package main import ( - "context" "encoding/json" "net/http" "reflect" @@ -17,9 +16,9 @@ func TestManualGPUInventoryCarriesWireFields(t *testing.T) { {"legacy", `[{"name":"Old","vram_bytes":8589934592}]`, `[{"name":"Old","memory_kind":"unavailable","vram_bytes":8589934592}]`}, } { t.Run(tc.name, func(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "gpu-node", Address: "node.local"} - m.nodes[entry.Name] = &trackedNode{entry: entry, status: ManualNodeStatus{ID: entry.Name, Address: entry.Address}} + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "gpu-peer") + entry := peer.track(m, "gpu-node") rt.set(http.MethodGet, "node.local:14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { return httpJSON(http.StatusOK, `{"GPUs":`+tc.gpus+`}`) }) @@ -57,9 +56,9 @@ func TestManualTelemetryMixedMeasuredGPUReadings(t *testing.T) { {"unknown", `[{"name":"Unknown"}]`, false, 0}, } { t.Run(tc.name, func(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "gpu-node", Address: "node.local"} - m.nodes[entry.Name] = &trackedNode{entry: entry, status: ManualNodeStatus{ID: entry.Name, Address: entry.Address}} + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "gpu-peer") + entry := peer.track(m, "gpu-node") rt.set(http.MethodGet, "node.local:14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { data, err := json.Marshal(struct { GPUs json.RawMessage `json:"GPUs"` @@ -70,7 +69,7 @@ func TestManualTelemetryMixedMeasuredGPUReadings(t *testing.T) { } return httpJSON(http.StatusOK, string(data)) }) - m.probeNodeTelemetry(context.Background(), entry.Name, entry) + m.probeNodeTelemetry(m.ctx, entry.Name, entry) msg := readCaptureUntil(t, rw, methodIs("node/telemetry")) got := decodeParams[NodeTelemetryNotification](t, msg) if got.GPUUtilizationPct != tc.want || got.TelemetryValid != tc.valid { diff --git a/services/nvpair-manual-nodes/inventory_fixture_test.go b/services/nvpair-manual-nodes/inventory_fixture_test.go new file mode 100644 index 000000000..5394725e0 --- /dev/null +++ b/services/nvpair-manual-nodes/inventory_fixture_test.go @@ -0,0 +1,154 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "io" + "net" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "sync/atomic" + "testing" + "time" + + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" +) + +func manualTestContext(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + t.Cleanup(cancel) + return ctx +} + +func bindTestManager(t *testing.T, m *Manager) { + t.Helper() + m.cancel() + m.ctx, m.cancel = context.WithTimeout(t.Context(), 10*time.Second) + t.Cleanup(func() { + m.cancel() + m.mu.Lock() + for _, node := range m.nodes { + if node.cancelTelemetry != nil { + node.cancelTelemetry() + } + } + m.mu.Unlock() + m.closeIdleTelemetry() + }) +} + +// inventoryPeer serves the fixture's payloads only after checking the caller's +// pin. Node-info advertises this same authenticated endpoint for both services. +// Payloads are supplied through rt so tests can change availability between reads. +type inventoryPeer struct { + entry ManualEntry + server *httptest.Server + principal string + polls chan string + newConns atomic.Int32 +} + +func newInventoryPeer(t *testing.T, m *Manager, rt *fakeRoundTripper, alias, principal string) *inventoryPeer { + t.Helper() + selfDir := newClusterSelfDir(t) + peerDir := t.TempDir() + clustertrusttest.WriteKeypair(t, peerDir, principal) + clustertrusttest.WriteAdmission(t, peerDir, "cluster-1", 1) + writePinFromCert(t, selfDir, principal, filepath.Join(peerDir, "node.crt")) + writePinFromCert(t, peerDir, "uuid-self", filepath.Join(selfDir, "node.crt")) + m.mesh = clustertrust.Open(selfDir) + peerMesh := clustertrust.Open(peerDir) + p := &inventoryPeer{principal: principal, polls: make(chan string, 64)} + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := peerMesh.VerifyClientPin(r); !ok { + http.Error(w, "forbidden", http.StatusForbidden) + return + } + port := "14322" + if r.URL.Path == "/v1/node-info" { + port = "14318" + select { + case p.polls <- r.Header.Get("X-NVPAIR-Poll"): + default: + } + } + req := r.Clone(r.Context()) + req.URL.Host = net.JoinHostPort(alias, port) + resp, err := rt.RoundTrip(req) + if err != nil { + http.Error(w, "fixture unavailable", http.StatusServiceUnavailable) + return + } + defer resp.Body.Close() + if r.URL.Path == "/v1/node-info" && resp.StatusCode == http.StatusOK { + var info NodeInfoResponse + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + http.Error(w, "invalid fixture", http.StatusBadGateway) + return + } + info.HostUUID = principal + info.ClusterUUID = &p.principal + info.PeerMetadata = &noderec.PeerMetadataParams{HostUUID: principal, Services: map[noderec.ServiceKey]int{ + noderec.ServiceNodeInfo: p.entry.TLSPort, + noderec.ServiceEngineManager: p.entry.TLSPort, + noderec.ServiceOllama: 11434, + noderec.ServiceLMStudio: 1234, + noderec.ServiceOMLX: 1236, + }} + _ = json.NewEncoder(w).Encode(info) + return + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + srv.TLS = peerMesh.ServerTLSConfig() + srv.Config.ReadTimeout = 2 * time.Second + srv.Config.WriteTimeout = 2 * time.Second + _, portString, err := net.SplitHostPort(srv.Listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(portString) + if err != nil { + t.Fatal(err) + } + p.entry = ManualEntry{Address: "127.0.0.1", TLSPort: port, MTLS: true} + p.server = srv + srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateNew { + p.newConns.Add(1) + } + } + srv.StartTLS() + t.Cleanup(func() { srv.CloseClientConnections(); srv.Close() }) + return p +} + +func (p *inventoryPeer) track(m *Manager, name string) ManualEntry { + entry := p.entry + entry.Name = name + m.nodes[nodeID(entry)] = &trackedNode{entry: entry, status: ManualNodeStatus{ + ID: nodeID(entry), Address: entry.Address, ClusterUUID: p.principal, + HostUUID: p.principal, IdentityVerification: "verified", PeerRecordID: m.mesh.PeerRecordID(p.principal), + }} + return entry +} + +func receivePoll(t *testing.T, ctx context.Context, polls <-chan string) string { + t.Helper() + select { + case poll := <-polls: + return poll + case <-ctx.Done(): + t.Fatal("timed out waiting for poll intent") + return "" + } +} diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 7ccf25e43..0769ef042 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -82,6 +82,7 @@ const ( ) type NodeTelemetryNotification struct { + PeerRecordID string `json:"peerRecordId,omitempty"` ID string `json:"id"` HostUUID string `json:"hostUuid,omitempty"` GPUUtilizationPct uint32 `json:"gpuUtilizationPct"` @@ -143,12 +144,14 @@ type MemoryInfo struct { } type NodeInfoResponse struct { - GPUs []GPUInfo `json:"GPUs"` - CPU *CPUInfo `json:"cpu,omitempty"` - Memory *MemoryInfo `json:"memory,omitempty"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` - SampleSeq uint64 `json:"sampleSeq,omitempty"` + PeerMetadata *noderec.PeerMetadataParams `json:"peerMetadata,omitempty"` + PeerRecordID string `json:"-"` + GPUs []GPUInfo `json:"GPUs"` + CPU *CPUInfo `json:"cpu,omitempty"` + Memory *MemoryInfo `json:"memory,omitempty"` + TelemetryValid bool `json:"telemetryValid"` + MSSince int64 `json:"msSince"` + SampleSeq uint64 `json:"sampleSeq,omitempty"` // HostUUID is the remote node's stable per-host identity, reported by // node-info. We surface it so a manual node is tracked by the same permanent // identity as the rest of the fleet — including deduping with the same @@ -182,12 +185,15 @@ type ManualEntry struct { // 14318 otherwise) so existing UI that displays the port stays // correct. type ManualNodeStatus struct { - ID string `json:"id"` - Name string `json:"name,omitempty"` - Address string `json:"address"` - OllamaUp bool `json:"ollama_up"` - OllamaPort int `json:"ollama_port"` - OllamaModels []string `json:"ollama_models,omitempty"` + IdentityVerification string `json:"identityVerification,omitempty"` + PeerRecordID string `json:"peerRecordId,omitempty"` + PeerMetadata *noderec.PeerMetadataParams `json:"peerMetadata,omitempty"` + ID string `json:"id"` + Name string `json:"name,omitempty"` + Address string `json:"address"` + OllamaUp bool `json:"ollama_up"` + OllamaPort int `json:"ollama_port"` + OllamaModels []string `json:"ollama_models,omitempty"` // LM Studio is probed on its default OpenAI-API port the same way Ollama // is on 11434, so a manually-added node running LM Studio can be bridged // into lmstudio-proxy by a supervising broker. @@ -239,6 +245,7 @@ type trackedNode struct { entry ManualEntry status ManualNodeStatus fromCluster bool + peerSeen bool // consecutiveFails counts back-to-back probes where neither // service answered (OllamaUp && NodeInfoUp both false). Reset @@ -298,7 +305,9 @@ type Manager struct { // node is probed over cluster mTLS (presenting our leaf, accepting any // currently-pinned server cert) since a clustered peer's node-info is // pin-gated and serves no plaintext listener. - mesh *clustertrust.Mesh + mesh *clustertrust.Mesh + inventoryPoolOnce sync.Once + inventoryPool *clustertrust.PeerClientPool // emPort is the port a remote node's engine-manager surface is probed on. // It is engineManagerPort in production; a test points it at an ephemeral @@ -379,6 +388,8 @@ func (m *Manager) getTelemetryTLSConfigAnyClient() *http.Client { } func (m *Manager) closeIdleTelemetry() { + m.inventoryPoolOnce.Do(func() { m.inventoryPool = clustertrust.NewPeerClientPool(m.mesh, probeTimeout) }) + m.inventoryPool.CloseIdle() m.mu.Lock() if m.telemetryAnyTransport != nil { m.telemetryAnyTransport.CloseIdleConnections() @@ -671,58 +682,33 @@ func (m *Manager) probeNode(entry ManualEntry) { addr := entry.Address id := nodeID(entry) - // Pick scheme + port + client based on the entry's TLS hint. - // The operator decides which scheme this manual node uses; we - // don't probe both. TLSPort > 0 means HTTPS on that port via - // the TLS client (which carries the operator's client cert, - // if configured). Otherwise it's plain HTTP on the historical - // 14318. - scheme := "http" - nodeInfoPort := 14318 - probeClient := m.client - if entry.TLSPort > 0 { - scheme = "https" - nodeInfoPort = entry.TLSPort - probeClient = m.tlsClient - // Clustered: a TLS manual node is a cluster peer whose node-info is - // pin-gated mTLS with no plaintext listener. Dial it with our cluster - // leaf, accepting any currently-pinned server cert (a manual node has no - // cluster-uuid= TXT to key a specific pin on). Refresh first so a cluster - // joined, or a peer paired, after startup is seen; falls back to the BYO - // tlsClient while unclustered. - m.mesh.Refresh() - if cfg, ok := m.mesh.ClientTLSConfigAny(); ok { - probeClient = &http.Client{Timeout: probeTimeout, Transport: &http.Transport{TLSClientConfig: cfg, DisableKeepAlives: true}} - } + m.mu.RLock() + tracked, exists := m.nodes[id] + if !exists || tracked.entry != entry { + m.mu.RUnlock() + return } - nodeInfoUp, info := m.probeNodeInfo(probeClient, scheme, addr, nodeInfoPort) + prior := tracked.status + peerSeen := tracked.peerSeen || tracked.fromCluster || prior.PeerRecordID != "" + m.mu.RUnlock() + // Use the request start as the age anchor so transport time cannot make a + // remotely sampled observation look fresher than it is. nodeInfoReceived := m.clockNow() - if !nodeInfoUp && entry.TLSPort == 0 && m.mesh != nil && m.mesh.Clustered() { - m.mesh.Refresh() - if cfg, ok := m.mesh.ClientTLSConfigAny(); ok { - tlsClient := &http.Client{Timeout: probeTimeout, Transport: &http.Transport{TLSClientConfig: cfg, DisableKeepAlives: true}} - for _, port := range []int{14319, 14318} { - if up, tlsInfo := m.probeNodeInfo(tlsClient, "https", addr, port); up { - nodeInfoUp = true - info = tlsInfo - nodeInfoPort = port - nodeInfoReceived = m.clockNow() - break - } - } - } - } - - var clusterUUID string - if info.ClusterUUID != nil { + info, nodeInfoPort, nodeInfoUp := m.authenticatedNodeInfo(m.ctx, entry, prior.ClusterUUID) + clusterUUID := "" + emPort := 0 + if nodeInfoUp { clusterUUID = *info.ClusterUUID + emPort = info.PeerMetadata.Services[noderec.ServiceEngineManager] + peerSeen = true } - if clusterUUID == "" && m.mesh != nil && info.HostUUID != "" && m.mesh.HasPin(info.HostUUID) { - clusterUUID = info.HostUUID + if !peerSeen && entry.TLSPort == 0 && !entry.MTLS { + // Public PAIR presence can only deny the bare-engine bypass, never + // establish an identity or supply service inventory. + publicUp, _ := m.probeNodeInfo(m.client, "http", addr, 14318) + peerSeen = publicUp } - - useTLS := entry.TLSPort > 0 || (nodeInfoUp && scheme == "https") - emUp, emRes, engineNative := m.probeEngineManager(addr, clusterUUID, useTLS, m.engineNativeDue(id)) + emUp, emRes, engineNative := m.probePeerEngineManager(m.ctx, addr, clusterUUID, info.PeerRecordID, emPort, m.engineNativeDue(id)) finalModels := emRes.Models finalModelsByEngine := emRes.ModelsByEngine @@ -731,7 +717,7 @@ func (m *Manager) probeNode(entry ManualEntry) { // Plaintext engine probes are only valid for unclustered/non-mTLS nodes; // clustered peers reject non-loopback plaintext requests on their proxy // ports and publish their authoritative inventory via Engine Manager (:14322). - canProbePlaintextEngines := !entry.MTLS && !useTLS && clusterUUID == "" + canProbePlaintextEngines := !peerSeen && !entry.MTLS && entry.TLSPort == 0 var ( ollamaUp, lmStudioUp, omlxUp bool @@ -826,45 +812,71 @@ func (m *Manager) probeNode(entry ManualEntry) { } } + verification := "manual" + if peerSeen { + verification = "unverified" + } + if nodeInfoUp { + verification = "verified" + } newStatus := ManualNodeStatus{ - ID: id, - Name: entry.Name, - Address: addr, - OllamaUp: ollamaUp, - OllamaPort: 11434, - OllamaModels: ollamaModels, - LMStudioUp: lmStudioUp, - LMStudioPort: lmStudioPort, - LMStudioModels: lmStudioModels, - OMLXUp: omlxUp, - OMLXPort: omlxPort, - OMLXModels: omlxModels, - NodeInfoUp: nodeInfoUp, - NodeInfoPort: nodeInfoPort, - TLSEnabled: entry.TLSPort > 0, - MTLSRequired: entry.TLSPort > 0 && entry.MTLS, - GPUs: info.GPUs, - CPU: info.CPU, - Memory: info.Memory, - TelemetryValid: info.TelemetryValid, - MSSince: msSince, - SampleSeq: info.SampleSeq, - HostUUID: info.HostUUID, - ClusterUUID: clusterUUID, - Models: finalModels, - ModelsByEngine: finalModelsByEngine, - LoadedByEngine: finalLoadedByEngine, - EngineNative: engineNative, + IdentityVerification: verification, + PeerRecordID: info.PeerRecordID, + PeerMetadata: info.PeerMetadata, + ID: id, + Name: entry.Name, + Address: addr, + OllamaUp: ollamaUp, + OllamaPort: 11434, + OllamaModels: ollamaModels, + LMStudioUp: lmStudioUp, + LMStudioPort: lmStudioPort, + LMStudioModels: lmStudioModels, + OMLXUp: omlxUp, + OMLXPort: omlxPort, + OMLXModels: omlxModels, + NodeInfoUp: nodeInfoUp, + NodeInfoPort: nodeInfoPort, + TLSEnabled: entry.TLSPort > 0, + MTLSRequired: entry.TLSPort > 0 && entry.MTLS, + GPUs: info.GPUs, + CPU: info.CPU, + Memory: info.Memory, + TelemetryValid: info.TelemetryValid, + MSSince: msSince, + SampleSeq: info.SampleSeq, + HostUUID: info.HostUUID, + ClusterUUID: clusterUUID, + Models: finalModels, + ModelsByEngine: finalModelsByEngine, + LoadedByEngine: finalLoadedByEngine, + EngineNative: engineNative, + } + + if nodeInfoUp { + newStatus.OllamaPort = info.PeerMetadata.Services[noderec.ServiceOllama] + newStatus.LMStudioPort = info.PeerMetadata.Services[noderec.ServiceLMStudio] + newStatus.OMLXPort = info.PeerMetadata.Services[noderec.ServiceOMLX] + newStatus.OllamaUp = newStatus.OllamaUp && newStatus.OllamaPort > 0 + newStatus.LMStudioUp = newStatus.LMStudioUp && newStatus.LMStudioPort > 0 + newStatus.OMLXUp = newStatus.OMLXUp && newStatus.OMLXPort > 0 } - reachable := newStatus.OllamaUp || newStatus.LMStudioUp || newStatus.OMLXUp || newStatus.NodeInfoUp || emUp m.mu.Lock() tn, exists := m.nodes[id] - if !exists { + if !exists || tn != tracked || tn.entry != entry { m.mu.Unlock() return } + if newStatus.PeerRecordID != "" { + m.mesh.Refresh() + if newStatus.PeerRecordID != m.mesh.PeerRecordID(newStatus.ClusterUUID) { + m.mu.Unlock() + return + } + } + tn.peerSeen = peerSeen prev := tn.status prevFails := tn.consecutiveFails @@ -872,14 +884,8 @@ func (m *Manager) probeNode(entry ManualEntry) { // so a transient node-info blip (while Ollama stays reachable) doesn't rekey // a live node UUID -> manual-id -> UUID — the broker keys the discovery store // by HostUUID. - if !nodeInfoUp && newStatus.HostUUID == "" { - newStatus.HostUUID = prev.HostUUID - } - if newStatus.ClusterUUID == "" { - newStatus.ClusterUUID = prev.ClusterUUID - } - if newStatus.ClusterUUID == "" && m.mesh != nil && newStatus.HostUUID != "" && m.mesh.HasPin(newStatus.HostUUID) { - newStatus.ClusterUUID = newStatus.HostUUID + if newStatus.PeerRecordID != prev.PeerRecordID { + tn.engineNative = nil } // A node with no view keeps the one it had: the last-good rule the // node-scanner's engine-native sweep follows too. A node that never @@ -901,7 +907,7 @@ func (m *Manager) probeNode(entry ManualEntry) { curFails := tn.consecutiveFails m.mu.Unlock() - changed := prev.OllamaUp != newStatus.OllamaUp || + changed := prev.IdentityVerification != newStatus.IdentityVerification || prev.PeerRecordID != newStatus.PeerRecordID || prev.OllamaUp != newStatus.OllamaUp || prev.LMStudioUp != newStatus.LMStudioUp || prev.OMLXUp != newStatus.OMLXUp || prev.OMLXPort != newStatus.OMLXPort || @@ -990,153 +996,6 @@ type engineManagerModels struct { LoadedByEngine map[string][]string `json:"loadedByEngine"` } -// probeEngineManager checks nvpair-engine-manager's LAN /v1/models endpoint on addr:14322. -// When reachable, it provides the authoritative model list, per-engine attribution, and -// loaded-in-memory models across all supported engines (Ollama, LM Studio, oMLX) in a -// single sweep. Tries cluster mTLS first if clustered, then configured TLS client, then HTTP. -// -// When the caller asks for it, the engine's own view of itself is read as well, -// over the same transport the model list was just read over: the engine's -// counters are cluster data exactly like the model inventory, so a peer is read -// over cluster mTLS pinned to the principal it advertises, and the view is -// never read over a weaker transport than the list beside it. -func (m *Manager) probeEngineManager(addr, clusterUUID string, useTLS bool, wantEngineNative bool) (bool, engineManagerModels, *enginenative.Stats) { - if m.mesh != nil && m.mesh.Clustered() { - m.mesh.Refresh() - var cfg *tls.Config - var ok bool - if clusterUUID != "" && m.mesh.HasPin(clusterUUID) { - cfg, ok = m.mesh.ClientTLSConfig(clusterUUID) - } - if !ok { - cfg, ok = m.mesh.ClientTLSConfigAny() - } - if ok { - tlsClient := &http.Client{ - Timeout: probeTimeout, - Transport: &http.Transport{TLSClientConfig: cfg, DisableKeepAlives: true}, - } - if success, res := m.fetchEngineManagerModels(tlsClient, "https", addr, m.emPort); success { - return true, res, readEngineNative(tlsClient, "https", addr, m.emPort, wantEngineNative) - } - } - } - - if useTLS && m.tlsClient != nil { - if success, res := m.fetchEngineManagerModels(m.tlsClient, "https", addr, m.emPort); success { - return true, res, readEngineNative(m.tlsClient, "https", addr, m.emPort, wantEngineNative) - } - } - - if success, res := m.fetchEngineManagerModels(m.client, "http", addr, m.emPort); success { - return true, res, readEngineNative(m.client, "http", addr, m.emPort, wantEngineNative) - } - - return false, engineManagerModels{}, nil -} - -func (m *Manager) fetchEngineManagerModels(client *http.Client, scheme, addr string, port int) (bool, engineManagerModels) { - url := scheme + "://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/v1/models" - start := time.Now() - resp, err := client.Get(url) - if err != nil { - slog.Debug("manual probe engine-manager failed", - "addr", addr, "port", port, "scheme", scheme, "err", err, - "duration_ms", time.Since(start).Milliseconds()) - return false, engineManagerModels{} - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - slog.Debug("manual probe engine-manager non-OK", - "addr", addr, "port", port, "scheme", scheme, "status", resp.StatusCode, - "duration_ms", time.Since(start).Milliseconds()) - return false, engineManagerModels{} - } - var res engineManagerModels - if err := json.NewDecoder(io.LimitReader(resp.Body, modelsResponseLimit)).Decode(&res); err != nil { - slog.Debug("manual probe engine-manager decode failed", - "addr", addr, "port", port, "scheme", scheme, "err", err, - "duration_ms", time.Since(start).Milliseconds()) - return false, engineManagerModels{} - } - if len(res.Models) == 0 && len(res.ModelsByEngine) > 0 { - for _, mList := range res.ModelsByEngine { - res.Models = append(res.Models, mList...) - } - res.Models = mergeModels(res.Models) - } - slog.Debug("manual probe engine-manager ok", - "addr", addr, "port", port, "scheme", scheme, - "models", len(res.Models), "engines", len(res.ModelsByEngine), - "duration_ms", time.Since(start).Milliseconds()) - return true, res -} - -// readEngineNative reads the engine's own view of a manual node: the counters, -// the rates and the active models the node's engine-manager publishes at -// GET /v1/engine-stats. -// -// The read rides the client the model list was just read over, so the view is -// cluster data read exactly as the list beside it is: a peer is read over -// cluster mTLS pinned to the principal it advertises, and there is no -// plaintext fallback for a peer. -// -// The answer is PAIR's own type, so it is decoded as-is. A node whose engine -// publishes no admin API answers "unsupported", and so does an engine that -// could not be reached; neither is a view, so neither is carried. Every -// failure — an unreachable node, a non-200, a malformed answer — leaves the -// caller with no view: a node with no engine-native view is a normal state, -// not a fault. -// -// want is false when the view is not due for a fresh reading (it is read on -// its own slower cadence); the view is then simply not read. -func readEngineNative(client *http.Client, scheme, addr string, port int, want bool) *enginenative.Stats { - if !want { - return nil - } - url := scheme + "://" + net.JoinHostPort(addr, strconv.Itoa(port)) + "/v1/engine-stats" - start := time.Now() - resp, err := client.Get(url) - if err != nil { - slog.Debug("manual probe engine-stats failed", - "addr", addr, "port", port, "scheme", scheme, "err", err, - "duration_ms", time.Since(start).Milliseconds()) - return nil - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - slog.Debug("manual probe engine-stats non-OK", - "addr", addr, "port", port, "scheme", scheme, "status", resp.StatusCode, - "duration_ms", time.Since(start).Milliseconds()) - return nil - } - var stats enginenative.Stats - if err := json.NewDecoder(io.LimitReader(resp.Body, engineNativeResponseLimit)).Decode(&stats); err != nil { - slog.Debug("manual probe engine-stats decode failed", - "addr", addr, "port", port, "scheme", scheme, "err", err, - "duration_ms", time.Since(start).Milliseconds()) - return nil - } - // "unsupported" is the em surface's answer for an engine with no admin API - // and for an engine that could not be reached. Neither is a view of the - // engine, so neither is carried. - if !stats.Supported { - return nil - } - // The stamp says when the numbers were observed. The em surface stamps it - // with the reading it took; a peer that does not stamp it (an older one) - // gets the local observation time instead, so the field is never a lie - // about when it was read. - if stats.ObservedAt == 0 { - stats.ObservedAt = time.Now().UnixMilli() - } - slog.Debug("manual probe engine-stats ok", - "addr", addr, "port", port, "scheme", scheme, - "engine", stats.Engine, "requests", stats.Totals.Requests, - "duration_ms", time.Since(start).Milliseconds()) - return &stats -} - // engineNativeDue reports whether this node's engine view is due for a fresh // reading, and claims the reading when it is. // @@ -1406,6 +1265,8 @@ func (m *Manager) runTelemetryLoop(ctx context.Context, id string, entry ManualE } func (m *Manager) probeNodeTelemetry(ctx context.Context, id string, entry ManualEntry) { + ctx, cancel := context.WithTimeout(ctx, telemetryProbeTimeout) + defer cancel() if ctx.Err() != nil { return } @@ -1415,55 +1276,15 @@ func (m *Manager) probeNodeTelemetry(ctx context.Context, id string, entry Manua m.mu.RUnlock() return } - addr := entry.Address - scheme := "http" - nodeInfoPort := 14318 - probeClient := m.getTelemetryClient() - clusterUUID := tn.status.ClusterUUID - if clusterUUID == "" && m.mesh != nil && tn.status.HostUUID != "" && m.mesh.HasPin(tn.status.HostUUID) { - clusterUUID = tn.status.HostUUID - } - if entry.TLSPort > 0 { - scheme = "https" - nodeInfoPort = entry.TLSPort - probeClient = m.getTelemetryTLSClient() - if m.mesh != nil && m.mesh.Clustered() { - m.mesh.Refresh() - if clusterUUID != "" && m.telemetryPeerPool != nil { - if client, ok := m.telemetryPeerPool.Client(clusterUUID); ok { - probeClient = client - } - } else if anyClient := m.getTelemetryTLSConfigAnyClient(); anyClient != nil { - probeClient = anyClient - } - } - } + tracked := tn + trackedEntry := tn.entry + prior := tn.status m.mu.RUnlock() - - nodeInfoUp, info := m.probeNodeInfoWithPoll(ctx, probeClient, scheme, addr, nodeInfoPort, m.pollIntent()) - nodeInfoReceived := m.clockNow() - if !nodeInfoUp && entry.TLSPort == 0 && m.mesh != nil && m.mesh.Clustered() && ctx.Err() == nil { - m.mesh.Refresh() - var tlsClient *http.Client - if clusterUUID != "" && m.telemetryPeerPool != nil { - if client, ok := m.telemetryPeerPool.Client(clusterUUID); ok { - tlsClient = client - } - } - if tlsClient == nil { - tlsClient = m.getTelemetryTLSConfigAnyClient() - } - if tlsClient != nil { - for _, port := range []int{14319, 14318} { - if up, tlsInfo := m.probeNodeInfoWithPoll(ctx, tlsClient, "https", addr, port, m.pollIntent()); up { - nodeInfoUp = true - info = tlsInfo - nodeInfoReceived = m.clockNow() - break - } - } - } + if trackedEntry != entry || prior.PeerRecordID == "" { + return } + nodeInfoReceived := m.clockNow() + info, _, nodeInfoUp := m.authenticatedNodeInfo(ctx, entry, prior.ClusterUUID) if !nodeInfoUp || ctx.Err() != nil { return @@ -1471,23 +1292,12 @@ func (m *Manager) probeNodeTelemetry(ctx context.Context, id string, entry Manua m.mu.Lock() tn, exists = m.nodes[id] - if !exists { + m.mesh.Refresh() + if !exists || tn != tracked || tn.entry != entry || tn.status.PeerRecordID != info.PeerRecordID || prior.PeerRecordID != info.PeerRecordID || info.PeerRecordID != m.mesh.PeerRecordID(prior.ClusterUUID) { m.mu.Unlock() return } - if info.ClusterUUID != nil && *info.ClusterUUID != "" && tn.status.ClusterUUID == "" { - tn.status.ClusterUUID = *info.ClusterUUID - } - if tn.status.ClusterUUID == "" && m.mesh != nil && info.HostUUID != "" && m.mesh.HasPin(info.HostUUID) { - tn.status.ClusterUUID = info.HostUUID - } - if info.HostUUID != "" && tn.status.HostUUID == "" { - tn.status.HostUUID = info.HostUUID - } hostUUID := info.HostUUID - if hostUUID == "" { - hostUUID = tn.status.HostUUID - } m.mu.Unlock() var msSince int64 @@ -1518,6 +1328,7 @@ func (m *Manager) probeNodeTelemetry(ctx context.Context, id string, entry Manua } _ = m.codec.Notify("node/telemetry", NodeTelemetryNotification{ + PeerRecordID: info.PeerRecordID, ID: id, HostUUID: hostUUID, GPUUtilizationPct: utilization, diff --git a/services/nvpair-manual-nodes/manager_test.go b/services/nvpair-manual-nodes/manager_test.go index 6e2173b06..c0bdeef38 100644 --- a/services/nvpair-manual-nodes/manager_test.go +++ b/services/nvpair-manual-nodes/manager_test.go @@ -41,8 +41,12 @@ func (rw *captureRW) Read(_ []byte) (int, error) { func (rw *captureRW) Write(p []byte) (int, error) { cp := append([]byte(nil), p...) - rw.frames <- cp - return len(p), nil + select { + case rw.frames <- cp: + return len(p), nil + case <-time.After(2 * time.Second): + return 0, context.DeadlineExceeded + } } type fakeRoundTripper struct { @@ -79,12 +83,14 @@ func httpJSON(status int, body string) (*http.Response, error) { }, nil } -func newTestManager() (*Manager, *captureRW, *fakeRoundTripper) { +func newTestManager(t *testing.T) (*Manager, *captureRW, *fakeRoundTripper) { + t.Helper() rw := newCaptureRW() m, err := NewManager(NewCodec(rw), tlsClientOptions{}, nil) if err != nil { panic(err) } + bindTestManager(t, m) rt := newFakeRoundTripper() m.client = &http.Client{Transport: rt, Timeout: time.Second} m.tlsClient = &http.Client{Transport: rt, Timeout: time.Second} @@ -101,7 +107,11 @@ func newTestManager() (*Manager, *captureRW, *fakeRoundTripper) { func newPollRecordingServer(t *testing.T, polls chan<- string, tls bool) (host string, port int, client *http.Client) { t.Helper() handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - polls <- r.Header.Get("X-NVPAIR-Poll") + select { + case polls <- r.Header.Get("X-NVPAIR-Poll"): + case <-r.Context().Done(): + return + } _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "peer-uuid"}) }) var srv *httptest.Server @@ -110,7 +120,8 @@ func newPollRecordingServer(t *testing.T, polls chan<- string, tls bool) (host s } else { srv = httptest.NewServer(handler) } - t.Cleanup(srv.Close) + srv.Client().Timeout = time.Second + t.Cleanup(func() { srv.CloseClientConnections(); srv.Close() }) h, portStr, err := net.SplitHostPort(srv.Listener.Addr().String()) if err != nil { t.Fatalf("split listener address: %v", err) @@ -136,7 +147,7 @@ func TestProbeNodeInfoPollIntentFollowsSchedulingDemand(t *testing.T) { {"scheduling demand", true, "scheduling"}, } { t.Run(tc.name, func(t *testing.T) { - m, _, _ := newTestManager() + m, _, _ := newTestManager(t) polls := make(chan string, 1) host, port, client := newPollRecordingServer(t, polls, false) m.setSchedulingDemand(tc.demand) @@ -145,7 +156,7 @@ func TestProbeNodeInfoPollIntentFollowsSchedulingDemand(t *testing.T) { if !up || info.HostUUID != "peer-uuid" { t.Fatalf("probeNodeInfo = %v, %+v; want a successful peer response", up, info) } - if got := <-polls; got != tc.want { + if got := receivePoll(t, m.ctx, polls); got != tc.want { t.Fatalf("poll intent = %q, want %q", got, tc.want) } }) @@ -164,24 +175,17 @@ func TestProbeNodeTelemetryPollIntentFollowsSchedulingDemand(t *testing.T) { {"scheduling demand", true, "scheduling"}, } { t.Run(tc.name, func(t *testing.T) { - m, _, _ := newTestManager() - polls := make(chan string, 1) - host, port, client := newPollRecordingServer(t, polls, true) - m.telemetryTLSClient = client + m, _, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "peer-uuid") + configureHealthyNode(rt, "node.local", nil, sampleInfo()) m.setSchedulingDemand(tc.demand) // Registered directly rather than through addNode: addNode probes in // the background, and that probe would race this one for the header. const id = "manual:node.local" - entry := ManualEntry{Address: host, TLSPort: port} - m.mu.Lock() - m.nodes[id] = &trackedNode{ - entry: entry, - status: ManualNodeStatus{ID: id}, - } - m.mu.Unlock() + entry := peer.track(m, id) - m.probeNodeTelemetry(context.Background(), id, entry) - if got := <-polls; got != tc.want { + m.probeNodeTelemetry(m.ctx, id, entry) + if got := receivePoll(t, m.ctx, peer.polls); got != tc.want { t.Fatalf("poll intent = %q, want %q", got, tc.want) } }) @@ -191,14 +195,14 @@ func TestProbeNodeTelemetryPollIntentFollowsSchedulingDemand(t *testing.T) { // A freshly spawned worker has been told nothing; an unrenewed scheduling lease // is the safe default until the broker says a scheduler is waiting on telemetry. func TestFreshManagerProbesInTheBackground(t *testing.T) { - m, _, _ := newTestManager() + m, _, _ := newTestManager(t) polls := make(chan string, 1) host, port, client := newPollRecordingServer(t, polls, false) if up, _ := m.probeNodeInfo(client, "http", host, port); !up { t.Fatal("probeNodeInfo failed") } - if got := <-polls; got != "background" { + if got := receivePoll(t, m.ctx, polls); got != "background" { t.Fatalf("poll intent = %q, want background before the broker pushes any demand", got) } } @@ -206,7 +210,7 @@ func TestFreshManagerProbesInTheBackground(t *testing.T) { // The demand decision arrives as a broker notification, so dispatch must accept // it (rather than dropping it as an unknown notification) and apply it. func TestHandleSetSchedulingDemand(t *testing.T) { - m, _, _ := newTestManager() + m, _, _ := newTestManager(t) for _, tc := range []struct { name string json string @@ -248,6 +252,11 @@ func configureHealthyNode(rt *fakeRoundTripper, addr string, models []string, in return httpJSON(http.StatusOK, string(data)) }) + rt.set(http.MethodGet, net.JoinHostPort(addr, "14322"), "/v1/models", func(*http.Request) (*http.Response, error) { + data, _ := json.Marshal(engineManagerModels{Models: models, ModelsByEngine: map[string][]string{"ollama": models}}) + return httpJSON(http.StatusOK, string(data)) + }) + infoHost := net.JoinHostPort(addr, "14318") rt.set(http.MethodGet, infoHost, "/v1/node-info", func(*http.Request) (*http.Response, error) { data, _ := json.Marshal(info) @@ -282,7 +291,7 @@ func configureHealthyLMStudio(rt *fakeRoundTripper, addr string, models []string // server reports up with its model ids parsed from /v1/models, and an absent // one (no handler registered, so the request errors) reports down. func TestProbeLMStudioReportsModels(t *testing.T) { - m, _, rt := newTestManager() + m, _, rt := newTestManager(t) configureHealthyLMStudio(rt, "node.local", []string{"qwen2.5-7b", "llama-3.1-8b"}) up, models := m.probeLMStudio("node.local", lmStudioPort) @@ -325,7 +334,7 @@ func configureHealthyOMLX(rt *fakeRoundTripper, addr string, models []string) { // server reports up with its model ids parsed from /v1/models, a loopback-only // 403 proxy also reports up, and an absent one reports down. func TestProbeOMLXReportsModels(t *testing.T) { - m, _, rt := newTestManager() + m, _, rt := newTestManager(t) configureHealthyOMLX(rt, "node.local", []string{"mlx-qwen", "mlx-llama"}) up, models := m.probeOMLX("node.local", omlxPort) @@ -487,17 +496,20 @@ func TestNodeID(t *testing.T) { } func TestNodeAddRespondsWithInitialStatusThenDiscoversProbeResult(t *testing.T) { - m, rw, rt := newTestManager() + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") configureHealthyNode(rt, "node.local", []string{"llama3", "mistral"}, sampleInfo()) - m.handleMessage(requestMessage(1, "node/add", ManualEntry{Address: "node.local", Name: "lab"})) + entry := peer.entry + entry.Name = "lab" + m.handleMessage(requestMessage(1, "node/add", entry)) resp := readCaptureUntil(t, rw, responseWithID(1)) initial := decodeResult[ManualNodeStatus](t, resp) - if initial.ID != "lab" || initial.Address != "node.local" { + if initial.ID != "lab" || initial.Address != entry.Address { t.Fatalf("initial status = %+v", initial) } - if initial.OllamaPort != 11434 || initial.NodeInfoPort != 14318 { + if initial.OllamaPort != 11434 || initial.NodeInfoPort != entry.TLSPort { t.Fatalf("default ports not set: %+v", initial) } if initial.OllamaUp || initial.NodeInfoUp { @@ -530,7 +542,7 @@ func TestNodeAddRespondsWithInitialStatusThenDiscoversProbeResult(t *testing.T) } func TestNodeAddValidationErrors(t *testing.T) { - m, rw, _ := newTestManager() + m, rw, _ := newTestManager(t) m.handleMessage(requestMessageRaw(1, "node/add", json.RawMessage(`"bad"`))) resp := readCaptureFrame(t, rw) @@ -550,10 +562,11 @@ func TestNodeAddValidationErrors(t *testing.T) { } func TestNodesListReturnsCurrentStatuses(t *testing.T) { - m, rw, rt := newTestManager() + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") configureHealthyNode(rt, "node.local", []string{"llama3"}, sampleInfo()) - m.handleMessage(requestMessage(1, "node/add", ManualEntry{Address: "node.local"})) + m.handleMessage(requestMessage(1, "node/add", peer.entry)) _ = readCaptureUntil(t, rw, methodIs("node/discovered")) m.handleMessage(requestMessage(2, "nodes/list", nil)) @@ -573,7 +586,7 @@ func TestNodesListReturnsCurrentStatuses(t *testing.T) { } func TestNodeRemoveReturnsRemovedAndNotifies(t *testing.T) { - m, rw, _ := newTestManager() + m, rw, _ := newTestManager(t) m.nodes["lab"] = &trackedNode{ entry: ManualEntry{Name: "lab", Address: "node.local"}, status: ManualNodeStatus{ID: "lab", Address: "node.local", OllamaPort: 11434, NodeInfoPort: 14318}, @@ -607,20 +620,28 @@ func TestNodeRemoveReturnsRemovedAndNotifies(t *testing.T) { } func TestAddThenRemoveBeforeInitialProbeDoesNotRediscover(t *testing.T) { - m, rw, rt := newTestManager() + m, rw, rt := newTestManager(t) started := make(chan struct{}) release := make(chan struct{}) - rt.set(http.MethodGet, net.JoinHostPort("node.local", "11434"), "/", func(*http.Request) (*http.Response, error) { + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + }) + rt.set(http.MethodGet, net.JoinHostPort("node.local", "11434"), "/", func(req *http.Request) (*http.Response, error) { close(started) - <-release + select { + case <-release: + case <-req.Context().Done(): + return nil, req.Context().Err() + } return httpJSON(http.StatusOK, `{}`) }) rt.set(http.MethodGet, net.JoinHostPort("node.local", "11434"), "/api/tags", func(*http.Request) (*http.Response, error) { return httpJSON(http.StatusOK, `{"models":[]}`) }) - rt.set(http.MethodGet, net.JoinHostPort("node.local", "14318"), "/v1/node-info", func(*http.Request) (*http.Response, error) { - return httpJSON(http.StatusOK, `{"GPUs":[]}`) - }) status := m.addNode(ManualEntry{Address: "node.local", Name: "lab"}) if status.ID != "lab" { @@ -646,18 +667,19 @@ func TestAddThenRemoveBeforeInitialProbeDoesNotRediscover(t *testing.T) { } func TestProbeNodeEmitsUpdatedOnStateChange(t *testing.T) { - m, rw, rt := newTestManager() - m.nodes["lab"] = &trackedNode{entry: ManualEntry{Name: "lab", Address: "node.local"}, status: ManualNodeStatus{ID: "lab", Address: "node.local", OllamaPort: 11434, NodeInfoPort: 14318}} + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") + entry := peer.track(m, "lab") configureHealthyNode(rt, "node.local", []string{"llama3"}, sampleInfo()) - m.probeNode(ManualEntry{Name: "lab", Address: "node.local"}) + m.probeNode(entry) first := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) if len(first.OllamaModels) != 1 || first.OllamaModels[0] != "llama3" { t.Fatalf("first update models = %#v", first.OllamaModels) } configureHealthyNode(rt, "node.local", []string{"mistral"}, sampleInfo()) - m.probeNode(ManualEntry{Name: "lab", Address: "node.local"}) + m.probeNode(entry) second := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) if len(second.OllamaModels) != 1 || second.OllamaModels[0] != "mistral" { t.Fatalf("second update models = %#v", second.OllamaModels) @@ -665,8 +687,12 @@ func TestProbeNodeEmitsUpdatedOnStateChange(t *testing.T) { } func TestProbeNodeNoUpdateWhenStable(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} + m, rw, rt := newTestManager(t) + // Real TLS reads take variable time; hold age fixed while testing equality. + now := time.Now() + m.now = func() time.Time { return now } + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") + entry := peer.track(m, "lab") m.nodes["lab"] = &trackedNode{entry: entry, status: ManualNodeStatus{ID: "lab", Address: "node.local", OllamaPort: 11434, NodeInfoPort: 14318}} configureHealthyNode(rt, "node.local", []string{"llama3"}, sampleInfo()) @@ -678,7 +704,7 @@ func TestProbeNodeNoUpdateWhenStable(t *testing.T) { } func TestProbeFailuresClearAvailability(t *testing.T) { - m, rw, rt := newTestManager() + m, rw, rt := newTestManager(t) entry := ManualEntry{Name: "lab", Address: "node.local"} m.nodes["lab"] = &trackedNode{ entry: entry, @@ -712,26 +738,29 @@ func TestProbeFailuresClearAvailability(t *testing.T) { } } -// TestProbeFailurePreservesHostUUID: a node-info blip -// (while Ollama stays reachable) must NOT blank the learned HostUUID, or the -// broker would rekey the live node UUID -> manual-id -> UUID. -func TestProbeFailurePreservesHostUUID(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} +// A failed authenticated read revokes service eligibility and the UUID, even +// while the bare engine remains reachable. Recovery must prove identity again. +func TestProbeFailureClearsUnprovenHostUUID(t *testing.T) { + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") + entry := peer.track(m, "lab") m.nodes["lab"] = &trackedNode{entry: entry, status: ManualNodeStatus{ID: "lab", Address: "node.local", OllamaPort: 11434, NodeInfoPort: 14318}} info := sampleInfo() info.HostUUID = "node-uuid" configureHealthyNode(rt, "node.local", []string{"llama3"}, info) - // First probe learns the UUID. + // First probe proves the UUID over pinned mTLS. m.probeNode(entry) first := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) if first.HostUUID != "node-uuid" { t.Fatalf("first probe HostUUID = %q, want node-uuid", first.HostUUID) } - // node-info goes down while Ollama stays up: the UUID must be preserved. + // Keep the actual address's bare engine reachable as well: any accidental + // fallback after a failed PAIR proof must make the assertion fail. + configureHealthyNode(rt, entry.Address, []string{"llama3"}, info) + // A failed proof must clear identity and cannot bypass to a bare engine. rt.set(http.MethodGet, net.JoinHostPort("node.local", "14318"), "/v1/node-info", func(*http.Request) (*http.Response, error) { return nil, errors.New("node-info down") }) @@ -740,11 +769,11 @@ func TestProbeFailurePreservesHostUUID(t *testing.T) { if down.NodeInfoUp { t.Fatal("node-info should be down") } - if down.HostUUID != "node-uuid" { - t.Fatalf("HostUUID dropped on node-info failure: %q", down.HostUUID) + if down.HostUUID != "" || down.ClusterUUID != "" || down.PeerRecordID != "" || down.IdentityVerification != "unverified" || down.OllamaUp { + t.Fatalf("failed proof retained identity or engine availability: %+v", down) } - // node-info recovers: still the same UUID (no flap). + // Node-info recovers: the same UUID is eligible only after a fresh proof. configureHealthyNode(rt, "node.local", []string{"llama3"}, info) m.probeNode(entry) up := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/updated"))) @@ -754,8 +783,9 @@ func TestProbeFailurePreservesHostUUID(t *testing.T) { } func TestManualTelemetryAgeIncludesProbeDuration(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") + entry := peer.track(m, "lab") m.nodes["lab"] = &trackedNode{ entry: entry, status: ManualNodeStatus{ @@ -784,15 +814,12 @@ func TestManualTelemetryAgeIncludesProbeDuration(t *testing.T) { return httpJSON(http.StatusOK, string(data)) }) - ollamaHost := net.JoinHostPort("node.local", "11434") - rt.set(http.MethodGet, ollamaHost, "/", func(*http.Request) (*http.Response, error) { + emHost := net.JoinHostPort("node.local", "14322") + rt.set(http.MethodGet, emHost, "/v1/models", func(*http.Request) (*http.Response, error) { mu.Lock() simulatedTime = simulatedTime.Add(2500 * time.Millisecond) mu.Unlock() - return httpJSON(http.StatusOK, `{}`) - }) - rt.set(http.MethodGet, ollamaHost, "/api/tags", func(*http.Request) (*http.Response, error) { - return httpJSON(http.StatusOK, `{"models":[{"name":"llama3"}]}`) + return httpJSON(http.StatusOK, `{"models":["llama3"],"modelsByEngine":{"ollama":["llama3"]}}`) }) m.probeNode(entry) @@ -807,8 +834,9 @@ func TestManualTelemetryAgeIncludesProbeDuration(t *testing.T) { } func TestManualTelemetryAgeInvalidTelemetryZero(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") + entry := peer.track(m, "lab") m.nodes["lab"] = &trackedNode{ entry: entry, status: ManualNodeStatus{ @@ -837,15 +865,12 @@ func TestManualTelemetryAgeInvalidTelemetryZero(t *testing.T) { return httpJSON(http.StatusOK, string(data)) }) - ollamaHost := net.JoinHostPort("node.local", "11434") - rt.set(http.MethodGet, ollamaHost, "/", func(*http.Request) (*http.Response, error) { + emHost := net.JoinHostPort("node.local", "14322") + rt.set(http.MethodGet, emHost, "/v1/models", func(*http.Request) (*http.Response, error) { mu.Lock() simulatedTime = simulatedTime.Add(2500 * time.Millisecond) mu.Unlock() - return httpJSON(http.StatusOK, `{}`) - }) - rt.set(http.MethodGet, ollamaHost, "/api/tags", func(*http.Request) (*http.Response, error) { - return httpJSON(http.StatusOK, `{"models":[{"name":"llama3"}]}`) + return httpJSON(http.StatusOK, `{"models":["llama3"],"modelsByEngine":{"ollama":["llama3"]}}`) }) m.probeNode(entry) @@ -860,8 +885,9 @@ func TestManualTelemetryAgeInvalidTelemetryZero(t *testing.T) { } func TestManualTelemetryAgeSaturatesOnOverflow(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") + entry := peer.track(m, "lab") m.nodes["lab"] = &trackedNode{ entry: entry, status: ManualNodeStatus{ @@ -890,15 +916,12 @@ func TestManualTelemetryAgeSaturatesOnOverflow(t *testing.T) { return httpJSON(http.StatusOK, string(data)) }) - ollamaHost := net.JoinHostPort("node.local", "11434") - rt.set(http.MethodGet, ollamaHost, "/", func(*http.Request) (*http.Response, error) { + emHost := net.JoinHostPort("node.local", "14322") + rt.set(http.MethodGet, emHost, "/v1/models", func(*http.Request) (*http.Response, error) { mu.Lock() simulatedTime = simulatedTime.Add(2500 * time.Millisecond) mu.Unlock() - return httpJSON(http.StatusOK, `{}`) - }) - rt.set(http.MethodGet, ollamaHost, "/api/tags", func(*http.Request) (*http.Response, error) { - return httpJSON(http.StatusOK, `{"models":[{"name":"llama3"}]}`) + return httpJSON(http.StatusOK, `{"models":["llama3"],"modelsByEngine":{"ollama":["llama3"]}}`) }) m.probeNode(entry) @@ -913,8 +936,9 @@ func TestManualTelemetryAgeSaturatesOnOverflow(t *testing.T) { } func TestManualTelemetryAgeNeverDecreases(t *testing.T) { - m, rw, rt := newTestManager() - entry := ManualEntry{Name: "lab", Address: "node.local"} + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "node-uuid") + entry := peer.track(m, "lab") m.nodes["lab"] = &trackedNode{ entry: entry, status: ManualNodeStatus{ @@ -943,15 +967,12 @@ func TestManualTelemetryAgeNeverDecreases(t *testing.T) { return httpJSON(http.StatusOK, string(data)) }) - ollamaHost := net.JoinHostPort("node.local", "11434") - rt.set(http.MethodGet, ollamaHost, "/", func(*http.Request) (*http.Response, error) { + emHost := net.JoinHostPort("node.local", "14322") + rt.set(http.MethodGet, emHost, "/v1/models", func(*http.Request) (*http.Response, error) { mu.Lock() simulatedTime = simulatedTime.Add(-500 * time.Millisecond) mu.Unlock() - return httpJSON(http.StatusOK, `{}`) - }) - rt.set(http.MethodGet, ollamaHost, "/api/tags", func(*http.Request) (*http.Response, error) { - return httpJSON(http.StatusOK, `{"models":[{"name":"llama3"}]}`) + return httpJSON(http.StatusOK, `{"models":["llama3"],"modelsByEngine":{"ollama":["llama3"]}}`) }) m.probeNode(entry) @@ -987,7 +1008,7 @@ func TestCPUAndMemoryNilAwareEquality(t *testing.T) { } func TestUnknownMethodReturnsMethodNotFound(t *testing.T) { - m, rw, _ := newTestManager() + m, rw, _ := newTestManager(t) m.handleMessage(requestMessage(1, "bogus", nil)) resp := readCaptureFrame(t, rw) if resp.Error == nil || resp.Error.Code != -32601 { @@ -996,7 +1017,7 @@ func TestUnknownMethodReturnsMethodNotFound(t *testing.T) { } func TestLogSetLevelRequest(t *testing.T) { - m, rw, _ := newTestManager() + m, rw, _ := newTestManager(t) m.handleMessage(requestMessage(1, applog.SetLevelMethod, applog.SetLevelParams{Level: "debug"})) resp := readCaptureFrame(t, rw) result := decodeResult[map[string]string](t, resp) @@ -1013,6 +1034,12 @@ func TestLogSetLevelRequest(t *testing.T) { func TestShutdownRequestCancelsRun(t *testing.T) { server, client := net.Pipe() + if err := server.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatal(err) + } + if err := client.SetDeadline(time.Now().Add(10 * time.Second)); err != nil { + t.Fatal(err) + } defer server.Close() defer client.Close() @@ -1021,7 +1048,7 @@ func TestShutdownRequestCancelsRun(t *testing.T) { t.Fatalf("NewManager: %v", err) } done := make(chan error, 1) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(manualTestContext(t)) defer cancel() go func() { done <- mgr.Run(ctx) @@ -1050,7 +1077,7 @@ func TestShutdownRequestCancelsRun(t *testing.T) { } func TestNotificationIsIgnored(t *testing.T) { - m, rw, _ := newTestManager() + m, rw, _ := newTestManager(t) m.handleMessage(notificationMessage("node/add", ManualEntry{Address: "node.local"})) if got := m.listNodes(); len(got) != 0 { t.Fatalf("notification mutated state: %#v", got) @@ -1100,6 +1127,7 @@ func TestSyncClusterMembers(t *testing.T) { if err != nil { t.Fatalf("NewManager: %v", err) } + bindTestManager(t, m) m.syncClusterMembers() @@ -1177,7 +1205,8 @@ func TestAddNodeSupersedesClusterSeededEntryAtSameAddress(t *testing.T) { if err != nil { t.Fatalf("NewManager: %v", err) } - m.client = &http.Client{Transport: newFakeRoundTripper()} + bindTestManager(t, m) + m.client = &http.Client{Transport: newFakeRoundTripper(), Timeout: time.Second} m.syncClusterMembers() if got := len(m.listNodes()); got != 1 { @@ -1207,7 +1236,8 @@ func TestAddNodeSupersedesClusterSeededEntryAtSameAddress(t *testing.T) { } func TestProbeEngineManagerReportsModels(t *testing.T) { - m, _, rt := newTestManager() + m, _, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "uuid-node-1") infoHost := net.JoinHostPort("node.local", "14318") rt.set(http.MethodGet, infoHost, "/v1/node-info", func(*http.Request) (*http.Response, error) { @@ -1230,8 +1260,8 @@ func TestProbeEngineManagerReportsModels(t *testing.T) { }`) }) - m.addNode(ManualEntry{Address: "node.local", Name: "test-node"}) - m.probeNode(ManualEntry{Address: "node.local", Name: "test-node"}) + entry := peer.track(m, "test-node") + m.probeNode(entry) nodes := m.listNodes() if len(nodes) != 1 { @@ -1271,11 +1301,12 @@ func TestProbeEngineManagerReportsModels(t *testing.T) { } func TestProbeNodeSkipsPlaintextEngineProbesForClusteredNode(t *testing.T) { - m, _, rt := newTestManager() + m, _, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "clustered.local", "uuid-clustered-1") var plaintextProbes int for _, port := range []string{"11434", "1234", "1236"} { - host := net.JoinHostPort("clustered.local", port) + host := net.JoinHostPort(peer.entry.Address, port) rt.set(http.MethodGet, host, "/", func(*http.Request) (*http.Response, error) { plaintextProbes++ return httpJSON(http.StatusForbidden, `{"error":"cluster peers must use mTLS"}`) @@ -1291,10 +1322,12 @@ func TestProbeNodeSkipsPlaintextEngineProbesForClusteredNode(t *testing.T) { return httpJSON(http.StatusOK, `{"hostUuid":"uuid-clustered-1","clusterUuid":"uuid-clustered-1"}`) }) - entry := ManualEntry{Address: "clustered.local", Name: "clustered-peer", MTLS: true} - m.addNode(entry) + entry := peer.track(m, "clustered-peer") m.probeNode(entry) + if got := m.nodes[nodeID(entry)].status; got.IdentityVerification != "verified" || !got.NodeInfoUp { + t.Fatalf("expected proven PAIR peer: %+v", got) + } if plaintextProbes != 0 { t.Fatalf("expected 0 plaintext engine probes for clustered/mTLS node, got %d", plaintextProbes) } @@ -1342,6 +1375,9 @@ func writePipeRequest(t *testing.T, conn net.Conn, id int, method string, params t.Fatalf("marshal request: %v", err) } data = append(data, '\n') + if err := conn.SetWriteDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("set write deadline: %v", err) + } if _, err := conn.Write(data); err != nil { t.Fatalf("write request: %v", err) } @@ -1367,7 +1403,8 @@ func TestTelemetryIntervalForNodeDeterministicAndBounded(t *testing.T) { } func TestNodeTelemetryEmittedForReachableNodeAndNoneForUnreachable(t *testing.T) { - m, rw, rt := newTestManager() + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "reachable.local", "peer-uuid-1") m.telemetryInterval = func(_ string) time.Duration { return 10 * time.Millisecond } @@ -1382,9 +1419,9 @@ func TestNodeTelemetryEmittedForReachableNodeAndNoneForUnreachable(t *testing.T) {Name: "GPU 1", UtilizationPercent: gpuReading(uint32(77))}, } - var pollHeader string + var pollHeader atomic.Value rt.set(http.MethodGet, net.JoinHostPort("reachable.local", "14318"), "/v1/node-info", func(req *http.Request) (*http.Response, error) { - pollHeader = req.Header.Get("X-NVPAIR-Poll") + pollHeader.Store(req.Header.Get("X-NVPAIR-Poll")) data, _ := json.Marshal(info) return httpJSON(http.StatusOK, string(data)) }) @@ -1395,7 +1432,9 @@ func TestNodeTelemetryEmittedForReachableNodeAndNoneForUnreachable(t *testing.T) return httpJSON(http.StatusOK, `{"models":[]}`) }) - m.addNode(ManualEntry{Address: "reachable.local", Name: "reachable-node"}) + entry := peer.entry + entry.Name = "reachable-node" + m.addNode(entry) msg := readCaptureUntil(t, rw, methodIs("node/telemetry")) var telem NodeTelemetryNotification @@ -1422,8 +1461,8 @@ func TestNodeTelemetryEmittedForReachableNodeAndNoneForUnreachable(t *testing.T) } // No workload is queued or running in this test, so the broker has pushed no // scheduling demand and every poll is a background read. - if pollHeader != "background" { - t.Errorf("expected X-NVPAIR-Poll = background, got %q", pollHeader) + if pollHeader.Load() != "background" { + t.Errorf("expected X-NVPAIR-Poll = background, got %q", pollHeader.Load()) } // Remove reachable node and drain frames so its loop stops @@ -1440,12 +1479,16 @@ func TestNodeTelemetryEmittedForReachableNodeAndNoneForUnreachable(t *testing.T) return nil, errors.New("connection refused") }) - m.addNode(ManualEntry{Address: "unreachable.local", Name: "unreachable-node"}) + peer.server.CloseClientConnections() + peer.server.Close() + entry.Name = "unreachable-node" + m.addNode(entry) assertNoCaptureMethod(t, rw, "node/telemetry") } func TestNodeTelemetryLoopStopsOnRemove(t *testing.T) { - m, rw, rt := newTestManager() + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "peer-uuid-1") m.telemetryInterval = func(_ string) time.Duration { return 10 * time.Millisecond } @@ -1463,7 +1506,9 @@ func TestNodeTelemetryLoopStopsOnRemove(t *testing.T) { return httpJSON(http.StatusOK, `{"models":[]}`) }) - m.addNode(ManualEntry{Address: "node.local", Name: "node-1"}) + entry := peer.entry + entry.Name = "node-1" + m.addNode(entry) _ = readCaptureUntil(t, rw, methodIs("node/telemetry")) if !m.removeNode("node-1") { @@ -1481,70 +1526,23 @@ func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { } func TestTelemetryLoopReusesKeepAliveConnections(t *testing.T) { - var newConns atomic.Int32 - ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path == "/v1/node-info" { - w.Header().Set("Content-Type", "application/json") - info := sampleInfo() - info.HostUUID = "peer-uuid" - info.TelemetryValid = true - _ = json.NewEncoder(w).Encode(info) - return - } - http.NotFound(w, r) - })) - ts.Config.ConnState = func(c net.Conn, state http.ConnState) { - if state == http.StateNew { - newConns.Add(1) - } - } - ts.StartTLS() - defer ts.Close() - - host, portStr, err := net.SplitHostPort(ts.Listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - port, err := strconv.Atoi(portStr) - if err != nil { - t.Fatal(err) - } - - rw := newCaptureRW() - m, err := NewManager(NewCodec(rw), tlsClientOptions{}, nil) - if err != nil { - t.Fatal(err) - } - - // Route 10s topology probes to a dummy roundtripper so only telemetry connects to ts - dummy := newFakeRoundTripper() - m.client = &http.Client{Transport: dummy} - m.tlsClient = &http.Client{Transport: dummy} - - // Configure telemetryTLSClient with the test server's TLS config and keep-alives enabled - tlsTransport := telemetryTransport(ts.Client().Transport.(*http.Transport).TLSClientConfig) - m.telemetryTLSClient = &http.Client{ - Timeout: telemetryProbeTimeout, - Transport: tlsTransport, - } - - m.telemetryInterval = func(_ string) time.Duration { - return 10 * time.Millisecond - } - - m.addNode(ManualEntry{Address: host, TLSPort: port, Name: "test-node"}) - - // Collect several node/telemetry frames across ticks + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "node.local", "peer-uuid") + configureHealthyNode(rt, "node.local", nil, sampleInfo()) + m.telemetryInterval = func(string) time.Duration { return 10 * time.Millisecond } + // Start with a known binding so discovery's one-shot identity hint is not + // counted as a telemetry connection. The periodic reads must reuse one socket. + entry := peer.track(m, "test-node") + m.mu.Lock() + m.startTelemetryLoopLocked(nodeID(entry), m.nodes[nodeID(entry)]) + m.mu.Unlock() for i := 0; i < 4; i++ { _ = readCaptureUntil(t, rw, methodIs("node/telemetry")) } - if !m.removeNode("test-node") { t.Fatal("failed to remove node") } - - // With keep-alive connection reuse, only 1 connection is dialed - if got := newConns.Load(); got != 1 { + if got := peer.newConns.Load(); got != 1 { t.Fatalf("expected 1 connection created across multiple telemetry ticks, got %d", got) } } @@ -1555,6 +1553,7 @@ func TestTelemetryTimeoutBelowInterval(t *testing.T) { if err != nil { t.Fatal(err) } + bindTestManager(t, m) if m.telemetryClient.Timeout != 1500*time.Millisecond { t.Errorf("telemetryClient timeout = %v, want 1.5s", m.telemetryClient.Timeout) } @@ -1568,54 +1567,37 @@ func TestTelemetryTimeoutBelowInterval(t *testing.T) { } func TestTelemetryCancellationOnRemovalAbortsInFlightRequest(t *testing.T) { - m, rw, _ := newTestManager() - m.telemetryInterval = func(_ string) time.Duration { - return 5 * time.Millisecond - } - + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "slow.local", "slow-peer") + m.telemetryInterval = func(string) time.Duration { return 5 * time.Millisecond } inFlight := make(chan struct{}) reqAborted := make(chan struct{}) - - m.telemetryClient = &http.Client{ - Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - if strings.HasSuffix(req.URL.Path, "/v1/node-info") { - select { - case <-inFlight: - default: - close(inFlight) - } - <-req.Context().Done() - select { - case <-reqAborted: - default: - close(reqAborted) - } - return nil, req.Context().Err() - } - return httpJSON(http.StatusOK, `{}`) - }), - } - - m.addNode(ManualEntry{Address: "slow.local", Name: "slow-node"}) - - // Wait until the request is in-flight + rt.set(http.MethodGet, "slow.local:14318", "/v1/node-info", func(req *http.Request) (*http.Response, error) { + close(inFlight) + select { + case <-req.Context().Done(): + close(reqAborted) + return nil, req.Context().Err() + case <-m.ctx.Done(): + return nil, m.ctx.Err() + } + }) + entry := peer.track(m, "slow-node") + m.mu.Lock() + m.startTelemetryLoopLocked(nodeID(entry), m.nodes[nodeID(entry)]) + m.mu.Unlock() select { case <-inFlight: case <-time.After(2 * time.Second): t.Fatal("timed out waiting for telemetry request to enter in-flight") } - - // Removing the node must cancel its context and abort the in-flight request immediately if !m.removeNode("slow-node") { t.Fatal("failed to remove slow-node") } - select { case <-reqAborted: - // In-flight request was aborted immediately by context cancellation case <-time.After(500 * time.Millisecond): t.Fatal("in-flight telemetry request was not aborted on node removal") } - _ = readCaptureUntil(t, rw, methodIs("node/removed")) } diff --git a/services/nvpair-manual-nodes/peer_inventory.go b/services/nvpair-manual-nodes/peer_inventory.go new file mode 100644 index 000000000..6195796c7 --- /dev/null +++ b/services/nvpair-manual-nodes/peer_inventory.go @@ -0,0 +1,98 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "net" + "net/http" + "nvpair-shared/clustertrust" + "nvpair-shared/enginenative" + "nvpair-shared/noderec" + "sort" + "strconv" +) + +func (m *Manager) inventoryClient(principal string) (*http.Client, bool) { + m.mesh.Refresh() + m.inventoryPoolOnce.Do(func() { m.inventoryPool = clustertrust.NewPeerClientPool(m.mesh, probeTimeout) }) + return m.inventoryPool.Client(principal) +} + +func (m *Manager) authenticatedNodeInfo(ctx context.Context, entry ManualEntry, expected string) (NodeInfoResponse, int, bool) { + if ctx == nil { + ctx = context.Background() + } + if m.mesh == nil { + return NodeInfoResponse{}, 0, false + } + m.mesh.Refresh() + port := 14318 + if entry.TLSPort > 0 { + port = entry.TLSPort + } + url := "https://" + net.JoinHostPort(entry.Address, strconv.Itoa(port)) + "/v1/node-info" + if expected == "" { + cfg, ok := m.mesh.ClientTLSConfigAny() + if !ok { + return NodeInfoResponse{}, port, false + } + tr := &http.Transport{TLSClientConfig: cfg, DisableKeepAlives: true} + defer tr.CloseIdleConnections() + var hint NodeInfoResponse + principal, _, ok := clustertrust.ReadPeerInventory(ctx, m.mesh, &http.Client{Transport: tr, Timeout: probeTimeout}, "", url, m.pollIntent(), nodeInfoResponseLimit, &hint) + if !ok { + return NodeInfoResponse{}, port, false + } + expected = principal + } + client, ok := m.inventoryClient(expected) + if !ok { + return NodeInfoResponse{}, port, false + } + var info NodeInfoResponse + principal, binding, ok := clustertrust.ReadPeerInventory(ctx, m.mesh, client, expected, url, m.pollIntent(), nodeInfoResponseLimit, &info) + if !ok || info.HostUUID != principal || info.ClusterUUID == nil || *info.ClusterUUID != principal || info.PeerMetadata == nil || info.PeerMetadata.HostUUID != principal || info.PeerMetadata.Services[noderec.ServiceNodeInfo] != port { + return NodeInfoResponse{}, port, false + } + info.PeerRecordID = binding + return info, port, true +} + +func (m *Manager) probePeerEngineManager(ctx context.Context, addr, principal, binding string, port int, wantNative bool) (bool, engineManagerModels, *enginenative.Stats) { + if port <= 0 || binding == "" { + return false, engineManagerModels{}, nil + } + client, ok := m.inventoryClient(principal) + if !ok || m.mesh.PeerRecordID(principal) != binding { + return false, engineManagerModels{}, nil + } + url := "https://" + net.JoinHostPort(addr, strconv.Itoa(port)) + var models engineManagerModels + _, got, ok := clustertrust.ReadPeerInventory(ctx, m.mesh, client, principal, url+"/v1/models", "", modelsResponseLimit, &models) + if !ok || got != binding { + return false, engineManagerModels{}, nil + } + if len(models.Models) == 0 && len(models.ModelsByEngine) > 0 { + for _, inventory := range models.ModelsByEngine { + models.Models = mergeModels(models.Models, inventory) + } + sort.Strings(models.Models) + } + var native *enginenative.Stats + if wantNative { + var stats enginenative.Stats + _, got, ok := clustertrust.ReadPeerInventory(ctx, m.mesh, client, principal, url+"/v1/engine-stats", "", engineNativeResponseLimit, &stats) + if ok && got == binding && stats.Supported { + if stats.ObservedAt == 0 { + stats.ObservedAt = m.clockNow().UnixMilli() + } + native = &stats + } + } + if m.mesh.PeerRecordID(principal) != binding { + return false, engineManagerModels{}, nil + } + return true, models, native +} diff --git a/services/nvpair-manual-nodes/response_bounds_test.go b/services/nvpair-manual-nodes/response_bounds_test.go index 8e87b0f7f..44dcf93d8 100644 --- a/services/nvpair-manual-nodes/response_bounds_test.go +++ b/services/nvpair-manual-nodes/response_bounds_test.go @@ -37,7 +37,7 @@ func TestProbeNodeInfoRejectsAnOversizedAnswer(t *testing.T) { if err != nil { t.Fatal(err) } - m, _, _ := newTestManager() + m, _, _ := newTestManager(t) if up, _ := m.probeNodeInfo(srv.Client(), "http", host, port); up != tc.want { t.Fatalf("probeNodeInfo up = %v, want %v", up, tc.want) } diff --git a/services/nvpair-manual-nodes/verified_inventory_test.go b/services/nvpair-manual-nodes/verified_inventory_test.go new file mode 100644 index 000000000..59a248718 --- /dev/null +++ b/services/nvpair-manual-nodes/verified_inventory_test.go @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "net/http" + "testing" + + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" +) + +func TestPublicManualInventoryCannotClaimPinnedPrincipal(t *testing.T) { + dir := t.TempDir() + clustertrusttest.Join(t, dir, "test-cluster", "self", "peer") + m, _, rt := newTestManager(t) + m.mesh = clustertrust.Open(dir) + entry := ManualEntry{Address: "forged-peer"} + id := nodeID(entry) + m.nodes[id] = &trackedNode{entry: entry} + rt.set("GET", "forged-peer:14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { + return httpJSON(200, `{"hostUuid":"peer","clusterUuid":"peer","telemetryValid":true,"GPUs":[{"name":"forged","utilization_percent":99}]}`) + }) + rt.set("GET", "forged-peer:14322", "/v1/models", func(*http.Request) (*http.Response, error) { + return httpJSON(200, `{"models":["forged-model"],"modelsByEngine":{"omlx":["forged-model"]}}`) + }) + m.probeNode(entry) + got := m.nodes[id].status + if got.IdentityVerification != "unverified" || got.HostUUID != "" || got.ClusterUUID != "" || got.TelemetryValid || len(got.GPUs) > 0 || len(got.Models) > 0 { + t.Fatalf("public inventory accepted for service decisions: %+v", got) + } +} + +// A deliberately configured bare engine is manual inventory, not a PAIR peer. +// It may advertise models without acquiring a host identity or GPU telemetry. +func TestBareEnginesUseExplicitManualInventory(t *testing.T) { + m, rw, rt := newTestManager(t) + entry := ManualEntry{Name: "bare-engines", Address: "bare.local"} + rt.set(http.MethodGet, "bare.local:14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { + return httpJSON(http.StatusNotFound, `{}`) + }) + rt.set(http.MethodGet, "bare.local:11434", "/", func(*http.Request) (*http.Response, error) { + return httpJSON(http.StatusOK, `{}`) + }) + rt.set(http.MethodGet, "bare.local:11434", "/api/tags", func(*http.Request) (*http.Response, error) { + return httpJSON(http.StatusOK, `{"models":[{"name":"llama3"}]}`) + }) + configureHealthyLMStudio(rt, entry.Address, []string{"qwen"}) + configureHealthyOMLX(rt, entry.Address, []string{"mlx"}) + initial := m.addNode(entry) + if initial.OllamaPort != 11434 || initial.NodeInfoPort != 14318 { + t.Fatalf("default ports not set: %+v", initial) + } + got := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/discovered"))) + if got.IdentityVerification != "manual" || got.HostUUID != "" || got.ClusterUUID != "" || got.PeerRecordID != "" || got.NodeInfoUp || got.TelemetryValid || len(got.GPUs) != 0 { + t.Fatalf("bare engine acquired PAIR identity or telemetry: %+v", got) + } + if !got.OllamaUp || !got.LMStudioUp || !got.OMLXUp || len(got.Models) != 3 { + t.Fatalf("bare engine availability or models lost: %+v", got) + } + for engine, want := range map[string]string{"ollama": "llama3", "lmstudio": "qwen", "omlx": "mlx"} { + if models := got.ModelsByEngine[engine]; len(models) != 1 || models[0] != want { + t.Errorf("manual %s models = %v, want [%s]", engine, models, want) + } + } +} From e480656b96897af97729124104c32de4b29ddee9 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:38:56 +0200 Subject: [PATCH 05/18] fix(broker): gate routing and scheduling on current peer provenance Recheck proof bindings for relay delivery, scheduler snapshots and all telemetry, including unavailable samples and replay. Withdraw retired worker sources independently and prevent preview claims from replacing verified records. Publish owned service metadata to node-info and carry provenance into client snapshots and the terminal interface. Adapt NVIDIA/Personal-AI-Router#179 to source-aware relay and manual-node ownership. Add regressions for unproven routes and invalid telemetry replacing authenticated pressure. Co-authored-by: Kaylee Lubick Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- desktop/docs/services-api.md | 2 +- .../nvpair-tui/ui/node_provenance_test.go | 29 +++ services/nvpair-tui/ui/nodes.go | 18 +- services/nvpair-ui-broker/README.md | 12 +- services/nvpair-ui-broker/broker.go | 121 +++++---- services/nvpair-ui-broker/discovery.go | 117 ++++++--- services/nvpair-ui-broker/discovery_test.go | 156 ++++++++---- .../nvpair-ui-broker/enginenative_test.go | 10 +- services/nvpair-ui-broker/manualnodes.go | 162 ++++++------ .../manualnodes_enginenative_test.go | 10 +- services/nvpair-ui-broker/peer_inventory.go | 240 ++++++++++++++++++ .../relay/record_identity_test.go | 65 +++++ services/nvpair-ui-broker/relay/relay.go | 74 +++++- services/nvpair-ui-broker/relay/relay_test.go | 37 +-- services/nvpair-ui-broker/scheduler_test.go | 5 +- services/nvpair-ui-broker/telemetry.go | 15 +- services/nvpair-ui-broker/telemetry_test.go | 31 ++- .../verified_inventory_test.go | 166 ++++++++++++ services/nvpair-ui-broker/workers.go | 2 +- 19 files changed, 1014 insertions(+), 258 deletions(-) create mode 100644 services/nvpair-tui/ui/node_provenance_test.go create mode 100644 services/nvpair-ui-broker/peer_inventory.go create mode 100644 services/nvpair-ui-broker/relay/record_identity_test.go create mode 100644 services/nvpair-ui-broker/verified_inventory_test.go diff --git a/desktop/docs/services-api.md b/desktop/docs/services-api.md index c41cbfc3c..841f9c7f6 100644 --- a/desktop/docs/services-api.md +++ b/desktop/docs/services-api.md @@ -204,7 +204,7 @@ | `ready` | notification (we consume) | ✅ yes | **Dynamic / unresolved notify sites (verify by hand — `npm run service-contracts` prints the line numbers):** -- `method (var) (daemon.go, 3 sites)` +- `method (var) (daemon.go, 2 sites)` ## nvpair-node-settings diff --git a/services/nvpair-tui/ui/node_provenance_test.go b/services/nvpair-tui/ui/node_provenance_test.go new file mode 100644 index 000000000..2c1c784a4 --- /dev/null +++ b/services/nvpair-tui/ui/node_provenance_test.go @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "encoding/json" + "testing" +) + +func TestNodeProvenanceKeepsPreviewSeparateFromVerifiedPeer(t *testing.T) { + var nodes []availableNode + err := json.Unmarshal([]byte(`[{"id":"claim","hostUuid":"peer","peerRecordId":"preview:claim","identityVerification":"unverified","trusted":false},{"id":"peer","hostUuid":"peer","peerRecordId":"peer:proof","identityVerification":"verified","trusted":true}]`), &nodes) + if err != nil { + t.Fatal(err) + } + view := newNodesView(nil) + view.SetSize(100, 30) + view.setNodes(nodes) + if len(view.nodes) != 2 || view.nodes[0].PeerRecordID == view.nodes[1].PeerRecordID { + t.Fatal("preview collided with verified row") + } + if nodeStatus(nodes[0]) != "Unverified" || nodeStatus(nodes[1]) != "Connected" { + t.Fatal("provenance labels do not distinguish trust") + } + if nodeStatus(availableNode{IdentityVerification: "manual"}) != "Manual engine" { + t.Fatal("manual bypass is not separately labelled") + } +} diff --git a/services/nvpair-tui/ui/nodes.go b/services/nvpair-tui/ui/nodes.go index 8dfe7e104..74f7980eb 100644 --- a/services/nvpair-tui/ui/nodes.go +++ b/services/nvpair-tui/ui/nodes.go @@ -18,12 +18,14 @@ import ( // availableNode mirrors the broker's discovery boundary shape (the // discovery:get-nodes element and discovery:nodes-changed payload entry). type availableNode struct { - ID string `json:"id"` - HostUUID string `json:"hostUuid"` - Name string `json:"name"` - IPAddress string `json:"ipAddress"` - Port int `json:"port"` - LastSeen int64 `json:"lastSeen"` // Unix seconds + PeerRecordID string `json:"peerRecordId,omitempty"` + IdentityVerification string `json:"identityVerification,omitempty"` + ID string `json:"id"` + HostUUID string `json:"hostUuid"` + Name string `json:"name"` + IPAddress string `json:"ipAddress"` + Port int `json:"port"` + LastSeen int64 `json:"lastSeen"` // Unix seconds // Trusted: this node is a paired cluster peer of ours. Clustered: it belongs // to some cluster (advertises a cluster-uuid), whether or not we're paired // with it. Either one makes it non-invitable — an already-clustered peer @@ -203,6 +205,10 @@ func (v *nodesView) setNodes(nodes []availableNode) { // empty for an invitable standalone node. func nodeStatus(n availableNode) string { switch { + case n.IdentityVerification == "unverified": + return "Unverified" + case n.IdentityVerification == "manual": + return "Manual engine" case n.Trusted: return "Connected" case n.Clustered: diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 58a359eed..52dbbef98 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -114,7 +114,7 @@ Bidirectional newline-delimited JSON-RPC 2.0 — same conventions as every other | `--settings-path ` | `./nvpair-node-settings[.exe]` in the CWD | Explicit path to the `nvpair-node-settings` binary the broker spawns for the typed settings store. Same optional semantics as `--node-info-path` | | `--cluster-manager-path ` | `./nvpair-cluster-manager[.exe]` in the CWD | Explicit path to the `nvpair-cluster-manager` binary the broker spawns for cluster pairing / membership. Same optional semantics as `--node-info-path` | | `--scheduler-path ` | `./nvpair-job-scheduler[.exe]` in the CWD | Explicit path to the `nvpair-job-scheduler` binary the broker spawns for responsive, node-wide workload and GPU-pressure ranking. Same optional semantics as `--node-info-path` | -| `--cluster-dir ` | `cluster/` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) | Cluster config dir (`node.crt`/`node.key` + `trusted/`, minted by `nvpair-cluster-manager`). Threaded to the cluster-scoped workers (`nvpair-errors`, `nvpair-workload-manager`, `nvpair-node-scanner`, `nvpair-manual-nodes`, and `nvpair-engine-manager`), each of which derives its membership from it continuously — so a create, join, or leave takes effect in place and the broker does **not** restart them. The broker also passes the parent of this path to `nvpair-cluster-manager` as `--config-dir`, so the only writer of the cluster dir and the workers reading it cannot resolve different directories. `nvpair-node-info` is excluded — it stays plain HTTP even when clustered (see the repository-root `SECURITY.md`). Defaults so cluster mTLS auto-activates with nothing to pass; with an empty or cert-less dir this node is not a member, so it serves and dials no inter-node cluster traffic at all | +| `--cluster-dir ` | `cluster/` in the per-user data dir (`%LocalAppData%\Nvidia Corporation\Personal AI Router` on Windows, `~/.config/Nvidia Corporation/Personal AI Router` on Linux) | Cluster config dir (`node.crt`/`node.key` + `trusted/`, minted by `nvpair-cluster-manager`). Threaded to the cluster-scoped workers (`nvpair-errors`, `nvpair-workload-manager`, `nvpair-node-scanner`, `nvpair-manual-nodes`, and `nvpair-engine-manager`), each of which derives its membership from it continuously — so a create, join, or leave takes effect in place and the broker does **not** restart them. The broker also passes the parent of this path to `nvpair-cluster-manager` as `--config-dir`, so the only writer of the cluster dir and the workers reading it cannot resolve different directories. `nvpair-node-info` receives the directory too, serving pinned mTLS inventory alongside public hardware previews (see repository-root `SECURITY.md`). Defaults so cluster mTLS auto-activates with nothing to pass; with an empty or cert-less dir this node is not a member, so it serves and dials no inter-node cluster traffic at all | | `--log-level ` | _(env `NVPAIR_LOG_LEVEL` or `info`)_ | `debug` \| `info` \| `warn` \| `error` | | `--worker-health-interval ` | `30s` | Diagnostic health interval, minimum `1s`; deadline is the smaller of `5s` and the interval | | `--version` | | Print version and exit | @@ -132,7 +132,7 @@ Use the reported port, not a hardcoded endpoint. **`nvpair-node-scanner`** (the consolidated discovery daemon) is spawned first. It pushes `discovery:node-discovered`, `discovery:node-updated`, and `discovery:node-removed` notifications into the broker, which maintains them in an in-memory map keyed by `id`. Clients query that map via `discovery:get-nodes` and — once they've opted in via `discovery:subscribe` — receive a `discovery:nodes-changed` notification on every store mutation. The raw `discovery:node-*` notifications are never forwarded as-is. The scanner polls healthy node-info endpoints on a staggered two-second cadence, backs consecutive remote failures off to a 30-second cap, and emits compact `discovery:node-telemetry` observations containing maximum GPU utilization, validity, and age; these remain internal to broker scheduling. Those polls carry an `X-NVPAIR-Poll` intent the broker owns: `telemetry/set-scheduling-demand` tells the scanner (and `nvpair-manual-nodes`, which polls node-info on the same terms) whether any workload anywhere — queued, running, local, or a peer's — still needs fresh telemetry. While it does, the polls renew each node's five-second scheduling lease; when the last workload reaches a terminal state the broker pushes `{active:false}` and every poll becomes a background read that renews no lease, so each node falls back to its ten-second idle cadence. Both workers start inactive and are re-told on every respawn, so a restart cannot pin the fleet to a faster cadence by itself. The broker registers this node's local service ports (`ni`/`er`/`wl`/`cl`/`em`, plus `ol`/`lm`/`om` from the engine pollers) with the daemon over the same link, so the daemon can advertise them all in one `_nvpair-node` record. -**`nvpair-node-info`** is spawned next. It's a server, not an event source: it stands up the local `/v1/node-info` HTTP endpoint (GPU/CPU/memory inventory). It does not advertise itself — the broker registers its `ni` port with the scanner daemon, which carries it in the node record, and a peer's daemon fetches `/v1/node-info` over plain HTTP to enrich the node. The broker consumes observed-address notifications and correlates health responses from node-info's stdout. Spawning it is **optional**: if the binary can't be resolved (and no `--node-info-path` override was given) the broker logs a warning and continues serving discovery without it. +**`nvpair-node-info`** is spawned next. It's a server, not an event source: it stands up the local `/v1/node-info` HTTP endpoint (GPU/CPU/memory inventory). It does not advertise itself — the broker registers its `ni` port with the scanner daemon, which carries it in the node record, and a peer's daemon authenticates `/v1/node-info` over pinned cluster mTLS before using its hardware or service metadata. The broker consumes observed-address notifications and correlates health responses from node-info's stdout. Spawning it is **optional**: if the binary can't be resolved (and no `--node-info-path` override was given) the broker logs a warning and continues serving discovery without it. **Engine advertising.** The broker runs an internal 5 s poll loop against local Ollama at its configured backend port and LM Studio (`GET /v1/models`) and reconciles this node's engine registration with the scanner daemon: @@ -466,7 +466,9 @@ Field-by-field: | `ipAddress` | string | the node's authoritative `ip=` (else the best-ranked advertised address); empty string when none is usable. | | `port` | number | the node's `ni` (node-info) port from its `_nvpair-node` TXT, typically `14318`. | | `lastSeen` | number | **Unix seconds**. Updated on every `discovery:node-discovered` / `discovery:node-updated` event the broker receives from the daemon; not refreshed by `discovery:get-nodes` itself (a pure read). | -| `trusted` | bool | whether this node is a paired cluster peer (the daemon holds a pin for its `cluster-uuid`); false for non-cluster/unknown nodes. | +| `trusted` | bool | peer identity proven over pinned cluster mTLS with a current authorization-bound proof; a claimed pinned UUID alone never grants trust. | +| `peerRecordId` | string | opaque proof binding for verified peers, or independent `preview:` / `manual:` record key for unverified previews and explicit manual configurations. | +| `identityVerification` | string | `local`, `verified`, `unverified`, or `manual`. Unverified records are keyed by `peerRecordId`, never their claimed host UUID; desktop adoption of these additive fields is follow-up work. | | `clustered` | bool | whether this node belongs to some cluster (it advertises a `cluster-uuid`), independent of whether we're paired with it (`trusted`). A client uses it to suppress a cluster invite that an already-clustered peer would reject. Omitted (false) for standalone/unknown nodes. | | `models` | string[] | the node's available model names, enriched by the daemon from the node's engine-manager `em` endpoint (`GET /v1/models`). Omitted when the node advertises no engine-manager or no engine is running. | | `modelsByEngine` | object | the same models attributed to the engine that serves each (keyed by engine name, e.g. `ollama` / `lmstudio`). Additive alongside the flat `models` union. A present engine key with `[]` means its inventory was successfully queried and is empty; a missing key means it was not running/queryable. Omitted when no engine inventory was successfully reported. | @@ -899,10 +901,10 @@ Attach to a pre-existing endpoint: ## What this version intentionally does NOT do (yet) - **Engine-advertise control surface.** Engine registration is auto-driven only: the broker tracks local ollama / LM Studio on their fixed coordinates and registers `ol` / `lm` with the daemon while up. There's no manual-advertise RPC (custom service, port, name, or TXT), and no way to advertise anything other than the detected engines. -- **node-info control surface.** node-info is spawned and torn down with the broker, and the broker pushes it only two things over stdin: the log level, and this node's cluster principal (`nodeinfo:set-cluster-identity`, sent on spawn and on every membership or pin-set change, because node-info holds no cluster dir and so cannot read membership itself). Otherwise it's hands-off: the broker registers its port with the daemon (which enriches over plain HTTP) but doesn't pass through TLS material (`--cert` / `--key` / `--client-ca`) or a custom `--port`, and exposes no RPC to query or reconfigure it. It runs with its own defaults plus those two pushes. +- **node-info control surface.** The broker passes its cluster directory and local node ID, pushes demand and log-level controls, and publishes broker-owned service metadata via `nodeinfo:set-peer-metadata`. Public hardware previews remain available; only pinned mTLS inventory can feed service decisions. There is no public reconfiguration RPC or custom node-info port through the broker. - **Manual-node persistence across restarts.** `nvpair-manual-nodes` remains an in-memory worker; the broker owns `configs/manual-nodes.json`, clears stale live projections when the worker exits, and replays the durable entries into its replacement. - **Per-event push semantics.** `discovery:nodes-changed` always carries the full current snapshot, not a delta. For small N this is fine and lets the client treat the payload as authoritative without state reconciliation. `errors:update` is likewise a full snapshot. -- **BYO-TLS node-info discovery.** The daemon enriches peers over plain HTTP per the consolidated transport policy (node-info is plain on the broker path). Reading a node-info served over operator-configured BYO-TLS / mTLS is not supported through the broker; discovery works for the plain-HTTP case (the common one). +- **BYO-TLS peer authorization.** Operator CA trust and plaintext previews do not establish PAIR peer identity. Service inventory requires pinned cluster mTLS and matching authenticated metadata; older peers lacking that metadata remain previews until upgraded. - **Per-worker log-level granularity.** `log/set-level` fans the broker's single level out to every running child, but there's no way to set a different level per worker — it's one level for the whole process tree. - **AuthN / AuthZ.** Transport security relies on the parent's pipe / socket ACL. There is no per-message token. - **HTTP / WebSocket transport.** JSON-RPC over stdio or pipe only — no web bridge yet. Likely lives in a separate component if/when added. diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index cbfc1491b..b23a90b23 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -74,8 +74,10 @@ type VersionResult struct { // the rest of the NVPAIR codebase uses snake_case, so this is the // deliberate camelCase island at the broker's outer boundary. type AvailableNode struct { - ID string `json:"id"` - Name string `json:"name"` + PeerRecordID string `json:"peerRecordId,omitempty"` + IdentityVerification string `json:"identityVerification,omitempty"` + ID string `json:"id"` + Name string `json:"name"` // HostUUID is the node's stable per-host identity (the daemon's uuid= TXT). // It is invariant across a PC rename and unique per physical machine, so it // is the key clients should dedup/track nodes by; id/name stay the hostname @@ -258,19 +260,22 @@ type Broker struct { // goroutine) while request handlers and the log-level fan-out read // them (on the read-loop / reader goroutines). Every access therefore // goes through the get*/set* helpers under this lock. - workersMu sync.Mutex - scanner *scannerProcess - nodeInfo *nodeInfoProcess - proxy *proxyProcess - lmstudioProxy *proxyProcess - omlxProxy *proxyProcess - workloadMgr *workloadManagerProcess - errorsProc *errorsProcess - engineMgr *rpcWorker - manualNodes *rpcWorker - settings *rpcWorker - clusterMgr *clusterManagerProcess - scheduler *rpcWorker + workersMu sync.Mutex + scannerInventoryGeneration atomic.Uint64 + manualInventoryGeneration atomic.Uint64 + manualCallbackMu sync.Mutex + scanner *scannerProcess + nodeInfo *nodeInfoProcess + proxy *proxyProcess + lmstudioProxy *proxyProcess + omlxProxy *proxyProcess + workloadMgr *workloadManagerProcess + errorsProc *errorsProcess + engineMgr *rpcWorker + manualNodes *rpcWorker + settings *rpcWorker + clusterMgr *clusterManagerProcess + scheduler *rpcWorker // Per-worker supervisors own the (re)spawn + crash-detect + restart // lifecycle of each worker. nil when the worker's binary wasn't resolved @@ -508,6 +513,7 @@ func NewBroker(codec *Codec, paths workerPaths) *Broker { ollamaPortReady: make(chan struct{}), lmstudioPortReady: make(chan struct{}), } + b.relayDir.Authorize = b.authorizedDirectoryNode b.initWorkerRegistry() return b } @@ -523,6 +529,7 @@ func (b *Broker) registerService(p noderec.RegisterParams) { if sc := b.getScanner(); sc != nil { go sc.pushRegister(p) } + b.pushPeerMetadataToNodeInfo() } // unregisterService removes a local service from the cache and the daemon. @@ -533,6 +540,7 @@ func (b *Broker) unregisterService(svc noderec.ServiceKey) { if sc := b.getScanner(); sc != nil { go sc.pushUnregister(svc) } + b.pushPeerMetadataToNodeInfo() } // get*/set* are the workersMu-guarded accessors for the supervised worker @@ -706,19 +714,33 @@ func (b *Broker) getClusterMgr() *clusterManagerProcess { // starting a child process. var ( launchScannerProcess = func(b *Broker) (*scannerProcess, error) { + generation := b.scannerInventoryGeneration.Add(1) return startScanner( b.scannerPath, applog.LevelString(), b.store, b.relayDir, - b.failWorkloadsForNode, + func(uuid, name string) { + if !b.hasDecisionInventory(uuid) { + b.failWorkloadsForNode(uuid, name) + } + }, func(telemetry noderec.NodeTelemetry) { b.ingestTelemetry(sourceScanner, telemetry) }, func(hostUUID string) { b.removeTelemetry(sourceScanner, hostUUID) }, + b.authorizedDirectoryNode, + func() bool { return b.scannerInventoryGeneration.Load() == generation }, b.clusterDirArgs()..., ) } launchManualNodesProcess = func(b *Broker) (*rpcWorker, error) { - return startRPCWorker("manual-nodes", b.manualNodesPath, append(b.logLevelArgs(), b.clusterDirArgs()...), b.forwardManualNodesNotification) + generation := b.manualInventoryGeneration.Add(1) + return startRPCWorker("manual-nodes", b.manualNodesPath, append(b.logLevelArgs(), b.clusterDirArgs()...), func(method string, params json.RawMessage) { + b.manualCallbackMu.Lock() + defer b.manualCallbackMu.Unlock() + if b.manualInventoryGeneration.Load() == generation { + b.forwardManualNodesNotification(method, params) + } + }) } ) @@ -750,26 +772,11 @@ func (b *Broker) spawnScanner() (supervisedHandle, error) { } func (b *Broker) spawnNodeInfo() (supervisedHandle, error) { - // Deliberately NOT passed b.clusterDirArgs(): node-info stays plain HTTP even - // when clustered, and is the ONE documented exception to "the inter-node - // cluster data plane is always mTLS" (errors and workload-manager admit only - // pinned members, in either direction). Its GPU/CPU/memory inventory is the - // lowest-sensitivity inter-node surface and is read directly by consumers that - // hold no cluster identity — the desktop app's two-second telemetry poll and - // baseline LAN discovery — so gating it would remove node telemetry from the UI. - // The accepted risk and its compensating controls are recorded under - // the repository-root SECURITY.md. - // - // node-info still implements the cluster-gated mode (pass it --cluster-dir and - // it admits only pinned peers). It is kept, unenabled, on purpose: it is the - // swap point if the desktop telemetry read is ever solved another way. Enabling - // it here without solving that first WILL blank the UI's node cards. - // - // Do pass --node-id: node-info reports the broker's already-resolved UUID so - // /v1/node-info agrees with the identity the fleet keys on, even on a custom - // --cluster-dir data root (node-info gets no cluster-dir, so it would - // otherwise resolve the default root and could report a different UUID). - np, err := startNodeInfo(b.nodeInfoPath, applog.LevelString(), b.forwardNodeInfoNotification, "--node-id", b.nodeID) + // Keep the public hardware preview and enable pinned cluster mTLS on the + // same port. Services never use the preview as proof. --node-id aligns the + // inventory with this broker, including a custom cluster data root. + args := append([]string{"--node-id", b.nodeID}, b.clusterDirArgs()...) + np, err := startNodeInfo(b.nodeInfoPath, applog.LevelString(), b.forwardNodeInfoNotification, args...) if err != nil { return nil, err } @@ -783,6 +790,7 @@ func (b *Broker) spawnNodeInfo() (supervisedHandle, error) { // node-info binds the fixed :14318 (force_ports is inert), so the broker // knows its port. Idempotent across restarts. b.registerService(noderec.RegisterParams{Service: noderec.ServiceNodeInfo, Port: nodeInfoHTTPPort}) + b.pushPeerMetadataToNodeInfo() slog.Info("node-info started", "path", b.nodeInfoPath, "pid", np.cmd.Process.Pid) return np, nil } @@ -1025,7 +1033,7 @@ func (b *Broker) spawnJobScheduler() (supervisedHandle, error) { b.setScheduler(w) replayed := b.replayActiveWorkloadsToScheduler(w) telemetryReplayed := b.replayTelemetryToScheduler(w) - if err := w.Notify("discovery:nodes-changed", b.store.Snapshot()); err != nil { + if err := w.Notify("discovery:nodes-changed", b.schedulerNodes()); err != nil { slog.Warn("push discovery baseline to scheduler failed", "err", err) } b.schedulerFeedMu.Unlock() @@ -1209,9 +1217,8 @@ func (b *Broker) clusterManagerConfigDir() string { // inter-node mTLS (nvpair-errors, nvpair-workload-manager, nvpair-node-scanner, // nvpair-manual-nodes, nvpair-engine-manager) so each serves/dials pinned cluster // peers once this node is clustered; absent it, those workers stay on plain HTTP -// (or, for engine-manager's ec surface, don't bind it at all). nvpair-node-info is -// deliberately excluded — it stays plain HTTP even on a clustered node (see the -// spawnNodeInfo call site), so it is never passed --cluster-dir. +// (or, for engine-manager's ec surface, don't bind it at all). Node-info uses it +// for its authenticated inventory personality while retaining public previews. func (b *Broker) clusterDirArgs() []string { if b.clusterDir == "" { return nil @@ -1266,6 +1273,7 @@ func resolveLocalNodeID(clusterDir string) string { // are idempotent and run on their own goroutine, because they are // bounded-but-blocking writes and the caller is the cluster-manager's reader. func (b *Broker) applyClusterIdentityChange() { + b.withdrawUnauthorizedInventory() if b.clusterDir == "" { return } @@ -1517,11 +1525,15 @@ func (b *Broker) forwardManualNodesNotification(method string, params json.RawMe // the alias's key changed (node-info revealed its real UUID) the // old key is reprojected from a surviving alias or released. func (b *Broker) upsertManualNode(s manualNodeStatus) { - key := manualToEnriched(s).storeKey() + s = b.normalizeManualStatus(s) + incoming := manualToEnriched(s) + b.annotateManualTrust(&incoming, s) + key := incoming.storeKey() receivedAt := time.Now() b.manualMu.Lock() oldKey, existed := b.manualNodeKeys[s.ID] + priorProof := b.manualNodeStatuses[s.ID].status.PeerRecordID b.manualNodeKeys[s.ID] = key b.manualNodeStatuses[s.ID] = manualNodeStatusEntry{status: s, receivedAt: receivedAt} canonical, ok := b.survivingAliasLocked(key) @@ -1530,6 +1542,9 @@ func (b *Broker) upsertManualNode(s manualNodeStatus) { canonical = manualNodeStatusEntry{status: s, receivedAt: receivedAt} } + if existed && oldKey == key && priorProof != canonical.status.PeerRecordID { + b.removeTelemetry(sourceManual, key) + } en := manualToEnriched(canonical.status) b.annotateManualTrust(&en, canonical.status) b.store.Upsert(en, sourceManual) @@ -1588,8 +1603,12 @@ func (b *Broker) reprojectOrRelease(key string) { b.bridgeManualNode(survivor.status, key) return } - if b.store.Remove(key, sourceManual) && b.relayDir != nil { - b.relayDir.ApplySource(relay.SourceManual, noderec.NotifyNodeRemoved, noderec.DirectoryNode{HostUUID: key}) + b.store.Remove(key, sourceManual) + if b.relayDir != nil { + b.relayDir.ApplySource(relay.SourceManual, noderec.NotifyNodeRemoved, noderec.RecordSelector(key)) + } + if b.workloads != nil && !b.hasDecisionInventory(key) { + b.failWorkloadsForNode(key, key) } b.removeManualNodeFromProxies(key) b.removeTelemetry(sourceManual, key) @@ -1639,6 +1658,9 @@ func (b *Broker) survivingAliasLocked(key string) (manualNodeStatusEntry, bool) // the discovery store. The durable broker-owned list is left intact and is // replayed into the replacement worker after it starts. func (b *Broker) clearManualNodesState() { + b.manualCallbackMu.Lock() + defer b.manualCallbackMu.Unlock() + b.manualInventoryGeneration.Add(1) b.setManualNodes(nil) b.manualMu.Lock() keys := b.manualNodeKeys @@ -1653,8 +1675,12 @@ func (b *Broker) clearManualNodesState() { } seen[key] = true // Drop only the manual claim; a co-located scanner node keeps its record. - if b.store.Remove(key, sourceManual) && b.relayDir != nil { - b.relayDir.ApplySource(relay.SourceManual, noderec.NotifyNodeRemoved, noderec.DirectoryNode{HostUUID: key}) + b.store.Remove(key, sourceManual) + if b.relayDir != nil { + b.relayDir.ApplySource(relay.SourceManual, noderec.NotifyNodeRemoved, noderec.RecordSelector(key)) + } + if b.workloads != nil && !b.hasDecisionInventory(key) { + b.failWorkloadsForNode(key, key) } b.removeTelemetry(sourceManual, key) // Pull the now-orphaned node out of every proxy too, so inference @@ -2114,7 +2140,7 @@ func (b *Broker) fanDiscoveryToScheduler() { if sc == nil { return } - if err := sc.Notify("discovery:nodes-changed", b.store.Snapshot()); err != nil { + if err := sc.Notify("discovery:nodes-changed", b.schedulerNodes()); err != nil { slog.Warn("fan discovery:nodes-changed to scheduler failed", "err", err) } } @@ -2605,6 +2631,7 @@ func (b *Broker) forwardClusterManagerNotification(method string, params json.Ra // fewer refresh triggers than the record does — otherwise a peer converges on a // principal we have already stopped advertising. func (b *Broker) applyClusterTrustChange() { + b.withdrawUnauthorizedInventory() slog.Info("cluster pin set changed; re-deriving peer trust") if sc := b.getScanner(); sc != nil { go sc.reloadTrust() diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index b8fd1f054..809ec84cf 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -36,7 +36,11 @@ type ( ) type EnrichedNode struct { - ID string `json:"id"` + PeerRecordID string `json:"peerRecordId,omitempty"` + IdentityVerification string `json:"identityVerification,omitempty"` + ClusterUUID string `json:"-"` + ManualEngine bool `json:"-"` + ID string `json:"id"` // HostUUID is the node's stable per-host identity (the daemon's uuid= TXT). // It's the discovery-store key, so a PC rename — which changes the hostname // (ID) but not the UUID — updates the existing entry in place instead of @@ -108,21 +112,24 @@ func directoryToEnriched(n noderec.DirectoryNode) EnrichedNode { port = ni.Port } return EnrichedNode{ - ID: n.Name, - HostUUID: n.HostUUID, - Host: n.Name, - Port: port, - Addresses: addrs, - TXT: txt, - GPUs: n.GPUs, - CPU: n.CPU, - Memory: n.Memory, - Trusted: n.Trusted, - Clustered: n.Clustered(), - Models: n.Models, - ModelsByEngine: n.ModelsByEngine, - LoadedByEngine: n.LoadedByEngine, - EngineNative: n.EngineNative, + ID: n.Name, + HostUUID: n.HostUUID, + Host: n.Name, + Port: port, + Addresses: addrs, + TXT: txt, + GPUs: n.GPUs, + CPU: n.CPU, + Memory: n.Memory, + Trusted: n.Trusted, + PeerRecordID: n.PeerRecordID, + IdentityVerification: n.IdentityVerification, + ClusterUUID: n.ClusterUUID, + Clustered: n.Clustered(), + Models: n.Models, + ModelsByEngine: n.ModelsByEngine, + LoadedByEngine: n.LoadedByEngine, + EngineNative: n.EngineNative, } } @@ -133,7 +140,7 @@ func directoryToEnriched(n noderec.DirectoryNode) EnrichedNode { // change never duplicates or splits an entry. No id fallback: an empty // key is rejected by Upsert rather than silently keyed by name. func (n EnrichedNode) storeKey() string { - return n.HostUUID + return (noderec.DirectoryNode{HostUUID: n.HostUUID, PeerRecordID: n.PeerRecordID, IdentityVerification: n.IdentityVerification}).RecordKey() } // nodeSource identifies which feed contributed a store record. The same node @@ -321,7 +328,7 @@ func (s *discoveryStore) Remove(key string, source nodeSource) bool { func enrichedNodeEqual(a, b EnrichedNode) bool { if a.ID != b.ID || a.HostUUID != b.HostUUID || a.Host != b.Host || a.Port != b.Port || - a.Trusted != b.Trusted || a.Clustered != b.Clustered { + a.PeerRecordID != b.PeerRecordID || a.IdentityVerification != b.IdentityVerification || a.ManualEngine != b.ManualEngine || a.Trusted != b.Trusted || a.Clustered != b.Clustered { return false } if !slices.Equal(a.Addresses, b.Addresses) || !slices.Equal(a.TXT, b.TXT) || !slices.Equal(a.Models, b.Models) { @@ -401,20 +408,26 @@ func (sn storedNode) toAvailable() AvailableNode { if len(candidates) < 2 { candidates = nil } + hostUUID := n.HostUUID + if n.IdentityVerification == "manual" { + hostUUID = n.storeKey() + } return AvailableNode{ - ID: n.ID, - Name: n.ID, - HostUUID: n.HostUUID, - IPAddress: primary, - IPAddresses: candidates, - Port: n.Port, - LastSeen: sn.lastSeen.Unix(), - Trusted: n.Trusted, - Clustered: n.Clustered, - Models: n.Models, - ModelsByEngine: n.ModelsByEngine, - LoadedByEngine: n.LoadedByEngine, - EngineNative: n.EngineNative, + PeerRecordID: n.PeerRecordID, + IdentityVerification: n.IdentityVerification, + ID: n.ID, + Name: n.ID, + HostUUID: hostUUID, + IPAddress: primary, + IPAddresses: candidates, + Port: n.Port, + LastSeen: sn.lastSeen.Unix(), + Trusted: n.Trusted, + Clustered: n.Clustered, + Models: n.Models, + ModelsByEngine: n.ModelsByEngine, + LoadedByEngine: n.LoadedByEngine, + EngineNative: n.EngineNative, } } @@ -440,6 +453,9 @@ type scannerProcess struct { onNodeLost func(uuid, name string) onTelemetry func(noderec.NodeTelemetry) onTelemetryRemoved func(hostUUID string) + authorize func(noderec.DirectoryNode) bool + accept func() bool + notifyMu sync.Mutex // activity queues peer liveness reports for the scanner. It exists so the // proxy-reader goroutine that receives them never waits on the scanner's // pipe: reports arrive while inference is streaming, and a wedged scanner @@ -490,6 +506,8 @@ func startScanner( onNodeLost func(uuid, name string), onTelemetry func(noderec.NodeTelemetry), onTelemetryRemoved func(hostUUID string), + authorize func(noderec.DirectoryNode) bool, + accept func() bool, extraArgs ...string, ) (*scannerProcess, error) { cmd := exec.Command(binaryPath, append([]string{"--log-level", logLevel}, extraArgs...)...) @@ -520,6 +538,8 @@ func startScanner( onNodeLost: onNodeLost, onTelemetry: onTelemetry, onTelemetryRemoved: onTelemetryRemoved, + authorize: authorize, + accept: accept, activity: make(chan nodeActivityReport, activityQueueDepth), done: make(chan struct{}), } @@ -674,6 +694,11 @@ func (s *scannerProcess) pushUnregister(svc noderec.ServiceKey) { // is logged. Responses to broker-issued Calls are demuxed by the peer itself and // never reach here; any other notification is ignored. func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { + s.notifyMu.Lock() + defer s.notifyMu.Unlock() + if s.accept != nil && !s.accept() { + return + } switch method { case noderec.NotifyNodeDiscovered, noderec.NotifyNodeUpdated, noderec.NotifyNodeRemoved: var ev noderec.NodeEvent @@ -681,13 +706,38 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { slog.Warn("daemon emitted invalid node event", "method", method, "err", err) return } + if s.authorize != nil && !s.authorize(ev.Node) { + if s.store != nil { + s.store.mu.RLock() + prior := s.store.nodes[ev.Node.RecordKey()].scanner + s.store.mu.RUnlock() + if prior != nil && s.authorize(noderec.DirectoryNode{HostUUID: prior.HostUUID, ClusterUUID: prior.ClusterUUID, Trusted: prior.Trusted, PeerRecordID: prior.PeerRecordID, IdentityVerification: prior.IdentityVerification}) { + return + } + } + ev.Node.Trusted = false + if method != noderec.NotifyNodeRemoved { + ev.Node.PeerRecordID = noderec.PreviewRecordID(ev.Node.Name, ev.Node.HostUUID) + ev.Node.IdentityVerification = "unverified" + } + ev.Node.Models, ev.Node.ModelsByEngine, ev.Node.LoadedByEngine = nil, nil, nil + ev.Node.EngineNative = nil + } + if s.store != nil && method != noderec.NotifyNodeRemoved && ev.Node.ServicePeer() { + s.store.mu.RLock() + prior := s.store.nodes[ev.Node.HostUUID].scanner + s.store.mu.RUnlock() + if prior != nil && prior.PeerRecordID != ev.Node.PeerRecordID && s.onTelemetryRemoved != nil { + s.onTelemetryRemoved(ev.Node.HostUUID) + } + } // Fold into the relay directory (internal subscribers: proxies, mesh // services) and the client-facing store (discovery:get-nodes / // nodes-changed, merging manual nodes), carrying the trusted annotation. if s.relayDir != nil { s.relayDir.ApplySource(relay.SourceScanner, method, ev.Node) } - if method == noderec.NotifyNodeRemoved && s.onTelemetryRemoved != nil { + if method == noderec.NotifyNodeRemoved && ev.Node.IdentityVerification != "unverified" && ev.Node.IdentityVerification != "manual" && s.onTelemetryRemoved != nil { s.onTelemetryRemoved(ev.Node.HostUUID) } if s.store != nil { @@ -701,7 +751,8 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { // present, so we must not fail its jobs. Identify the departed // node by HostUUID, the value workloads are stamped/keyed with, // not the display hostname. - if s.store.Remove(directoryToEnriched(ev.Node).storeKey(), sourceScanner) && s.onNodeLost != nil { + gone := s.store.Remove(directoryToEnriched(ev.Node).storeKey(), sourceScanner) + if ev.Node.IdentityVerification != "unverified" && ev.Node.IdentityVerification != "manual" && (gone || s.authorize != nil) && s.onNodeLost != nil { s.onNodeLost(ev.Node.HostUUID, ev.Node.Name) } } else { diff --git a/services/nvpair-ui-broker/discovery_test.go b/services/nvpair-ui-broker/discovery_test.go index 35b6f0c33..c173bea6b 100644 --- a/services/nvpair-ui-broker/discovery_test.go +++ b/services/nvpair-ui-broker/discovery_test.go @@ -7,6 +7,8 @@ import ( "encoding/json" "testing" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" "nvpair-ui-broker/relay" ) @@ -18,8 +20,8 @@ func TestDiscoveryStoreRekeysOnRename(t *testing.T) { s := newDiscoveryStore() const uuid = "11111111-1111-1111-1111-111111111111" - s.Upsert(EnrichedNode{ID: "old-host", HostUUID: uuid}, sourceScanner) - s.Upsert(EnrichedNode{ID: "new-host", HostUUID: uuid}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "old-host", HostUUID: uuid}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "new-host", HostUUID: uuid}, sourceScanner) snap := s.Snapshot() if len(snap) != 1 { @@ -37,24 +39,31 @@ func TestDiscoveryStoreRekeysOnRename(t *testing.T) { // case: two different machines that share a hostname stay two entries. func TestDiscoveryStoreDistinctUUIDsSameName(t *testing.T) { s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "samename", HostUUID: "aaaaaaaa-0000-0000-0000-000000000001"}, sourceScanner) - s.Upsert(EnrichedNode{ID: "samename", HostUUID: "bbbbbbbb-0000-0000-0000-000000000002"}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "samename", HostUUID: "aaaaaaaa-0000-0000-0000-000000000001"}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "samename", HostUUID: "bbbbbbbb-0000-0000-0000-000000000002"}, sourceScanner) if got := len(s.Snapshot()); got != 2 { t.Fatalf("two machines sharing a hostname must not merge: got %d entries, want 2", got) } } // TestManualToEnrichedHostUUID verifies the manual-node ingestion boundary: a -// manual node keys by the remote's real hostUuid once node-info reports one, and -// by its manual id until then. +// manual node keys by the proven peer's hostUuid, never by an unproven claim. func TestManualToEnrichedHostUUID(t *testing.T) { - withUUID := manualToEnriched(manualNodeStatus{ID: "manual:10.0.0.9", HostUUID: "real-uuid"}) + b := newManualTestBroker(t, "real-uuid") + status := ownedManualStatus(t, b, manualNodeStatus{ID: "manual:10.0.0.9", HostUUID: "real-uuid"}) + withUUID := manualToEnriched(status) + b.annotateManualTrust(&withUUID, status) if withUUID.storeKey() != "real-uuid" { t.Fatalf("store key = %q, want the real hostUuid", withUUID.storeKey()) } + claimedUUID := manualToEnriched(manualNodeStatus{ID: "manual:10.0.0.9", HostUUID: "real-uuid"}) + wantManualKey := noderec.ManualRecordID("manual:10.0.0.9") + if claimedUUID.storeKey() != wantManualKey { + t.Fatalf("unproven hostUuid must use the manual config namespace: got %q, want %q", claimedUUID.storeKey(), wantManualKey) + } noUUID := manualToEnriched(manualNodeStatus{ID: "manual:10.0.0.9"}) - if noUUID.storeKey() != "manual:10.0.0.9" { - t.Fatalf("store key = %q, want the manual id fallback", noUUID.storeKey()) + if noUUID.storeKey() != wantManualKey { + t.Fatalf("store key = %q, want namespaced manual config key %q", noUUID.storeKey(), wantManualKey) } } @@ -79,8 +88,8 @@ func TestDiscoveryStoreSourceOwnership(t *testing.T) { // Manual-remove overlap: scanner + manual both claim uuid; removing the // manual claim must leave the scanner's node in the snapshot. s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid, Port: 14318}, sourceScanner) - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid, Port: 14318}, sourceManual) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid, Port: 14318}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid, Port: 14318}, sourceManual) if got := len(s.Snapshot()); got != 1 { t.Fatalf("shared uuid should be one record, got %d", got) } @@ -95,8 +104,8 @@ func TestDiscoveryStoreSourceOwnership(t *testing.T) { // Symmetric: removing the scanner claim leaves the manual node. s = newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceScanner) - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceManual) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid}, sourceManual) s.Remove(uuid, sourceScanner) if got := len(s.Snapshot()); got != 1 { t.Fatalf("removing the scanner claim evicted the manual node (got %d entries)", got) @@ -109,8 +118,8 @@ func TestDiscoveryStoreSourceOwnership(t *testing.T) { func TestDiscoveryStoreScannerProjectionWins(t *testing.T) { const uuid = "shared-uuid" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "manual-host", HostUUID: uuid, Trusted: false}, sourceManual) - s.Upsert(EnrichedNode{ID: "mdns-host", HostUUID: uuid, Trusted: true}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "manual-host", HostUUID: uuid, Trusted: false}, sourceManual) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "mdns-host", HostUUID: uuid, Trusted: true}, sourceScanner) snap := s.Snapshot() if len(snap) != 1 { t.Fatalf("want one record, got %d", len(snap)) @@ -126,22 +135,48 @@ func TestDiscoveryStoreScannerProjectionWins(t *testing.T) { } } -func newManualTestBroker() *Broker { - return &Broker{ +func newManualTestBroker(t *testing.T, peers ...string) *Broker { + t.Helper() + dir := t.TempDir() + clustertrusttest.Join(t, dir, "fixture-cluster", "local-node", peers...) + b := &Broker{ + nodeID: "local-node", + mesh: clustertrust.Open(dir), manualNodeKeys: make(map[string]string), manualNodeStatuses: make(map[string]manualNodeStatusEntry), store: newDiscoveryStore(), telemetry: newTelemetryCache(), relayDir: relay.NewDirectory(), } + b.relayDir.Authorize = b.authorizedDirectoryNode + return b +} + +// ownedManualStatus models a broker-owned worker's authenticated status, using +// this test mesh's current pin record. TLS authentication itself is exercised by +// the cross-service proof tests, not these broker projection tests. +func ownedManualStatus(t *testing.T, b *Broker, status manualNodeStatus) manualNodeStatus { + t.Helper() + status.ClusterUUID = status.HostUUID + status.PeerRecordID = b.mesh.PeerRecordID(status.HostUUID) + if status.PeerRecordID == "" { + t.Fatalf("missing fixture pin for %q", status.HostUUID) + } + status.PeerMetadata = &noderec.PeerMetadataParams{ + HostUUID: status.HostUUID, + Name: status.ID, + Services: map[noderec.ServiceKey]int{}, + } + return status } -func manualStatus(id, addr, uuid string) manualNodeStatus { - return manualNodeStatus{ID: id, Address: addr, HostUUID: uuid, NodeInfoPort: 14318} +func manualStatus(t *testing.T, b *Broker, id, addr, uuid string) manualNodeStatus { + t.Helper() + return ownedManualStatus(t, b, manualNodeStatus{ID: id, Address: addr, HostUUID: uuid, NodeInfoPort: 14318}) } func TestManualNodeBridgesToRelayDir(t *testing.T) { - b := newManualTestBroker() + b := newManualTestBroker(t, "remote-node-uuid") const uuid = "remote-node-uuid" status := manualNodeStatus{ @@ -155,6 +190,13 @@ func TestManualNodeBridgesToRelayDir(t *testing.T) { OllamaPort: 11434, } + status = ownedManualStatus(t, b, status) + status.PeerMetadata.Services = map[noderec.ServiceKey]int{ + noderec.ServiceEngineControl: 14323, + noderec.ServiceEngineManager: 14322, + noderec.ServiceNodeInfo: 14318, + noderec.ServiceOllama: 11434, + } b.upsertManualNode(status) // Check relayDir @@ -195,7 +237,7 @@ func TestManualNodeBridgesToRelayDir(t *testing.T) { } func TestManualNodeModelsAttribution(t *testing.T) { - b := newManualTestBroker() + b := newManualTestBroker(t, "remote-node-uuid") const uuid = "remote-node-uuid" status := manualNodeStatus{ @@ -217,6 +259,13 @@ func TestManualNodeModelsAttribution(t *testing.T) { }, } + status = ownedManualStatus(t, b, status) + status.PeerMetadata.Services = map[noderec.ServiceKey]int{ + noderec.ServiceNodeInfo: 14318, + noderec.ServiceOllama: 11434, + noderec.ServiceLMStudio: 1234, + noderec.ServiceOMLX: 1236, + } b.upsertManualNode(status) // Check discovery store snapshot @@ -225,6 +274,20 @@ func TestManualNodeModelsAttribution(t *testing.T) { t.Fatalf("expected 1 node in store snapshot, got %d", len(snap)) } sn := snap[0] + raw, err := json.Marshal(sn) + if err != nil { + t.Fatalf("marshal available node: %v", err) + } + var identity struct { + PeerRecordID string `json:"peerRecordId"` + IdentityVerification string `json:"identityVerification"` + } + if err := json.Unmarshal(raw, &identity); err != nil { + t.Fatalf("decode available node identity: %v", err) + } + if identity.PeerRecordID != status.PeerRecordID || identity.IdentityVerification != "verified" { + t.Fatalf("available node wire identity = %+v, want current peer record and verified marker", identity) + } if len(sn.Models) != 3 || sn.Models[0] != "llama3:8b" || sn.Models[1] != "qwen2.5:7b" || sn.Models[2] != "mlx-qwen" { t.Errorf("unexpected Models in snapshot: %#v", sn.Models) } @@ -259,7 +322,7 @@ func TestManualNodeModelsAttribution(t *testing.T) { if len(rn.ModelsByEngine["omlx"]) != 1 { t.Errorf("unexpected ModelsByEngine in relayDir: %#v", rn.ModelsByEngine) } - // Also ensure ServiceOllama, ServiceLMStudio, and ServiceOMLX were inferred from ModelsByEngine + // Authenticated metadata supplies the engine ports; model names do not. if svc, ok := rn.Services[noderec.ServiceOllama]; !ok || svc.Port != 11434 { t.Errorf("ServiceOllama = %+v, want port 11434", svc) } @@ -288,9 +351,9 @@ func TestManualAliasesShareKeyUntilLastRemoved(t *testing.T) { {"remove-B-first", "alias-b", "alias-a", "10.0.0.1"}, } { t.Run(tc.name, func(t *testing.T) { - b := newManualTestBroker() - b.upsertManualNode(manualStatus("alias-a", "10.0.0.1", uuid)) - b.upsertManualNode(manualStatus("alias-b", "10.0.0.2", uuid)) + b := newManualTestBroker(t, "shared-uuid") + b.upsertManualNode(manualStatus(t, b, "alias-a", "10.0.0.1", uuid)) + b.upsertManualNode(manualStatus(t, b, "alias-b", "10.0.0.2", uuid)) if got := len(b.store.Snapshot()); got != 1 { t.Fatalf("two aliases for one machine should be one record, got %d", got) } @@ -320,14 +383,14 @@ func TestManualAliasesShareKeyUntilLastRemoved(t *testing.T) { // stay — reprojected from the alias that still owns it — alongside the rekeyed // alias's new record. func TestManualRekeyReprojectsSharedOldKey(t *testing.T) { - b := newManualTestBroker() + b := newManualTestBroker(t, "shared-uuid", "alias-a-uuid") const shared = "shared-uuid" - b.upsertManualNode(manualStatus("alias-a", "10.0.0.1", shared)) - b.upsertManualNode(manualStatus("alias-b", "10.0.0.2", shared)) + b.upsertManualNode(manualStatus(t, b, "alias-a", "10.0.0.1", shared)) + b.upsertManualNode(manualStatus(t, b, "alias-b", "10.0.0.2", shared)) // alias-a rekeys onto its own UUID; alias-b still owns the shared key. - b.upsertManualNode(manualStatus("alias-a", "10.0.0.1", "alias-a-uuid")) + b.upsertManualNode(manualStatus(t, b, "alias-a", "10.0.0.1", "alias-a-uuid")) snap := b.store.Snapshot() if len(snap) != 2 { @@ -353,11 +416,11 @@ func TestManualRekeyReprojectsSharedOldKey(t *testing.T) { // refreshManualNodesTrust passes must project a deterministic canonical alias // without emitting spurious discovery store change notifications. func TestManualAliasesDoNotOscillateOnUpsertOrTrustRefresh(t *testing.T) { - b := newManualTestBroker() + b := newManualTestBroker(t, "shared-uuid") const shared = "shared-uuid" - b.upsertManualNode(manualStatus("alias-b", "10.0.0.1", shared)) - b.upsertManualNode(manualStatus("alias-a", "10.0.0.1", shared)) + b.upsertManualNode(manualStatus(t, b, "alias-b", "10.0.0.1", shared)) + b.upsertManualNode(manualStatus(t, b, "alias-a", "10.0.0.1", shared)) snap := b.store.Snapshot() if len(snap) != 1 || snap[0].ID != "alias-a" { @@ -371,7 +434,7 @@ func TestManualAliasesDoNotOscillateOnUpsertOrTrustRefresh(t *testing.T) { // refreshManualNodesTrust across map iterations must not flip the projected // ID or emit store changes. for i := 0; i < 10; i++ { - b.upsertManualNode(manualStatus("alias-b", "10.0.0.1", shared)) + b.upsertManualNode(manualStatus(t, b, "alias-b", "10.0.0.1", shared)) b.refreshManualNodesTrust() } @@ -387,12 +450,13 @@ func TestManualAliasesDoNotOscillateOnUpsertOrTrustRefresh(t *testing.T) { // A failed, lexically first alias must not hide a working address for the same // host. Operational failover may change the address, but not the display identity. func TestManualAliasesRetainReachableRouteWithStableIdentity(t *testing.T) { - b := newManualTestBroker() + b := newManualTestBroker(t, "shared-uuid") const shared = "shared-uuid" - first := manualStatus("alias-a", "192.0.2.1", shared) + first := manualStatus(t, b, "alias-a", "192.0.2.1", shared) first.NodeInfoUp = true first.OllamaUp = true first.OllamaModels = []string{"test-model"} + first.PeerMetadata.Services[noderec.ServiceOllama] = 11434 second := first second.ID = "alias-b" second.Address = "192.0.2.2" @@ -421,7 +485,7 @@ func TestManualAliasesRetainReachableRouteWithStableIdentity(t *testing.T) { } assertRoute("192.0.2.1") - b.upsertManualNode(manualStatus("alias-a", "192.0.2.1", shared)) + b.upsertManualNode(manualStatus(t, b, "alias-a", "192.0.2.1", shared)) assertRoute("192.0.2.2") for i := 0; i < 3; i++ { b.upsertManualNode(second) @@ -430,7 +494,7 @@ func TestManualAliasesRetainReachableRouteWithStableIdentity(t *testing.T) { } // A healthy node-info endpoint alone must not displace a known engine route. - metadataOnly := manualStatus("alias-a", "192.0.2.1", shared) + metadataOnly := manualStatus(t, b, "alias-a", "192.0.2.1", shared) metadataOnly.NodeInfoUp = true b.upsertManualNode(metadataOnly) assertRoute("192.0.2.2") @@ -443,7 +507,7 @@ func TestManualAliasesRetainReachableRouteWithStableIdentity(t *testing.T) { func TestDiscoveryStoreRemoveWrongSourceNoop(t *testing.T) { const uuid = "u" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid}, sourceScanner) if s.Remove(uuid, sourceManual) { // manual never claimed it t.Fatal("removing an unowned source must report false (nothing removed)") } @@ -457,8 +521,8 @@ func TestDiscoveryStoreRemoveWrongSourceNoop(t *testing.T) { func TestDiscoveryStoreRemoveReportsFinalClaim(t *testing.T) { const uuid = "shared-uuid" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceScanner) - s.Upsert(EnrichedNode{ID: "host", HostUUID: uuid}, sourceManual) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid}, sourceScanner) + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid}, sourceManual) if s.Remove(uuid, sourceScanner) { t.Fatal("removing one of two claims must report false (node still present)") @@ -475,7 +539,7 @@ func TestDiscoveryStoreRemoveReportsFinalClaim(t *testing.T) { // the given display name and stable host UUID. func nodeRemovedFrame(t *testing.T, name, uuid string) json.RawMessage { t.Helper() - b, err := json.Marshal(noderec.NodeEvent{Node: noderec.DirectoryNode{Name: name, HostUUID: uuid}}) + b, err := json.Marshal(noderec.NodeEvent{Node: noderec.DirectoryNode{IdentityVerification: "local", Name: name, HostUUID: uuid}}) if err != nil { t.Fatalf("marshal node event: %v", err) } @@ -489,8 +553,8 @@ func nodeRemovedFrame(t *testing.T, name, uuid string) json.RawMessage { func TestHandleNotifyNodeLostOnlyOnFinalClaim(t *testing.T) { const uuid = "peer-uuid" store := newDiscoveryStore() - store.Upsert(EnrichedNode{ID: "peer-friendly-name", HostUUID: uuid}, sourceScanner) - store.Upsert(EnrichedNode{ID: "peer-friendly-name", HostUUID: uuid}, sourceManual) + store.Upsert(EnrichedNode{IdentityVerification: "local", ID: "peer-friendly-name", HostUUID: uuid}, sourceScanner) + store.Upsert(EnrichedNode{IdentityVerification: "local", ID: "peer-friendly-name", HostUUID: uuid}, sourceManual) var calls int sp := &scannerProcess{store: store, onNodeLost: func(string, string) { calls++ }} @@ -512,7 +576,7 @@ func TestHandleNotifyNodeLostUsesHostUUID(t *testing.T) { const uuid = "peer-uuid" const name = "peer-friendly-name" store := newDiscoveryStore() - store.Upsert(EnrichedNode{ID: name, HostUUID: uuid}, sourceScanner) + store.Upsert(EnrichedNode{IdentityVerification: "local", ID: name, HostUUID: uuid}, sourceScanner) var gotUUID, gotName string var calls int @@ -567,14 +631,14 @@ func TestDiscoveryStoreMergesManualAddressesIntoScannerNode(t *testing.T) { const uuid = "shared-uuid-123" s := newDiscoveryStore() - s.Upsert(EnrichedNode{ + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid, Addresses: []string{"192.168.1.50"}, TXT: []string{"ip=192.168.1.50"}, }, sourceScanner) - s.Upsert(EnrichedNode{ + s.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host", HostUUID: uuid, Addresses: []string{"100.64.1.2"}, diff --git a/services/nvpair-ui-broker/enginenative_test.go b/services/nvpair-ui-broker/enginenative_test.go index 2f3d8e17f..902c4386c 100644 --- a/services/nvpair-ui-broker/enginenative_test.go +++ b/services/nvpair-ui-broker/enginenative_test.go @@ -40,7 +40,7 @@ func TestBrokerCarriesTheEngineNativeView(t *testing.T) { }, ObservedAt: 1716998400000, } - node := noderec.DirectoryNode{ + node := noderec.DirectoryNode{IdentityVerification: "local", HostUUID: "uuid-a", Name: "host-a", IP: "192.0.2.10", @@ -104,7 +104,7 @@ func TestEngineNativeViewChangeIsAnnounced(t *testing.T) { Totals: enginenative.Totals{Requests: 2}, ObservedAt: 1000, } - store.Upsert(EnrichedNode{ID: "host-a", HostUUID: "uuid-a", EngineNative: view}, sourceScanner) + store.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host-a", HostUUID: "uuid-a", EngineNative: view}, sourceScanner) if changes != 1 { t.Fatalf("the first sighting reported %d changes, want 1", changes) } @@ -113,7 +113,7 @@ func TestEngineNativeViewChangeIsAnnounced(t *testing.T) { // The observation is still newer, so it is still a change. same := *view same.ObservedAt = 2000 - store.Upsert(EnrichedNode{ID: "host-a", HostUUID: "uuid-a", EngineNative: &same}, sourceScanner) + store.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host-a", HostUUID: "uuid-a", EngineNative: &same}, sourceScanner) if changes != 2 { t.Fatalf("a newer observation reported %d changes, want 2", changes) } @@ -121,14 +121,14 @@ func TestEngineNativeViewChangeIsAnnounced(t *testing.T) { // The engine's own counters moved: that is a change too. moved := same moved.Totals.Requests = 3 - store.Upsert(EnrichedNode{ID: "host-a", HostUUID: "uuid-a", EngineNative: &moved}, sourceScanner) + store.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host-a", HostUUID: "uuid-a", EngineNative: &moved}, sourceScanner) if changes != 3 { t.Fatalf("a changed engine-native view reported %d changes, want 3", changes) } // The same reading again — same numbers, same stamp — is not a change: // nothing about it moved. - store.Upsert(EnrichedNode{ID: "host-a", HostUUID: "uuid-a", EngineNative: &moved}, sourceScanner) + store.Upsert(EnrichedNode{IdentityVerification: "local", ID: "host-a", HostUUID: "uuid-a", EngineNative: &moved}, sourceScanner) if changes != 3 { t.Fatalf("an unchanged engine-native view reported %d changes, want 3", changes) } diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index f5ba498b8..e74e40b46 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -16,8 +16,6 @@ import ( "nvpair-shared/appdir" "nvpair-shared/enginenative" "nvpair-shared/noderec" - - "nvpair-ui-broker/relay" ) // manualNodeStatus is the subset of nvpair-manual-nodes' ManualNodeStatus the @@ -28,25 +26,28 @@ import ( // (bridgeManualNode); the rest project into the discovery store via // manualToEnriched. type manualNodeStatus struct { - ID string `json:"id"` - Address string `json:"address"` - OllamaUp bool `json:"ollama_up"` - OllamaPort int `json:"ollama_port"` - OllamaModels []string `json:"ollama_models,omitempty"` - LMStudioUp bool `json:"lmstudio_up"` - LMStudioPort int `json:"lmstudio_port"` - LMStudioModels []string `json:"lmstudio_models,omitempty"` - OMLXUp bool `json:"omlx_up"` - OMLXPort int `json:"omlx_port"` - OMLXModels []string `json:"omlx_models,omitempty"` - NodeInfoUp bool `json:"node_info_up"` - NodeInfoPort int `json:"node_info_port"` - GPUs []GPUInfo `json:"gpus"` - CPU *CPUInfo `json:"cpu"` - Memory *MemoryInfo `json:"memory"` - TelemetryValid bool `json:"telemetryValid"` - MSSince int64 `json:"msSince"` - SampleSeq uint64 `json:"sampleSeq,omitempty"` + IdentityVerification string `json:"identityVerification,omitempty"` + PeerRecordID string `json:"peerRecordId,omitempty"` + PeerMetadata *noderec.PeerMetadataParams `json:"peerMetadata,omitempty"` + ID string `json:"id"` + Address string `json:"address"` + OllamaUp bool `json:"ollama_up"` + OllamaPort int `json:"ollama_port"` + OllamaModels []string `json:"ollama_models,omitempty"` + LMStudioUp bool `json:"lmstudio_up"` + LMStudioPort int `json:"lmstudio_port"` + LMStudioModels []string `json:"lmstudio_models,omitempty"` + OMLXUp bool `json:"omlx_up"` + OMLXPort int `json:"omlx_port"` + OMLXModels []string `json:"omlx_models,omitempty"` + NodeInfoUp bool `json:"node_info_up"` + NodeInfoPort int `json:"node_info_port"` + GPUs []GPUInfo `json:"gpus"` + CPU *CPUInfo `json:"cpu"` + Memory *MemoryInfo `json:"memory"` + TelemetryValid bool `json:"telemetryValid"` + MSSince int64 `json:"msSince"` + SampleSeq uint64 `json:"sampleSeq,omitempty"` // HostUUID is the remote's stable per-host identity, learned from its // node-info /v1/node-info. It lets a manual node key by the same permanent // identity as mDNS-discovered nodes (and dedup with itself when the same @@ -70,6 +71,7 @@ type manualNodeStatusEntry struct { } type manualNodeTelemetryParams struct { + PeerRecordID string `json:"peerRecordId,omitempty"` ID string `json:"id"` HostUUID string `json:"hostUuid,omitempty"` GPUUtilizationPct uint32 `json:"gpuUtilizationPct"` @@ -238,6 +240,7 @@ func manualNodeTelemetry(status manualNodeStatus, hostUUID string) noderec.NodeT } return noderec.NodeTelemetry{ HostUUID: hostUUID, + PeerRecordID: status.PeerRecordID, GPUUtilizationPct: utilization, TelemetryValid: status.TelemetryValid, MSSince: status.MSSince, @@ -259,6 +262,7 @@ func (b *Broker) ingestManualNodeTelemetry(t manualNodeTelemetryParams) { } b.ingestTelemetryAt(sourceManual, noderec.NodeTelemetry{ HostUUID: key, + PeerRecordID: t.PeerRecordID, GPUUtilizationPct: t.GPUUtilizationPct, TelemetryValid: t.TelemetryValid, MSSince: t.MSSince, @@ -281,9 +285,18 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { // fallback. Once the real UUID is learned, a manually-added machine // that's also discovered over mDNS collapses to the one hostUuid-keyed entry. hostUUID := s.HostUUID - if hostUUID == "" { + if hostUUID == "" || s.PeerRecordID == "" { hostUUID = s.ID } + recordID := s.PeerRecordID + verification := "verified" + if recordID == "" { + recordID = noderec.ManualRecordID(s.ID) + verification = "unverified" + if s.HostUUID == "" && s.ClusterUUID == "" && s.IdentityVerification != "unverified" { + verification = "manual" + } + } models := s.Models if len(models) == 0 { models = mergeModels(s.OllamaModels, s.LMStudioModels, s.OMLXModels) @@ -293,18 +306,22 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { modelsByEngine = manualModelsByEngine(s) } en := EnrichedNode{ - ID: s.ID, - HostUUID: hostUUID, - Host: s.Address, - Port: s.NodeInfoPort, - GPUs: s.GPUs, - CPU: s.CPU, - Memory: s.Memory, - Models: models, - ModelsByEngine: modelsByEngine, - LoadedByEngine: s.LoadedByEngine, - EngineNative: s.EngineNative, - Clustered: s.ClusterUUID != "", + ID: s.ID, + HostUUID: hostUUID, + Host: s.Address, + Port: s.NodeInfoPort, + GPUs: s.GPUs, + CPU: s.CPU, + Memory: s.Memory, + Models: models, + ModelsByEngine: modelsByEngine, + LoadedByEngine: s.LoadedByEngine, + EngineNative: s.EngineNative, + Clustered: s.ClusterUUID != "", + PeerRecordID: recordID, + IdentityVerification: verification, + ClusterUUID: s.ClusterUUID, + ManualEngine: verification == "manual", } if s.Address != "" { en.Addresses = []string{s.Address} @@ -328,31 +345,32 @@ func (b *Broker) annotateManualTrust(en *EnrichedNode, s manualNodeStatus) { return } b.mesh.Refresh() - if (s.ClusterUUID != "" && b.mesh.HasPin(s.ClusterUUID)) || - (en.HostUUID != "" && b.mesh.HasPin(en.HostUUID)) { - en.Trusted = true + en.Trusted = s.HostUUID == s.ClusterUUID && s.PeerMetadata != nil && s.PeerMetadata.HostUUID == s.HostUUID && b.currentPeerProof(s.ClusterUUID, s.PeerRecordID) + if !en.Trusted && en.IdentityVerification == "verified" { + en.IdentityVerification = "unverified" + en.PeerRecordID = noderec.ManualRecordID(s.ID) + } + if en.Trusted { en.Clustered = true + en.IdentityVerification = "verified" } } func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.DirectoryNode { + if s.PeerRecordID != "" && s.PeerMetadata != nil && s.HostUUID == s.ClusterUUID && b.currentPeerProof(s.ClusterUUID, s.PeerRecordID) { + services := make(map[noderec.ServiceKey]noderec.ServiceStatus, len(s.PeerMetadata.Services)) + for service, port := range s.PeerMetadata.Services { + services[service] = noderec.ServiceStatus{Port: port} + } + return noderec.DirectoryNode{HostUUID: s.HostUUID, Name: s.PeerMetadata.Name, IP: s.Address, IPs: []string{s.Address}, ClusterUUID: s.ClusterUUID, Trusted: true, IdentityVerification: "verified", PeerRecordID: s.PeerRecordID, Services: services, GPUs: s.GPUs, CPU: s.CPU, Memory: s.Memory, Models: s.Models, ModelsByEngine: s.ModelsByEngine, LoadedByEngine: s.LoadedByEngine, EngineNative: s.EngineNative, LastSeen: time.Now().Unix()} + } hostUUID := s.HostUUID if hostUUID == "" { hostUUID = key } trusted := false clusterUUID := s.ClusterUUID - if b.mesh != nil { - b.mesh.Refresh() - if clusterUUID != "" && b.mesh.HasPin(clusterUUID) { - trusted = true - } else if hostUUID != "" && b.mesh.HasPin(hostUUID) { - trusted = true - if clusterUUID == "" { - clusterUUID = hostUUID - } - } - } + // Public/manual IDs never become peer proofs merely because a pin exists. services := make(map[noderec.ServiceKey]noderec.ServiceStatus) if s.NodeInfoUp { port := s.NodeInfoPort @@ -420,22 +438,28 @@ func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.D models = mergeModels(s.OllamaModels, s.LMStudioModels, s.OMLXModels) } + verification := "unverified" + if s.HostUUID == "" && s.ClusterUUID == "" && s.IdentityVerification != "unverified" { + verification = "manual" + } return noderec.DirectoryNode{ - HostUUID: hostUUID, - Name: s.ID, - IP: s.Address, - IPs: ips, - ClusterUUID: clusterUUID, - Trusted: trusted, - Services: services, - GPUs: s.GPUs, - CPU: s.CPU, - Memory: s.Memory, - Models: models, - ModelsByEngine: modelsByEngine, - LoadedByEngine: s.LoadedByEngine, - EngineNative: s.EngineNative, - LastSeen: time.Now().Unix(), + PeerRecordID: noderec.ManualRecordID(s.ID), + IdentityVerification: verification, + HostUUID: hostUUID, + Name: s.ID, + IP: s.Address, + IPs: ips, + ClusterUUID: clusterUUID, + Trusted: trusted, + Services: services, + GPUs: s.GPUs, + CPU: s.CPU, + Memory: s.Memory, + Models: models, + ModelsByEngine: modelsByEngine, + LoadedByEngine: s.LoadedByEngine, + EngineNative: s.EngineNative, + LastSeen: time.Now().Unix(), } } @@ -458,13 +482,7 @@ func (b *Broker) refreshManualNodesTrust() { b.manualMu.Unlock() for _, entry := range entries { - en := manualToEnriched(entry.status) - key := en.storeKey() - b.annotateManualTrust(&en, entry.status) - b.store.Upsert(en, sourceManual) - if b.relayDir != nil { - b.relayDir.ApplySource(relay.SourceManual, noderec.NotifyNodeUpdated, b.manualToDirectoryNode(entry.status, key)) - } + b.upsertManualNode(entry.status) } } @@ -582,7 +600,9 @@ func (b *Broker) bridgeToProxy(p *proxyProcess, engine string, s manualNodeStatu if p == nil { return } - if up && s.Address != "" && port > 0 { + // Proven PAIR peers travel exclusively through the authenticated relay. + // Only independent, explicitly configured bare engines use this bypass. + if up && s.IdentityVerification != "unverified" && s.HostUUID == "" && s.ClusterUUID == "" && s.PeerRecordID == "" && s.Address != "" && port > 0 { node := proxyManualNode{ ID: key, Host: s.Address, diff --git a/services/nvpair-ui-broker/manualnodes_enginenative_test.go b/services/nvpair-ui-broker/manualnodes_enginenative_test.go index 4b07ef462..2b772011c 100644 --- a/services/nvpair-ui-broker/manualnodes_enginenative_test.go +++ b/services/nvpair-ui-broker/manualnodes_enginenative_test.go @@ -23,7 +23,7 @@ import ( // discovery:get-nodes snapshot is taken from, and the relay directory the // discovery:nodes-changed stream is taken from. func TestManualNodeCarriesTheEngineNativeView(t *testing.T) { - b := newManualTestBroker() + b := newManualTestBroker(t, "remote-node-uuid") const uuid = "remote-node-uuid" view := &enginenative.Stats{ @@ -44,6 +44,7 @@ func TestManualNodeCarriesTheEngineNativeView(t *testing.T) { EngineNative: view, } + status = ownedManualStatus(t, b, status) b.upsertManualNode(status) // The client-facing snapshot (discovery:get-nodes) carries the view. @@ -74,8 +75,8 @@ func TestManualNodeCarriesTheEngineNativeView(t *testing.T) { // TestManualNodeWithoutAnEngineNativeViewCarriesNone pins that a node with no // view carries none: the field is absent, not empty. func TestManualNodeWithoutAnEngineNativeViewCarriesNone(t *testing.T) { - b := newManualTestBroker() - b.upsertManualNode(manualStatus("Retirement-Plan", "100.107.205.77", "remote-node-uuid")) + b := newManualTestBroker(t, "remote-node-uuid") + b.upsertManualNode(manualStatus(t, b, "Retirement-Plan", "100.107.205.77", "remote-node-uuid")) snap := b.store.Snapshot() if len(snap) != 1 { @@ -97,7 +98,7 @@ func TestManualNodeWithoutAnEngineNativeViewCarriesNone(t *testing.T) { // reads: the node record the broker publishes for a manual node carries the // same engineNative field a discovered node's record carries. func TestManualNodeEngineNativeViewIsOnTheNodeRecord(t *testing.T) { - b := newManualTestBroker() + b := newManualTestBroker(t, "remote-node-uuid") view := &enginenative.Stats{Engine: enginenative.EngineOmlx, Supported: true} status := manualNodeStatus{ ID: "Retirement-Plan", @@ -108,6 +109,7 @@ func TestManualNodeEngineNativeViewIsOnTheNodeRecord(t *testing.T) { NodeInfoPort: 14318, EngineNative: view, } + status = ownedManualStatus(t, b, status) node := b.manualToDirectoryNode(status, "remote-node-uuid") if node.EngineNative == nil { t.Fatal("the manual node's record carries no engine-native view") diff --git a/services/nvpair-ui-broker/peer_inventory.go b/services/nvpair-ui-broker/peer_inventory.go new file mode 100644 index 000000000..b05855747 --- /dev/null +++ b/services/nvpair-ui-broker/peer_inventory.go @@ -0,0 +1,240 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "log/slog" + "nvpair-shared/noderec" + "nvpair-ui-broker/relay" + "os" + "sort" +) + +func (b *Broker) pushPeerMetadataToNodeInfo() { + np := b.getNodeInfo() + if np == nil || b.regCache == nil { + return + } + // Serialize snapshot creation with writes so concurrent registrations cannot + // publish an older snapshot after a newer one. + np.stdinMu.Lock() + defer np.stdinMu.Unlock() + name, _ := os.Hostname() + meta := noderec.PeerMetadataParams{HostUUID: b.nodeID, Name: name, Services: make(map[noderec.ServiceKey]int)} + for _, p := range b.regCache.Snapshot() { + meta.Services[p.Service] = p.Port + } + if err := np.codec.Notify(noderec.MethodSetPeerMetadata, meta); err != nil { + slog.Debug("failed to publish local peer metadata", "err", err) + } +} + +func (b *Broker) normalizeManualStatus(s manualNodeStatus) manualNodeStatus { + if s.PeerRecordID == "" { + return s + } + if s.HostUUID == s.ClusterUUID && s.PeerMetadata != nil && s.PeerMetadata.HostUUID == s.HostUUID && b.currentPeerProof(s.ClusterUUID, s.PeerRecordID) { + return s + } + // A retired worker proof may remain as an unverified configuration preview, + // but never under the peer's host UUID and never as a bare-engine bypass. + s.PeerRecordID = "" + s.IdentityVerification = "unverified" + s.PeerMetadata = nil + s.NodeInfoUp = false + s.OllamaUp = false + s.LMStudioUp = false + s.OMLXUp = false + s.Models, s.OllamaModels, s.LMStudioModels, s.OMLXModels = nil, nil, nil, nil + s.ModelsByEngine, s.LoadedByEngine = nil, nil + s.GPUs, s.CPU, s.Memory = nil, nil, nil + s.EngineNative = nil + s.TelemetryValid = false + return s +} + +func (b *Broker) authorizedDirectoryNode(n noderec.DirectoryNode) bool { + if n.IdentityVerification == "local" { + return n.HostUUID != "" && n.HostUUID == b.nodeID + } + return n.ServicePeer() && n.HostUUID == n.ClusterUUID && b.currentPeerProof(n.ClusterUUID, n.PeerRecordID) +} + +func (b *Broker) decisionInventory(n EnrichedNode) bool { + if n.IdentityVerification == "local" { + return n.HostUUID != "" && n.HostUUID == b.nodeID + } + if n.ManualEngine { + return true + } + return n.IdentityVerification == "verified" && n.Trusted && n.HostUUID == n.ClusterUUID && b.currentPeerProof(n.ClusterUUID, n.PeerRecordID) +} + +// schedulerNodes never uses the public preview projection as a node universe. +func (b *Broker) schedulerNodes() []AvailableNode { + if b.store == nil { + return nil + } + b.store.mu.RLock() + records := make([]storedNode, 0, len(b.store.nodes)) + for _, sn := range b.store.nodes { + records = append(records, sn) + } + b.store.mu.RUnlock() + out := make([]AvailableNode, 0, len(records)) + for _, sn := range records { + if sn.scanner != nil && !b.decisionInventory(*sn.scanner) { + sn.scanner = nil + } + if sn.manual != nil && !b.decisionInventory(*sn.manual) { + sn.manual = nil + } + if sn.claimed() { + out = append(out, sn.toAvailable()) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].HostUUID < out[j].HostUUID }) + return out +} + +func (b *Broker) hasDecisionInventory(hostUUID string) bool { + if b.store == nil { + return false + } + b.store.mu.RLock() + sn := b.store.nodes[hostUUID] + b.store.mu.RUnlock() + return (sn.scanner != nil && b.decisionInventory(*sn.scanner)) || (sn.manual != nil && b.decisionInventory(*sn.manual)) +} + +func (b *Broker) authorizedTelemetry(source nodeSource, hostUUID, proof string) bool { + if b.store == nil { + return false + } + b.store.mu.RLock() + sn := b.store.nodes[hostUUID] + var n *EnrichedNode + if source == sourceScanner { + n = sn.scanner + } else { + n = sn.manual + } + b.store.mu.RUnlock() + if n == nil || n.ManualEngine { + return false + } + if n.IdentityVerification == "local" { + return hostUUID != "" && hostUUID == b.nodeID + } + return proof != "" && n.PeerRecordID == proof && b.decisionInventory(*n) +} + +// Withdraw each stale source independently; an authorized source from the other +// worker must survive. Rechecking every incoming proof also rejects late frames. +func (b *Broker) withdrawUnauthorizedInventory() { + if b.store == nil { + return + } + type stale struct { + node EnrichedNode + source nodeSource + } + var staleNodes []stale + b.store.mu.RLock() + for _, sn := range b.store.nodes { + if sn.scanner != nil && sn.scanner.IdentityVerification == "verified" && !b.decisionInventory(*sn.scanner) { + staleNodes = append(staleNodes, stale{*sn.scanner, sourceScanner}) + } + if sn.manual != nil && sn.manual.IdentityVerification == "verified" && !b.decisionInventory(*sn.manual) { + staleNodes = append(staleNodes, stale{*sn.manual, sourceManual}) + } + } + b.store.mu.RUnlock() + for _, entry := range staleNodes { + n := entry.node + if !b.store.withdrawProof(entry.source, entry.node.PeerRecordID, n) { + continue + } + b.removeTelemetry(entry.source, n.HostUUID) + if b.relayDir != nil { + source := relay.SourceScanner + if entry.source == sourceManual { + source = relay.SourceManual + } + b.relayDir.ApplySource(source, noderec.NotifyNodeRemoved, noderec.DirectoryNode{HostUUID: n.HostUUID, PeerRecordID: entry.node.PeerRecordID, IdentityVerification: "verified"}) + } + b.store.mu.RLock() + current := b.store.nodes[n.HostUUID] + b.store.mu.RUnlock() + if b.workloads != nil && (current.scanner == nil || !b.decisionInventory(*current.scanner)) && (current.manual == nil || !b.decisionInventory(*current.manual)) { + b.failWorkloadsForNode(n.HostUUID, n.ID) + } + } +} + +func (s *discoveryStore) withdrawProof(source nodeSource, proof string, replacement EnrichedNode) bool { + s.mu.Lock() + sn := s.nodes[replacement.HostUUID] + prior := sn.scanner + if source == sourceManual { + prior = sn.manual + } + if prior == nil || prior.PeerRecordID != proof { + s.mu.Unlock() + return false + } + sn.setSource(source, nil) + if sn.claimed() { + s.nodes[replacement.HostUUID] = sn + } else { + delete(s.nodes, replacement.HostUUID) + } + onChange := s.onChange + s.mu.Unlock() + if onChange != nil { + onChange() + } + return true +} + +func (b *Broker) clearScannerInventory() { + sp := b.getScanner() + if sp != nil { + sp.notifyMu.Lock() + defer sp.notifyMu.Unlock() + } + b.scannerInventoryGeneration.Add(1) + b.setScanner(nil) + if b.store == nil { + return + } + b.store.mu.RLock() + var nodes []EnrichedNode + for _, sn := range b.store.nodes { + if sn.scanner != nil { + nodes = append(nodes, *sn.scanner) + } + } + b.store.mu.RUnlock() + for _, n := range nodes { + gone := b.store.Remove(n.storeKey(), sourceScanner) + if b.relayDir != nil { + b.relayDir.ApplySource(relay.SourceScanner, noderec.NotifyNodeRemoved, noderec.DirectoryNode{HostUUID: n.HostUUID, PeerRecordID: n.PeerRecordID, IdentityVerification: n.IdentityVerification}) + } + if n.IdentityVerification == "verified" || n.IdentityVerification == "local" { + b.removeTelemetry(sourceScanner, n.HostUUID) + } + if gone && b.workloads != nil && (n.IdentityVerification == "verified" || n.IdentityVerification == "local") { + b.failWorkloadsForNode(n.HostUUID, n.ID) + } + } +} + +func (b *Broker) currentPeerProof(principal, recordID string) bool { + if b.mesh == nil || recordID == "" { + return false + } + b.mesh.Refresh() + return b.mesh.PeerRecordID(principal) == recordID +} diff --git a/services/nvpair-ui-broker/relay/record_identity_test.go b/services/nvpair-ui-broker/relay/record_identity_test.go new file mode 100644 index 000000000..b4c64ef98 --- /dev/null +++ b/services/nvpair-ui-broker/relay/record_identity_test.go @@ -0,0 +1,65 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package relay + +import ( + "testing" + + "nvpair-shared/noderec" +) + +// A claimed host UUID must not let an independent preview overwrite or remove +// the verified peer. These are projection fixtures, not TLS proof fixtures. +func TestDirectoryPreviewRemovalPreservesVerifiedRecord(t *testing.T) { + d := NewDirectory() + verified := noderec.DirectoryNode{ + HostUUID: "peer", ClusterUUID: "peer", Name: "owned-peer", + IdentityVerification: "verified", PeerRecordID: "current-pin-record", Trusted: true, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, + Models: []string{"owned-model"}, + } + preview := noderec.DirectoryNode{ + HostUUID: "peer", Name: "unproven-claim", IdentityVerification: "unverified", + PeerRecordID: noderec.PreviewRecordID("unproven-claim", "peer"), + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 9999}}, + Models: []string{"unproven-model"}, + } + d.Apply(noderec.NotifyNodeDiscovered, verified) + d.Apply(noderec.NotifyNodeDiscovered, preview) + if all := d.Snapshot(""); len(all) != 2 { + t.Fatalf("verified record and independent preview must coexist, got %+v", all) + } + assertVerified := func() { + t.Helper() + nodes := d.Snapshot(noderec.ServiceOllama) + if len(nodes) != 1 || nodes[0].Name != "owned-peer" || nodes[0].PeerRecordID != "current-pin-record" || nodes[0].Services[noderec.ServiceOllama].Port != 11434 || len(nodes[0].Models) != 1 || nodes[0].Models[0] != "owned-model" { + t.Fatalf("verified service inventory was replaced by a claimed UUID: %+v", nodes) + } + } + assertVerified() + d.Apply(noderec.NotifyNodeRemoved, preview) + if all := d.Snapshot(""); len(all) != 1 || all[0].IdentityVerification != "verified" { + t.Fatalf("preview removal must retain the verified record: %+v", all) + } + assertVerified() +} + +func TestDirectoryWithdrawalUsesOldProofToken(t *testing.T) { + d := NewDirectory() + node := noderec.DirectoryNode{ + HostUUID: "peer", ClusterUUID: "peer", IdentityVerification: "verified", + PeerRecordID: "old-pin-record", Trusted: true, + } + d.Apply(noderec.NotifyNodeDiscovered, node) + wrongToken := node + wrongToken.PeerRecordID = "replacement-pin-record" + d.Apply(noderec.NotifyNodeRemoved, wrongToken) + if nodes := d.Snapshot(""); len(nodes) != 1 || nodes[0].PeerRecordID != "old-pin-record" { + t.Fatalf("a different proof token removed the old record: %+v", nodes) + } + d.Apply(noderec.NotifyNodeRemoved, node) + if nodes := d.Snapshot(""); len(nodes) != 0 { + t.Fatalf("withdrawal using the old record's token left inventory: %+v", nodes) + } +} diff --git a/services/nvpair-ui-broker/relay/relay.go b/services/nvpair-ui-broker/relay/relay.go index 5e2a5a0fd..fa6f8b904 100644 --- a/services/nvpair-ui-broker/relay/relay.go +++ b/services/nvpair-ui-broker/relay/relay.go @@ -143,7 +143,7 @@ func (e directoryEntry) projected() (noderec.DirectoryNode, bool) { } func directoryNodeEqual(a, b noderec.DirectoryNode) bool { - if a.HostUUID != b.HostUUID || a.Name != b.Name || a.IP != b.IP || a.ClusterUUID != b.ClusterUUID || a.Trusted != b.Trusted { + if a.PeerRecordID != b.PeerRecordID || a.IdentityVerification != b.IdentityVerification || a.HostUUID != b.HostUUID || a.Name != b.Name || a.IP != b.IP || a.ClusterUUID != b.ClusterUUID || a.Trusted != b.Trusted { return false } if !slices.Equal(a.IPs, b.IPs) || !slices.Equal(a.Models, b.Models) { @@ -199,11 +199,13 @@ func directoryNodeEqual(a, b noderec.DirectoryNode) bool { // subscriber set. It's fed by the daemon's node-* events via Apply and queried // by discovery:get-nodes via Snapshot. type Directory struct { - mu sync.Mutex - entries map[string]directoryEntry - nodes map[string]noderec.DirectoryNode - subs map[int]*Subscriber - nextID int + // Authorize rechecks proof bindings at delivery, including after pin changes. + Authorize func(noderec.DirectoryNode) bool + mu sync.Mutex + entries map[string]directoryEntry + nodes map[string]noderec.DirectoryNode + subs map[int]*Subscriber + nextID int } func NewDirectory() *Directory { @@ -247,6 +249,10 @@ func (d *Directory) Deliver(sub *Subscriber) { func (d *Directory) filteredLocked(f noderec.SubscribeParams) []noderec.DirectoryNode { out := make([]noderec.DirectoryNode, 0, len(d.nodes)) for _, n := range d.nodes { + if n.IdentityVerification == "verified" && d.Authorize != nil && !d.Authorize(n) { + continue + } + n = d.serviceView(n) if f.Matches(n) { out = append(out, n) } @@ -255,6 +261,35 @@ func (d *Directory) filteredLocked(f noderec.SubscribeParams) []noderec.Director return out } +// serviceView retains public preview/invitation hints but never lets their +// advertised engines, model inventories or service ports become service targets. +func (d *Directory) serviceView(n noderec.DirectoryNode) noderec.DirectoryNode { + allowed := n.ServicePeer() + if d.Authorize != nil { + allowed = d.Authorize(n) + } + if allowed { + return n + } + n.Trusted = false + if n.PeerRecordID == "" { + n.PeerRecordID = noderec.PreviewRecordID(n.Name, n.HostUUID) + } + if n.IdentityVerification != "manual" { + n.IdentityVerification = "unverified" + } + services := make(map[noderec.ServiceKey]noderec.ServiceStatus) + for _, key := range []noderec.ServiceKey{noderec.ServiceNodeInfo, noderec.ServiceCluster} { + if service, ok := n.Services[key]; ok { + services[key] = service + } + } + n.Services = services + n.Models, n.ModelsByEngine, n.LoadedByEngine = nil, nil, nil + n.EngineNative = nil + return n +} + // Unsubscribe removes a subscriber. func (d *Directory) Unsubscribe(id int) { d.mu.Lock() @@ -274,14 +309,23 @@ func (d *Directory) Apply(method string, node noderec.DirectoryNode) { // fallbacks, preventing oscillation. Deliveries to subscribers are only made when // the projected node state actually changes. func (d *Directory) ApplySource(source Source, method string, node noderec.DirectoryNode) { - if node.HostUUID == "" { + key := node.RecordKey() + if key == "" { return } d.mu.Lock() - entry := d.entries[node.HostUUID] - prevNode, hadPrev := d.nodes[node.HostUUID] + entry := d.entries[key] + prevNode, hadPrev := d.nodes[key] if method == noderec.NotifyNodeRemoved { + prior := entry.scanner + if source == SourceManual { + prior = entry.manual + } + if node.PeerRecordID != "" && prior != nil && prior.PeerRecordID != node.PeerRecordID { + d.mu.Unlock() + return + } if source == SourceScanner { entry.scanner = nil } else { @@ -299,12 +343,12 @@ func (d *Directory) ApplySource(source Source, method string, node noderec.Direc proj, exists := entry.projected() var changed bool if exists { - d.entries[node.HostUUID] = entry - d.nodes[node.HostUUID] = proj + d.entries[key] = entry + d.nodes[key] = proj changed = !hadPrev || !directoryNodeEqual(prevNode, proj) } else { - delete(d.entries, node.HostUUID) - delete(d.nodes, node.HostUUID) + delete(d.entries, key) + delete(d.nodes, key) changed = hadPrev } @@ -333,6 +377,10 @@ func (d *Directory) Snapshot(filter noderec.ServiceKey) []noderec.DirectoryNode d.mu.Lock() out := make([]noderec.DirectoryNode, 0, len(d.nodes)) for _, n := range d.nodes { + if n.IdentityVerification == "verified" && d.Authorize != nil && !d.Authorize(n) { + continue + } + n = d.serviceView(n) if filter != "" && !n.HasService(filter) { continue } diff --git a/services/nvpair-ui-broker/relay/relay_test.go b/services/nvpair-ui-broker/relay/relay_test.go index 597931425..f9cb44423 100644 --- a/services/nvpair-ui-broker/relay/relay_test.go +++ b/services/nvpair-ui-broker/relay/relay_test.go @@ -77,10 +77,10 @@ func ids(nodes []noderec.DirectoryNode) []string { } func olNode(id string) noderec.DirectoryNode { - return noderec.DirectoryNode{HostUUID: id, Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}} + return noderec.DirectoryNode{HostUUID: id, IdentityVerification: "local", Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}} } func erNode(id string) noderec.DirectoryNode { - return noderec.DirectoryNode{HostUUID: id, Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceErrors: {Port: 14319}}} + return noderec.DirectoryNode{HostUUID: id, IdentityVerification: "local", Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceErrors: {Port: 14319}}} } func TestDirectorySubscribeInitialSnapshot(t *testing.T) { @@ -207,11 +207,12 @@ func TestDirectoryScannerPrecedenceOverManual(t *testing.T) { d.Subscribe(&Subscriber{Filter: noderec.SubscribeParams{}, Send: sub.send}) scannerNode := noderec.DirectoryNode{ - HostUUID: "uuid-1", - Name: "workstation.local", - IP: "192.168.1.50", - IPs: []string{"192.168.1.50"}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, + HostUUID: "uuid-1", + IdentityVerification: "local", + Name: "workstation.local", + IP: "192.168.1.50", + IPs: []string{"192.168.1.50"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, } d.ApplySource(SourceScanner, noderec.NotifyNodeDiscovered, scannerNode) if len(sub.snaps) != 1 { @@ -223,11 +224,12 @@ func TestDirectoryScannerPrecedenceOverManual(t *testing.T) { // Manual update for the same hostUuid should NOT overwrite scanner's name, but should merge manual IP. manualNode := noderec.DirectoryNode{ - HostUUID: "uuid-1", - Name: "100.100.100.1", - IP: "100.100.100.1", - IPs: []string{"100.100.100.1"}, - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, + HostUUID: "uuid-1", + IdentityVerification: "local", + Name: "100.100.100.1", + IP: "100.100.100.1", + IPs: []string{"100.100.100.1"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, } d.ApplySource(SourceManual, noderec.NotifyNodeDiscovered, manualNode) if len(sub.snaps) != 2 { @@ -258,11 +260,12 @@ func TestDirectoryScannerPrecedenceOverManual(t *testing.T) { // engineNativeNode is a directory node carrying the engine's own view. func engineNativeNode(view *enginenative.Stats) noderec.DirectoryNode { return noderec.DirectoryNode{ - HostUUID: "uuid-a", - Name: "host-a", - IP: "192.0.2.10", - Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14320}}, - EngineNative: view, + HostUUID: "uuid-a", + IdentityVerification: "local", + Name: "host-a", + IP: "192.0.2.10", + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceEngineManager: {Port: 14320}}, + EngineNative: view, } } diff --git a/services/nvpair-ui-broker/scheduler_test.go b/services/nvpair-ui-broker/scheduler_test.go index f74dcd42e..95fb3a182 100644 --- a/services/nvpair-ui-broker/scheduler_test.go +++ b/services/nvpair-ui-broker/scheduler_test.go @@ -28,7 +28,8 @@ func TestSchedulerFeedBaselinePrecedesConcurrentLiveWorkload(t *testing.T) { _ = schedulerSide.Close() }) worker := &rpcWorker{peer: NewPeer(NewCodec(brokerSide))} - b := &Broker{workloads: workloadstore.New(), telemetry: newTelemetryCache()} + b := &Broker{nodeID: "a", store: newDiscoveryStore(), workloads: workloadstore.New(), telemetry: newTelemetryCache()} + b.store.Upsert(EnrichedNode{HostUUID: "a", IdentityVerification: "local"}, sourceScanner) b.workloads.Apply(storeIncoming("active", "host", "ollama", "run", "running", "a")) b.workloads.Apply(storeIncoming("historic", "host", "ollama", "run", "completed", "a")) b.telemetry.Upsert(sourceScanner, noderec.NodeTelemetry{ @@ -48,7 +49,7 @@ func TestSchedulerFeedBaselinePrecedesConcurrentLiveWorkload(t *testing.T) { close(feedLocked) replayed := b.replayActiveWorkloadsToScheduler(worker) b.replayTelemetryToScheduler(worker) - _ = worker.Notify("discovery:nodes-changed", []AvailableNode{{HostUUID: "a"}}) + _ = worker.Notify("discovery:nodes-changed", b.schedulerNodes()) b.schedulerFeedMu.Unlock() b.workloadEmitMu.Unlock() initDone <- replayed diff --git a/services/nvpair-ui-broker/telemetry.go b/services/nvpair-ui-broker/telemetry.go index f2282d1cb..d57c7782d 100644 --- a/services/nvpair-ui-broker/telemetry.go +++ b/services/nvpair-ui-broker/telemetry.go @@ -158,6 +158,9 @@ func (b *Broker) ingestTelemetryAt(source nodeSource, value noderec.NodeTelemetr if b.telemetry == nil { return } + if !b.authorizedTelemetry(source, value.HostUUID, value.PeerRecordID) { + return + } projected, ok := b.telemetry.Upsert(source, value, receivedAt) if ok { b.fanTelemetryToScheduler(projected) @@ -175,8 +178,13 @@ func (b *Broker) removeTelemetry(source nodeSource, hostUUID string) { } func (b *Broker) fanTelemetryToScheduler(telemetry noderec.NodeTelemetry) { + queuedAt := time.Now() b.schedulerFeedMu.Lock() defer b.schedulerFeedMu.Unlock() + if !b.authorizedTelemetry(sourceScanner, telemetry.HostUUID, telemetry.PeerRecordID) && !b.authorizedTelemetry(sourceManual, telemetry.HostUUID, telemetry.PeerRecordID) { + telemetry = noderec.NodeTelemetry{HostUUID: telemetry.HostUUID} + } + telemetry = observedTelemetryAt(telemetryObservation{value: telemetry, receivedAt: queuedAt}, time.Now()) scheduler := b.getScheduler() if scheduler == nil { @@ -195,7 +203,12 @@ func (b *Broker) replayTelemetryToScheduler(scheduler *rpcWorker) int { return 0 } replayed := 0 - for _, telemetry := range b.telemetry.Snapshot(time.Now()) { + snapshotAt := time.Now() + for _, telemetry := range b.telemetry.Snapshot(snapshotAt) { + telemetry = observedTelemetryAt(telemetryObservation{value: telemetry, receivedAt: snapshotAt}, time.Now()) + if !b.authorizedTelemetry(sourceScanner, telemetry.HostUUID, telemetry.PeerRecordID) && !b.authorizedTelemetry(sourceManual, telemetry.HostUUID, telemetry.PeerRecordID) { + continue + } if err := scheduler.Notify(schedulerwire.MethodTelemetry, telemetry); err != nil { slog.Warn("replay telemetry to scheduler failed", "host_uuid", telemetry.HostUUID, "err", err) break diff --git a/services/nvpair-ui-broker/telemetry_test.go b/services/nvpair-ui-broker/telemetry_test.go index 64836c92e..59150c88a 100644 --- a/services/nvpair-ui-broker/telemetry_test.go +++ b/services/nvpair-ui-broker/telemetry_test.go @@ -102,7 +102,8 @@ func TestReplayTelemetryToSchedulerIncludesCurrentAge(t *testing.T) { _ = schedulerSide.Close() }) worker := &rpcWorker{peer: NewPeer(NewCodec(brokerSide))} - b := &Broker{telemetry: newTelemetryCache()} + b := &Broker{nodeID: "node-a", store: newDiscoveryStore(), telemetry: newTelemetryCache()} + b.store.Upsert(EnrichedNode{HostUUID: "node-a", IdentityVerification: "local"}, sourceScanner) b.telemetry.Upsert(sourceScanner, noderec.NodeTelemetry{ HostUUID: "node-a", GPUUtilizationPct: 45, @@ -132,12 +133,13 @@ func TestReplayTelemetryToSchedulerIncludesCurrentAge(t *testing.T) { } func TestManualTelemetryReprojectsSurvivingAliasAtOriginalAge(t *testing.T) { - b := newManualTestBroker() - first := manualStatus("first", "10.0.0.1", "node-a") + b := newManualTestBroker(t, "node-a") + first := manualStatus(t, b, "first", "10.0.0.1", "node-a") b.upsertManualNode(first) rawFirst, err := json.Marshal(manualNodeTelemetryParams{ ID: "first", + PeerRecordID: first.PeerRecordID, HostUUID: "node-a", GPUUtilizationPct: 25, TelemetryValid: true, @@ -148,7 +150,7 @@ func TestManualTelemetryReprojectsSurvivingAliasAtOriginalAge(t *testing.T) { } b.forwardManualNodesNotification("node/telemetry", rawFirst) - second := manualStatus("second", "10.0.0.2", "node-a") + second := manualStatus(t, b, "second", "10.0.0.2", "node-a") b.upsertManualNode(second) b.removeManualNode("second") @@ -202,8 +204,8 @@ func TestManualNodeTelemetryPreservesSampleSeq(t *testing.T) { } func TestManualNodeTelemetryIngestedWithoutNodesChanged(t *testing.T) { - b := newManualTestBroker() - node := manualStatus("manual-node-1", "10.0.0.1", "node-uuid-1") + b := newManualTestBroker(t, "node-uuid-1") + node := manualStatus(t, b, "manual-node-1", "10.0.0.1", "node-uuid-1") b.upsertManualNode(node) // Step 1: Status updates must NOT feed scheduler telemetry @@ -222,6 +224,7 @@ func TestManualNodeTelemetryIngestedWithoutNodesChanged(t *testing.T) { // Send node/telemetry telemParams := manualNodeTelemetryParams{ ID: "manual-node-1", + PeerRecordID: node.PeerRecordID, HostUUID: "node-uuid-1", GPUUtilizationPct: 88, TelemetryValid: true, @@ -278,6 +281,7 @@ func TestManualNodeTelemetryIngestedWithoutNodesChanged(t *testing.T) { // Verify mismatched hostUuid is dropped mismatchedParams := manualNodeTelemetryParams{ ID: "manual-node-1", + PeerRecordID: node.PeerRecordID, HostUUID: "different-uuid-999", GPUUtilizationPct: 99, TelemetryValid: true, @@ -290,6 +294,21 @@ func TestManualNodeTelemetryIngestedWithoutNodesChanged(t *testing.T) { t.Fatalf("mismatched hostUuid telemetry was not dropped: %+v", snapMismatched) } + // Even the registered ID and matching hostUuid cannot authenticate a sample + // without the current peer record proof. + unprovenParams := telemParams + unprovenParams.PeerRecordID = "" + unprovenParams.GPUUtilizationPct = 99 + rawUnproven, err := json.Marshal(unprovenParams) + if err != nil { + t.Fatalf("marshal unproven telemetry: %v", err) + } + b.forwardManualNodesNotification("node/telemetry", rawUnproven) + snapUnproven := b.telemetry.Snapshot(time.Now()) + if len(snapUnproven) != 1 || snapUnproven[0].GPUUtilizationPct != 88 || snapUnproven[0].SampleSeq != 99 { + t.Fatalf("unproven telemetry replaced the authenticated sample: %+v", snapUnproven) + } + // Verify removal cleans up what was ingested leaving no cached telemetry b.removeManualNode("manual-node-1") snapRemoved := b.telemetry.Snapshot(time.Now()) diff --git a/services/nvpair-ui-broker/verified_inventory_test.go b/services/nvpair-ui-broker/verified_inventory_test.go new file mode 100644 index 000000000..a4e55b294 --- /dev/null +++ b/services/nvpair-ui-broker/verified_inventory_test.go @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" + "nvpair-ui-broker/relay" + "nvpair-ui-broker/workloadstore" + "testing" + "time" +) + +func inventoryTestBroker(t *testing.T) *Broker { + t.Helper() + dir := t.TempDir() + clustertrusttest.Join(t, dir, "test-cluster", "self", "peer") + b := &Broker{nodeID: "self", clusterDir: dir, mesh: clustertrust.Open(dir), store: newDiscoveryStore(), relayDir: relay.NewDirectory(), telemetry: newTelemetryCache(), workloads: workloadstore.New(), manualNodeKeys: make(map[string]string), manualNodeStatuses: make(map[string]manualNodeStatusEntry)} + b.relayDir.Authorize = b.authorizedDirectoryNode + return b +} +func verifiedDirectoryForTest(b *Broker) noderec.DirectoryNode { + return noderec.DirectoryNode{HostUUID: "peer", ClusterUUID: "peer", Name: "peer", IP: "192.0.2.2", Trusted: true, IdentityVerification: "verified", PeerRecordID: b.mesh.PeerRecordID("peer"), Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceNodeInfo: {Port: 14318}, noderec.ServiceOllama: {Port: 11434}, noderec.ServiceLMStudio: {Port: 1234}, noderec.ServiceOMLX: {Port: 1236}, noderec.ServiceEngineControl: {Port: 14323}}, Models: []string{"model"}} +} + +func TestClaimedManualPrincipalIsNotTrustedOrScheduled(t *testing.T) { + b := inventoryTestBroker(t) + s := manualNodeStatus{ID: "manual:preview", Address: "192.0.2.2", HostUUID: "peer", ClusterUUID: "peer", NodeInfoUp: true, OllamaUp: true, OllamaModels: []string{"forged"}, TelemetryValid: true} + en := manualToEnriched(s) + b.annotateManualTrust(&en, s) + if en.Trusted { + t.Fatal("a reported pinned ID granted trust without an mTLS proof") + } + b.upsertManualNode(s) + if got := b.schedulerNodes(); len(got) != 0 { + t.Fatalf("unproven inventory in scheduler: %+v", got) + } + for _, service := range []noderec.ServiceKey{noderec.ServiceOllama, noderec.ServiceLMStudio, noderec.ServiceOMLX, noderec.ServiceEngineControl} { + if got := b.relayDir.Snapshot(service); len(got) != 0 { + t.Fatalf("unproven %s target: %+v", service, got) + } + } + b.ingestTelemetry(sourceManual, noderec.NodeTelemetry{HostUUID: "peer", TelemetryValid: true, GPUUtilizationPct: 99}) + if len(b.telemetry.Snapshot(time.Now())) != 0 { + t.Fatal("unproven telemetry entered scheduler cache") + } +} + +// Invalid readings still affect scheduler state: they must carry the same +// provenance as valid samples, rather than clearing a proven peer's pressure. +func TestUnprovenInvalidTelemetryCannotReplaceAuthenticatedSample(t *testing.T) { + for name, source := range map[string]nodeSource{"scanner": sourceScanner, "manual": sourceManual} { + t.Run(name, func(t *testing.T) { + b := inventoryTestBroker(t) + node := verifiedDirectoryForTest(b) + b.store.Upsert(directoryToEnriched(node), source) + b.ingestTelemetry(source, noderec.NodeTelemetry{HostUUID: "peer", PeerRecordID: node.PeerRecordID, TelemetryValid: true, GPUUtilizationPct: 73, SampleSeq: 8}) + b.ingestTelemetry(source, noderec.NodeTelemetry{HostUUID: "peer", TelemetryValid: false, GPUUtilizationPct: 99, SampleSeq: 900}) + got := b.telemetry.Snapshot(time.Now()) + if len(got) != 1 || !got[0].TelemetryValid || got[0].GPUUtilizationPct != 73 || got[0].SampleSeq != 8 { + t.Fatalf("unproven invalid sample replaced authenticated pressure: %+v", got) + } + // A current peer may legitimately report an unavailable sample. + b.ingestTelemetry(source, noderec.NodeTelemetry{HostUUID: "peer", PeerRecordID: node.PeerRecordID, TelemetryValid: false, SampleSeq: 9}) + got = b.telemetry.Snapshot(time.Now()) + if len(got) != 1 || got[0].TelemetryValid || got[0].SampleSeq != 9 { + t.Fatalf("authenticated invalid sample rejected: %+v", got) + } + }) + } +} + +func TestReplayInvalidTelemetryRequiresCurrentProof(t *testing.T) { + b := inventoryTestBroker(t) + node := verifiedDirectoryForTest(b) + b.store.Upsert(directoryToEnriched(node), sourceScanner) + b.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: "peer", PeerRecordID: node.PeerRecordID, SampleSeq: 19}) + var out bytes.Buffer + worker := &rpcWorker{peer: NewPeer(NewCodec(&out))} + if got := b.replayTelemetryToScheduler(worker); got != 1 || out.Len() == 0 { + t.Fatalf("current peer's unavailable sample was not replayed: count=%d", got) + } + out.Reset() + clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") + if got := b.replayTelemetryToScheduler(worker); got != 0 || out.Len() != 0 { + t.Fatalf("retired invalid sample reached scheduler: count=%d frame=%s", got, out.String()) + } +} + +func TestVerifiedInventoryWithdrawnOnPinAndAdmissionChanges(t *testing.T) { + for _, change := range []string{"pin", "admission"} { + t.Run(change, func(t *testing.T) { + b := inventoryTestBroker(t) + node := verifiedDirectoryForTest(b) + b.store.Upsert(directoryToEnriched(node), sourceScanner) + b.relayDir.Apply(noderec.NotifyNodeDiscovered, node) + b.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: "peer", PeerRecordID: node.PeerRecordID, TelemetryValid: true, GPUUtilizationPct: 17}) + if len(b.schedulerNodes()) != 1 || len(b.relayDir.Snapshot(noderec.ServiceOMLX)) != 1 || len(b.telemetry.Snapshot(time.Now())) != 1 { + t.Fatal("current proof not accepted") + } + if change == "pin" { + clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") + } else { + clustertrusttest.WriteAdmission(t, b.clusterDir, "test-cluster", 2) + } + // Cached relay/scheduler reads independently recheck authorization, before + // the worker has reacted to the notification. + if len(b.schedulerNodes()) != 0 || len(b.relayDir.Snapshot(noderec.ServiceOllama)) != 0 { + t.Fatal("stale proof remained service eligible") + } + b.withdrawUnauthorizedInventory() + b.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: "peer", PeerRecordID: node.PeerRecordID, TelemetryValid: true, GPUUtilizationPct: 99}) + for _, sample := range b.telemetry.Snapshot(time.Now()) { + if sample.TelemetryValid { + t.Fatal("retired telemetry revived") + } + } + }) + } +} + +func TestUnprovenScannerClaimCannotReplaceVerifiedPeer(t *testing.T) { + b := inventoryTestBroker(t) + node := verifiedDirectoryForTest(b) + sp := &scannerProcess{store: b.store, relayDir: b.relayDir, authorize: b.authorizedDirectoryNode, onTelemetryRemoved: func(host string) { b.removeTelemetry(sourceScanner, host) }} + send := func(method string, n noderec.DirectoryNode) { + data, _ := json.Marshal(noderec.NodeEvent{Node: n}) + sp.handleNotify(method, data) + } + send(noderec.NotifyNodeDiscovered, node) + forged := node + forged.PeerRecordID = noderec.PreviewRecordID(forged.Name, forged.HostUUID) + forged.IdentityVerification = "unverified" + forged.Trusted = false + forged.Models = []string{"forged"} + b.ingestTelemetry(sourceScanner, noderec.NodeTelemetry{HostUUID: "peer", PeerRecordID: node.PeerRecordID, TelemetryValid: true}) + send(noderec.NotifyNodeUpdated, forged) + if len(b.store.Snapshot()) != 2 { + t.Fatal("preview did not get an independent record key") + } + send(noderec.NotifyNodeRemoved, forged) + if len(b.store.Snapshot()) != 1 || len(b.telemetry.Snapshot(time.Now())) != 1 { + t.Fatal("preview removal affected verified source state") + } + got := b.schedulerNodes() + if len(got) != 1 || len(got[0].Models) != 1 || got[0].Models[0] != "model" { + t.Fatalf("public claim displaced verified peer: %+v", got) + } +} + +func TestScannerRestartWithdrawsProofButPreservesIndependentManualEngine(t *testing.T) { + b := inventoryTestBroker(t) + node := verifiedDirectoryForTest(b) + b.store.Upsert(directoryToEnriched(node), sourceScanner) + b.relayDir.Apply(noderec.NotifyNodeDiscovered, node) + b.upsertManualNode(manualNodeStatus{ID: "manual:engine", Address: "192.0.2.3", OMLXUp: true, OMLXPort: 1236, OMLXModels: []string{"bare-model"}}) + b.clearScannerInventory() + got := b.schedulerNodes() + if len(got) != 1 || got[0].HostUUID != noderec.ManualRecordID("manual:engine") || len(got[0].Models) != 1 || got[0].Models[0] != "bare-model" { + t.Fatalf("restart sources = %+v", got) + } +} diff --git a/services/nvpair-ui-broker/workers.go b/services/nvpair-ui-broker/workers.go index ccdfbea72..b0619414d 100644 --- a/services/nvpair-ui-broker/workers.go +++ b/services/nvpair-ui-broker/workers.go @@ -38,7 +38,7 @@ func (b *Broker) initWorkerRegistry() { spawn func() (supervisedHandle, error) clear func() }{ - {"scanner", "nvpair-node-scanner", &b.scannerPath, &b.scannerSup, b.spawnScanner, func() { b.setScanner(nil) }}, + {"scanner", "nvpair-node-scanner", &b.scannerPath, &b.scannerSup, b.spawnScanner, b.clearScannerInventory}, {"errors", "nvpair-errors", &b.errorsPath, &b.errorsSup, b.spawnErrors, func() { b.setErrors(nil) }}, {"settings", "nvpair-node-settings", &b.settingsPath, &b.settingsSup, b.spawnSettings, func() { b.setSettings(nil) }}, {"engine-manager", "nvpair-engine-manager", &b.engineMgrPath, &b.engineMgrSup, b.spawnEngineMgr, func() { b.setEngineMgr(nil); b.clearEngineRelaySub() }}, From 8af50ab31e30da42860a01c7d4167e5290fb2a6c Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 01:38:56 +0200 Subject: [PATCH 06/18] docs(security): document verified inventory boundaries and restart risk Describe authenticated inventory, independent previews and older-peer compatibility. Disclose the manual-worker restart downgrade risk and the desktop identity and durable manual-kind follow-ups. Bump node-info, scanner and manual-nodes for additive provenance and security behavior; retain already-unreleased broker and other component bumps relative to v0.8.0. Document the adaptation of NVIDIA/Personal-AI-Router#179. Co-authored-by: Kaylee Lubick Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- SECURITY.md | 20 +++++-- docs/architecture.mdx | 116 ++++++++++++++++++++++------------------- services/versions.json | 6 +-- 3 files changed, 80 insertions(+), 62 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index cd92a694b..4432cfd3c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -87,10 +87,22 @@ credential store. ### LAN Metadata and Overlay Addresses -In the **broker-managed deployment**, `/v1/node-info` serves unauthenticated -plaintext hardware/utilization metadata even after pairing. Standalone node-info -with `--cluster-dir` can instead use membership-gated mTLS; do not assume that -mode is active in the desktop bundle. +In the **broker-managed deployment**, `/v1/node-info` retains an unauthenticated +plaintext hardware/utilization preview. It also serves inventory and broker-owned +service metadata over pinned cluster mTLS on the same port. Services grant peer +trust and use remote inventory for routing or scheduling only after that mTLS +read proves the exact current certificate and matching response identity. Public +previews have independent record keys and an explicit `unverified` marker; a +claimed host UUID cannot replace or remove a verified record. Proofs are bound +to the local admission and both certificates and are withdrawn on authorization +changes or worker retirement. Older peers without authenticated service metadata +remain previews, not service targets. Explicitly configured bare-engine manual +routes remain a separately labelled bypass and supply no peer telemetry. +Manual-entry kind is not yet persisted: after a manual-nodes worker restart, +an address previously added for a PAIR peer can be routed as an unauthenticated +plaintext bare engine if no PAIR endpoint answers there, including when another +machine has taken that address. Persisting PAIR-vs-bare-engine intent is required +to prevent that downgrade across worker restarts. PAIR can advertise/rank Tailscale CGNAT addresses (`100.64.0.0/10`) and use known peer address fallbacks. This is not arbitrary tailnet peer enumeration, and diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 9abe98c63..6653aa857 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -241,17 +241,18 @@ Not every surface is gated, and the exceptions are deliberate: | Workload replication, error synchronization | Mutual TLS | | Cluster membership | Mutual TLS, after pairing | | Pairing itself | Plaintext, authenticated by the PIN | -| Host and GPU telemetry | Plaintext HTTP, **not** authenticated | +| Service inventory and scheduling telemetry | Pinned cluster mutual TLS | +| Public hardware preview | Plaintext HTTP, **not** authenticated; display-only | Pairing is plaintext because no trust exists yet at that point. The six-digit PIN is what authenticates the exchange that establishes the pinned certificates. -Node telemetry is a considered trade-off, not an oversight. It is treated as the -lowest-sensitivity inter-node surface and is read by consumers that are not -cluster members, so it stays plaintext even on a clustered node. The practical -consequence is that hostname, hardware inventory, and utilization for a node -running PAIR are readable by anything on the same subnet that asks. Do not run -PAIR on a network where that is unacceptable. +Public hardware previews remain readable by nonmembers over plaintext HTTP, +including after pairing. They are explicitly unverified and never authorize +routing, scheduling, or peer trust. Services separately authenticate node-info +and its broker-owned service metadata over pinned cluster mTLS on the same port. +The public preview still exposes hostname, hardware inventory, and utilization +to devices on the subnet; do not run PAIR where that disclosure is unacceptable. ### Two Personalities on One Port @@ -756,8 +757,11 @@ of being crammed into TXT. PAIR identifies every node by a stable UUID, never by its hostname. An mDNS instance name defaults to the hostname, so two machines that share one -would silently merge into a single node. Every service therefore advertises a -per-host UUID, and the directory is keyed by it. +must not merge. Verified peers use their authenticated host UUID as the directory +key. Unverified discovery previews use a separate opaque `peerRecordId` and +`identityVerification: "unverified"`; their claimed host UUID is never a storage +key and cannot overwrite or remove a verified peer. Explicit manual engine +configurations occupy their own namespace too. That UUID is the correlation key across the entire system: discovery entries, cluster membership, workload attribution, error reports, telemetry, and proxy @@ -803,60 +807,62 @@ or prove that these failures are prevented. ### Node Enrichment -Announcements establish that a node exists and which ports it serves. PAIR -fetches the interesting facts afterwards, for each node, over HTTP: - -- GPU, CPU, and memory inventory from its node-info port -- Its model list from its engine-manager port, as a flat union, a per-engine - breakdown, and the per-engine set of models currently loaded in memory - -Each enrichment keeps a last-good value, so one failed fetch dims nothing. A -node's card does not blank out because a single poll timed out. - -With a cluster directory configured, a browsed peer whose `cluster-uuid=` matches -a pin this node holds is annotated as trusted, which is how the interface can -distinguish a paired node from a stranger it can merely see. +Announcements supply candidate endpoints, not identity proofs or authoritative +service ports. Before granting `trusted`, the scanner fetches node-info over +pinned cluster mTLS, checks the exact current server certificate, and requires +matching certificate, response, and broker-owned metadata identities. Only that +metadata supplies service ports. The endpoint which completed authentication is +promoted; other advertised addresses remain hints for subsequent authentication. + +The scanner then reads models, per-engine attribution, loaded models, engine +self-reports and compact scheduling telemetry over pinned mTLS. Completed reads +recheck their authorization after decoding, reject redirects, and enforce body +limits. Each proof is bound to the local admission, local leaf and exact peer pin +record, so pin rotation, readmission or membership loss cannot revive old cached +inventory. Public hardware previews remain separately keyed and untrusted. + +Older peers without authenticated service metadata stay visible as previews but +cannot contribute service inventory until upgraded. Desktop adoption of the +additive record-key/provenance fields is a follow-up; routing and scheduling are +already enforced in the Go services. ### Node Eviction -The scanner scans every five seconds and tolerates three consecutive misses -before it treats a node as gone. Even then it does not drop the node right away. -It runs these checks and keeps the node if any of them answers: - -- Inference the node served in the last minute, which the proxies report -- Recent telemetry from that node -- A direct request to the node's node-info endpoint -- A probe of the engine-manager port - -The activity check comes first because a node under load is the one most likely -to miss an announcement, and evicting it would pull a working machine out of the -routing pool at its busiest. - -The remaining checks deliberately avoid the inference proxy ports. Those ports -exist to serve requests, and using them as a liveness test would put connection -attempts on the serving path every time multicast dropped a packet. - -Multicast is lossy, and Wi-Fi, VPN adapters, and sleeping network interfaces all -produce gaps that have nothing to do with a node's health. Without that probe a -momentary gap would evict a perfectly healthy node and take its models out of the -routing pool. - -Discovery holds record replacement to the same standard. A node can reappear with -a new identity, for example after a reset. The old entry then looks superseded, -and leaving both in place would list the machine twice. But a matching address -and hostname arrive in an unauthenticated announcement that anything on the -network can send, so those signals only mark the old record as a suspect. -Discovery confirms the node's identity before dropping that record. A stale -duplicate therefore cannot linger, and nothing can displace a peer by claiming -its address. +Public candidates retain the browser's missed-announcement and reachability +policy. Their disappearance removes only their preview record, never a peer +whose UUID they claim. Verified peers have an independent lifetime: a 15-second +refresh authenticates their inventory again, and failed authentication expires +the record after 45 seconds without a successful inventory read. Recent pinned +inference activity can keep a busy peer present while its control plane is +saturated. Pin or admission changes withdraw the proof immediately rather than +waiting for that liveness deadline. + +The broker also rechecks proof bindings when publishing service targets, +initializing the scheduler, and accepting or replaying telemetry. Retired worker +callbacks cannot restore the old worker's records, and scanner retirement clears +its sources without deleting a still-authenticated manual source. Withdrawal +removes the verified record with its original proof token; it does not turn that +record into a preview under the same host key. ### Manual Nodes Some networks block or filter multicast, so discovery is not the only path in. `nvpair-manual-nodes` takes an address you enter directly and probes it on a -fixed interval, and a manual node that answers is folded into the same directory -as a discovered one. It is initially keyed by the address you typed, and re-keyed -to the peer's real UUID as soon as that node reports it. +fixed interval. A PAIR peer enters the verified directory only after pinned +cluster mTLS proves its current identity and service metadata; public HTTP or +operator CA trust cannot substitute for that proof. Its configuration keeps an +independent manual record key until authentication succeeds. There is no +plaintext downgrade for peer hardware, models, engine state or scheduling data. +Explicitly user-added bare engines keep the separate manual routing bypass and +never establish a PAIR peer identity or provide scheduling telemetry. Recognizing +a PAIR endpoint prevents a bare-engine downgrade for that configuration's lifetime +in the manual-nodes worker, but this kind is not persisted. After that worker +restarts, a previously added PAIR address can become an unauthenticated plaintext +bare-engine route if no PAIR endpoint answers there, including if another machine +answers instead. Persisting each entry's PAIR-vs-bare-engine kind in +`manual-nodes.json` is a follow-up, alongside desktop handling of the additive +`peerRecordId` and `identityVerification` fields without keying previews by a +claimed `hostUuid`. ## Observability and Distributed Tracing diff --git a/services/versions.json b/services/versions.json index a1711d886..44677892d 100644 --- a/services/versions.json +++ b/services/versions.json @@ -5,9 +5,9 @@ "components": { "ollama-proxy": "1.4.1", "lmstudio-proxy": "1.4.1", - "nvpair-node-info": "1.3.0", - "nvpair-node-scanner": "1.2.1", - "nvpair-manual-nodes": "1.2.2", + "nvpair-node-info": "1.4.0", + "nvpair-node-scanner": "1.3.0", + "nvpair-manual-nodes": "1.3.0", "nvpair-workload-manager": "1.3.1", "nvpair-errors": "1.1.1", "nvpair-node-settings": "3.2.0", From ef6a85c4441f656da34cea797ebe6867fae10689 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:36:33 +0200 Subject: [PATCH 07/18] fix(clustertrust): require active admission for inventory proofs Retained peer pins can keep the transport clustered after admission is cleared. Require an active non-retired admission for proof bindings so fresh, pooled and cached inventory cannot survive withdrawal. Cover absent, cleared and retired admission records with retained pins. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-node-scanner/README.md | 2 +- .../shared/clustertrust/inventory_test.go | 65 +++++++++++++++++++ services/shared/clustertrust/membership.go | 5 +- services/shared/clustertrust/mesh.go | 2 +- 4 files changed, 70 insertions(+), 4 deletions(-) diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index 8ae598598..298e5b7b2 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -22,7 +22,7 @@ Concretely, the daemon: cadence. It emits only host UUID, maximum GPU utilization, validity, and source-reported sample age; it does not mutate the public directory for each telemetry sample. -- **Grants trust only after proof:** `trusted` requires a completed pinned mTLS inventory read. `peerRecordId` binds the proof to the local admission, local certificate and exact peer pin record. A pin by itself never promotes a claim. Trust changes withdraw the verified record using its original token rather than clearing the token and turning it into a preview under the host key. +- **Grants trust only after proof:** `trusted` requires a completed pinned mTLS inventory read. `peerRecordId` binds the proof to an active, non-retired local admission, local certificate and exact peer pin record. Retained pins after admission withdrawal cannot authorize fresh or cached inventory, even over an existing connection. A pin by itself never promotes a claim. Trust changes withdraw the verified record using its original token rather than clearing the token and turning it into a preview under the host key. - **Advertises live membership:** the same reload re-derives this node's own `cluster-uuid=` from the cluster dir and re-advertises when it differs from what is being published, so joining or leaving a cluster takes effect without a restart. - **Public-preview anti-flap:** a candidate has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. - **A scan that loses everything is treated as our own fault:** when more than one node is known and a scan returns *none* of them, no node is penalized for it — for up to six consecutive scans (~30s), after which they count normally. Independent machines do not leave the network in the same five-second window, but a process too starved to drain its multicast socket within `scanTimeout` hears silence from all of them at once; without this, one saturated machine dropped its entire cluster, idle peers included, in a single reconcile. The suppression needs two or more known nodes to engage: losing a lone peer looks the same whether it left or we stopped listening, so that case falls through to the ordinary threshold. diff --git a/services/shared/clustertrust/inventory_test.go b/services/shared/clustertrust/inventory_test.go index e93032185..3cc514296 100644 --- a/services/shared/clustertrust/inventory_test.go +++ b/services/shared/clustertrust/inventory_test.go @@ -126,6 +126,71 @@ func TestReadPeerInventoryRejectsRedirectsAndOversizedJSON(t *testing.T) { t.Fatal("authenticated read followed plaintext redirect") } } + +// Retained pins are not admission: teardown can clear admission before pin cleanup. +func TestInventoryProofRequiresActiveNonRetiredAdmission(t *testing.T) { + for _, state := range []string{"absent", "cleared", "retired"} { + for _, transport := range []string{"fresh", "pooled", "inflight"} { + t.Run(state+"/"+transport, func(t *testing.T) { + dir, self, peer := inventoryPair(t) + withdraw := func() { + path := filepath.Join(dir, "admission.json") + if state == "absent" { + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + return + } + body := `{"counter":1,"retired":true}` + if state == "retired" { + body = `{"counter":1,"clusterId":"test-cluster","epoch":1,"retired":true}` + } + if err := os.WriteFile(path, []byte(body), 0600); err != nil { + t.Fatal(err) + } + } + var requests int + srv := inventoryServer(t, peer, func(w http.ResponseWriter, _ *http.Request) { + requests++ + if transport == "inflight" && requests == 2 { + withdraw() + } + _, _ = w.Write([]byte(`{"value":42}`)) + }) + pool := NewPeerClientPool(self, time.Second) + defer pool.CloseIdle() + client, _ := pool.Client("peer") + read := func() bool { + var value struct{ Value int } + _, _, ok := ReadPeerInventory(t.Context(), self, client, "peer", srv.URL, "", 1024, &value) + return ok + } + if !read() { + t.Fatal("initial authorized read failed") + } + if transport == "fresh" { + pool.CloseIdle() + } + if transport != "inflight" { + withdraw() + } + if read() { + t.Error("withdrawn admission issued a fresh inventory proof") + } + self.Refresh() + for _, principal := range []string{"self", "peer"} { + if self.PeerRecordID(principal) != "" { + t.Errorf("withdrawn admission still authorizes cached %s proof", principal) + } + } + if self.PeerCount() != 1 { + t.Fatal("test must retain peer pin after withdrawal") + } + }) + } + } +} + func TestPeerRecordIDRetainsAdmissionOnReadFailureButNotClear(t *testing.T) { dir := t.TempDir() clustertrusttest.Join(t, dir, "test-cluster", "self") diff --git a/services/shared/clustertrust/membership.go b/services/shared/clustertrust/membership.go index 678cc6034..0845a9b4b 100644 --- a/services/shared/clustertrust/membership.go +++ b/services/shared/clustertrust/membership.go @@ -15,14 +15,15 @@ const admissionFileName = "admission.json" // admissionRecord mirrors the subset of admission.json that answers "does this // node currently belong to a cluster". nvpair-cluster-manager owns the full -// schema (a monotonic counter, retirement marker, etc.); the read side only -// needs the active pair. A live admission carries both a clusterId and a nonzero +// schema (a monotonic counter, etc.); the read side needs the active pair and +// retirement marker. A live admission carries both a clusterId and a nonzero // epoch; teardown (leave/removal) clears both while deliberately keeping the // counter, so absence of an active pair is the authoritative "no longer a // member" signal — even though the node.key/node.crt keypair persists. type admissionRecord struct { ClusterID string `json:"clusterId"` Epoch uint64 `json:"epoch"` + Retired bool `json:"retired,omitempty"` } // readAdmission returns the durable membership and epoch, or an empty record diff --git a/services/shared/clustertrust/mesh.go b/services/shared/clustertrust/mesh.go index 5314d1cb0..14b125854 100644 --- a/services/shared/clustertrust/mesh.go +++ b/services/shared/clustertrust/mesh.go @@ -253,7 +253,7 @@ func (m *Mesh) PeerRecordID(uuid string) string { } m.mu.RLock() defer m.mu.RUnlock() - if !m.clustered || m.id == nil { + if !m.clustered || m.id == nil || m.admission.ClusterID == "" || m.admission.Epoch == 0 || m.admission.Retired { return "" } peer := selfLeafDER(m.id) From 6286a3208e902da56b9ac6dd8e4ddd33f214c731 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:39:39 +0200 Subject: [PATCH 08/18] fix(clustertrust): expire certificate-bound inventory proofs Check local and pinned peer certificate validity whenever a proof binding is used, and recheck the response certificate at acceptance. This disqualifies cached routing inventory and telemetry when a certificate expires, including responses crossing expiry on pooled connections. Add invalid-window, in-flight expiry and broker cache/replay regressions. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-node-scanner/README.md | 2 +- .../verified_inventory_test.go | 33 +++++++- .../shared/clustertrust/inventory_test.go | 82 ++++++++++++++++++- services/shared/clustertrust/mesh.go | 19 ++++- .../clustertrusttest/clustertrusttest.go | 17 +++- 5 files changed, 144 insertions(+), 9 deletions(-) diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index 298e5b7b2..c29b6f604 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -22,7 +22,7 @@ Concretely, the daemon: cadence. It emits only host UUID, maximum GPU utilization, validity, and source-reported sample age; it does not mutate the public directory for each telemetry sample. -- **Grants trust only after proof:** `trusted` requires a completed pinned mTLS inventory read. `peerRecordId` binds the proof to an active, non-retired local admission, local certificate and exact peer pin record. Retained pins after admission withdrawal cannot authorize fresh or cached inventory, even over an existing connection. A pin by itself never promotes a claim. Trust changes withdraw the verified record using its original token rather than clearing the token and turning it into a preview under the host key. +- **Grants trust only after proof:** `trusted` requires a completed pinned mTLS inventory read. `peerRecordId` binds the proof to an active, non-retired local admission, local certificate and exact peer pin record. Retained pins after admission withdrawal cannot authorize fresh or cached inventory, even over an existing connection. Both local and peer certificates must be within their validity windows when accepting or rechecking proofs; expiry also disqualifies cached inventory and telemetry. A pin by itself never promotes a claim. Trust changes withdraw the verified record using its original token rather than clearing the token and turning it into a preview under the host key. - **Advertises live membership:** the same reload re-derives this node's own `cluster-uuid=` from the cluster dir and re-advertises when it differs from what is being published, so joining or leaving a cluster takes effect without a restart. - **Public-preview anti-flap:** a candidate has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. - **A scan that loses everything is treated as our own fault:** when more than one node is known and a scan returns *none* of them, no node is penalized for it — for up to six consecutive scans (~30s), after which they count normally. Independent machines do not leave the network in the same five-second window, but a process too starved to drain its multicast socket within `scanTimeout` hears silence from all of them at once; without this, one saturated machine dropped its entire cluster, idle peers included, in a single reconcile. The suppression needs two or more known nodes to engage: losing a lone peer looks the same whether it left or we stopped listening, so that case falls through to the ordinary threshold. diff --git a/services/nvpair-ui-broker/verified_inventory_test.go b/services/nvpair-ui-broker/verified_inventory_test.go index a4e55b294..ea4fe1491 100644 --- a/services/nvpair-ui-broker/verified_inventory_test.go +++ b/services/nvpair-ui-broker/verified_inventory_test.go @@ -11,6 +11,8 @@ import ( "nvpair-shared/noderec" "nvpair-ui-broker/relay" "nvpair-ui-broker/workloadstore" + "os" + "path/filepath" "testing" "time" ) @@ -92,9 +94,26 @@ func TestReplayInvalidTelemetryRequiresCurrentProof(t *testing.T) { } func TestVerifiedInventoryWithdrawnOnPinAndAdmissionChanges(t *testing.T) { - for _, change := range []string{"pin", "admission"} { + for _, change := range []string{"pin", "admission", "expiry"} { t.Run(change, func(t *testing.T) { b := inventoryTestBroker(t) + deadline := time.Now().Truncate(time.Second).Add(2 * time.Second) + if change == "expiry" { + peerDir := t.TempDir() + clustertrusttest.WriteKeypairWithValidity(t, peerDir, "peer", time.Now().Add(-time.Hour), deadline) + cert, err := os.ReadFile(filepath.Join(peerDir, "node.crt")) + if err != nil { + t.Fatal(err) + } + pin, err := json.Marshal(map[string]string{"nodeUuid": "peer", "certPem": string(cert)}) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(b.clusterDir, "trusted", "peer.json"), pin, 0600); err != nil { + t.Fatal(err) + } + b.mesh.Refresh() + } node := verifiedDirectoryForTest(b) b.store.Upsert(directoryToEnriched(node), sourceScanner) b.relayDir.Apply(noderec.NotifyNodeDiscovered, node) @@ -104,8 +123,18 @@ func TestVerifiedInventoryWithdrawnOnPinAndAdmissionChanges(t *testing.T) { } if change == "pin" { clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") - } else { + } else if change == "admission" { clustertrusttest.WriteAdmission(t, b.clusterDir, "test-cluster", 2) + } else { + <-time.After(time.Until(deadline.Add(time.Millisecond))) + } + if b.authorizedTelemetry(sourceScanner, "peer", node.PeerRecordID) { + t.Error("stale proof still authorizes telemetry") + } + var out bytes.Buffer + worker := &rpcWorker{peer: NewPeer(NewCodec(&out))} + if got := b.replayTelemetryToScheduler(worker); got != 0 || out.Len() != 0 { + t.Error("stale cached telemetry replayed") } // Cached relay/scheduler reads independently recheck authorization, before // the worker has reacted to the notification. diff --git a/services/shared/clustertrust/inventory_test.go b/services/shared/clustertrust/inventory_test.go index 3cc514296..a1a24f3d7 100644 --- a/services/shared/clustertrust/inventory_test.go +++ b/services/shared/clustertrust/inventory_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "net/http/httptrace" "nvpair-shared/clustertrusttest" "os" "path/filepath" @@ -17,10 +18,16 @@ import ( ) func inventoryPair(t *testing.T) (string, *Mesh, *Mesh) { + t.Helper() + return inventoryPairWithPeerValidity(t, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)) +} + +func inventoryPairWithPeerValidity(t *testing.T, notBefore, notAfter time.Time) (string, *Mesh, *Mesh) { t.Helper() selfDir, peerDir := t.TempDir(), t.TempDir() clustertrusttest.Join(t, selfDir, "test-cluster", "self") - clustertrusttest.Join(t, peerDir, "test-cluster", "peer") + clustertrusttest.WriteKeypairWithValidity(t, peerDir, "peer", notBefore, notAfter) + clustertrusttest.WriteAdmission(t, peerDir, "test-cluster", 1) pin := func(dir, uuid, certDir string) { cert, err := os.ReadFile(filepath.Join(certDir, "node.crt")) if err != nil { @@ -56,6 +63,79 @@ func inventoryServer(t *testing.T, mesh *Mesh, handler http.HandlerFunc) *httpte return srv } +func TestInventoryProofRejectsCertificatesOutsideValidity(t *testing.T) { + now := time.Now() + for _, tc := range []struct { + name string + before, after time.Time + }{ + {"expired", now.Add(-time.Hour), now.Add(-time.Minute)}, + {"not-yet-valid", now.Add(time.Hour), now.Add(2 * time.Hour)}, + } { + t.Run(tc.name, func(t *testing.T) { + _, self, peer := inventoryPairWithPeerValidity(t, tc.before, tc.after) + id, _ := peer.identity() + if self.VerifyPeerCertificate("peer", id.Cert.Leaf) { + t.Error("invalid leaf accepted as proof") + } + if self.PeerRecordID("peer") != "" || peer.PeerRecordID("peer") != "" || peer.PeerRecordID("self") != "" { + t.Error("invalid peer or local leaf authorized cached proof") + } + srv := inventoryServer(t, peer, func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write([]byte(`{"value":42}`)) }) + pool := NewPeerClientPool(self, time.Second) + defer pool.CloseIdle() + client, _ := pool.Client("peer") + var value struct{ Value int } + if _, _, ok := ReadPeerInventory(t.Context(), self, client, "peer", srv.URL, "", 1024, &value); ok { + t.Error("invalid leaf authorized inventory read") + } + }) + } +} + +func TestInventoryProofExpiresOnPooledAndInflightReads(t *testing.T) { + deadline := time.Now().Truncate(time.Second).Add(2 * time.Second) + _, self, peer := inventoryPairWithPeerValidity(t, time.Now().Add(-time.Hour), deadline) + var hold atomic.Bool + srv := inventoryServer(t, peer, func(w http.ResponseWriter, _ *http.Request) { + if hold.Load() { + <-time.After(time.Until(deadline.Add(time.Millisecond))) + } + _, _ = w.Write([]byte(`{"value":42}`)) + }) + pool := NewPeerClientPool(self, 4*time.Second) + defer pool.CloseIdle() + client, _ := pool.Client("peer") + var reused atomic.Bool + ctx := httptrace.WithClientTrace(t.Context(), &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) { reused.Store(info.Reused) }}) + read := func() bool { + var value struct{ Value int } + _, _, ok := ReadPeerInventory(ctx, self, client, "peer", srv.URL, "", 1024, &value) + return ok + } + if !read() { + t.Fatal("valid certificate rejected") + } + proof := self.PeerRecordID("peer") + hold.Store(true) + if read() { + t.Error("read crossing expiry accepted on existing connection") + } + if !reused.Load() { + t.Fatal("expiry regression did not exercise a pooled connection") + } + if self.PeerRecordID("peer") != "" { + t.Errorf("cached inventory proof %s remains authorized after expiry", proof) + } + id, _ := peer.identity() + if self.VerifyPeerCertificate("peer", id.Cert.Leaf) { + t.Error("expired peer certificate accepted") + } + if read() { + t.Error("pooled read after expiry accepted") + } +} + func TestReadPeerInventoryRequiresCurrentExactAuthorization(t *testing.T) { for _, change := range []string{"none", "pin", "admission"} { t.Run(change, func(t *testing.T) { diff --git a/services/shared/clustertrust/mesh.go b/services/shared/clustertrust/mesh.go index 14b125854..0f1ce6d50 100644 --- a/services/shared/clustertrust/mesh.go +++ b/services/shared/clustertrust/mesh.go @@ -15,6 +15,7 @@ import ( "path/filepath" "strings" "sync" + "time" ) // ClusterUUIDTXTKey is the mDNS TXT key carrying a node's cluster principal — @@ -256,12 +257,17 @@ func (m *Mesh) PeerRecordID(uuid string) string { if !m.clustered || m.id == nil || m.admission.ClusterID == "" || m.admission.Epoch == 0 || m.admission.Retired { return "" } + now := time.Now() + if !validLeafDER(selfLeafDER(m.id), now) { + return "" + } peer := selfLeafDER(m.id) if uuid != m.id.NodeUUID { m.trust.mu.RLock() peer = bytes.Clone(m.trust.raw[uuid]) + valid := validLeafDER(m.trust.ders[uuid], now) m.trust.mu.RUnlock() - if len(peer) == 0 { + if len(peer) == 0 || !valid { return "" } } @@ -272,7 +278,16 @@ func (m *Mesh) PeerRecordID(uuid string) string { // VerifyPeerCertificate rechecks the current exact pin after a response, since // pooled connections and in-flight requests can outlive their authorization. func (m *Mesh) VerifyPeerCertificate(uuid string, cert *x509.Certificate) bool { - return cert != nil && UUIDFromCert(cert) == uuid && m.matchDER(uuid, cert.Raw) + return validLeaf(cert, time.Now()) && UUIDFromCert(cert) == uuid && m.matchDER(uuid, cert.Raw) +} + +func validLeafDER(der []byte, now time.Time) bool { + cert, err := x509.ParseCertificate(der) + return err == nil && validLeaf(cert, now) +} + +func validLeaf(cert *x509.Certificate, now time.Time) bool { + return cert != nil && !now.Before(cert.NotBefore) && !now.After(cert.NotAfter) } // matchDER is the server-side gate: accept der for uuid when it byte-for-byte diff --git a/services/shared/clustertrusttest/clustertrusttest.go b/services/shared/clustertrusttest/clustertrusttest.go index 60cc293e9..51a1dad36 100644 --- a/services/shared/clustertrusttest/clustertrusttest.go +++ b/services/shared/clustertrusttest/clustertrusttest.go @@ -34,11 +34,17 @@ const nodeURISANPrefix = "urn:nvpair:node:" // clusterDir. The keypair is the durable identity: it deliberately outlives a // membership, so writing it alone does NOT make a node clustered (see Join). func WriteKeypair(t *testing.T, clusterDir, uuid string) { + t.Helper() + WriteKeypairWithValidity(t, clusterDir, uuid, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)) +} + +// WriteKeypairWithValidity writes a leaf with controlled validity boundaries. +func WriteKeypairWithValidity(t *testing.T, clusterDir, uuid string, notBefore, notAfter time.Time) { t.Helper() if err := os.MkdirAll(clusterDir, 0o700); err != nil { t.Fatal(err) } - certPEM, keyPEM := mintLeaf(t, uuid) + certPEM, keyPEM := mintLeafWithValidity(t, uuid, notBefore, notAfter) if err := os.WriteFile(filepath.Join(clusterDir, "node.crt"), certPEM, 0o644); err != nil { t.Fatal(err) } @@ -109,6 +115,11 @@ func Join(t *testing.T, clusterDir, clusterID, selfUUID string, peerUUIDs ...str // mintLeaf returns a self-signed PEM certificate/key pair whose URI SAN carries // uuid as the node principal. func mintLeaf(t *testing.T, uuid string) (certPEM, keyPEM []byte) { + t.Helper() + return mintLeafWithValidity(t, uuid, time.Now().Add(-time.Hour), time.Now().Add(time.Hour)) +} + +func mintLeafWithValidity(t *testing.T, uuid string, notBefore, notAfter time.Time) (certPEM, keyPEM []byte) { t.Helper() pub, priv, err := ed25519.GenerateKey(rand.Reader) if err != nil { @@ -121,8 +132,8 @@ func mintLeaf(t *testing.T, uuid string) (certPEM, keyPEM []byte) { tmpl := &x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: uuid}, - NotBefore: time.Now().Add(-time.Hour), - NotAfter: time.Now().Add(time.Hour), + NotBefore: notBefore, + NotAfter: notAfter, URIs: []*url.URL{uri}, } der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, pub, priv) From 8640a0e2e918939532886bb108e4fa65273e5422 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:44:35 +0200 Subject: [PATCH 09/18] fix(broker): serialize manual trust replay with retirement Hold the worker callback mutex across trust-refresh snapshot and replay, and tag statuses with their worker generation. Reject retired-generation snapshots and advance launch generations under the same mutex. Controlled replay regressions cover retirement, removal and newer callbacks without resurrecting stale routing inventory. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-ui-broker/README.md | 2 +- services/nvpair-ui-broker/broker.go | 6 +- services/nvpair-ui-broker/manualnodes.go | 8 +- .../verified_inventory_test.go | 97 +++++++++++++++++++ 4 files changed, 109 insertions(+), 4 deletions(-) diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 52dbbef98..8f23d0776 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -902,7 +902,7 @@ Attach to a pre-existing endpoint: - **Engine-advertise control surface.** Engine registration is auto-driven only: the broker tracks local ollama / LM Studio on their fixed coordinates and registers `ol` / `lm` with the daemon while up. There's no manual-advertise RPC (custom service, port, name, or TXT), and no way to advertise anything other than the detected engines. - **node-info control surface.** The broker passes its cluster directory and local node ID, pushes demand and log-level controls, and publishes broker-owned service metadata via `nodeinfo:set-peer-metadata`. Public hardware previews remain available; only pinned mTLS inventory can feed service decisions. There is no public reconfiguration RPC or custom node-info port through the broker. -- **Manual-node persistence across restarts.** `nvpair-manual-nodes` remains an in-memory worker; the broker owns `configs/manual-nodes.json`, clears stale live projections when the worker exits, and replays the durable entries into its replacement. +- **Manual-node persistence across restarts.** `nvpair-manual-nodes` remains an in-memory worker; the broker owns `configs/manual-nodes.json`, clears stale live projections when the worker exits, and replays the durable entries into its replacement. Trust-refresh snapshot/replay is serialized with worker callbacks and retirement; status snapshots from retired worker generations cannot republish inventory. - **Per-event push semantics.** `discovery:nodes-changed` always carries the full current snapshot, not a delta. For small N this is fine and lets the client treat the payload as authoritative without state reconciliation. `errors:update` is likewise a full snapshot. - **BYO-TLS peer authorization.** Operator CA trust and plaintext previews do not establish PAIR peer identity. Service inventory requires pinned cluster mTLS and matching authenticated metadata; older peers lacking that metadata remain previews until upgraded. - **Per-worker log-level granularity.** `log/set-level` fans the broker's single level out to every running child, but there's no way to set a different level per worker — it's one level for the whole process tree. diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index b23a90b23..1d0096582 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -733,7 +733,9 @@ var ( ) } launchManualNodesProcess = func(b *Broker) (*rpcWorker, error) { + b.manualCallbackMu.Lock() generation := b.manualInventoryGeneration.Add(1) + b.manualCallbackMu.Unlock() return startRPCWorker("manual-nodes", b.manualNodesPath, append(b.logLevelArgs(), b.clusterDirArgs()...), func(method string, params json.RawMessage) { b.manualCallbackMu.Lock() defer b.manualCallbackMu.Unlock() @@ -1535,11 +1537,11 @@ func (b *Broker) upsertManualNode(s manualNodeStatus) { oldKey, existed := b.manualNodeKeys[s.ID] priorProof := b.manualNodeStatuses[s.ID].status.PeerRecordID b.manualNodeKeys[s.ID] = key - b.manualNodeStatuses[s.ID] = manualNodeStatusEntry{status: s, receivedAt: receivedAt} + b.manualNodeStatuses[s.ID] = manualNodeStatusEntry{generation: b.manualInventoryGeneration.Load(), status: s, receivedAt: receivedAt} canonical, ok := b.survivingAliasLocked(key) b.manualMu.Unlock() if !ok { - canonical = manualNodeStatusEntry{status: s, receivedAt: receivedAt} + canonical = manualNodeStatusEntry{generation: b.manualInventoryGeneration.Load(), status: s, receivedAt: receivedAt} } if existed && oldKey == key && priorProof != canonical.status.PeerRecordID { diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index e74e40b46..d7c3bc668 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -66,6 +66,7 @@ type manualNodeStatus struct { } type manualNodeStatusEntry struct { + generation uint64 status manualNodeStatus receivedAt time.Time } @@ -464,6 +465,11 @@ func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.D } func (b *Broker) refreshManualNodesTrust() { + // Snapshot and replay are one operation relative to worker callbacks and + // retirement. Never let a copied status outlive its worker generation. + b.manualCallbackMu.Lock() + defer b.manualCallbackMu.Unlock() + generation := b.manualInventoryGeneration.Load() b.manualMu.Lock() seenKeys := make(map[string]struct{}, len(b.manualNodeKeys)) entries := make([]manualNodeStatusEntry, 0, len(b.manualNodeKeys)) @@ -475,7 +481,7 @@ func (b *Broker) refreshManualNodesTrust() { continue } seenKeys[key] = struct{}{} - if canonical, ok := b.survivingAliasLocked(key); ok { + if canonical, ok := b.survivingAliasLocked(key); ok && canonical.generation == generation { entries = append(entries, canonical) } } diff --git a/services/nvpair-ui-broker/verified_inventory_test.go b/services/nvpair-ui-broker/verified_inventory_test.go index ea4fe1491..970eb147c 100644 --- a/services/nvpair-ui-broker/verified_inventory_test.go +++ b/services/nvpair-ui-broker/verified_inventory_test.go @@ -13,6 +13,7 @@ import ( "nvpair-ui-broker/workloadstore" "os" "path/filepath" + "sync/atomic" "testing" "time" ) @@ -181,6 +182,102 @@ func TestUnprovenScannerClaimCannotReplaceVerifiedPeer(t *testing.T) { } } +// Pause replay after its first publication, then race retirement or worker +// callbacks against the still-copied second entry. The newest operation wins. +func TestManualTrustRefreshSerializesWithWorkerChanges(t *testing.T) { + for _, operation := range []string{"retire", "remove", "update"} { + t.Run(operation, func(t *testing.T) { + b := inventoryTestBroker(t) + for _, id := range []string{"a", "b"} { + clustertrusttest.WritePeerPin(t, b.clusterDir, id) + } + b.mesh.Refresh() + statusFor := func(id, model string) manualNodeStatus { + return manualNodeStatus{ID: id, HostUUID: id, ClusterUUID: id, PeerRecordID: b.mesh.PeerRecordID(id), Address: "192.0.2.2", OMLXUp: true, OMLXModels: []string{model}, PeerMetadata: &noderec.PeerMetadataParams{HostUUID: id, Name: id, Services: map[noderec.ServiceKey]int{noderec.ServiceOMLX: 1236}}} + } + for _, id := range []string{"a", "b"} { + b.upsertManualNode(statusFor(id, "old")) + } + // Force replay to publish changes while preserving its status inputs. + for _, key := range b.manualNodeKeys { + b.store.Remove(key, sourceManual) + } + entered, release := make(chan struct{}), make(chan struct{}) + var publications atomic.Int32 + b.store.onChange = func() { + if publications.Add(1) == 1 { + close(entered) + <-release + } + } + refreshed := make(chan struct{}) + go func() { b.refreshManualNodesTrust(); close(refreshed) }() + select { + case <-entered: + case <-time.After(2 * time.Second): + t.Fatal("trust replay did not publish") + } + changed := make(chan struct{}) + go func() { + defer close(changed) + if operation == "retire" { + b.clearManualNodesState() + return + } + b.manualCallbackMu.Lock() + defer b.manualCallbackMu.Unlock() + for _, id := range []string{"a", "b"} { + method := "node/removed" + status := statusFor(id, "new") + if operation == "update" { + method = "node/updated" + } + data, _ := json.Marshal(status) + b.forwardManualNodesNotification(method, data) + } + }() + select { + case <-changed: + t.Error("worker change overtook paused trust replay") + case <-time.After(50 * time.Millisecond): + } + close(release) + <-refreshed + <-changed + got := b.schedulerNodes() + if operation != "update" { + if len(got) != 0 || len(b.relayDir.Snapshot(noderec.ServiceOMLX)) != 0 { + t.Fatalf("withdrawn manual inventory resurrected: %+v", got) + } + } else { + if len(got) != 2 { + t.Fatalf("updated nodes: %+v", got) + } + for _, n := range got { + if len(n.Models) != 1 || n.Models[0] != "new" { + t.Errorf("old snapshot replaced worker update: %+v", n) + } + } + } + }) + } +} + +func TestManualTrustRefreshRejectsRetiredGeneration(t *testing.T) { + b := inventoryTestBroker(t) + b.upsertManualNode(manualNodeStatus{ID: "old", Address: "192.0.2.2", OMLXUp: true, OMLXModels: []string{"old"}}) + // Simulate an old snapshot retained across the generation boundary. It must + // not publish into the replacement generation even if its pins remain valid. + for _, key := range b.manualNodeKeys { + b.store.Remove(key, sourceManual) + } + b.manualInventoryGeneration.Add(1) + b.refreshManualNodesTrust() + if len(b.schedulerNodes()) != 0 { + t.Fatal("retired generation snapshot republished") + } +} + func TestScannerRestartWithdrawsProofButPreservesIndependentManualEngine(t *testing.T) { b := inventoryTestBroker(t) node := verifiedDirectoryForTest(b) From 10913bb77e1b1b958c25611004247c7fa4ffd633 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:50:07 +0200 Subject: [PATCH 10/18] fix(scanner): conditionally expire verified inventory atomically Remove an expired directory snapshot only when the complete record is still current under the directory lock. Serialize verified publication and enrichment apply/notify with removal, cache cleanup and withdrawal. Controlled interleavings cover same-second refreshes and a publication racing a paused withdrawal, preserving fresh inventory and notification order. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-node-scanner/README.md | 2 +- services/nvpair-node-scanner/daemon.go | 11 ++- services/nvpair-node-scanner/directory.go | 13 +++ .../inventory_lifetime_test.go | 86 +++++++++++++++++++ .../nvpair-node-scanner/peer_inventory.go | 8 +- 5 files changed, 114 insertions(+), 6 deletions(-) diff --git a/services/nvpair-node-scanner/README.md b/services/nvpair-node-scanner/README.md index c29b6f604..3e00e89e8 100644 --- a/services/nvpair-node-scanner/README.md +++ b/services/nvpair-node-scanner/README.md @@ -22,7 +22,7 @@ Concretely, the daemon: cadence. It emits only host UUID, maximum GPU utilization, validity, and source-reported sample age; it does not mutate the public directory for each telemetry sample. -- **Grants trust only after proof:** `trusted` requires a completed pinned mTLS inventory read. `peerRecordId` binds the proof to an active, non-retired local admission, local certificate and exact peer pin record. Retained pins after admission withdrawal cannot authorize fresh or cached inventory, even over an existing connection. Both local and peer certificates must be within their validity windows when accepting or rechecking proofs; expiry also disqualifies cached inventory and telemetry. A pin by itself never promotes a claim. Trust changes withdraw the verified record using its original token rather than clearing the token and turning it into a preview under the host key. +- **Grants trust only after proof:** `trusted` requires a completed pinned mTLS inventory read. `peerRecordId` binds the proof to an active, non-retired local admission, local certificate and exact peer pin record. Retained pins after admission withdrawal cannot authorize fresh or cached inventory, even over an existing connection. Both local and peer certificates must be within their validity windows when accepting or rechecking proofs; expiry also disqualifies cached inventory and telemetry. A pin by itself never promotes a claim. Trust changes withdraw the verified record using its original token rather than clearing the token and turning it into a preview under the host key. Expiry atomically removes only the exact observed record; verified publication, enrichment updates and withdrawal notifications are ordered so a stale expiry cannot delete or withdraw a refreshed peer. - **Advertises live membership:** the same reload re-derives this node's own `cluster-uuid=` from the cluster dir and re-advertises when it differs from what is being published, so joining or leaving a cluster takes effect without a restart. - **Public-preview anti-flap:** a candidate has to be absent from every scan for the browser's full miss threshold (twelve scans at 5s, a full minute) before it is evicted, and from ~15s in it is TCP-probed on its advertised service ports on *every* scan — any answer resets the counter and recovers it outright. So neither a momentary multicast gap nor a sustained one drops a healthy node, and eviction rests on a run of ~10 consecutive failures rather than one dial on the deadline. The window is sized deliberately: a node saturated by its own inference load starves its control plane, so it stops answering mDNS *and* stops accepting probe connections for as long as the load runs, while still serving requests. Two signals can also vouch for a node without any probe — a node-info enrichment in the last 10s, and inference response bytes from it in the last 60s (`discovery:node-activity`, relayed by the broker from whichever proxy saw the bytes). The second is the only liveness evidence that gets stronger the busier a node is. - **A scan that loses everything is treated as our own fault:** when more than one node is known and a scan returns *none* of them, no node is penalized for it — for up to six consecutive scans (~30s), after which they count normally. Independent machines do not leave the network in the same five-second window, but a process too starved to drain its multicast socket within `scanTimeout` hears silence from all of them at once; without this, one saturated machine dropped its entire cluster, idle peers included, in a single reconcile. The suppression needs two or more known nodes to engage: losing a lone peer looks the same whether it left or we stopped listening, so that case falls through to the ordinary threshold. diff --git a/services/nvpair-node-scanner/daemon.go b/services/nvpair-node-scanner/daemon.go index 67139e53f..789ced50f 100644 --- a/services/nvpair-node-scanner/daemon.go +++ b/services/nvpair-node-scanner/daemon.go @@ -61,8 +61,11 @@ type daemon struct { modelsHTTP *http.Client // modelsClients is the long-lived per-peer mTLS pool for LAN /v1/models // fetches. A throwaway Transport every 15s paid a full handshake each time. - modelsClients *clustertrust.PeerClientPool - peerInfoOnce sync.Once + modelsClients *clustertrust.PeerClientPool + peerInfoOnce sync.Once + // inventoryMu orders verified publication, conditional removal, cache cleanup + // and their notifications. Network probes run outside this lock. + inventoryMu sync.Mutex candidateMu sync.Mutex candidates map[string]noderec.DirectoryNode peerInfoClients *clustertrust.PeerClientPool @@ -2048,6 +2051,8 @@ func (d *daemon) refreshNodeModelsCandidates(hostUUID, guardIP string, hosts []s return false } models, byEngine, loadedByEngine := inventory.models, inventory.byEngine, inventory.loadedByEngine + d.inventoryMu.Lock() + defer d.inventoryMu.Unlock() if !d.localNode(hostUUID) { after, exists := d.dir.get(hostUUID) if !exists || before.PeerRecordID != after.PeerRecordID || !d.currentPeer(after) { @@ -2192,6 +2197,8 @@ func (d *daemon) refreshNodeEngineNative(ctx context.Context, hostUUID, guardIP if !fetched { return false } + d.inventoryMu.Lock() + defer d.inventoryMu.Unlock() if !d.localNode(hostUUID) { after, exists := d.dir.get(hostUUID) if !exists || before.PeerRecordID != after.PeerRecordID || !d.currentPeer(after) { diff --git a/services/nvpair-node-scanner/directory.go b/services/nvpair-node-scanner/directory.go index dece7dd40..ac852def9 100644 --- a/services/nvpair-node-scanner/directory.go +++ b/services/nvpair-node-scanner/directory.go @@ -126,6 +126,19 @@ func (d *directory) replaceIfCurrent(old, replacement noderec.DirectoryNode) boo return true } +// removeIfCurrent atomically removes only the exact snapshot that expired. +// A refresh within the same LastSeen second is still a different record. +func (d *directory) removeIfCurrent(old noderec.DirectoryNode) bool { + d.mu.Lock() + defer d.mu.Unlock() + current, ok := d.nodes[old.RecordKey()] + if !ok || !reflect.DeepEqual(current, old) { + return false + } + delete(d.nodes, old.RecordKey()) + return true +} + // remove drops a node by hostUuid and reports whether it existed. func (d *directory) remove(hostUUID string) (existed bool) { d.mu.Lock() diff --git a/services/nvpair-node-scanner/inventory_lifetime_test.go b/services/nvpair-node-scanner/inventory_lifetime_test.go index 0b35c61b4..eaffeeedb 100644 --- a/services/nvpair-node-scanner/inventory_lifetime_test.go +++ b/services/nvpair-node-scanner/inventory_lifetime_test.go @@ -4,7 +4,11 @@ package main import ( + "bytes" + "encoding/json" "net" + "net/http" + "runtime" "strconv" "testing" "time" @@ -12,6 +16,88 @@ import ( "nvpair-shared/noderec" ) +// A refresh in the same timestamp second can change inventory without changing +// the proof, IP or LastSeen. Expiry must compare the entire observed record. +func TestVerifiedExpiryPreservesConcurrentRefresh(t *testing.T) { + h := newTelemetryHarness(t) + h.peer("peer", func(*http.Request) NodeInfoResponse { return NodeInfoResponse{HostUUID: "peer"} }) + old, _ := h.d.dir.get("peer") + fresh := old + fresh.Models = []string{"fresh-model"} + capture := ¬ifyCapture{} + h.d.codec = NewCodec(capture) + // Hold the directory lock while expiry starts with its old snapshot, then + // publish the refreshed record before permitting its conditional removal. + h.d.dir.mu.Lock() + started, done := make(chan struct{}), make(chan struct{}) + go func() { close(started); h.d.expireVerified(old); close(done) }() + <-started + h.d.dir.nodes["peer"] = fresh + h.d.dir.mu.Unlock() + <-done + got, ok := h.d.dir.get("peer") + if !ok || len(got.Models) != 1 || got.Models[0] != "fresh-model" { + t.Fatalf("stale expiry deleted refreshed inventory: %+v, exists=%v", got, ok) + } + if len(capture.nodes(t)) != 0 { + t.Fatal("stale expiry withdrew refreshed peer telemetry") + } +} + +func TestVerifiedExpiryOrdersWithdrawalBeforeNewPublication(t *testing.T) { + h := newTelemetryHarness(t) + h.peer("peer", func(*http.Request) NodeInfoResponse { return NodeInfoResponse{HostUUID: "peer"} }) + old, _ := h.d.dir.get("peer") + fresh := old + fresh.Models = []string{"fresh-model"} + capture := ¬ifyCapture{} + h.d.codec = NewCodec(capture) + // Pause expiry after directory removal but before cache cleanup/withdrawal. + h.d.infoMu.Lock() + expired := make(chan struct{}) + go func() { h.d.expireVerified(old); close(expired) }() + deadline := time.Now().Add(2 * time.Second) + for { + if _, exists := h.d.dir.get("peer"); !exists { + break + } + if time.Now().After(deadline) { + h.d.infoMu.Unlock() + t.Fatal("expiry did not remove old record") + } + runtime.Gosched() + } + published := make(chan struct{}) + go func() { h.d.publishVerified(fresh, ""); close(published) }() + select { + case <-published: + t.Error("publication overtook paused withdrawal") + case <-time.After(50 * time.Millisecond): + } + h.d.infoMu.Unlock() + <-expired + <-published + capture.mu.Lock() + raw := bytes.Clone(capture.frames) + capture.mu.Unlock() + lines := bytes.Split(bytes.TrimSpace(raw), []byte("\n")) + if len(lines) != 2 { + t.Fatalf("expected withdrawal and update, got %s", raw) + } + for i, want := range []string{noderec.NotifyNodeRemoved, noderec.NotifyNodeUpdated} { + var msg Message + if err := json.Unmarshal(lines[i], &msg); err != nil { + t.Fatal(err) + } + if msg.Method != want { + t.Errorf("event %d=%s, want %s; fresh inventory would be withdrawn", i, msg.Method, want) + } + } + if got, ok := h.d.dir.get("peer"); !ok || len(got.Models) != 1 || got.Models[0] != "fresh-model" { + t.Fatal("new publication lost") + } +} + // Browser churn must not reset the independent verified peer's retry budget. func TestBrowseChurnPreservesVerifiedTelemetryRetry(t *testing.T) { d, server, _ := secureInventoryFixture(t, "uuid-peer") diff --git a/services/nvpair-node-scanner/peer_inventory.go b/services/nvpair-node-scanner/peer_inventory.go index d810728b4..81eda6c61 100644 --- a/services/nvpair-node-scanner/peer_inventory.go +++ b/services/nvpair-node-scanner/peer_inventory.go @@ -79,6 +79,8 @@ func (d *daemon) candidateSnapshot() []noderec.DirectoryNode { } func (d *daemon) publishVerified(node noderec.DirectoryNode, candidateKey string) bool { + d.inventoryMu.Lock() + defer d.inventoryMu.Unlock() if !d.currentPeer(node) { return false } @@ -116,11 +118,11 @@ func (d *daemon) publishVerified(node noderec.DirectoryNode, candidateKey string } func (d *daemon) expireVerified(node noderec.DirectoryNode) { - current, exists := d.dir.get(node.HostUUID) - if !exists || current.PeerRecordID != node.PeerRecordID || current.LastSeen != node.LastSeen || current.IP != node.IP { + d.inventoryMu.Lock() + defer d.inventoryMu.Unlock() + if !d.dir.removeIfCurrent(node) { return } - d.dir.remove(node.HostUUID) d.forget(node.HostUUID) d.emit(noderec.NotifyNodeRemoved, node) } From 2ab83daa41e107f719634033228c7786efa0c5e1 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 07:51:12 +0200 Subject: [PATCH 11/18] test(scanner): restore authenticated telemetry failover assertions Exercise refreshTelemetryOnce against the mutually pinned TLS fixture with an unreachable first address. Assert emitted peer identity, proof and GPU utilization, and retain the shared remembered-address selection assertion. The regression fails when telemetry is restricted to its first address. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- .../nvpair-node-scanner/telemetry_test.go | 32 ++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/services/nvpair-node-scanner/telemetry_test.go b/services/nvpair-node-scanner/telemetry_test.go index 257402305..f19c3f91b 100644 --- a/services/nvpair-node-scanner/telemetry_test.go +++ b/services/nvpair-node-scanner/telemetry_test.go @@ -104,11 +104,35 @@ func TestRefreshNodeTelemetryRejectsMismatchedIdentity(t *testing.T) { } func TestRefreshTelemetryFailsOverToAnAnsweringAddress(t *testing.T) { h := newTelemetryHarness(t) - host, port := h.peer("node-a", func(*http.Request) NodeInfoResponse { - return NodeInfoResponse{HostUUID: "node-a", TelemetryValid: true} + host, _ := h.peer("node-a", func(*http.Request) NodeInfoResponse { + return NodeInfoResponse{HostUUID: "node-a", TelemetryValid: true, GPUs: []GPUInfo{{Name: "GPU 0", UtilizationPercent: gpuReading(uint32(61))}}} }) - if !h.d.refreshNodeTelemetryCandidates(t.Context(), "node-a", []string{"127.0.0.2", host}, port) { - t.Fatal("answering second address ignored") + node, _ := h.d.dir.get("node-a") + // This loopback address has no listener; only the second address can answer + // with the pinned peer certificate. Exercise the whole sweep, not a helper. + node.IP = "127.0.0.2" + node.IPs = []string{"127.0.0.2", host} + h.d.dir.upsert(node) + capture := ¬ifyCapture{} + h.d.codec = NewCodec(capture) + h.d.refreshTelemetryOnce(t.Context()) + capture.mu.Lock() + data := append([]byte(nil), capture.frames...) + capture.mu.Unlock() + var message Message + if err := json.Unmarshal(data, &message); err != nil { + t.Fatalf("decode notification: %v (output %q)", err, data) + } + var got noderec.NodeTelemetry + if err := json.Unmarshal(message.Params, &got); err != nil { + t.Fatal(err) + } + if message.Method != noderec.NotifyNodeTelemetry || got.HostUUID != "node-a" || got.GPUUtilizationPct != 61 || !got.TelemetryValid || got.PeerRecordID != node.PeerRecordID { + t.Fatalf("telemetry = %+v, want authenticated node-a at 61%% from its second address", got) + } + key := hostKey{hostUUID: "node-a", service: noderec.ServiceNodeInfo} + if remembered := h.d.enrichHosts.get(key); remembered != host { + t.Fatalf("remembered address = %q, want %q", remembered, host) } } func TestTelemetryIntervalForNodeIsStableAndBounded(t *testing.T) { From cb118dda0f8e62c68fb5ac45e20a853528be3026 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:04:36 +0200 Subject: [PATCH 12/18] test(broker): distinguish bare engines from unverified PAIR previews The manual bridge fixtures answered public PAIR node-info, which now intentionally denies the unauthenticated bare-engine bypass. Hold the node-info port but return 404 for bare Ollama and LM Studio, and expect the namespaced manual routing key and manual inventory label. Replace the unproven learned-UUID routing expectation with preview regressions for absent and claimed UUIDs. Check independent peerRecordId and identityVerification, no peer trust or models, and exclusion from both proxies across a re-probe interval. Verified UUID keying remains covered by TestManualToEnrichedHostUUID and TestManualNodeBridgesToRelayDir in the broker module. No runtime behavior or versions change. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/tests/broker_management_test.go | 229 +++++++++++++++-------- 1 file changed, 152 insertions(+), 77 deletions(-) diff --git a/services/tests/broker_management_test.go b/services/tests/broker_management_test.go index 967165749..7544fbdc1 100644 --- a/services/tests/broker_management_test.go +++ b/services/tests/broker_management_test.go @@ -3,9 +3,8 @@ // Cross-process tests for the two broker-management hotfix legs: // -// A. the broker bridges a reachable supervised manual node into -// ollama-proxy (node/add-manual), so inference can route to a node -// that never appears over mDNS; and +// A. the broker bridges explicitly configured bare engines into the proxies, +// but keeps unverified PAIR nodes as non-routable previews; and // B. the broker accepts a client-originated errors:report and forwards it // into nvpair-errors, reflected in the next errors:update. // @@ -25,6 +24,8 @@ import ( "time" "nvpair-shared/appdir" + "nvpair-shared/jsonrpc" + "nvpair-shared/noderec" ) // fakeOllama stands up a minimal Ollama-looking HTTP server on the fixed @@ -91,10 +92,11 @@ func persistLMStudioProxyPort(t *testing.T, port int) string { } // fakeNodeInfo stands up a minimal node-info server on the fixed port the -// manual prober hits (14318): GET /v1/node-info returns 200 with the given -// hostUuid, so a manual node pointed at it learns its stable UUID. Skips the -// test if the port can't be bound. -func fakeNodeInfo(t *testing.T, hostUUID string) func() { +// manual prober hits (14318). A non-nil hostUUID serves an unclustered PAIR +// preview, never an identity proof. Nil returns 404 while still holding the +// port, so a real local PAIR cannot answer for a bare-engine fixture. +// Skips the test if the port can't be bound. +func fakeNodeInfo(t *testing.T, hostUUID *string) func() { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:14318") if err != nil { @@ -102,8 +104,12 @@ func fakeNodeInfo(t *testing.T, hostUUID string) func() { } mux := http.NewServeMux() mux.HandleFunc("/v1/node-info", func(w http.ResponseWriter, r *http.Request) { + if hostUUID == nil { + http.NotFound(w, r) + return + } w.Header().Set("Content-Type", "application/json") - _, _ = io.WriteString(w, fmt.Sprintf(`{"GPUs":[],"hostUuid":%q}`, hostUUID)) + _, _ = io.WriteString(w, fmt.Sprintf(`{"GPUs":[],"hostUuid":%q}`, *hostUUID)) }) srv := &http.Server{Handler: mux} go func() { _ = srv.Serve(ln) }() @@ -127,7 +133,7 @@ func proxyNodesHas(t *testing.T, raw json.RawMessage, id string) bool { t.Helper() var res proxyNodesResult if err := json.Unmarshal(raw, &res); err != nil { - return false + t.Fatalf("decode proxy nodes/list: %v", err) } for _, n := range res.Nodes { if n.ID == id { @@ -137,21 +143,60 @@ func proxyNodesHas(t *testing.T, raw json.RawMessage, id string) bool { return false } -// TestBrokerBridgesManualNodeIntoProxy is the leg-A repro: with the broker -// supervising both nvpair-manual-nodes and ollama-proxy, a manual node whose -// Ollama is reachable must be bridged into the proxy (node/add-manual) so it -// shows up in proxy:nodes/list and can be routed to — even though it never -// appears over mDNS. -func TestBrokerBridgesManualNodeIntoProxy(t *testing.T) { +type manualInventoryNode struct { + ID string `json:"id"` + HostUUID string `json:"hostUuid"` + PeerRecordID string `json:"peerRecordId"` + IdentityVerification string `json:"identityVerification"` + Trusted bool `json:"trusted"` + Models []string `json:"models"` +} + +func brokerNodesResult(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, method string) json.RawMessage { + t.Helper() + msg := callRPC(t, stdin, msgs, method, nil, 5*time.Second) + if msg.Error != nil { + t.Fatalf("%s failed: %+v", method, msg.Error) + } + return msg.Result +} + +func manualInventory(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, nodeName string) (manualInventoryNode, bool) { + t.Helper() + var res struct { + Nodes []manualInventoryNode `json:"nodes"` + } + if err := json.Unmarshal(brokerNodesResult(t, stdin, msgs, "discovery:get-nodes"), &res); err != nil { + t.Fatalf("decode discovery:get-nodes: %v", err) + } + for _, n := range res.Nodes { + if n.ID == nodeName { + return n, true + } + } + return manualInventoryNode{}, false +} + +func assertManualInventory(t *testing.T, stdin io.Writer, msgs <-chan jsonrpc.Message, nodeName, verification string) { + t.Helper() + n, ok := manualInventory(t, stdin, msgs, nodeName) + if !ok || n.IdentityVerification != verification || n.PeerRecordID != noderec.ManualRecordID(nodeName) || n.Trusted { + t.Fatalf("manual inventory = %+v (present=%v), want independent %s record without peer trust", n, ok, verification) + } +} + +// An explicitly configured bare Ollama remains a labelled manual routing bypass, +// even though it never appears over mDNS and supplies no PAIR identity proof. +func TestBrokerBridgesBareManualOllamaIntoProxy(t *testing.T) { if portBusy(11435) { t.Skip("ollama-proxy port 11435 already in use; skipping") } stopOllama := fakeOllama(t) // skips if 11434 unavailable t.Cleanup(stopOllama) - // Own node-info as well as the engine: a running local PAIR must not - // supply its cluster principal/UUID to this deliberately unclustered node. - t.Cleanup(fakeNodeInfo(t, "")) + // Hold node-info without answering as PAIR, preventing a real local PAIR + // from turning this bare engine into an unverified peer preview. + t.Cleanup(fakeNodeInfo(t, nil)) stdin, msgs, _, cleanup := startBrokerWith(t, "--manual-nodes-path", manualNodesBin, @@ -181,9 +226,15 @@ func TestBrokerBridgesManualNodeIntoProxy(t *testing.T) { if !ok { t.Fatal("broker stream closed before the manual node bridged into the proxy") } - if msg.Method == "" && msg.ID != nil && proxyNodesHas(t, msg.Result, nodeName) { - t.Logf("manual node %q bridged into proxy nodes/list", nodeName) - return + if msg.Method == "" && msg.ID != nil { + if msg.Error != nil { + t.Fatalf("broker request failed: %+v", msg.Error) + } + if proxyNodesHas(t, msg.Result, noderec.ManualRecordID(nodeName)) { + assertManualInventory(t, stdin, msgs, nodeName, "manual") + t.Logf("bare manual Ollama %q bridged into proxy nodes/list", nodeName) + return + } } case <-ticker.C: reqID++ @@ -194,74 +245,92 @@ func TestBrokerBridgesManualNodeIntoProxy(t *testing.T) { } } -// TestBrokerBridgesManualNodeUnderLearnedUUID: once a -// manual node's node-info reports its stable hostUuid, the proxy candidate must -// be keyed by that UUID — the same operational key the discovery store and -// scheduler use — so scheduler priority (node/set-priority) and scheduledOn -// resolve to it. The candidate must therefore appear in proxy:nodes/list under -// the learned hostUuid, not the user-supplied manual name. -func TestBrokerBridgesManualNodeUnderLearnedUUID(t *testing.T) { +// Public PAIR presence denies the bare-engine bypass, with or without a claimed +// UUID. Neither the manual alias, preview record nor claimed UUID may route. +// Verified UUID keying is covered by TestManualToEnrichedHostUUID and +// TestManualNodeBridgesToRelayDir in nvpair-ui-broker; this unclustered fixture +// deliberately cannot supply a pinned mTLS proof or authenticated metadata. +func TestBrokerKeepsUnclusteredManualPAIRAsUnverifiedPreview(t *testing.T) { if portBusy(11435) { t.Skip("ollama-proxy port 11435 already in use; skipping") } - stopOllama := fakeOllama(t) // skips if 11434 unavailable - t.Cleanup(stopOllama) - stopNodeInfo := fakeNodeInfo(t, "learned-host-uuid") // skips if 14318 unavailable - t.Cleanup(stopNodeInfo) - - stdin, msgs, _, cleanup := startBrokerWith(t, - "--manual-nodes-path", manualNodesBin, - "--proxy-path", proxyBin, - ) - t.Cleanup(cleanup) - - waitForMethod(t, msgs, "app:ready", 10*time.Second) + t.Cleanup(fakeOllama(t)) + t.Cleanup(fakeLMStudio(t)) + + for _, tc := range []struct { + name string + hostUUID string + }{ + {name: "no-uuid"}, + {name: "claimed-uuid", hostUUID: "unverified-claimed-host-uuid"}, + } { + t.Run(tc.name, func(t *testing.T) { + configDir := persistLMStudioProxyPort(t, freePort(t)) + t.Cleanup(fakeNodeInfo(t, &tc.hostUUID)) + stdin, msgs, _, cleanup := startBrokerWithConfigDir(t, configDir, + "--manual-nodes-path", manualNodesBin, + "--proxy-path", proxyBin, + "--lmstudio-proxy-path", lmstudioProxyBin, + ) + t.Cleanup(cleanup) + waitForMethod(t, msgs, "app:ready", 10*time.Second) + + nodeName := "xproc-manual-pair-" + tc.name + added := callRPC(t, stdin, msgs, "node/add", map[string]string{"address": "127.0.0.1", "name": nodeName}, 5*time.Second) + if added.Error != nil { + t.Fatalf("node/add failed: %+v", added.Error) + } - const nodeName = "xproc-manual-learned-uuid" - addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":940,"method":"node/add","params":{"address":"127.0.0.1","name":%q}}`, nodeName) + "\n" - if _, err := stdin.Write([]byte(addReq)); err != nil { - t.Fatalf("write node/add: %v", err) - } + deadline := time.NewTimer(30 * time.Second) + defer deadline.Stop() + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + var previewSince time.Time + for { + n, ok := manualInventory(t, stdin, msgs, nodeName) + if ok && n.IdentityVerification == "unverified" { + if n.PeerRecordID != noderec.ManualRecordID(nodeName) || n.Trusted || len(n.Models) != 0 { + t.Fatalf("PAIR preview acquired peer trust or routing inventory: %+v", n) + } + if tc.hostUUID != "" && (n.HostUUID == tc.hostUUID || n.PeerRecordID == tc.hostUUID) { + t.Fatalf("unverified UUID became the preview identity: %+v", n) + } + if previewSince.IsZero() { + previewSince = time.Now() + } + } else if !previewSince.IsZero() { + t.Fatalf("unverified preview disappeared or changed kind: %+v (present=%v)", n, ok) + } - deadline := time.After(25 * time.Second) - ticker := time.NewTicker(1 * time.Second) - defer ticker.Stop() - reqID := 941 - sendReq(t, stdin, reqID, "proxy:nodes/list") - for { - select { - case msg, ok := <-msgs: - if !ok { - t.Fatal("broker stream closed before the manual node bridged under its learned UUID") - } - if msg.Method == "" && msg.ID != nil { - if proxyNodesHas(t, msg.Result, "learned-host-uuid") { - t.Logf("manual node bridged into proxy under learned hostUuid") + for _, method := range []string{"proxy:nodes/list", "lmstudio-proxy:nodes/list"} { + raw := brokerNodesResult(t, stdin, msgs, method) + for _, key := range []string{nodeName, noderec.ManualRecordID(nodeName), tc.hostUUID} { + if key != "" && proxyNodesHas(t, raw, key) { + t.Fatalf("unverified PAIR node routed as %q in %s", key, method) + } + } + } + // Observe a full 10s manual-worker re-probe after the preview + // appears, rather than passing on an empty startup snapshot. + if !previewSince.IsZero() && time.Since(previewSince) >= 12*time.Second { return } - if proxyNodesHas(t, msg.Result, nodeName) { - t.Fatalf("proxy candidate keyed by manual name %q, not the learned hostUuid (scheduler would not match)", nodeName) + select { + case <-ticker.C: + case <-deadline.C: + t.Fatal("timed out waiting for a stable unverified PAIR preview") } } - case <-ticker.C: - reqID++ - sendReq(t, stdin, reqID, "proxy:nodes/list") - case <-deadline: - t.Fatalf("timed out waiting for manual node under learned hostUuid in proxy:nodes/list") - } + }) } } -// TestBrokerBridgesManualNodeIntoLMStudioProxy is the LM Studio counterpart of -// the Ollama bridge test: with the broker supervising both nvpair-manual-nodes and -// lmstudio-proxy, a manual node whose LM Studio is reachable must be bridged -// into lmstudio-proxy (node/add-manual) so it shows up in -// lmstudio-proxy:nodes/list — even though it never appears over mDNS. -func TestBrokerBridgesManualNodeIntoLMStudioProxy(t *testing.T) { +// Bare LM Studio has the same explicit manual bypass as bare Ollama. +func TestBrokerBridgesBareManualLMStudioIntoProxy(t *testing.T) { configDir := persistLMStudioProxyPort(t, freePort(t)) stopLM := fakeLMStudio(t) // skips if 1234 unavailable t.Cleanup(stopLM) - t.Cleanup(fakeNodeInfo(t, "")) + t.Cleanup(fakeNodeInfo(t, nil)) stdin, msgs, _, cleanup := startBrokerWithConfigDir(t, configDir, "--manual-nodes-path", manualNodesBin, @@ -288,9 +357,15 @@ func TestBrokerBridgesManualNodeIntoLMStudioProxy(t *testing.T) { if !ok { t.Fatal("broker stream closed before the manual node bridged into lmstudio-proxy") } - if msg.Method == "" && msg.ID != nil && proxyNodesHas(t, msg.Result, nodeName) { - t.Logf("manual node %q bridged into lmstudio-proxy nodes/list", nodeName) - return + if msg.Method == "" && msg.ID != nil { + if msg.Error != nil { + t.Fatalf("broker request failed: %+v", msg.Error) + } + if proxyNodesHas(t, msg.Result, noderec.ManualRecordID(nodeName)) { + assertManualInventory(t, stdin, msgs, nodeName, "manual") + t.Logf("bare manual LM Studio %q bridged into proxy nodes/list", nodeName) + return + } } case <-ticker.C: reqID++ From 212e9f7f604cf46819c1e6321f9d1afa0ff6535a Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:26:32 +0200 Subject: [PATCH 13/18] fix(broker): retire workloads when unverified previews disappear Do not exempt preview withdrawal or scanner retirement from cleaning up already tracked workloads. Preserve jobs when a proven claim survives for the same participant, and reject new active events against known preview-only inventory without blocking terminal reconciliation or participants not yet discovered locally. Add in-module regressions for preview-only loss, surviving verified claims, scanner retirement and preview workload admission. All new failing cases were observed on the unfixed broker. Keep the existing unreleased broker 2.5.0 bump relative to released 2.4.1. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-ui-broker/README.md | 1 + services/nvpair-ui-broker/broker.go | 6 ++ services/nvpair-ui-broker/discovery.go | 2 +- services/nvpair-ui-broker/peer_inventory.go | 28 +++++- .../nvpair-ui-broker/workload_preview_test.go | 87 +++++++++++++++++++ 5 files changed, 122 insertions(+), 2 deletions(-) create mode 100644 services/nvpair-ui-broker/workload_preview_test.go diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index 8f23d0776..adada0eb5 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -156,6 +156,7 @@ Two classes of proxy notification are **not** re-emitted under the `proxy:` name - **Outbound (proxy -> broker -> manager -> peers).** When any proxy emits a `workload:started` / `workload:completed` / `workload:errored`, the broker stamps the local stable `hostUuid` onto `params.workloadInfo.originatedFrom`, applies the transition to its authoritative workload store, and fans the accepted update to the scheduler before forwarding the original lifecycle frame to the manager. The manager broadcasts it to peer nodes. With no manager supervised the event still updates local scheduling and subscribed clients, but is not broadcast. - **Inbound (peers -> manager -> broker).** The manager translates peer-origin lifecycle events into `workloads:upsert` and peer-origin removals into `workloads:remove` on stdout. The broker applies each accepted transition to the same store, fans it to the scheduler, and relays it to clients subscribed via `workloads:subscribe`. +- **Preview lifecycle.** Active workload events cannot name a known unverified preview as origin or executor; terminal events remain accepted to retire existing state. Relayed participants not yet discovered locally are not treated as previews. Removing a preview (including scanner retirement) still fails previously tracked jobs when no verified or explicit bare-engine claim survives for that participant. A preview claiming a verified peer's UUID cannot fail that peer's jobs. - **Local echo.** Local-origin proxy workloads are also emitted to the same `workloads:*` client stream (lifecycle translated to `workloads:upsert`), so a subscribed client sees a coherent cluster-wide view — its own workloads alongside peers'. **`nvpair-job-scheduler`** consumes the accepted workload stream, compact GPU telemetry, and discovery snapshot. It smooths fresh utilization into pressure 0–3, uses neutral pressure 1 for invalid/missing/older-than-10-second samples, and orders by `pending + gpuPressure`, then pressure, then stable UUID. Load is node-wide across Ollama, LM Studio, and oMLX because all contend for the same resources. Each engine-specific `schedule:priority` carries `{engine,nodes,ranks}` and refreshes when order, pending counts, or pressure changes. The broker caches, generation-orders, and replays the full `{nodes,ranks}` snapshot to the matching proxy, where a newly delivered snapshot resets optimistic reservation deltas. On scheduler spawn/restart the broker replays active workloads and telemetry before discovery, then resumes all three live feeds. diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 1d0096582..68a355e49 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -2804,6 +2804,12 @@ func (b *Broker) applyWorkloadEvent(method string, params json.RawMessage, infer if !ok { return false } + // A peer may relay jobs whose other participants we have not yet + // discovered, but a known preview is not a workload participant. + // Still accept terminal events so existing state can be retired. + if !inferred && !workloadstore.IsTerminal(in.State) && (b.onlyUnprovenWorkloadInventory(in.Origin) || b.onlyUnprovenWorkloadInventory(in.ScheduledOn)) { + return false + } if inferred { if !unchangedSince.IsZero() { return b.workloads.ApplyInferredUnchangedSince(in, unchangedSince) diff --git a/services/nvpair-ui-broker/discovery.go b/services/nvpair-ui-broker/discovery.go index 809ec84cf..eb9b8f316 100644 --- a/services/nvpair-ui-broker/discovery.go +++ b/services/nvpair-ui-broker/discovery.go @@ -752,7 +752,7 @@ func (s *scannerProcess) handleNotify(method string, params json.RawMessage) { // node by HostUUID, the value workloads are stamped/keyed with, // not the display hostname. gone := s.store.Remove(directoryToEnriched(ev.Node).storeKey(), sourceScanner) - if ev.Node.IdentityVerification != "unverified" && ev.Node.IdentityVerification != "manual" && (gone || s.authorize != nil) && s.onNodeLost != nil { + if (gone || (ev.Node.ServicePeer() && s.authorize != nil)) && s.onNodeLost != nil { s.onNodeLost(ev.Node.HostUUID, ev.Node.Name) } } else { diff --git a/services/nvpair-ui-broker/peer_inventory.go b/services/nvpair-ui-broker/peer_inventory.go index b05855747..c77b35d34 100644 --- a/services/nvpair-ui-broker/peer_inventory.go +++ b/services/nvpair-ui-broker/peer_inventory.go @@ -108,6 +108,32 @@ func (b *Broker) hasDecisionInventory(hostUUID string) bool { return (sn.scanner != nil && b.decisionInventory(*sn.scanner)) || (sn.manual != nil && b.decisionInventory(*sn.manual)) } +// onlyUnprovenWorkloadInventory rejects known previews (including their +// independent record keys), without requiring every participant of a relayed +// workload to have been discovered locally. A preview cannot shadow a proven +// record or an explicit bare-engine configuration for the same participant. +func (b *Broker) onlyUnprovenWorkloadInventory(id string) bool { + if id == "" || id == b.nodeID || b.store == nil { + return false + } + b.store.mu.RLock() + var candidates []EnrichedNode + for _, sn := range b.store.nodes { + for _, n := range []*EnrichedNode{sn.scanner, sn.manual} { + if n != nil && (n.HostUUID == id || n.storeKey() == id) { + candidates = append(candidates, *n) + } + } + } + b.store.mu.RUnlock() + for _, n := range candidates { + if b.decisionInventory(n) { + return false + } + } + return len(candidates) != 0 +} + func (b *Broker) authorizedTelemetry(source nodeSource, hostUUID, proof string) bool { if b.store == nil { return false @@ -225,7 +251,7 @@ func (b *Broker) clearScannerInventory() { if n.IdentityVerification == "verified" || n.IdentityVerification == "local" { b.removeTelemetry(sourceScanner, n.HostUUID) } - if gone && b.workloads != nil && (n.IdentityVerification == "verified" || n.IdentityVerification == "local") { + if gone && b.workloads != nil && !b.hasDecisionInventory(n.HostUUID) { b.failWorkloadsForNode(n.HostUUID, n.ID) } } diff --git a/services/nvpair-ui-broker/workload_preview_test.go b/services/nvpair-ui-broker/workload_preview_test.go new file mode 100644 index 000000000..c90681a8b --- /dev/null +++ b/services/nvpair-ui-broker/workload_preview_test.go @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "testing" + + "nvpair-shared/noderec" +) + +func TestPreviewWithdrawalRetiresAlreadyTrackedWorkloads(t *testing.T) { + for _, verifiedSurvives := range []bool{false, true} { + t.Run(map[bool]string{false: "preview-only", true: "verified-survives"}[verifiedSurvives], func(t *testing.T) { + b := inventoryTestBroker(t) + preview := noderec.DirectoryNode{HostUUID: "peer", Name: "unproven", IdentityVerification: "unverified", PeerRecordID: noderec.PreviewRecordID("unproven", "peer")} + b.store.Upsert(directoryToEnriched(preview), sourceScanner) + if verifiedSurvives { + b.store.Upsert(directoryToEnriched(verifiedDirectoryForTest(b)), sourceManual) + } + // Seed a previously accepted record: the withdrawal must clean up + // even state admitted before preview admission was guarded. + b.workloads.Apply(storeIncoming("7", "peer", "ollama", "r1", "running", "peer")) + sp := &scannerProcess{store: b.store, authorize: b.authorizedDirectoryNode, onNodeLost: func(uuid, name string) { + if !b.hasDecisionInventory(uuid) { + b.failWorkloadsForNode(uuid, name) + } + }} + params, err := json.Marshal(noderec.NodeEvent{Node: preview}) + if err != nil { + t.Fatal(err) + } + sp.handleNotify(noderec.NotifyNodeRemoved, params) + want := "failed" + if verifiedSurvives { + want = "running" + } + if got, ok := b.workloads.Get("peer", "7"); !ok || got.State != want { + t.Fatalf("workload after preview withdrawal = %+v, want %s", got, want) + } + }) + } +} + +func TestScannerRetirementRetiresPreviewWorkloads(t *testing.T) { + b := inventoryTestBroker(t) + b.store.Upsert(EnrichedNode{HostUUID: "peer", ID: "preview", IdentityVerification: "unverified", PeerRecordID: noderec.PreviewRecordID("preview", "peer")}, sourceScanner) + b.workloads.Apply(storeIncoming("7", "peer", "ollama", "r1", "running", "peer")) + b.clearScannerInventory() + if got, ok := b.workloads.Get("peer", "7"); !ok || got.State != "failed" { + t.Fatalf("workload after scanner retirement = %+v, want failed", got) + } +} + +func TestActiveWorkloadsCannotUseUnverifiedPreview(t *testing.T) { + for _, field := range []string{"origin", "executor", "preview-key"} { + t.Run(field, func(t *testing.T) { + b := inventoryTestBroker(t) + preview := noderec.DirectoryNode{HostUUID: "peer", Name: "unproven", IdentityVerification: "unverified", PeerRecordID: noderec.PreviewRecordID("unproven", "peer")} + b.store.Upsert(directoryToEnriched(preview), sourceScanner) + origin, executor := "self", "peer" + if field == "origin" { + origin, executor = "peer", "self" + } else if field == "preview-key" { + executor = preview.RecordKey() + } + incoming := storeIncoming("7", origin, "ollama", "r1", "running", executor) + params, err := json.Marshal(map[string]json.RawMessage{"workloadInfo": incoming.Info}) + if err != nil { + t.Fatal(err) + } + b.forwardWorkloadManagerNotification("workloads:upsert", params) + if _, ok := b.workloads.Get(origin, "7"); ok { + t.Fatal("active workload was tracked against an unverified preview") + } + // A proven record sharing the claimed UUID must not be shadowed + // by an unrelated preview. The preview key itself stays forbidden. + b.store.Upsert(directoryToEnriched(verifiedDirectoryForTest(b)), sourceManual) + b.forwardWorkloadManagerNotification("workloads:upsert", params) + _, accepted := b.workloads.Get(origin, "7") + if accepted != (field != "preview-key") { + t.Fatalf("verified admission = %v for %s", accepted, field) + } + }) + } +} From 7bd921a5fa451a4f437e4df4494a2990644f9128 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:30:19 +0200 Subject: [PATCH 14/18] test(services): exercise verified multi-node inventory and workloads Upgrade the model enrichment, periodic refresh, workload restart, node-loss and outbound fixtures to real admitted cross-pinned peers. Serve authenticated node-info metadata with the exact certificate principal and service ports before model reads or broadcasts become eligible. Preserve all enrichment, deletion, running/terminal rehydration and node-loss assertions. Keep a plaintext lookalike preview alongside the proven model peer and assert that it never contributes model inventory or receives a model read. Cross-process execution is intentionally left to Linux CI; local validation is three-platform go vet and formatting only. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/tests/broker_test.go | 20 +++-- services/tests/cluster_data_plane_test.go | 54 +++---------- services/tests/models_http_test.go | 54 ++++++++----- services/tests/models_refresh_test.go | 34 ++------ services/tests/verified_peer_test.go | 79 +++++++++++++++++++ .../tests/workload_identity_interop_test.go | 4 +- services/tests/workload_interop_test.go | 47 +++-------- 7 files changed, 159 insertions(+), 133 deletions(-) create mode 100644 services/tests/verified_peer_test.go diff --git a/services/tests/broker_test.go b/services/tests/broker_test.go index c947be6fd..d64656171 100644 --- a/services/tests/broker_test.go +++ b/services/tests/broker_test.go @@ -38,14 +38,18 @@ import ( // availableNode mirrors the broker's external camelCase wire shape (see // AvailableNode in nvpair-ui-broker). type availableNode struct { - ID string `json:"id"` - Name string `json:"name"` - IPAddress string `json:"ipAddress"` - Port int `json:"port"` - LastSeen int64 `json:"lastSeen"` - Models []string `json:"models,omitempty"` - ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` - LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` + HostUUID string `json:"hostUuid"` + Trusted bool `json:"trusted"` + PeerRecordID string `json:"peerRecordId"` + IdentityVerification string `json:"identityVerification"` + ID string `json:"id"` + Name string `json:"name"` + IPAddress string `json:"ipAddress"` + Port int `json:"port"` + LastSeen int64 `json:"lastSeen"` + Models []string `json:"models,omitempty"` + ModelsByEngine map[string][]string `json:"modelsByEngine,omitempty"` + LoadedByEngine map[string][]string `json:"loadedByEngine,omitempty"` } type availableNodesResult struct { diff --git a/services/tests/cluster_data_plane_test.go b/services/tests/cluster_data_plane_test.go index 901e6900e..77f5ee3b4 100644 --- a/services/tests/cluster_data_plane_test.go +++ b/services/tests/cluster_data_plane_test.go @@ -15,7 +15,6 @@ package tests import ( - "crypto/tls" "encoding/json" "fmt" "io" @@ -30,8 +29,6 @@ import ( "nvpair-shared/clustertrust" "nvpair-shared/jsonrpc" "nvpair-shared/netpick" - - "github.com/grandcat/zeroconf" ) const ( @@ -52,9 +49,14 @@ type interNodeCluster struct { } func newInterNodeCluster(t *testing.T) *interNodeCluster { + t.Helper() + return newInterNodeClusterWithPeer(t, "data-plane-peer") +} + +func newInterNodeClusterWithPeer(t *testing.T, peerUUID string) *interNodeCluster { t.Helper() nodeDir, peerDir := t.TempDir(), t.TempDir() - const nodeUUID, peerUUID = "data-plane-node", "data-plane-peer" + const nodeUUID = "data-plane-node" nodeCert := mintClusterIdentity(t, nodeDir, nodeUUID) peerCert := mintClusterIdentity(t, peerDir, peerUUID) writePin(t, nodeDir, peerUUID, peerCert) // the node under test trusts the peer @@ -81,25 +83,13 @@ func (c *interNodeCluster) clientAsPeer(t *testing.T) *http.Client { } // startStubClusterPeer runs a stub inter-node peer for this cluster over cluster -// mTLS and advertises it as a single _nvpair-node record carrying wl= and -// its cluster principal. The principal is what lets the node under test resolve -// its pin for the stub: the scanner annotates the browsed record as trusted and -// the workload-manager keys its pinned client on it. Without mTLS and that TXT -// key, the stub is simply not a routable cluster target and is never dialed. -// -// It returns the channel of frames the stub received. instance/hostUUID keep the -// record distinct from the node under test's own. -func (c *interNodeCluster) startStubClusterPeer(t *testing.T, instance, hostUUID string) <-chan jsonrpc.Message { +// mTLS. Authenticated node-info proves the exact pinned principal and publishes +// its workload service port; mDNS alone never grants service eligibility. +// It returns the channel of frames the stub received. +func (c *interNodeCluster) startStubClusterPeer(t *testing.T, instance string) <-chan jsonrpc.Message { t.Helper() received := make(chan jsonrpc.Message, 128) - mux := http.NewServeMux() - mux.HandleFunc("/v1/workloads/events", func(w http.ResponseWriter, r *http.Request) { - // The sender must present the leaf this peer pins — the outbound half of - // the gate, so a broadcast is authenticated in both directions. - if _, ok := c.peer.VerifyClientPin(r); !ok { - http.Error(w, "forbidden: not a pinned cluster peer", http.StatusForbidden) - return - } + c.startVerifiedPeer(t, instance, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { body, _ := io.ReadAll(r.Body) var msg jsonrpc.Message if json.Unmarshal(body, &msg) == nil { @@ -109,27 +99,7 @@ func (c *interNodeCluster) startStubClusterPeer(t *testing.T, instance, hostUUID } } w.WriteHeader(http.StatusOK) - }) - - // Listen on all interfaces: zeroconf advertises the host's real interface - // IP(s), so the manager dials the peer there, not at 127.0.0.1. - ln, err := net.Listen("tcp", ":0") - if err != nil { - t.Fatalf("stub peer listen: %v", err) - } - port := ln.Addr().(*net.TCPAddr).Port - srv := &http.Server{Handler: mux} - go srv.Serve(tls.NewListener(ln, c.peer.ServerTLSConfig())) - t.Cleanup(func() { _ = srv.Close() }) - - txt := []string{"v=1", "uuid=" + hostUUID, fmt.Sprintf("wl=%d", port), - clustertrust.ClusterUUIDTXTKey + "=" + c.peerUUID} - adv, err := zeroconf.Register(instance, nodeRecordService, testDomain, port, txt, nil) - if err != nil { - t.Fatalf("register stub peer: %v", err) - } - t.Cleanup(adv.Shutdown) - t.Logf("stub cluster peer advertising %s (wl=%d, cluster-uuid=%s)", nodeRecordService, port, c.peerUUID) + })) return received } diff --git a/services/tests/models_http_test.go b/services/tests/models_http_test.go index 20e3708e3..ad1226921 100644 --- a/services/tests/models_http_test.go +++ b/services/tests/models_http_test.go @@ -1,13 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -// Cross-process test for the models-http path: a node advertises an -// engine-manager (em) service on its _nvpair-node record, and the promoted -// discovery daemon enriches the node's model list by fetching /v1/models over -// plain HTTP from that em port — replacing the retired models= TXT key. We stand -// in for engine-manager with a tiny HTTP server, advertise em= + ip so the -// daemon dials it over loopback, and assert the enriched list surfaces on the -// broker's discovery:get-nodes response. +// Cross-process model enrichment through pinned cluster mTLS: discovery first +// proves the peer's node-info identity and service map, then fetches /v1/models +// from that authenticated engine-manager port. An unauthenticated lookalike +// remains a preview without service inventory. package tests import ( @@ -17,6 +14,7 @@ import ( "net/http/httptest" "net/url" "strconv" + "sync/atomic" "testing" "time" @@ -25,8 +23,8 @@ import ( func TestModelsHTTPEnrichment(t *testing.T) { requireMulticast(t) - // Stand-in engine-manager HTTP surface serving the model list. - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Stand-in engine-manager inventory, served only after pinned mTLS proof. + models := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/v1/models" { http.NotFound(w, r) return @@ -44,6 +42,18 @@ func TestModelsHTTPEnrichment(t *testing.T) { "lmstudio": {}, }, }) + }) + const instance = "xproc-models-1" + fx := newInterNodeClusterWithPeer(t, instance+"-uuid") + fx.startVerifiedPeer(t, instance, models, nil) + + // This lookalike has only public HTTP and a claimed principal, not the key. + var unprovenModelReads atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/models" { + unprovenModelReads.Add(1) + } + models.ServeHTTP(w, r) })) t.Cleanup(srv.Close) u, err := url.Parse(srv.URL) @@ -55,21 +65,23 @@ func TestModelsHTTPEnrichment(t *testing.T) { t.Fatalf("stub port: %v", err) } - const instance = "xproc-models-1" - // Advertise a _nvpair-node with em= and ip=127.0.0.1 so the daemon - // fetches the model list from the stub over loopback (ip= wins over the mDNS - // address in the daemon's enrichment target selection). - txt := []string{"v=1", "uuid=" + instance + "-uuid", "ip=127.0.0.1", fmt.Sprintf("em=%d", emPort)} - zsrv, err := zeroconf.Register(instance, nodeRecordService, testDomain, emPort, txt, nil) + const preview = "xproc-models-preview" + txt := []string{"v=1", "uuid=" + fx.peerUUID, "cluster-uuid=" + fx.peerUUID, "ip=127.0.0.1", fmt.Sprintf("ni=%d", emPort), fmt.Sprintf("em=%d", emPort)} + zsrv, err := zeroconf.Register(preview, nodeRecordService, testDomain, emPort, txt, nil) if err != nil { - t.Fatalf("register %s: %v", instance, err) + t.Fatalf("register %s: %v", preview, err) } t.Cleanup(zsrv.Shutdown) - t.Logf("advertising %s @ %s (em=%d)", instance, nodeRecordService, emPort) - - stdin, msgs, cleanup := startBroker(t) + stdin, msgs, cleanup := startBrokerProcInCluster(t, fx.nodeDir, "--scanner-path", scannerBin) t.Cleanup(cleanup) + pollForNode(t, stdin, msgs, preview, 30*time.Second, func(n availableNode) bool { + if n.IdentityVerification != "unverified" || n.Trusted || len(n.Models) != 0 || unprovenModelReads.Load() != 0 { + t.Fatalf("unauthenticated stub supplied service inventory: %+v, model reads=%d", n, unprovenModelReads.Load()) + } + return true + }) + // Poll get-nodes until the node appears carrying the enriched model list. deadline := time.After(30 * time.Second) ticker := time.NewTicker(1500 * time.Millisecond) @@ -86,8 +98,12 @@ func TestModelsHTTPEnrichment(t *testing.T) { var res availableNodesResult if json.Unmarshal(msg.Result, &res) == nil { if n, found := findNode(res.Nodes, instance); found && + n.IdentityVerification == "verified" && n.Trusted && n.HostUUID == fx.peerUUID && modelsMatch(n.Models) && modelsByEngineMatch(n.ModelsByEngine) && loadedByEngineMatch(n.LoadedByEngine) { + if unprovenModelReads.Load() != 0 { + t.Fatal("discovery fetched model inventory from the unauthenticated stub") + } return // enriched flat + per-engine + loaded lists surfaced } } diff --git a/services/tests/models_refresh_test.go b/services/tests/models_refresh_test.go index 9bd318cf3..3b4a26053 100644 --- a/services/tests/models_refresh_test.go +++ b/services/tests/models_refresh_test.go @@ -12,18 +12,12 @@ package tests import ( "encoding/json" - "fmt" "io" "net/http" - "net/http/httptest" - "net/url" - "strconv" "sync" "testing" "time" - "github.com/grandcat/zeroconf" - "nvpair-shared/jsonrpc" ) @@ -59,34 +53,18 @@ func TestModelsPeriodicRefreshConvergesWithoutMDNSChange(t *testing.T) { // Start with an empty inventory (engine present but serving no models yet). stub := &swapModels{} stub.set(map[string]any{"models": []string{}}) - srv := httptest.NewServer(stub.handler()) - t.Cleanup(srv.Close) - u, err := url.Parse(srv.URL) - if err != nil { - t.Fatalf("parse stub url: %v", err) - } - emPort, err := strconv.Atoi(u.Port()) - if err != nil { - t.Fatalf("stub port: %v", err) - } - const instance = "xproc-models-refresh-1" - // Advertise em= + ip=127.0.0.1 once and never touch the record again, - // so the only thing that changes during the test is the /v1/models body. - txt := []string{"v=1", "uuid=" + instance + "-uuid", "ip=127.0.0.1", fmt.Sprintf("em=%d", emPort)} - zsrv, err := zeroconf.Register(instance, nodeRecordService, testDomain, emPort, txt, nil) - if err != nil { - t.Fatalf("register %s: %v", instance, err) - } - t.Cleanup(zsrv.Shutdown) - t.Logf("advertising %s @ %s (em=%d), models initially empty", instance, nodeRecordService, emPort) + fx := newInterNodeClusterWithPeer(t, instance+"-uuid") + // Keep mDNS, identity and authenticated service metadata fixed; only the + // pinned mTLS /v1/models response changes during all three phases. + fx.startVerifiedPeer(t, instance, stub.handler(), nil) - stdin, msgs, cleanup := startBroker(t) + stdin, msgs, cleanup := startBrokerProcInCluster(t, fx.nodeDir, "--scanner-path", scannerBin) t.Cleanup(cleanup) // Phase 1: the node is discovered and enriched, but with no models. pollForNode(t, stdin, msgs, instance, 30*time.Second, func(n availableNode) bool { - return len(n.Models) == 0 + return n.IdentityVerification == "verified" && n.Trusted && n.HostUUID == fx.peerUUID && len(n.Models) == 0 }) t.Log("node discovered with an empty inventory; now serving models without an mDNS change") diff --git a/services/tests/verified_peer_test.go b/services/tests/verified_peer_test.go new file mode 100644 index 000000000..68d37b5fd --- /dev/null +++ b/services/tests/verified_peer_test.go @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package tests + +import ( + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "strconv" + "sync" + "testing" + + "github.com/grandcat/zeroconf" + + "nvpair-shared/clustertrust" + "nvpair-shared/noderec" +) + +// startVerifiedPeer uses the same synthetic admission and exact cross-pins as +// the inter-node data-plane tests. Discovery must prove this peer through a +// real mTLS node-info read before consuming its service map. All inventory and +// workload requests also require the broker's pinned client leaf. +// The returned stop withdraws both the advertisement and the answering server. +func (c *interNodeCluster) startVerifiedPeer(t *testing.T, instance string, models, workloads http.Handler) func() { + t.Helper() + ln, err := net.Listen("tcp", ":0") + if err != nil { + t.Fatalf("verified peer listen: %v", err) + } + _, portString, err := net.SplitHostPort(ln.Addr().String()) + if err != nil { + t.Fatal(err) + } + port, err := strconv.Atoi(portString) + if err != nil { + t.Fatal(err) + } + services := map[noderec.ServiceKey]int{noderec.ServiceNodeInfo: port} + txt := []string{"v=1", "uuid=" + c.peerUUID, "ip=127.0.0.1", fmt.Sprintf("ni=%d", port), clustertrust.ClusterUUIDTXTKey + "=" + c.peerUUID} + mux := http.NewServeMux() + mux.HandleFunc("/v1/node-info", func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{ + "hostUuid": c.peerUUID, "clusterUuid": c.peerUUID, + "peerMetadata": noderec.PeerMetadataParams{HostUUID: c.peerUUID, Name: instance, Services: services}, + "GPUs": []any{}, "telemetryValid": false, + }) + }) + if models != nil { + services[noderec.ServiceEngineManager] = port + txt = append(txt, fmt.Sprintf("em=%d", port)) + mux.Handle("/v1/models", models) + } + if workloads != nil { + services[noderec.ServiceWorkload] = port + txt = append(txt, fmt.Sprintf("wl=%d", port)) + mux.Handle("/v1/workloads/events", workloads) + } + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if _, ok := c.peer.VerifyClientPin(r); !ok { + http.Error(w, "not a pinned peer", http.StatusForbidden) + return + } + mux.ServeHTTP(w, r) + })} + go func() { _ = srv.Serve(tls.NewListener(ln, c.peer.ServerTLSConfig())) }() + t.Cleanup(func() { _ = srv.Close() }) + adv, err := zeroconf.Register(instance, nodeRecordService, testDomain, port, txt, nil) + if err != nil { + t.Fatalf("advertise verified peer: %v", err) + } + var once sync.Once + stop := func() { once.Do(func() { adv.Shutdown(); _ = srv.Close() }) } + t.Cleanup(stop) + t.Logf("advertising pinned peer %s (principal=%s, ni=%d)", instance, c.peerUUID, port) + return stop +} diff --git a/services/tests/workload_identity_interop_test.go b/services/tests/workload_identity_interop_test.go index d6afd0c6b..cf979bd1d 100644 --- a/services/tests/workload_identity_interop_test.go +++ b/services/tests/workload_identity_interop_test.go @@ -190,7 +190,7 @@ func TestWorkloadManagerRehydratesActiveWorkloadOnRestart(t *testing.T) { // manager fans out to it. The inter-node interface is cluster mTLS only, so the // stub must hold a cluster identity the node under test pins. fx := newInterNodeCluster(t) - received := fx.startStubClusterPeer(t, "rehydrate-wm-peer", "rehydrate-peer-uuid") + received := fx.startStubClusterPeer(t, "rehydrate-wm-peer") stdin, msgs, stderr, cleanup := startBrokerWithDirs(t, t.TempDir(), fx.nodeDir, "--proxy-path", proxyBin, @@ -292,7 +292,7 @@ func TestWorkloadManagerRehydratesRecentTerminalOnRestart(t *testing.T) { ollamaPort := portOfURL(t, ollama.URL) fx := newInterNodeCluster(t) - received := fx.startStubClusterPeer(t, "rehydrate-term-peer", "rehydrate-term-peer-uuid") + received := fx.startStubClusterPeer(t, "rehydrate-term-peer") stdin, msgs, stderr, cleanup := startBrokerWithDirs(t, t.TempDir(), fx.nodeDir, "--proxy-path", proxyBin, diff --git a/services/tests/workload_interop_test.go b/services/tests/workload_interop_test.go index bfbbc978e..c8b6f91b8 100644 --- a/services/tests/workload_interop_test.go +++ b/services/tests/workload_interop_test.go @@ -35,8 +35,6 @@ import ( "time" "nvpair-shared/jsonrpc" - - "github.com/grandcat/zeroconf" ) const workloadMgrService = "_nvpair-workload-manager._tcp" @@ -290,7 +288,8 @@ func TestWorkloadFailedOnNodeLoss(t *testing.T) { const peerNode = "nodeloss-peer" const peerUUID = "nodeloss-peer-uuid" - fx := newInterNodeCluster(t) + fx := newInterNodeClusterWithPeer(t, peerUUID) + stopPeer := fx.startVerifiedPeer(t, peerNode, nil, nil) stdin, msgs, cleanup := startBrokerProcInCluster(t, fx.nodeDir, "--scanner-path", scannerBin, "--workload-manager-path", workloadMgrBin, @@ -309,27 +308,11 @@ func TestWorkloadFailedOnNodeLoss(t *testing.T) { } t.Log("subscribed to workloads stream") - // Advertise the peer as an identity-only _nvpair-node record (no service - // ports), so once its advertisement is withdrawn the scanner's liveness - // probe has nothing to dial and evicts it cleanly. A distinct uuid keeps - // it from colliding with this host's own record. - peerTXT := []string{"v=1", "uuid=" + peerUUID, "ip=127.0.0.1"} - peerAdv, err := zeroconf.Register(peerNode, nodeRecordService, testDomain, 14999, peerTXT, nil) - if err != nil { - t.Fatalf("register peer node: %v", err) - } - stopAdv := func() { - if peerAdv != nil { - peerAdv.Shutdown() - peerAdv = nil - } - } - t.Cleanup(stopAdv) - t.Logf("advertising peer %q as %s", peerNode, nodeRecordService) - - // The peer must be in the broker's directory before its removal can - // trigger a sweep, so wait until discovery has folded it in. - pollNodeListed(t, stdin, msgs, peerNode, 1000, 30*time.Second, nil) + // Require the authenticated peer, not merely its mDNS preview, before + // admitting a workload against it. Its pin and answering leaf agree. + pollForNode(t, stdin, msgs, peerNode, 30*time.Second, func(n availableNode) bool { + return n.IdentityVerification == "verified" && n.Trusted && n.HostUUID == peerUUID + }) t.Logf("peer %q discovered", peerNode) // A peer-origin running workload (scheduled on that same peer) POSTed to @@ -345,15 +328,11 @@ func TestWorkloadFailedOnNodeLoss(t *testing.T) { } t.Log("peer workload tracked as running") - // Pull the peer off the network. The scanner holds a node through - // shared/discovery's full miss threshold (12 scans at 5s, a full minute) with - // a failed liveness probe on every scan from ~15s in, and only then evicts it, - // at which point the broker marks the workload it can no longer hear about. - // The window is deliberately long enough to outlast a node saturated by its - // own inference load, so this deadline has to clear it with room for the - // probes and scan jitter. - stopAdv() - t.Log("peer advertisement withdrawn; awaiting node eviction + workload failure") + // Pull the advertisement AND authenticated endpoint off the network. + // A verified peer has its own proof-refresh lifetime, independent of the + // preview. Its withdrawal must still fail the tracked workload. + stopPeer() + t.Log("peer withdrawn; awaiting verified inventory eviction + workload failure") failed := waitForWorkloadEvent(t, msgs, "workloads:upsert", "wl-nodeloss-1", 2*time.Minute) if failed.WorkloadInfo.State != "failed" { @@ -390,7 +369,7 @@ func TestWorkloadManagerOutboundBroadcast(t *testing.T) { // _nvpair-node record and pushes it to the subscribed workload-manager's relay // peer source, which is the path that replaced the per-service browse. fx := newInterNodeCluster(t) - received := fx.startStubClusterPeer(t, "stub-wm-peer", "stub-wm-peer-uuid") + received := fx.startStubClusterPeer(t, "stub-wm-peer") stdin, msgs, cleanup := startBrokerProcInCluster(t, fx.nodeDir, "--scanner-path", scannerBin, From 7ee1039a44ad69b97e4a059b26e79ae67972e1b4 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:05:40 +0200 Subject: [PATCH 15/18] test(scanner): guard verified expiry after a same-address wipe Replace a proven peer's key-bearing listener with a fresh public identity at the identical endpoint. Keep the verified record through a transient proof failure, then require its original-token withdrawal after the grace period despite a live TCP port. A multicast miss alone must preserve a peer that still proves its identity. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- .../inventory_lifetime_test.go | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/services/nvpair-node-scanner/inventory_lifetime_test.go b/services/nvpair-node-scanner/inventory_lifetime_test.go index eaffeeedb..f086a6dc2 100644 --- a/services/nvpair-node-scanner/inventory_lifetime_test.go +++ b/services/nvpair-node-scanner/inventory_lifetime_test.go @@ -131,6 +131,61 @@ func TestBrowseChurnPreservesVerifiedTelemetryRetry(t *testing.T) { } } +// A wipe replaces the key-bearing endpoint, not merely a hostUuid field under +// the old certificate. An answering TCP port and a new public identity must not +// extend the independent verified record's lifetime. +func TestVerifiedPeerWipedAtSameAddressExpiresDespitePublicResponder(t *testing.T) { + d, server, _ := secureInventoryFixture(t, "uuid-peer") + addr := server.Listener.Addr().String() + host, portString, _ := net.SplitHostPort(addr) + port, _ := strconv.Atoi(portString) + raw := RawNode{ID: "genuine", Addresses: []string{host}, TXT: []string{ + "v=1", "uuid=uuid-peer", "cluster-uuid=uuid-peer", "ip=" + host, "ni=" + portString, + }} + d.onBrowse(DiscoveryEvent{Type: "discovered", Node: raw}) + original, ok := d.dir.get("uuid-peer") + if !ok || !original.ServicePeer() { + t.Fatal("fixture did not establish an authenticated peer") + } + // A multicast miss alone must not withdraw the proven incarnation. + d.onBrowse(DiscoveryEvent{Type: "removed", Node: raw}) + d.refreshNodeClusterIdentity("uuid-peer", host, port) + if node, ok := d.dir.get("uuid-peer"); !ok || !node.ServicePeer() { + t.Fatal("multicast loss evicted a peer that still proves its identity") + } + server.Close() + listener, err := net.Listen("tcp", addr) + if err != nil { + t.Fatal(err) + } + wiped := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(NodeInfoResponse{HostUUID: "new-uuid-after-wipe"}) + })} + t.Cleanup(func() { _ = wiped.Close() }) + go func() { _ = wiped.Serve(listener) }() + if info, ok := d.probeIdentity(host, port); !ok || info.HostUUID != "new-uuid-after-wipe" { + t.Fatalf("wiped endpoint must remain live under a fresh identity: %+v, ok=%v", info, ok) + } + + capture := ¬ifyCapture{} + d.codec = NewCodec(capture) + d.refreshNodeClusterIdentity("uuid-peer", host, port) + if _, ok := d.dir.get("uuid-peer"); !ok || len(capture.nodes(t)) != 0 { + t.Fatal("one failed proof must not bypass the verified-peer grace period") + } + stale, _ := d.dir.get("uuid-peer") + stale.LastSeen = time.Now().Add(-time.Minute).Unix() + d.dir.upsert(stale) + d.refreshNodeClusterIdentity("uuid-peer", host, port) + if _, ok := d.dir.get("uuid-peer"); ok { + t.Fatal("a new identity at the same live endpoint kept stale verified inventory alive") + } + events := capture.nodes(t) + if len(events) != 1 || events[0].HostUUID != "uuid-peer" || events[0].PeerRecordID != original.PeerRecordID || events[0].IdentityVerification != "verified" { + t.Fatalf("withdrawal did not name the original verified record: %+v", events) + } +} + func TestFailedProofExpiresVerifiedRecordWithOriginalToken(t *testing.T) { d, server, mismatch := secureInventoryFixture(t, "uuid-peer") host, portString, _ := net.SplitHostPort(server.Listener.Addr().String()) From 9a53ed68435fd6eb35b3d45a7879614a52af9415 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:05:40 +0200 Subject: [PATCH 16/18] test(services): restore verified and preview ghost-node coverage The plaintext ghost fixture now produces an independently keyed preview through node-updated, so waiting for node-discovered silently skipped working Linux discovery. Exercise the verified record with the admitted, cross-pinned mTLS fixture, then replace its listener with a wiped public identity at the exact same address. Retain a separate preview test for the public identity-confirming liveness probe, and fail missing discovery after the multicast prerequisite succeeds. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/tests/ghost_node_test.go | 106 ++++++++++++++++++++++----- services/tests/verified_peer_test.go | 7 ++ 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/services/tests/ghost_node_test.go b/services/tests/ghost_node_test.go index a50bae8c1..c604969a4 100644 --- a/services/tests/ghost_node_test.go +++ b/services/tests/ghost_node_test.go @@ -23,7 +23,70 @@ import ( ) // TestScannerEvictsRecordSupersededAtItsAddress is the cross-process regression -// test for the duplicate ("ghost") node a reset machine leaves behind. +// test for the duplicate ("ghost") verified node a reset machine leaves behind. +// Pinned mTLS inventory refresh, not the public browser's TCP guard, owns this +// record's lifetime. A wiped machine loses the old key and cluster admission, +// but still answers public node-info under its fresh identity on the same port. +func TestScannerEvictsRecordSupersededAtItsAddress(t *testing.T) { + requireMulticast(t) + const ( + originalUUID = "11111111-aaaa-4aaa-8aaa-000000000001" + wipedUUID = "22222222-bbbb-4bbb-8bbb-000000000002" + ) + fx := newInterNodeClusterWithPeer(t, originalUUID) + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := ln.Addr().String() + stop := fx.startVerifiedPeerOn(t, "ghost-verified-peer", nil, nil, ln) + events := startScannerForGhostTest(t, fx.nodeDir) + original, ok := awaitNodeEvent(events, noderec.NotifyNodeUpdated, originalUUID, 30*time.Second) + if !ok { + t.Fatalf("verified peer record was never discovered after requireMulticast succeeded") + } + if !original.ServicePeer() || original.IdentityVerification != "verified" || original.PeerRecordID == "" { + t.Fatalf("fixture did not prove the original peer: %+v", original) + } + + // Drop the old advertisement and key-bearing listener, then bring up the + // wiped, unclustered machine at the identical address. A live public HTTP + // response proves this test cannot pass merely because a TCP port closed. + stop() + replacement, err := net.Listen("tcp", addr) + if err != nil { + t.Fatalf("rebind wiped peer at %s: %v", addr, err) + } + srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]any{"hostUuid": wipedUUID, "clusterUuid": "", "GPUs": []any{}}) + })} + t.Cleanup(func() { _ = srv.Close() }) + go func() { _ = srv.Serve(replacement) }() + client := &http.Client{Timeout: time.Second} + resp, err := client.Get("http://" + addr + "/v1/node-info") + if err != nil { + t.Fatalf("wiped peer must still answer at the original endpoint: %v", err) + } + var info struct { + HostUUID string `json:"hostUuid"` + } + err = json.NewDecoder(resp.Body).Decode(&info) + _ = resp.Body.Close() + if err != nil || info.HostUUID != wipedUUID { + t.Fatalf("wiped identity = %q, decode error = %v", info.HostUUID, err) + } + removed, ok := awaitNodeEvent(events, noderec.NotifyNodeRemoved, originalUUID, 2*time.Minute) + if !ok { + t.Fatalf("scanner kept %s after a fresh identity answered at its address", originalUUID) + } + if removed.PeerRecordID != original.PeerRecordID || removed.IdentityVerification != "verified" { + t.Fatalf("withdrawal did not identify the original verified record: %+v", removed) + } +} + +// TestScannerEvictsPreviewSupersededAtItsAddress covers the separate public +// candidate lifetime. Unlike a verified peer, it uses the browser miss threshold +// and identity-confirming public liveness probe. // // A machine whose appdata is wiped mints a fresh hostUuid but keeps its LAN // address and re-binds the same fixed service ports. The scanner's @@ -37,7 +100,7 @@ import ( // responder publishes the record, and a local HTTP server plays its node-info. // Flipping that server's reported hostUuid is exactly the signal a wipe // produces, and is what the probe must notice. -func TestScannerEvictsRecordSupersededAtItsAddress(t *testing.T) { +func TestScannerEvictsPreviewSupersededAtItsAddress(t *testing.T) { if testing.Short() { t.Skip("multicast + miss-threshold timing; skipped under -short") } @@ -92,12 +155,17 @@ func TestScannerEvictsRecordSupersededAtItsAddress(t *testing.T) { } }() - events := startScannerForGhostTest(t) + events := startScannerForGhostTest(t, t.TempDir()) - // The scanner must see the peer before there is anything to evict. A slow - // or lossy multicast environment is an inconclusive run, not a failure. - if !awaitNodeEvent(events, noderec.NotifyNodeDiscovered, originalUUID, 30*time.Second) { - t.Skip("peer record was never discovered; multicast is unavailable in this environment") + // requireMulticast already made the environment decision. Discovery failure + // here is a regression, not grounds for an inconclusive self-skip. + key := noderec.PreviewRecordID(peerInstance, originalUUID) + preview, ok := awaitNodeEvent(events, noderec.NotifyNodeUpdated, key, 30*time.Second) + if !ok { + t.Fatalf("preview record was never discovered after requireMulticast succeeded") + } + if preview.IdentityVerification != "unverified" || preview.Trusted || preview.PeerRecordID != key { + t.Fatalf("plaintext fixture unexpectedly proved an identity: %+v", preview) } // The wipe: the record stops being advertised under the old identity, and @@ -111,7 +179,7 @@ func TestScannerEvictsRecordSupersededAtItsAddress(t *testing.T) { // inference load — so this deadline is generous by design. The identity probe // runs on every scan from ~15s in, so the mismatch is detected long before the // eviction it eventually authorizes. - if !awaitNodeEvent(events, noderec.NotifyNodeRemoved, originalUUID, 2*time.Minute) { + if _, ok := awaitNodeEvent(events, noderec.NotifyNodeRemoved, key, 2*time.Minute); !ok { t.Fatalf("scanner never evicted %s after its address began answering as %s; "+ "a record whose machine was re-identified must not be kept alive by a TCP probe", originalUUID, wipedUUID) @@ -123,13 +191,14 @@ func TestScannerEvictsRecordSupersededAtItsAddress(t *testing.T) { // no subscription is needed. stdin is held open for the process lifetime — // closing it ends the scanner's read loop. // -// --cluster-dir is a throwaway temp dir, as everywhere else in this package. +// --cluster-dir is a throwaway identity (empty or synthetically admitted), as +// everywhere else in this package. // Without it the scanner resolves the real per-user app dir: it would read (and // mint into) the developer's own identity files and advertise their machine's // real cluster identity on the LAN while the test runs. -func startScannerForGhostTest(t *testing.T) <-chan jsonrpc.Message { +func startScannerForGhostTest(t *testing.T, clusterDir string) <-chan jsonrpc.Message { t.Helper() - cmd := testCommand(t, scannerBin, "--cluster-dir", t.TempDir()) + cmd := testCommand(t, scannerBin, "--cluster-dir", clusterDir) cmd.Stderr = os.Stderr stdin, err := cmd.StdinPipe() @@ -151,16 +220,15 @@ func startScannerForGhostTest(t *testing.T) <-chan jsonrpc.Message { return startMsgReader(stdout) } -// awaitNodeEvent reports whether the named discovery:node-* event for hostUUID -// arrives before the deadline. Other events (this node's own record, the peer's -// enrichment updates) are ignored. -func awaitNodeEvent(events <-chan jsonrpc.Message, method, hostUUID string, timeout time.Duration) bool { +// awaitNodeEvent matches the independent record key, never a preview's claimed +// host UUID. Other events (self, enrichment, other candidates) are ignored. +func awaitNodeEvent(events <-chan jsonrpc.Message, method, recordKey string, timeout time.Duration) (noderec.DirectoryNode, bool) { deadline := time.After(timeout) for { select { case msg, ok := <-events: if !ok { - return false + return noderec.DirectoryNode{}, false } if msg.Method != method { continue @@ -169,11 +237,11 @@ func awaitNodeEvent(events <-chan jsonrpc.Message, method, hostUUID string, time if err := json.Unmarshal(msg.Params, &ev); err != nil { continue } - if ev.Node.HostUUID == hostUUID { - return true + if ev.Node.RecordKey() == recordKey { + return ev.Node, true } case <-deadline: - return false + return noderec.DirectoryNode{}, false } } } diff --git a/services/tests/verified_peer_test.go b/services/tests/verified_peer_test.go index 68d37b5fd..880403c59 100644 --- a/services/tests/verified_peer_test.go +++ b/services/tests/verified_peer_test.go @@ -30,6 +30,13 @@ func (c *interNodeCluster) startVerifiedPeer(t *testing.T, instance string, mode if err != nil { t.Fatalf("verified peer listen: %v", err) } + return c.startVerifiedPeerOn(t, instance, models, workloads, ln) +} + +// startVerifiedPeerOn lets a wipe regression reuse the exact listener address +// after stopping the original, authenticated incarnation. +func (c *interNodeCluster) startVerifiedPeerOn(t *testing.T, instance string, models, workloads http.Handler, ln net.Listener) func() { + t.Helper() _, portString, err := net.SplitHostPort(ln.Addr().String()) if err != nil { t.Fatal(err) From 8ff7c5ec79148693fa4fc46ab25b053591764dd3 Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:05:40 +0200 Subject: [PATCH 17/18] test(services): fail worker prerequisites after multicast admission Remove the remaining post-requireMulticast self-skips in active and terminal workload rehydration and the shared workload-manager startup helper. A required port collision must fail those tests rather than silently erase their runtime coverage. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/tests/workload_identity_interop_test.go | 4 ++-- services/tests/workload_interop_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/tests/workload_identity_interop_test.go b/services/tests/workload_identity_interop_test.go index cf979bd1d..5cb91a904 100644 --- a/services/tests/workload_identity_interop_test.go +++ b/services/tests/workload_identity_interop_test.go @@ -160,7 +160,7 @@ func TestWorkloadCrossEngineIdentityDistinct(t *testing.T) { func TestWorkloadManagerRehydratesActiveWorkloadOnRestart(t *testing.T) { requireMulticast(t) if portBusy(11435) || portBusy(14320) { - t.Skip("ollama-proxy (11435) or workload-manager (14320) port already in use; skipping") + t.Fatalf("ollama-proxy (11435) or workload-manager (14320) port already in use after requireMulticast succeeded") } // Fake Ollama that accepts the request then blocks, keeping the proxied @@ -277,7 +277,7 @@ func TestWorkloadManagerRehydratesActiveWorkloadOnRestart(t *testing.T) { func TestWorkloadManagerRehydratesRecentTerminalOnRestart(t *testing.T) { requireMulticast(t) if portBusy(11435) || portBusy(14320) { - t.Skip("ollama-proxy (11435) or workload-manager (14320) port already in use; skipping") + t.Fatalf("ollama-proxy (11435) or workload-manager (14320) port already in use after requireMulticast succeeded") } // Fake Ollama that completes immediately, so the workload reaches a terminal diff --git a/services/tests/workload_interop_test.go b/services/tests/workload_interop_test.go index c8b6f91b8..45d0a09a2 100644 --- a/services/tests/workload_interop_test.go +++ b/services/tests/workload_interop_test.go @@ -79,7 +79,7 @@ func startBrokerProcInCluster(t *testing.T, clusterDir string, args ...string) ( t.Helper() for _, arg := range args { if arg == "--workload-manager-path" && portBusy(14320) { - t.Skip("workload-manager inter-node port 14320 already in use; skipping") + t.Fatalf("workload-manager inter-node port 14320 already in use; cannot exercise the requested worker") } } args = append([]string{"--cluster-dir", clusterDir}, args...) From 59cd814e33fa5933ad6c6e2243f89f145c8c43cb Mon Sep 17 00:00:00 2001 From: Jared Ping <13906915+JaredP94@users.noreply.github.com> Date: Tue, 3 Nov 2026 11:15:34 +0200 Subject: [PATCH 18/18] fix(broker): preserve local workloads across peer proof renewal Peer certificate renewal changes the pin before discovery publishes the renewed proof. The preview-only workload guard treated that inventory gap as an unverified remote claim and discarded authoritative lifecycle reports from our own proxies. Track workload provenance from the supervised worker path, exempt local proxy reports from preview admission, and retain stable participant IDs and existing loss and withdrawal sweeps. Remote claims remain guarded even when their payload names the local origin. Cover all three proxies, client and scheduler delivery, and cleanup after renewal, withdrawal and scanner retirement. Keep the broker at its already-unreleased 2.5.0 version; no wire payload changes. Signed-off-by: Jared Ping <13906915+JaredP94@users.noreply.github.com> --- services/nvpair-ui-broker/README.md | 2 +- services/nvpair-ui-broker/broker.go | 44 +++-- .../nvpair-ui-broker/node_metrics_test.go | 22 +-- .../nvpair-ui-broker/workload_preview_test.go | 151 ++++++++++++++++++ 4 files changed, 191 insertions(+), 28 deletions(-) diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index adada0eb5..ef958d7e5 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -156,7 +156,7 @@ Two classes of proxy notification are **not** re-emitted under the `proxy:` name - **Outbound (proxy -> broker -> manager -> peers).** When any proxy emits a `workload:started` / `workload:completed` / `workload:errored`, the broker stamps the local stable `hostUuid` onto `params.workloadInfo.originatedFrom`, applies the transition to its authoritative workload store, and fans the accepted update to the scheduler before forwarding the original lifecycle frame to the manager. The manager broadcasts it to peer nodes. With no manager supervised the event still updates local scheduling and subscribed clients, but is not broadcast. - **Inbound (peers -> manager -> broker).** The manager translates peer-origin lifecycle events into `workloads:upsert` and peer-origin removals into `workloads:remove` on stdout. The broker applies each accepted transition to the same store, fans it to the scheduler, and relays it to clients subscribed via `workloads:subscribe`. -- **Preview lifecycle.** Active workload events cannot name a known unverified preview as origin or executor; terminal events remain accepted to retire existing state. Relayed participants not yet discovered locally are not treated as previews. Removing a preview (including scanner retirement) still fails previously tracked jobs when no verified or explicit bare-engine claim survives for that participant. A preview claiming a verified peer's UUID cannot fail that peer's jobs. +- **Preview lifecycle.** Remote active workload claims cannot name a known unverified preview as origin or executor; terminal events remain accepted to retire existing state. Local proxy lifecycle reports are authoritative about routed work and remain accepted while discovery asynchronously refreshes a peer's certificate proof, preserving the stable origin/executor IDs rather than binding jobs to a transient proof or preview key. This exception is selected by the supervised local-proxy path, never by a claimed origin in the payload. Relayed participants not yet discovered locally are not treated as previews. Removing a preview (including scanner retirement) still fails previously tracked jobs when no verified or explicit bare-engine claim survives for that participant. A preview claiming a verified peer's UUID cannot fail that peer's jobs. - **Local echo.** Local-origin proxy workloads are also emitted to the same `workloads:*` client stream (lifecycle translated to `workloads:upsert`), so a subscribed client sees a coherent cluster-wide view — its own workloads alongside peers'. **`nvpair-job-scheduler`** consumes the accepted workload stream, compact GPU telemetry, and discovery snapshot. It smooths fresh utilization into pressure 0–3, uses neutral pressure 1 for invalid/missing/older-than-10-second samples, and orders by `pending + gpuPressure`, then pressure, then stable UUID. Load is node-wide across Ollama, LM Studio, and oMLX because all contend for the same resources. Each engine-specific `schedule:priority` carries `{engine,nodes,ranks}` and refreshes when order, pending counts, or pressure changes. The broker caches, generation-orders, and replays the full `{nodes,ranks}` snapshot to the matching proxy, where a newly delivered snapshot resets optimistic reservation deltas. On scheduler spawn/restart the broker replays active workloads and telemetry before discovery, then resumes all three live feeds. diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index 68a355e49..c8eea34d1 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -2316,9 +2316,9 @@ func (b *Broker) routeProxyWorkload(method string, params json.RawMessage) { // same translation the manager applies to peer-origin lifecycle events, spec // 7.0); a removal stays a removal. if method == "workloads:remove" { - b.emitWorkloadEvent("workloads:remove", stamped) + b.emitWorkloadEventProvenance("workloads:remove", stamped, workloadLocalProxy, noSightingGuard) } else { - b.emitWorkloadEvent("workloads:upsert", stamped) + b.emitWorkloadEventProvenance("workloads:upsert", stamped, workloadLocalProxy, noSightingGuard) } if wm := b.getWorkloadMgr(); wm != nil { if err := wm.Forward(method, stamped); err != nil { @@ -2700,9 +2700,9 @@ func (b *Broker) ingestProgress(frame inferencemetrics.ProgressFrame) { // emitWorkloadEvent applies a workloads:* notification to the authoritative // store and, only when it is a real forward change, fans it to the scheduler -// and re-emits it to a subscribed client. Both the inbound relay (peer-origin -// events off the manager's stdout) and the local echo (local-origin proxy -// workloads) funnel through here. Dropping stale/regressive upserts at the +// and re-emits it to a subscribed client. This entry point handles remote +// claims; local proxy reports enter the same provenance-aware emitter with +// local authority instead. Dropping stale/regressive upserts at the // store is what stops an out-of-order "running" (arriving after a workload's // terminal event) from resurrecting it on the client stream and in the // scheduler's view. The stream is off by default, so an unsubscribed client @@ -2712,7 +2712,7 @@ func (b *Broker) ingestProgress(frame inferencemetrics.ProgressFrame) { // client-visible order matches the applied order even when the node-loss sweep // (a separate goroutine) races a relayed event for the same workload. func (b *Broker) emitWorkloadEvent(method string, params json.RawMessage) { - b.emitWorkloadEventProvenance(method, params, false, noSightingGuard) + b.emitWorkloadEventProvenance(method, params, workloadRemote, noSightingGuard) } // emitWorkloadEventInferred is emitWorkloadEvent for a locally-inferred @@ -2720,7 +2720,7 @@ func (b *Broker) emitWorkloadEvent(method string, params json.RawMessage) { // records the state as inferred so a later authoritative event from the origin // (e.g. the node was only briefly unreachable) reconciles it away. func (b *Broker) emitWorkloadEventInferred(method string, params json.RawMessage) { - b.emitWorkloadEventProvenance(method, params, true, noSightingGuard) + b.emitWorkloadEventProvenance(method, params, workloadInferred, noSightingGuard) } // emitWorkloadEventStale is emitWorkloadEventInferred for the staleness sweep, @@ -2729,7 +2729,7 @@ func (b *Broker) emitWorkloadEventInferred(method string, params json.RawMessage // guess is already known to be wrong and is dropped rather than briefly showing // running work as failed. func (b *Broker) emitWorkloadEventStale(method string, params json.RawMessage, seenAt time.Time) { - b.emitWorkloadEventProvenance(method, params, true, seenAt) + b.emitWorkloadEventProvenance(method, params, workloadInferred, seenAt) } // noSightingGuard means "apply unconditionally" — every caller except the @@ -2737,11 +2737,23 @@ func (b *Broker) emitWorkloadEventStale(method string, params json.RawMessage, s // always carries a real sighting. var noSightingGuard time.Time -func (b *Broker) emitWorkloadEventProvenance(method string, params json.RawMessage, inferred bool, unchangedSince time.Time) { +// Provenance comes from the supervised worker path, never a payload's origin +// field. Local proxies report what they actually routed; discovery proof refresh +// is asynchronous and must not suppress those reports or change participant IDs. +// These jobs remain subject to the same node-loss and withdrawal sweeps. +type workloadProvenance uint8 + +const ( + workloadRemote workloadProvenance = iota + workloadLocalProxy + workloadInferred +) + +func (b *Broker) emitWorkloadEventProvenance(method string, params json.RawMessage, source workloadProvenance, unchangedSince time.Time) { b.workloadEmitMu.Lock() defer b.workloadEmitMu.Unlock() - if !b.applyWorkloadEvent(method, params, inferred, unchangedSince) { + if !b.applyWorkloadEvent(method, params, source, unchangedSince) { return } b.recomputeDemand() @@ -2791,7 +2803,7 @@ type workloadsInitialResult struct { // or duplicate events return false, so they're neither fanned nor forwarded); // a workloads:remove drops the entry and always propagates. Malformed or // id-less params are ignored (false). Callers hold workloadEmitMu. -func (b *Broker) applyWorkloadEvent(method string, params json.RawMessage, inferred bool, unchangedSince time.Time) bool { +func (b *Broker) applyWorkloadEvent(method string, params json.RawMessage, source workloadProvenance, unchangedSince time.Time) bool { switch method { case "workloads:upsert": var env struct { @@ -2804,13 +2816,13 @@ func (b *Broker) applyWorkloadEvent(method string, params json.RawMessage, infer if !ok { return false } - // A peer may relay jobs whose other participants we have not yet - // discovered, but a known preview is not a workload participant. - // Still accept terminal events so existing state can be retired. - if !inferred && !workloadstore.IsTerminal(in.State) && (b.onlyUnprovenWorkloadInventory(in.Origin) || b.onlyUnprovenWorkloadInventory(in.ScheduledOn)) { + // Remote claims cannot establish active state against known previews. + // Local proxy reports are authoritative even during a proof refresh; + // terminal events from either source can still retire existing state. + if source == workloadRemote && !workloadstore.IsTerminal(in.State) && (b.onlyUnprovenWorkloadInventory(in.Origin) || b.onlyUnprovenWorkloadInventory(in.ScheduledOn)) { return false } - if inferred { + if source == workloadInferred { if !unchangedSince.IsZero() { return b.workloads.ApplyInferredUnchangedSince(in, unchangedSince) } diff --git a/services/nvpair-ui-broker/node_metrics_test.go b/services/nvpair-ui-broker/node_metrics_test.go index a4d9b0405..a41c1516b 100644 --- a/services/nvpair-ui-broker/node_metrics_test.go +++ b/services/nvpair-ui-broker/node_metrics_test.go @@ -81,7 +81,7 @@ func nodeMetricsWorkloadEvent(id, state, scheduledOn string) json.RawMessage { // totals and into the per-model breakdown. func TestNodeMetricsAccumulateFromAcceptedWorkload(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), workloadRemote, noSightingGuard) snap := b.getNodeMetrics() if len(snap.Nodes) != 1 { t.Fatalf("nodes: %+v", snap) @@ -140,7 +140,7 @@ const omlxWorkloadInfoLiteral = `{"id":"17","model":"omlx-fixture-model","engine func TestNodeMetricsAcceptsTheProxyWorkloadFrame(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) event := json.RawMessage(`{"workloadInfo":` + omlxWorkloadInfoLiteral + `}`) - if !b.applyWorkloadEvent("workloads:upsert", event, false, noSightingGuard) { + if !b.applyWorkloadEvent("workloads:upsert", event, workloadRemote, noSightingGuard) { t.Fatal("the workload event must be accepted") } @@ -186,8 +186,8 @@ func TestNodeMetricsAcceptsTheProxyWorkloadFrame(t *testing.T) { func TestDuplicateWorkloadDoesNotDoubleCount(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) event := nodeMetricsWorkloadEvent("1", "completed", "node-b") - b.applyWorkloadEvent("workloads:upsert", event, false, noSightingGuard) - b.applyWorkloadEvent("workloads:upsert", event, false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", event, workloadRemote, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", event, workloadRemote, noSightingGuard) snap := b.getNodeMetrics() if len(snap.Nodes) != 1 { t.Fatalf("nodes: %+v", snap) @@ -205,7 +205,7 @@ func TestDuplicateWorkloadDoesNotDoubleCount(t *testing.T) { // reach the accumulator. func TestInferredWorkloadDoesNotCount(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), true, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), workloadInferred, noSightingGuard) snap := b.getNodeMetrics() if len(snap.Nodes) != 0 { t.Fatalf("an inferred event must not be counted: %+v", snap) @@ -217,7 +217,7 @@ func TestInferredWorkloadDoesNotCount(t *testing.T) { // came from is the only attribution available. func TestNodeMetricsAttributesToOriginWhenNotScheduled(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", ""), false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", ""), workloadRemote, noSightingGuard) snap := b.getNodeMetrics() if len(snap.Nodes) != 1 { t.Fatalf("nodes: %+v", snap) @@ -233,11 +233,11 @@ func TestNodeMetricsAttributesToOriginWhenNotScheduled(t *testing.T) { // carries the measurement. func TestNodeMetricsCountsTerminalWorkloadsOnly(t *testing.T) { b := NewBroker(NewCodec(writeOnlyReadWriter{io.Discard}), workerPaths{}) - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "running", "node-b"), false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "running", "node-b"), workloadRemote, noSightingGuard) if snap := b.getNodeMetrics(); len(snap.Nodes) != 0 { t.Fatalf("a running workload is not a served request: %+v", snap) } - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), workloadRemote, noSightingGuard) snap := b.getNodeMetrics() if len(snap.Nodes) != 1 { t.Fatalf("nodes: %+v", snap) @@ -255,7 +255,7 @@ func TestNodeMetricsCountsTerminalWorkloadsOnly(t *testing.T) { func TestNodeMetricsRPCSurface(t *testing.T) { var out syncBuffer b := NewBroker(NewCodec(writeOnlyReadWriter{&out}), workerPaths{}) - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), workloadRemote, noSightingGuard) id := json.RawMessage(`1`) b.handleMessage(&Message{JSONRPC: "2.0", ID: &id, Method: "metrics:get-node-metrics"}) @@ -330,7 +330,7 @@ func TestNodeMetricsToleratesAMissingStore(t *testing.T) { func TestNodeMetricsPushIsGatedOnSubscription(t *testing.T) { var out syncBuffer b := NewBroker(NewCodec(writeOnlyReadWriter{&out}), workerPaths{}) - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent("1", "completed", "node-b"), workloadRemote, noSightingGuard) b.flushNodeMetrics() if len(out.Frames()) != 0 { @@ -368,7 +368,7 @@ func TestNodeMetricsPushIsCoalesced(t *testing.T) { setMetricsSubscribed(b, true) for i := 0; i < 5; i++ { - b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent(string(rune('a'+i)), "completed", "node-b"), false, noSightingGuard) + b.applyWorkloadEvent("workloads:upsert", nodeMetricsWorkloadEvent(string(rune('a'+i)), "completed", "node-b"), workloadRemote, noSightingGuard) } ctx, cancel := context.WithCancel(context.Background()) diff --git a/services/nvpair-ui-broker/workload_preview_test.go b/services/nvpair-ui-broker/workload_preview_test.go index c90681a8b..ed1d418c3 100644 --- a/services/nvpair-ui-broker/workload_preview_test.go +++ b/services/nvpair-ui-broker/workload_preview_test.go @@ -4,12 +4,163 @@ package main import ( + "bytes" "encoding/json" "testing" + "nvpair-shared/clustertrusttest" "nvpair-shared/noderec" + "nvpair-ui-broker/workloadstore" ) +// Rotating the pin before discovery publishes its renewed proof must not +// discard the local proxy's authoritative report about an already routed job. +func TestLocalWorkloadNotLostDuringPeerCertificateRenewal(t *testing.T) { + for _, engine := range []string{"ollama", "lmstudio", "omlx"} { + for _, departure := range []string{"renewed-then-lost", "trust-withdrawal", "scanner-retirement"} { + t.Run(engine+"/"+departure, func(t *testing.T) { + b := inventoryTestBroker(t) + var client, scheduler bytes.Buffer + b.codec = NewCodec(&client) + b.workloadsSubscribed = true + b.setScheduler(&rpcWorker{peer: NewPeer(NewCodec(&scheduler))}) + node := verifiedDirectoryForTest(b) + b.store.Upsert(directoryToEnriched(node), sourceScanner) + if !b.hasDecisionInventory("peer") { + t.Fatal("initial peer proof was not accepted") + } + clustertrusttest.WritePeerPin(t, b.clusterDir, "peer") + b.mesh.Refresh() + if b.hasDecisionInventory("peer") { + t.Fatal("old proof remained current after certificate renewal") + } + + for _, event := range []struct{ method, state string }{{"workload:submitted", "queued"}, {"workload:started", "running"}} { + params, err := json.Marshal(map[string]any{"workloadInfo": map[string]any{ + "id": "7", "engine": engine, "runId": "r1", "state": event.state, + "scheduledOn": "peer", "createdAt": 1, "model": "m", + }}) + if err != nil { + t.Fatal(err) + } + b.routeProxyWorkloadMethod(event.method, params) + got, ok := b.workloads.Get("self", "7") + if !ok || got.State != event.state || got.Origin != "self" || got.ScheduledOn != "peer" { + t.Fatalf("local %s lost or rekeyed during renewal: %+v", event.method, got) + } + } + + switch departure { + case "renewed-then-lost": + node = verifiedDirectoryForTest(b) + b.store.Upsert(directoryToEnriched(node), sourceScanner) + if !b.hasDecisionInventory("peer") { + t.Fatal("renewed proof was not accepted") + } + sp := &scannerProcess{store: b.store, authorize: b.authorizedDirectoryNode, onNodeLost: func(uuid, name string) { + if !b.hasDecisionInventory(uuid) { + b.failWorkloadsForNode(uuid, name) + } + }} + params, err := json.Marshal(noderec.NodeEvent{Node: node}) + if err != nil { + t.Fatal(err) + } + sp.handleNotify(noderec.NotifyNodeRemoved, params) + case "trust-withdrawal": + clustertrusttest.RemovePeerPin(t, b.clusterDir, "peer") + b.withdrawUnauthorizedInventory() + case "scanner-retirement": + b.clearScannerInventory() + } + if got, ok := b.workloads.Get("self", "7"); !ok || got.State != "failed" { + t.Fatalf("local workload after %s = %+v, want failed", departure, got) + } + for name, output := range map[string]*bytes.Buffer{"client": &client, "scheduler": &scheduler} { + dec := json.NewDecoder(output) + for _, state := range []string{"queued", "running", "failed"} { + var msg Message + if err := dec.Decode(&msg); err != nil { + t.Fatalf("%s missing %s notification: %v", name, state, err) + } + var env struct { + WorkloadInfo json.RawMessage `json:"workloadInfo"` + } + if err := json.Unmarshal(msg.Params, &env); err != nil { + t.Fatal(err) + } + got, ok := workloadstore.ParseIncoming(env.WorkloadInfo) + if msg.Method != "workloads:upsert" || !ok || got.State != state || got.Origin != "self" || got.ScheduledOn != "peer" { + t.Fatalf("%s notification = %s %+v, want %s with stable participants", name, msg.Method, got, state) + } + } + if dec.More() { + t.Fatalf("%s received extra workload notifications", name) + } + } + }) + } + } +} + +// A relayed claim cannot opt into local-proxy authority by naming our node as +// its origin, even when it describes the same job a local proxy could report. +func TestRemoteWorkloadClaimRejectedAgainstUnprovenInventory(t *testing.T) { + for _, inventory := range []string{"preview", "renewing"} { + t.Run(inventory, func(t *testing.T) { + b := inventoryTestBroker(t) + if inventory == "preview" { + b.store.Upsert(EnrichedNode{HostUUID: "peer", ID: "unproven", IdentityVerification: "unverified", PeerRecordID: noderec.PreviewRecordID("unproven", "peer")}, sourceScanner) + } else { + b.store.Upsert(directoryToEnriched(verifiedDirectoryForTest(b)), sourceScanner) + clustertrusttest.WritePeerPin(t, b.clusterDir, "peer") + b.mesh.Refresh() + } + var client, scheduler bytes.Buffer + b.codec = NewCodec(&client) + b.workloadsSubscribed = true + b.setScheduler(&rpcWorker{peer: NewPeer(NewCodec(&scheduler))}) + incoming := storeIncoming("7", "self", "ollama", "r1", "running", "peer") + params, err := json.Marshal(map[string]json.RawMessage{"workloadInfo": incoming.Info}) + if err != nil { + t.Fatal(err) + } + b.forwardWorkloadManagerNotification("workloads:upsert", params) + if _, ok := b.workloads.Get("self", "7"); ok || client.Len() != 0 || scheduler.Len() != 0 { + t.Fatal("unverified remote claim reached the catalog, client or scheduler") + } + }) + } +} + +func TestLocalProxyWorkloadOnPreviewFailsOnWithdrawal(t *testing.T) { + b := inventoryTestBroker(t) + preview := noderec.DirectoryNode{HostUUID: "peer", Name: "unproven", IdentityVerification: "unverified", PeerRecordID: noderec.PreviewRecordID("unproven", "peer")} + b.store.Upsert(directoryToEnriched(preview), sourceScanner) + incoming := storeIncoming("7", "self", "omlx", "r1", "running", "peer") + params, err := json.Marshal(map[string]json.RawMessage{"workloadInfo": incoming.Info}) + if err != nil { + t.Fatal(err) + } + b.routeProxyWorkloadMethod("workload:started", params) + if got, ok := b.workloads.Get("self", "7"); !ok || got.State != "running" || got.ScheduledOn != "peer" { + t.Fatalf("local proxy report was not tracked by stable participant: %+v", got) + } + sp := &scannerProcess{store: b.store, authorize: b.authorizedDirectoryNode, onNodeLost: func(uuid, name string) { + if !b.hasDecisionInventory(uuid) { + b.failWorkloadsForNode(uuid, name) + } + }} + params, err = json.Marshal(noderec.NodeEvent{Node: preview}) + if err != nil { + t.Fatal(err) + } + sp.handleNotify(noderec.NotifyNodeRemoved, params) + if got, ok := b.workloads.Get("self", "7"); !ok || got.State != "failed" { + t.Fatalf("local proxy workload after preview loss = %+v, want failed", got) + } +} + func TestPreviewWithdrawalRetiresAlreadyTrackedWorkloads(t *testing.T) { for _, verifiedSurvives := range []bool{false, true} { t.Run(map[bool]string{false: "preview-only", true: "verified-survives"}[verifiedSurvives], func(t *testing.T) {