Keep nodes added by address persistent with explicit PAIR or engine intent

SECURITY.md

@@ -98,11 +98,15 @@ to the local admission and both certificates and are withdrawn on authorization
 changes or worker retirement. Older peers without authenticated service metadata
 remain previews, not service targets. Explicitly configured bare-engine manual
 routes remain a separately labelled bypass and supply no peer telemetry.
-Manual-entry kind is not yet persisted: after a manual-nodes worker restart,
-an address previously added for a PAIR peer can be routed as an unauthenticated
-plaintext bare engine if no PAIR endpoint answers there, including when another
-machine has taken that address. Persisting PAIR-vs-bare-engine intent is required
-to prevent that downgrade across worker restarts.
+The broker persists each manual entry's `pair` or `engine` kind in the canonical
+`configs/manual-nodes.json`. A `pair` entry never downgrades to a plaintext engine,
+including after worker/broker restarts or address reassignment. Entries without a
+historical kind migrate conservatively on disk to `pair`; resuming a bare-engine
+route requires an explicit **Engine only** choice. An `engine` entry trusts no
+PAIR claims even if a PAIR endpoint answers at that address. An authenticated
+pairing completion can associate a manual hostname with the pinned peer solely
+for later cleanup. This saved association does not mark the entry verified or
+supply routing, identity, inventory or telemetry authority.
 
 PAIR can advertise/rank Tailscale CGNAT addresses (`100.64.0.0/10`) and use known
 peer address fallbacks. This is not arbitrary tailnet peer enumeration, and

desktop/docs/frontend-api.md

@@ -52,6 +52,12 @@ They do not imply a WebSocket connection. Browser clients are not supported.
 - `onRemove(callback)` reports node removals.
 - `onMembersChanged(callback)` reports a complete membership snapshot.
 
+### `pairApi.manualNodes`
+
+- `list()` returns all tracked manual nodes with kind (`pair` | `engine`), address, name, and identity verification.
+- `add({ address, kind, name?, tls_port?, mtls? })` adds a manual node entry with explicit kind (`pair` or `engine`).
+- `remove(id)` removes a manual node entry from durable storage.
+
 ### `pairApi.cluster`
 
 - `getInitial()` returns identity, settings, membership, and all live inbound invites.

desktop/docs/service-contract-exceptions.json

@@ -1,6 +1,7 @@
 {
     "$comment": "Intentionally unintegrated JSON-RPC methods for desktop ↔ services contract checks. Each ignoredMethods entry needs a current technical reason. Edit by hand; run npm run service-contracts:write to refresh docs/services-api.md after services/ changes.",
     "ignoredMethods": {
+        "cluster:pairing-completed": "Cluster-manager-to-broker internal lifecycle association, emitted only after authenticated pairing commit under the roster ordering lock with the original dial address, authenticated peer UUID, cluster ID and admission epoch. The broker validates the current admission, rejects removed/obsolete completions, and persists cleanup-only manual aliases and never forwards this event or treats it as inventory, identity verification, or routing authority.",
         "engine:settings-request": "Worker-to-broker internal live permission read. Only settings/get-engine-allow-lan-bind is allowed; the broker returns a correlated engine:settings-reply directly to the originating worker, never the renderer.",
         "engine:settings-reply": "Broker-to-worker internal response for the live LAN-bind permission read. Missing, malformed, failed or timed-out values deny non-loopback local start overrides.",
         "schedule:priority": "Broker-internal. nvpair-job-scheduler emits a per-engine priority snapshot ({engine, nodes, ranks?} — ordered node ids plus each node's pending count) to nvpair-ui-broker, which forwards it to ollama-proxy and lmstudio-proxy via node/set-priority; each proxy then layers its own optimistic burst reservations on that baseline. PAIR consumes the routing result rather than this notification, and must not mirror the pending/reservation accounting.",

desktop/docs/services-api.md

@@ -94,6 +94,7 @@
 | `cluster:invite-canceled` | notification (we consume) | ✅ yes |
 | `cluster:invite-expired` | notification (we consume) | ✅ yes |
 | `cluster:invite-received` | notification (we consume) | ✅ yes |
+| `cluster:pairing-completed` | notification (we consume) | ➖ ignored |
 | `cluster:trust-changed` | notification (we consume) | ➖ ignored |
 | `nodes:changed` | notification (we consume) | ✅ yes |
 | `cluster:cancel-invite` | request (we call) | ✅ yes |

desktop/docs/services-parity.md

@@ -117,8 +117,9 @@ Two limitations follow from the current discovery and node-info contract:
   to a placeholder until the backend reports OS on discovery or node-info.
 
 Manual nodes use the broker's `node/add`, `node/remove`, and `nodes/list`
-surface. Electron persists user entries and replays them after broker startup so
-they survive worker restarts.
+surface. The Go broker canonically persists entries with explicit kind ('pair'
+vs 'engine') across restarts, while desktop exposes manual-node bridge channels
+and settings UI.
 
 ## Routing and inference
 

desktop/src/electron/service-bridge/empty-handlers.ts

@@ -4,6 +4,7 @@
 import type { WsInvokeChannel, WsInvokeRequest, WsInvokeResponse } from '@/shared/types/ws-channels'
 import type { ClusterInitialSnapshot } from '@/shared/types/bootstrap'
 import type { ClusterNode, ClusterNodeIdentity, Invite } from '@/shared/types/cluster'
+import type { ManualNodeKind, ManualNodeStatus } from '@/shared/types/manual-nodes'
 import type { EngineType } from '@/shared/types/engines'
 import type { ServiceError } from '@/shared/types/errors'
 import type { NodeInferenceMetricsSnapshot } from '@/shared/types/metrics'
@@ -989,10 +990,7 @@ async function handleNodeRemoveMember(
     const selfId = state.getSelfId()
     const isSelfLeave = selfId !== null && payload.nodeId === selfId
 
-    // The broker owns manual-node persistence and resolves a stable node UUID
-    // back to the manual alias expected by nvpair-manual-nodes. For a purely
-    // discovered peer this remains a harmless no-op.
-    supervisor.callProcess('broker', 'node/remove', { id: payload.nodeId }).catch(() => {})
+    // The broker forgets associated manual addresses only after successful unpair.
 
     if (isSelfLeave) {
         // Self-departure is `cluster:leave`, not `nodes:remove` (the cluster-manager
@@ -1015,6 +1013,115 @@ async function handleNodeRemoveMember(
     }
 }
 
+function parseStringArray(value: JsonValue | undefined): string[] {
+    if (!Array.isArray(value)) return []
+    const result: string[] = []
+    for (const item of value) {
+        if (typeof item === 'string') {
+            result.push(item)
+        }
+    }
+    return result
+}
+
+function parseManualNodeStatus(raw: JsonValue | undefined): ManualNodeStatus | null {
+    const obj = objectValue(raw)
+    if (!obj) return null
+    const id = stringValue(obj.id)
+    const address = stringValue(obj.address)
+    if (!id || !address) return null
+
+    if (obj.kind !== 'engine' && obj.kind !== 'pair') return null
+    const kind: ManualNodeKind = obj.kind
+    const name = stringValue(obj.name)
+    const identityVerification = stringValue(obj.identityVerification)
+    const peerRecordId = stringValue(obj.peerRecordId)
+    const hostUuid = stringValue(obj.hostUuid)
+    const clusterUuid = stringValue(obj.clusterUuid)
+
+    const status: ManualNodeStatus = {
+        id,
+        address,
+        kind,
+        ...(name ? { name } : {}),
+        ...(identityVerification ? { identityVerification } : {}),
+        ...(peerRecordId ? { peerRecordId } : {}),
+        ...(hostUuid ? { hostUuid } : {}),
+        ...(clusterUuid ? { clusterUuid } : {}),
+        ollama_up: booleanValue(obj.ollama_up),
+        ollama_port: numberValue(obj.ollama_port),
+        lmstudio_up: booleanValue(obj.lmstudio_up),
+        lmstudio_port: numberValue(obj.lmstudio_port),
+        omlx_up: booleanValue(obj.omlx_up),
+        omlx_port: numberValue(obj.omlx_port),
+        node_info_up: booleanValue(obj.node_info_up),
+        node_info_port: numberValue(obj.node_info_port),
+        tls_enabled: booleanValue(obj.tls_enabled),
+        mtls_required: booleanValue(obj.mtls_required),
+        models: parseStringArray(obj.models)
+    }
+    return status
+}
+
+function parseManualNodeList(raw: JsonValue | undefined): ManualNodeStatus[] {
+    const obj = objectValue(raw)
+    if (!obj) return []
+    const rawNodes = Array.isArray(obj.nodes) ? obj.nodes : []
+    const result: ManualNodeStatus[] = []
+    for (const item of rawNodes) {
+        if (objectValue(item)?.from_cluster === true) continue
+        const parsed = parseManualNodeStatus(item)
+        if (parsed) {
+            result.push(parsed)
+        }
+    }
+    return result
+}
+
+async function handleManualNodesList(): Promise<ManualNodeStatus[]> {
+    const supervisor = getModularSupervisor()
+    if (!supervisor.hasProcess('broker')) {
+        return []
+    }
+    const result = await supervisor.callProcess('broker', 'nodes/list')
+    return parseManualNodeList(result)
+}
+
+async function handleManualNodesAdd(
+    payload?: WsInvokeRequest<'nodes:manual-add'>
+): Promise<ManualNodeStatus> {
+    if (!payload?.address) {
+        throw new Error('Address is required')
+    }
+    const supervisor = getModularSupervisor()
+    const result = await supervisor.callProcess('broker', 'node/add', {
+        address: payload.address,
+        ...(payload.name ? { name: payload.name } : {}),
+        kind: payload.kind,
+        ...(payload.tls_port ? { tls_port: payload.tls_port } : {}),
+        ...(payload.mtls !== undefined ? { mtls: payload.mtls } : {})
+    })
+    const status = parseManualNodeStatus(result)
+    if (!status) {
+        throw new Error('Invalid response from node/add')
+    }
+    return status
+}
+
+async function handleManualNodesRemove(
+    payload?: WsInvokeRequest<'nodes:manual-remove'>
+): Promise<{ removed: boolean }> {
+    if (!payload?.id) {
+        throw new Error('Node ID is required')
+    }
+    const supervisor = getModularSupervisor()
+    const result = await supervisor.callProcess('broker', 'node/remove', { id: payload.id })
+    const obj = objectValue(result)
+    return {
+        removed: booleanValue(obj?.removed)
+    }
+}
+
 function telemetryObject(value: JsonValue | undefined): JsonObject {
     return objectValue(value) ?? {}
 }
@@ -1166,6 +1273,9 @@ const EMPTY_SERVICE_BRIDGE_HANDLERS: BridgeHandlerMap = {
     'nodes:remove-member': payload => handleNodeRemoveMember(payload),
     'nodes:install-macmon': payload =>
         payload ? getModularSupervisor().installMacmon(payload.nodeId) : 'failed',
+    'nodes:manual-list': () => handleManualNodesList(),
+    'nodes:manual-add': payload => handleManualNodesAdd(payload),
+    'nodes:manual-remove': payload => handleManualNodesRemove(payload),
 
     'discovery:get-nodes': () => getModularBridgeState().getAvailableNodes(),
 

desktop/src/shared/types/manual-nodes.ts

@@ -0,0 +1,41 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+export type ManualNodeKind = 'pair' | 'engine'
+
+export interface ManualNodeAddRequest {
+    address: string
+    name?: string
+    kind: ManualNodeKind
+    tls_port?: number
+    mtls?: boolean
+}
+
+export interface ManualNodeRemoveRequest {
+    id: string
+}
+
+export interface ManualNodeStatus {
+    id: string
+    address: string
+    name?: string
+    kind: ManualNodeKind
+    identityVerification?: string
+    peerRecordId?: string
+    hostUuid?: string
+    clusterUuid?: string
+    ollama_up?: boolean
+    ollama_port?: number
+    ollama_models?: string[]
+    lmstudio_up?: boolean
+    lmstudio_port?: number
+    lmstudio_models?: string[]
+    omlx_up?: boolean
+    omlx_port?: number
+    omlx_models?: string[]
+    node_info_up?: boolean
+    node_info_port?: number
+    tls_enabled?: boolean
+    mtls_required?: boolean
+    models?: string[]
+}

desktop/src/shared/types/ws-channels.ts

@@ -48,6 +48,11 @@ import type {
     ClusterNode,
     Invite
 } from '@/shared/types/cluster'
+import type {
+    ManualNodeAddRequest,
+    ManualNodeRemoveRequest,
+    ManualNodeStatus
+} from '@/shared/types/manual-nodes'
 import type {
     TelemetryConfig,
     TelemetryConfigRequest,
@@ -94,6 +99,11 @@ export interface WsInvokeChannelMap {
         response: MacmonInstallOutcome
     }
 
+    // Manual nodes (relayed to nvpair-manual-nodes via the broker)
+    'nodes:manual-list': { request: void; response: ManualNodeStatus[] }
+    'nodes:manual-add': { request: ManualNodeAddRequest; response: ManualNodeStatus }
+    'nodes:manual-remove': { request: ManualNodeRemoveRequest; response: { removed: boolean } }
+
     // Discovery
     'discovery:get-nodes': { request: void; response: AvailableNode[] }
 

desktop/src/ui/api/pair-api.ts

@@ -20,6 +20,7 @@ import type {
 import type { Workload, WorkloadLiveProgress } from '@/shared/types/workloads'
 import type { WorkerStatus } from '@/shared/types/workers'
 import type { AppInitialSnapshot, ClusterInitialSnapshot } from '@/shared/types/bootstrap'
+import type { ManualNodeAddRequest, ManualNodeStatus } from '@/shared/types/manual-nodes'
 import type {
     TelemetryConfig,
     TelemetryConfigRequest,
@@ -153,6 +154,15 @@ export interface IMetricsApi {
     onNodeMetrics(callback: (snapshot: NodeInferenceMetricsSnapshot) => void): () => void
 }
 
+interface IManualNodesApi {
+    /** Fetch all tracked manual nodes. */
+    list(): Promise<ManualNodeStatus[]>
+    /** Add a manual node with explicit kind ('pair' | 'engine'). */
+    add(params: ManualNodeAddRequest): Promise<ManualNodeStatus>
+    /** Remove a manual node by id from durable storage. */
+    remove(id: string): Promise<{ removed: boolean }>
+}
+
 // ---------------------------------------------------------------------------
 // Composite API
 // ---------------------------------------------------------------------------
@@ -165,6 +175,7 @@ export interface IPairApi {
     app: IAppApi
     connection: IConnectionApi
     nodes: INodesApi
+    manualNodes: IManualNodesApi
     cluster: IClusterApi
     discovery: IDiscoveryApi
     engines: IEngineApi
@@ -202,6 +213,11 @@ export function createPairApi(transport: ServiceTransport): IPairApi {
             onRemove: cb => transport.subscribePush('nodes:remove', cb),
             onMembersChanged: cb => transport.subscribePush('nodes:changed', cb)
         },
+        manualNodes: {
+            list: () => transport.invoke('nodes:manual-list'),
+            add: params => transport.invoke('nodes:manual-add', params),
+            remove: id => transport.invoke('nodes:manual-remove', { id })
+        },
         cluster: {
             getInitial: () => transport.invoke('cluster:get-initial'),
             inviteNode: ipAddress => transport.invoke('cluster:invite-node', { ipAddress }),