diff --git a/SECURITY.md b/SECURITY.md index 4432cfd3c..d62041cd0 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -98,11 +98,15 @@ to the local admission and both certificates and are withdrawn on authorization changes or worker retirement. Older peers without authenticated service metadata remain previews, not service targets. Explicitly configured bare-engine manual routes remain a separately labelled bypass and supply no peer telemetry. -Manual-entry kind is not yet persisted: after a manual-nodes worker restart, -an address previously added for a PAIR peer can be routed as an unauthenticated -plaintext bare engine if no PAIR endpoint answers there, including when another -machine has taken that address. Persisting PAIR-vs-bare-engine intent is required -to prevent that downgrade across worker restarts. +The broker persists each manual entry's `pair` or `engine` kind in the canonical +`configs/manual-nodes.json`. A `pair` entry never downgrades to a plaintext engine, +including after worker/broker restarts or address reassignment. Entries without a +historical kind migrate conservatively on disk to `pair`; resuming a bare-engine +route requires an explicit **Engine only** choice. An `engine` entry trusts no +PAIR claims even if a PAIR endpoint answers at that address. An authenticated +pairing completion can associate a manual hostname with the pinned peer solely +for later cleanup. This saved association does not mark the entry verified or +supply routing, identity, inventory or telemetry authority. PAIR can advertise/rank Tailscale CGNAT addresses (`100.64.0.0/10`) and use known peer address fallbacks. This is not arbitrary tailnet peer enumeration, and diff --git a/desktop/docs/frontend-api.md b/desktop/docs/frontend-api.md index 4d8c66b1b..9ba2a0ccb 100644 --- a/desktop/docs/frontend-api.md +++ b/desktop/docs/frontend-api.md @@ -52,6 +52,12 @@ They do not imply a WebSocket connection. Browser clients are not supported. - `onRemove(callback)` reports node removals. - `onMembersChanged(callback)` reports a complete membership snapshot. +### `pairApi.manualNodes` + +- `list()` returns all tracked manual nodes with kind (`pair` | `engine`), address, name, and identity verification. +- `add({ address, kind, name?, tls_port?, mtls? })` adds a manual node entry with explicit kind (`pair` or `engine`). +- `remove(id)` removes a manual node entry from durable storage. + ### `pairApi.cluster` - `getInitial()` returns identity, settings, membership, and all live inbound invites. diff --git a/desktop/docs/service-contract-exceptions.json b/desktop/docs/service-contract-exceptions.json index 9f11b8fa0..d207852af 100644 --- a/desktop/docs/service-contract-exceptions.json +++ b/desktop/docs/service-contract-exceptions.json @@ -1,6 +1,7 @@ { "$comment": "Intentionally unintegrated JSON-RPC methods for desktop ↔ services contract checks. Each ignoredMethods entry needs a current technical reason. Edit by hand; run npm run service-contracts:write to refresh docs/services-api.md after services/ changes.", "ignoredMethods": { + "cluster:pairing-completed": "Cluster-manager-to-broker internal lifecycle association, emitted only after authenticated pairing commit under the roster ordering lock with the original dial address, authenticated peer UUID, cluster ID and admission epoch. The broker validates the current admission, rejects removed/obsolete completions, and persists cleanup-only manual aliases and never forwards this event or treats it as inventory, identity verification, or routing authority.", "engine:settings-request": "Worker-to-broker internal live permission read. Only settings/get-engine-allow-lan-bind is allowed; the broker returns a correlated engine:settings-reply directly to the originating worker, never the renderer.", "engine:settings-reply": "Broker-to-worker internal response for the live LAN-bind permission read. Missing, malformed, failed or timed-out values deny non-loopback local start overrides.", "schedule:priority": "Broker-internal. nvpair-job-scheduler emits a per-engine priority snapshot ({engine, nodes, ranks?} — ordered node ids plus each node's pending count) to nvpair-ui-broker, which forwards it to ollama-proxy and lmstudio-proxy via node/set-priority; each proxy then layers its own optimistic burst reservations on that baseline. PAIR consumes the routing result rather than this notification, and must not mirror the pending/reservation accounting.", diff --git a/desktop/docs/services-api.md b/desktop/docs/services-api.md index 841f9c7f6..c3d635aa4 100644 --- a/desktop/docs/services-api.md +++ b/desktop/docs/services-api.md @@ -94,6 +94,7 @@ | `cluster:invite-canceled` | notification (we consume) | ✅ yes | | `cluster:invite-expired` | notification (we consume) | ✅ yes | | `cluster:invite-received` | notification (we consume) | ✅ yes | +| `cluster:pairing-completed` | notification (we consume) | ➖ ignored | | `cluster:trust-changed` | notification (we consume) | ➖ ignored | | `nodes:changed` | notification (we consume) | ✅ yes | | `cluster:cancel-invite` | request (we call) | ✅ yes | diff --git a/desktop/docs/services-parity.md b/desktop/docs/services-parity.md index de5aa0986..c69e0e8b5 100644 --- a/desktop/docs/services-parity.md +++ b/desktop/docs/services-parity.md @@ -117,8 +117,9 @@ Two limitations follow from the current discovery and node-info contract: to a placeholder until the backend reports OS on discovery or node-info. Manual nodes use the broker's `node/add`, `node/remove`, and `nodes/list` -surface. Electron persists user entries and replays them after broker startup so -they survive worker restarts. +surface. The Go broker canonically persists entries with explicit kind ('pair' +vs 'engine') across restarts, while desktop exposes manual-node bridge channels +and settings UI. ## Routing and inference diff --git a/desktop/src/electron/service-bridge/empty-handlers.ts b/desktop/src/electron/service-bridge/empty-handlers.ts index 1a4c6303d..e8e2bdc7b 100644 --- a/desktop/src/electron/service-bridge/empty-handlers.ts +++ b/desktop/src/electron/service-bridge/empty-handlers.ts @@ -4,6 +4,7 @@ import type { WsInvokeChannel, WsInvokeRequest, WsInvokeResponse } from '@/shared/types/ws-channels' import type { ClusterInitialSnapshot } from '@/shared/types/bootstrap' import type { ClusterNode, ClusterNodeIdentity, Invite } from '@/shared/types/cluster' +import type { ManualNodeKind, ManualNodeStatus } from '@/shared/types/manual-nodes' import type { EngineType } from '@/shared/types/engines' import type { ServiceError } from '@/shared/types/errors' import type { NodeInferenceMetricsSnapshot } from '@/shared/types/metrics' @@ -989,10 +990,7 @@ async function handleNodeRemoveMember( const selfId = state.getSelfId() const isSelfLeave = selfId !== null && payload.nodeId === selfId - // The broker owns manual-node persistence and resolves a stable node UUID - // back to the manual alias expected by nvpair-manual-nodes. For a purely - // discovered peer this remains a harmless no-op. - supervisor.callProcess('broker', 'node/remove', { id: payload.nodeId }).catch(() => {}) + // The broker forgets associated manual addresses only after successful unpair. if (isSelfLeave) { // Self-departure is `cluster:leave`, not `nodes:remove` (the cluster-manager @@ -1015,6 +1013,115 @@ async function handleNodeRemoveMember( } } +function parseStringArray(value: JsonValue | undefined): string[] { + if (!Array.isArray(value)) return [] + const result: string[] = [] + for (const item of value) { + if (typeof item === 'string') { + result.push(item) + } + } + return result +} + +function parseManualNodeStatus(raw: JsonValue | undefined): ManualNodeStatus | null { + const obj = objectValue(raw) + if (!obj) return null + const id = stringValue(obj.id) + const address = stringValue(obj.address) + if (!id || !address) return null + + if (obj.kind !== 'engine' && obj.kind !== 'pair') return null + const kind: ManualNodeKind = obj.kind + const name = stringValue(obj.name) + const identityVerification = stringValue(obj.identityVerification) + const peerRecordId = stringValue(obj.peerRecordId) + const hostUuid = stringValue(obj.hostUuid) + const clusterUuid = stringValue(obj.clusterUuid) + + const status: ManualNodeStatus = { + id, + address, + kind, + ...(name ? { name } : {}), + ...(identityVerification ? { identityVerification } : {}), + ...(peerRecordId ? { peerRecordId } : {}), + ...(hostUuid ? { hostUuid } : {}), + ...(clusterUuid ? { clusterUuid } : {}), + ollama_up: booleanValue(obj.ollama_up), + ollama_port: numberValue(obj.ollama_port), + lmstudio_up: booleanValue(obj.lmstudio_up), + lmstudio_port: numberValue(obj.lmstudio_port), + omlx_up: booleanValue(obj.omlx_up), + omlx_port: numberValue(obj.omlx_port), + node_info_up: booleanValue(obj.node_info_up), + node_info_port: numberValue(obj.node_info_port), + tls_enabled: booleanValue(obj.tls_enabled), + mtls_required: booleanValue(obj.mtls_required), + models: parseStringArray(obj.models) + } + return status +} + +function parseManualNodeList(raw: JsonValue | undefined): ManualNodeStatus[] { + const obj = objectValue(raw) + if (!obj) return [] + const rawNodes = Array.isArray(obj.nodes) ? obj.nodes : [] + const result: ManualNodeStatus[] = [] + for (const item of rawNodes) { + if (objectValue(item)?.from_cluster === true) continue + const parsed = parseManualNodeStatus(item) + if (parsed) { + result.push(parsed) + } + } + return result +} + +async function handleManualNodesList(): Promise { + const supervisor = getModularSupervisor() + if (!supervisor.hasProcess('broker')) { + return [] + } + const result = await supervisor.callProcess('broker', 'nodes/list') + return parseManualNodeList(result) +} + +async function handleManualNodesAdd( + payload?: WsInvokeRequest<'nodes:manual-add'> +): Promise { + if (!payload?.address) { + throw new Error('Address is required') + } + const supervisor = getModularSupervisor() + const result = await supervisor.callProcess('broker', 'node/add', { + address: payload.address, + ...(payload.name ? { name: payload.name } : {}), + kind: payload.kind, + ...(payload.tls_port ? { tls_port: payload.tls_port } : {}), + ...(payload.mtls !== undefined ? { mtls: payload.mtls } : {}) + }) + const status = parseManualNodeStatus(result) + if (!status) { + throw new Error('Invalid response from node/add') + } + return status +} + +async function handleManualNodesRemove( + payload?: WsInvokeRequest<'nodes:manual-remove'> +): Promise<{ removed: boolean }> { + if (!payload?.id) { + throw new Error('Node ID is required') + } + const supervisor = getModularSupervisor() + const result = await supervisor.callProcess('broker', 'node/remove', { id: payload.id }) + const obj = objectValue(result) + return { + removed: booleanValue(obj?.removed) + } +} + function telemetryObject(value: JsonValue | undefined): JsonObject { return objectValue(value) ?? {} } @@ -1166,6 +1273,9 @@ const EMPTY_SERVICE_BRIDGE_HANDLERS: BridgeHandlerMap = { 'nodes:remove-member': payload => handleNodeRemoveMember(payload), 'nodes:install-macmon': payload => payload ? getModularSupervisor().installMacmon(payload.nodeId) : 'failed', + 'nodes:manual-list': () => handleManualNodesList(), + 'nodes:manual-add': payload => handleManualNodesAdd(payload), + 'nodes:manual-remove': payload => handleManualNodesRemove(payload), 'discovery:get-nodes': () => getModularBridgeState().getAvailableNodes(), diff --git a/desktop/src/shared/types/manual-nodes.ts b/desktop/src/shared/types/manual-nodes.ts new file mode 100644 index 000000000..a224cdd8b --- /dev/null +++ b/desktop/src/shared/types/manual-nodes.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export type ManualNodeKind = 'pair' | 'engine' + +export interface ManualNodeAddRequest { + address: string + name?: string + kind: ManualNodeKind + tls_port?: number + mtls?: boolean +} + +export interface ManualNodeRemoveRequest { + id: string +} + +export interface ManualNodeStatus { + id: string + address: string + name?: string + kind: ManualNodeKind + identityVerification?: string + peerRecordId?: string + hostUuid?: string + clusterUuid?: string + ollama_up?: boolean + ollama_port?: number + ollama_models?: string[] + lmstudio_up?: boolean + lmstudio_port?: number + lmstudio_models?: string[] + omlx_up?: boolean + omlx_port?: number + omlx_models?: string[] + node_info_up?: boolean + node_info_port?: number + tls_enabled?: boolean + mtls_required?: boolean + models?: string[] +} diff --git a/desktop/src/shared/types/ws-channels.ts b/desktop/src/shared/types/ws-channels.ts index 111a34719..566266aec 100644 --- a/desktop/src/shared/types/ws-channels.ts +++ b/desktop/src/shared/types/ws-channels.ts @@ -48,6 +48,11 @@ import type { ClusterNode, Invite } from '@/shared/types/cluster' +import type { + ManualNodeAddRequest, + ManualNodeRemoveRequest, + ManualNodeStatus +} from '@/shared/types/manual-nodes' import type { TelemetryConfig, TelemetryConfigRequest, @@ -94,6 +99,11 @@ export interface WsInvokeChannelMap { response: MacmonInstallOutcome } + // Manual nodes (relayed to nvpair-manual-nodes via the broker) + 'nodes:manual-list': { request: void; response: ManualNodeStatus[] } + 'nodes:manual-add': { request: ManualNodeAddRequest; response: ManualNodeStatus } + 'nodes:manual-remove': { request: ManualNodeRemoveRequest; response: { removed: boolean } } + // Discovery 'discovery:get-nodes': { request: void; response: AvailableNode[] } diff --git a/desktop/src/ui/api/pair-api.ts b/desktop/src/ui/api/pair-api.ts index 9d93f331b..23ce5b3b6 100644 --- a/desktop/src/ui/api/pair-api.ts +++ b/desktop/src/ui/api/pair-api.ts @@ -20,6 +20,7 @@ import type { import type { Workload, WorkloadLiveProgress } from '@/shared/types/workloads' import type { WorkerStatus } from '@/shared/types/workers' import type { AppInitialSnapshot, ClusterInitialSnapshot } from '@/shared/types/bootstrap' +import type { ManualNodeAddRequest, ManualNodeStatus } from '@/shared/types/manual-nodes' import type { TelemetryConfig, TelemetryConfigRequest, @@ -153,6 +154,15 @@ export interface IMetricsApi { onNodeMetrics(callback: (snapshot: NodeInferenceMetricsSnapshot) => void): () => void } +interface IManualNodesApi { + /** Fetch all tracked manual nodes. */ + list(): Promise + /** Add a manual node with explicit kind ('pair' | 'engine'). */ + add(params: ManualNodeAddRequest): Promise + /** Remove a manual node by id from durable storage. */ + remove(id: string): Promise<{ removed: boolean }> +} + // --------------------------------------------------------------------------- // Composite API // --------------------------------------------------------------------------- @@ -165,6 +175,7 @@ export interface IPairApi { app: IAppApi connection: IConnectionApi nodes: INodesApi + manualNodes: IManualNodesApi cluster: IClusterApi discovery: IDiscoveryApi engines: IEngineApi @@ -202,6 +213,11 @@ export function createPairApi(transport: ServiceTransport): IPairApi { onRemove: cb => transport.subscribePush('nodes:remove', cb), onMembersChanged: cb => transport.subscribePush('nodes:changed', cb) }, + manualNodes: { + list: () => transport.invoke('nodes:manual-list'), + add: params => transport.invoke('nodes:manual-add', params), + remove: id => transport.invoke('nodes:manual-remove', { id }) + }, cluster: { getInitial: () => transport.invoke('cluster:get-initial'), inviteNode: ipAddress => transport.invoke('cluster:invite-node', { ipAddress }), diff --git a/desktop/src/ui/components/AddNodeModal.tsx b/desktop/src/ui/components/AddNodeModal.tsx index c6329bbae..4c1502091 100644 --- a/desktop/src/ui/components/AddNodeModal.tsx +++ b/desktop/src/ui/components/AddNodeModal.tsx @@ -21,9 +21,12 @@ import { nodeRecordKey, nodeVerificationLabel } from '@/shared/utils/node-record-key' +import { InlineErrorBanner } from '@/ui/components/InlineErrorBanner' import { useBlurOnOpen } from '@/ui/hooks/useBlurOnOpen' import { useInvitePairing } from '@/ui/hooks/useInvitePairing' import { useInvitablePeers } from '@/ui/hooks/useInvitablePeers' +import type { ManualNodeKind } from '@/shared/types/manual-nodes' +import getErrorString from '@/shared/utils/get-error-string' interface AddNodeModalProps { open: boolean @@ -32,24 +35,45 @@ interface AddNodeModalProps { export function AddNodeModal({ open, onOpenChange }: AddNodeModalProps) { useBlurOnOpen(open) - const [manualIp, setManualIp] = useState('') + const [kind, setKind] = useState('pair') + const [manualAddress, setManualAddress] = useState('') + const [error, setError] = useState(null) + const [submitting, setSubmitting] = useState(false) const pairing = useInvitePairing() const nodesThatCanBeAdded = useInvitablePeers() const handleOpenChange = useCallback( (next: boolean) => { - setManualIp('') + setManualAddress('') + setKind('pair') + setError(null) + setSubmitting(false) pairing.reset() onOpenChange(next) }, [onOpenChange, pairing] ) - const handleManualInvite = useCallback(() => { - const ip = manualIp.trim() - if (!ip) return - void pairing.start(ip) - }, [manualIp, pairing]) + const handleAddManualNode = useCallback(async () => { + const address = manualAddress.trim() + if (!address) return + setError(null) + setSubmitting(true) + try { + if (kind === 'pair') { + // Save reachability before pairing; failed pairing keeps an unverified entry. + await window.pairApi.manualNodes.add({ address, kind: 'pair' }) + void pairing.start(address) + } else { + await window.pairApi.manualNodes.add({ address, kind: 'engine' }) + handleOpenChange(false) + } + } catch (err) { + setError(getErrorString(err)) + } finally { + setSubmitting(false) + } + }, [manualAddress, kind, pairing, handleOpenChange]) const showPairing = pairing.invite !== null || pairing.error !== null const inviteInFlight = pairing.submitting || pairing.invite?.state === 'pending' @@ -61,7 +85,7 @@ export function AddNodeModal({ open, onOpenChange }: AddNodeModalProps) { handleOpenChange(false)}> Add node - {pairing.submitting && ( + {(pairing.submitting || submitting) && ( )} @@ -77,25 +101,71 @@ export function AddNodeModal({ open, onOpenChange }: AddNodeModalProps) { /> ) : ( <> + {error && } + + + Node type + + + + + {kind === 'engine' && ( + + Manual bypass: PAIR trusts no PAIR claims from this + engine. A PAIR machine’s engine is reachable only if its + owner enabled LAN engine binding. + + )} + + - + { - if (event.key === 'Enter') handleManualInvite() + if (event.key === 'Enter') { + void handleAddManualNode() + } }} - disabled={inviteInFlight} + disabled={inviteInFlight || submitting} /> diff --git a/desktop/src/ui/components/ClusterSettings/ClusterSettings.tsx b/desktop/src/ui/components/ClusterSettings/ClusterSettings.tsx index 671589fd7..a98cd5fc2 100644 --- a/desktop/src/ui/components/ClusterSettings/ClusterSettings.tsx +++ b/desktop/src/ui/components/ClusterSettings/ClusterSettings.tsx @@ -11,6 +11,7 @@ import { useClusterInvitationsStore } from '@/ui/stores/cluster-invitations.stor import { useInvitablePeers } from '@/ui/hooks/useInvitablePeers' import ActiveNode from './ActiveNode' import AvailableNode from './AvailableNode' +import ManualEntriesSection from './ManualEntriesSection' import PendingInviteCard from './PendingInviteCard' import { InlineErrorBanner } from '@/ui/components/InlineErrorBanner' @@ -154,6 +155,8 @@ export default function ClusterSettings() { ))} )} + + ) } diff --git a/desktop/src/ui/components/ClusterSettings/ManualEntriesSection.tsx b/desktop/src/ui/components/ClusterSettings/ManualEntriesSection.tsx new file mode 100644 index 000000000..d1ff8d34b --- /dev/null +++ b/desktop/src/ui/components/ClusterSettings/ManualEntriesSection.tsx @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useCallback, useEffect, useState } from 'react' +import { Button, Flex, Stack, Text } from '@nvidia/foundations-react-core' +import type { ManualNodeStatus } from '@/shared/types/manual-nodes' +import getErrorString from '@/shared/utils/get-error-string' +import { InlineErrorBanner } from '@/ui/components/InlineErrorBanner' +import ManualNodeItem from './ManualNodeItem' + +export default function ManualEntriesSection() { + const [manualNodes, setManualNodes] = useState([]) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + const refresh = useCallback(async () => { + if (!window.pairApi?.manualNodes) return + try { + setLoading(true) + const list = await window.pairApi.manualNodes.list() + setManualNodes(list) + setError(null) + } catch (err) { + setError(getErrorString(err)) + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + void refresh() + }, [refresh]) + + return ( + + + Manual entries + + {loading && ( + + )} + + + {error && } + + {manualNodes.length === 0 ? ( +
+ + No manual entries added yet. Use “Add node” to add a PAIR + machine or bare engine by address. + +
+ ) : ( + manualNodes.map(node => ( + + )) + )} + + ) +} diff --git a/desktop/src/ui/components/ClusterSettings/ManualNodeItem.tsx b/desktop/src/ui/components/ClusterSettings/ManualNodeItem.tsx new file mode 100644 index 000000000..60794e329 --- /dev/null +++ b/desktop/src/ui/components/ClusterSettings/ManualNodeItem.tsx @@ -0,0 +1,109 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { useState } from 'react' +import { Badge, Button, Flex, Stack, Text } from '@nvidia/foundations-react-core' +import type { ManualNodeStatus } from '@/shared/types/manual-nodes' +import { InvitePairingPanel } from '@/ui/components/InvitePairingPanel' +import { useInvitePairing } from '@/ui/hooks/useInvitePairing' +import getErrorString from '@/shared/utils/get-error-string' + +interface ManualNodeItemProps { + node: ManualNodeStatus + onChanged: () => void + onError: (error: string | null) => void +} + +export default function ManualNodeItem({ node, onChanged, onError }: ManualNodeItemProps) { + const [removing, setRemoving] = useState(false) + const pairing = useInvitePairing() + const showPairing = pairing.invite !== null || pairing.error !== null + const inviteInFlight = pairing.submitting || pairing.invite?.state === 'pending' + const isVerified = node.identityVerification === 'verified' + + const handleRemove = async () => { + try { + setRemoving(true) + onError(null) + const result = await window.pairApi.manualNodes.remove(node.id) + if (!result.removed) { + onError(`Failed to remove manual node ${node.id}`) + return + } + onChanged() + } catch (err) { + onError(getErrorString(err)) + } finally { + setRemoving(false) + } + } + + if (showPairing) { + return ( +
+ void pairing.cancel()} + onDone={() => { + pairing.reset() + onChanged() + }} + /> +
+ ) + } + + return ( +
+ + + + + {node.name || node.address} + + + {node.kind === 'pair' ? 'PAIR machine' : 'Engine only'} + + {node.kind === 'pair' ? ( + + {isVerified ? 'Verified' : 'Unverified'} + + ) : ( + + Manual bypass + + )} + + + {node.address} + + + + + {node.kind === 'pair' && !isVerified && ( + + )} + + + +
+ ) +} diff --git a/desktop/tests/modular/manual-nodes-bridge.test.ts b/desktop/tests/modular/manual-nodes-bridge.test.ts new file mode 100644 index 000000000..10afa3433 --- /dev/null +++ b/desktop/tests/modular/manual-nodes-bridge.test.ts @@ -0,0 +1,182 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { getModularSupervisor } from '@/electron/service-bridge/modular-supervisor' +import type { getModularBridgeState } from '@/electron/service-bridge/modular-state' + +type Supervisor = ReturnType +type BridgeState = ReturnType + +const mocks = vi.hoisted(() => ({ + state: { + getSelfId: vi.fn(() => 'self-uuid') + }, + supervisor: { + hasProcess: vi.fn(() => true), + callProcess: vi.fn() + } +})) + +vi.mock('@/electron/service-bridge/modular-supervisor', () => ({ + getModularSupervisor: () => mocks.supervisor +})) +vi.mock('@/electron/service-bridge/modular-state', () => ({ + getModularBridgeState: () => mocks.state, + isProxyEngine: () => false, + isUpstreamUnreachableError: () => false, + parseServiceErrors: () => [] +})) +vi.mock('@/electron/model-hub', () => ({ + getEngineHubModels: vi.fn(), + lookupEngineHubModel: vi.fn() +})) + +import { handleServiceBridgeInvoke } from '@/electron/service-bridge/empty-handlers' + +describe('manual-nodes service bridge handlers', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.supervisor.hasProcess.mockReturnValue(true) + }) + + describe('nodes:manual-list', () => { + it('calls broker nodes/list and parses nodes correctly', async () => { + mocks.supervisor.callProcess.mockResolvedValue({ + nodes: [ + { + id: 'node-1', + address: '192.168.1.50', + name: 'my-peer', + kind: 'pair', + identityVerification: 'verified', + ollama_up: true, + ollama_port: 11434, + models: ['llama3'] + }, + { + id: 'node-2', + address: 'engine.local', + kind: 'engine', + lmstudio_up: true, + lmstudio_port: 1234 + } + ] + }) + + const result = await handleServiceBridgeInvoke('nodes:manual-list', undefined) + + expect(mocks.supervisor.callProcess).toHaveBeenCalledWith('broker', 'nodes/list') + expect(result).toHaveLength(2) + expect(result[0]).toEqual({ + id: 'node-1', + address: '192.168.1.50', + name: 'my-peer', + kind: 'pair', + identityVerification: 'verified', + ollama_up: true, + ollama_port: 11434, + lmstudio_up: false, + lmstudio_port: 0, + omlx_up: false, + omlx_port: 0, + node_info_up: false, + node_info_port: 0, + tls_enabled: false, + mtls_required: false, + models: ['llama3'] + }) + expect(result[1].kind).toBe('engine') + expect(result[1].lmstudio_up).toBe(true) + }) + + it('returns empty array when supervisor has no broker process', async () => { + mocks.supervisor.hasProcess.mockReturnValue(false) + + const result = await handleServiceBridgeInvoke('nodes:manual-list', undefined) + + expect(result).toEqual([]) + expect(mocks.supervisor.callProcess).not.toHaveBeenCalled() + }) + }) + + describe('nodes:manual-add', () => { + it('relays node/add to broker and returns parsed status', async () => { + mocks.supervisor.callProcess.mockResolvedValue({ + id: 'manual:192.168.1.100', + address: '192.168.1.100', + kind: 'pair', + identityVerification: 'unverified' + }) + + const result = await handleServiceBridgeInvoke('nodes:manual-add', { + address: '192.168.1.100', + kind: 'pair' + }) + + expect(mocks.supervisor.callProcess).toHaveBeenCalledWith('broker', 'node/add', { + address: '192.168.1.100', + kind: 'pair' + }) + expect(result.id).toBe('manual:192.168.1.100') + expect(result.kind).toBe('pair') + expect(result.identityVerification).toBe('unverified') + }) + + it('relays engine kind with optional fields', async () => { + mocks.supervisor.callProcess.mockResolvedValue({ + id: 'remote-engine', + address: '10.0.0.5', + name: 'remote-engine', + kind: 'engine' + }) + + const result = await handleServiceBridgeInvoke('nodes:manual-add', { + address: '10.0.0.5', + name: 'remote-engine', + kind: 'engine', + tls_port: 14319, + mtls: true + }) + + expect(mocks.supervisor.callProcess).toHaveBeenCalledWith('broker', 'node/add', { + address: '10.0.0.5', + name: 'remote-engine', + kind: 'engine', + tls_port: 14319, + mtls: true + }) + expect(result.kind).toBe('engine') + }) + + it('throws if address is missing', async () => { + await expect( + handleServiceBridgeInvoke('nodes:manual-add', { + address: '', + kind: 'pair' + }) + ).rejects.toThrow('Address is required') + }) + }) + + describe('nodes:manual-remove', () => { + it('relays node/remove to broker and returns removed boolean', async () => { + mocks.supervisor.callProcess.mockResolvedValue({ removed: true }) + + const result = await handleServiceBridgeInvoke('nodes:manual-remove', { + id: 'manual:192.168.1.100' + }) + + expect(mocks.supervisor.callProcess).toHaveBeenCalledWith('broker', 'node/remove', { + id: 'manual:192.168.1.100' + }) + expect(result).toEqual({ removed: true }) + }) + + it('throws if node id is missing', async () => { + await expect( + handleServiceBridgeInvoke('nodes:manual-remove', { id: '' }) + ).rejects.toThrow('Node ID is required') + }) + }) +}) diff --git a/desktop/tests/modular/manual-nodes-ui.test.ts b/desktop/tests/modular/manual-nodes-ui.test.ts new file mode 100644 index 000000000..219dd3f64 --- /dev/null +++ b/desktop/tests/modular/manual-nodes-ui.test.ts @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// @vitest-environment jsdom + +import { act, createElement } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { AddNodeModal } from '@/ui/components/AddNodeModal' +import ManualNodeItem from '@/ui/components/ClusterSettings/ManualNodeItem' +import ManualEntriesSection from '@/ui/components/ClusterSettings/ManualEntriesSection' +import type { ManualNodeStatus } from '@/shared/types/manual-nodes' + +let container: HTMLDivElement +let root: Root +const api = { + manualNodes: { list: vi.fn(), add: vi.fn(), remove: vi.fn() }, + cluster: { + inviteNode: vi.fn(), + inviteStatus: vi.fn(), + cancelInvite: vi.fn(), + abandonIfSolo: vi.fn() + }, + discovery: { + getNodes: vi.fn().mockResolvedValue([]), + onNodesChanged: vi.fn().mockReturnValue(() => {}) + } +} + +function button(label: string): HTMLButtonElement { + const found = Array.from(container.querySelectorAll('button')).find( + b => b.textContent === label + ) + if (!found) throw new Error(`Missing button ${label}`) + return found +} + +async function address(value: string) { + const input = container.querySelector('input') + if (!input) throw new Error('Missing address input') + await act(async () => { + Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, 'value')?.set?.call( + input, + value + ) + input.dispatchEvent(new Event('input', { bubbles: true })) + }) +} + +const pairNode: ManualNodeStatus = { + id: 'manual:overlay.example', + address: 'overlay.example', + kind: 'pair', + identityVerification: 'unverified' +} +const engineNode: ManualNodeStatus = { + id: 'manual:engine.example', + address: 'engine.example', + kind: 'engine', + identityVerification: 'manual' +} + +describe('manual entries UI', () => { + beforeEach(() => { + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + HTMLDialogElement.prototype.showModal = function showModal() { + this.open = true + } + HTMLDialogElement.prototype.close = function close() { + this.open = false + } + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + vi.stubGlobal('pairApi', api) + api.manualNodes.list.mockResolvedValue([]) + api.manualNodes.add.mockResolvedValue(pairNode) + api.manualNodes.remove.mockResolvedValue({ removed: true }) + api.cluster.inviteNode.mockResolvedValue({ + inviteId: 'invite', + state: 'pending', + pin: '123456' + }) + }) + afterEach(() => { + act(() => root.unmount()) + container.remove() + vi.resetAllMocks() + vi.unstubAllGlobals() + }) + + it('persists PAIR intent before pairing and keeps the hostname as entered', async () => { + let saved: ((status: ManualNodeStatus) => void) | undefined + api.manualNodes.add.mockImplementationOnce( + () => + new Promise(resolve => { + saved = resolve + }) + ) + await act(async () => { + root.render(createElement(AddNodeModal, { open: true, onOpenChange: vi.fn() })) + }) + await address(' Workstation.tail.example ') + await act(async () => button('Invite').click()) + expect(api.manualNodes.add).toHaveBeenCalledWith({ + address: 'Workstation.tail.example', + kind: 'pair' + }) + expect(api.cluster.inviteNode).not.toHaveBeenCalled() + await act(async () => saved?.(pairNode)) + expect(api.cluster.inviteNode).toHaveBeenCalledWith('Workstation.tail.example') + }) + + it('registers an explicitly chosen engine without PIN pairing', async () => { + const onOpenChange = vi.fn() + await act(async () => { + root.render(createElement(AddNodeModal, { open: true, onOpenChange })) + }) + await act(async () => button('Engine only').click()) + expect(container.textContent).toContain('trusts no PAIR claims') + await address('engine.example') + await act(async () => button('Add engine').click()) + expect(api.manualNodes.add).toHaveBeenCalledWith({ + address: 'engine.example', + kind: 'engine' + }) + expect(api.cluster.inviteNode).not.toHaveBeenCalled() + expect(onOpenChange).toHaveBeenCalledWith(false) + }) + + it('does not start pairing when persistence fails', async () => { + api.manualNodes.add.mockRejectedValueOnce(new Error('Address was not saved')) + await act(async () => { + root.render(createElement(AddNodeModal, { open: true, onOpenChange: vi.fn() })) + }) + await address('overlay.example') + await act(async () => button('Invite').click()) + expect(container.textContent).toContain('Address was not saved') + expect(api.cluster.inviteNode).not.toHaveBeenCalled() + }) + + it('keeps the saved PAIR entry when pairing fails', async () => { + api.cluster.inviteNode.mockRejectedValueOnce(new Error('Peer offline')) + await act(async () => { + root.render(createElement(AddNodeModal, { open: true, onOpenChange: vi.fn() })) + }) + await address('overlay.example') + await act(async () => button('Invite').click()) + expect(container.textContent).toContain('Peer offline') + expect(api.manualNodes.add).toHaveBeenCalledOnce() + expect(api.manualNodes.remove).not.toHaveBeenCalled() + }) + + it('lists entries, labels their kind and retries unverified PAIR pairing', async () => { + api.manualNodes.list.mockResolvedValue([pairNode, engineNode]) + await act(async () => root.render(createElement(ManualEntriesSection))) + expect(container.textContent).toContain('overlay.example') + expect(container.textContent).toContain('engine.example') + expect(container.textContent).toContain('Unverified') + expect(container.textContent).toContain('Manual bypass') + await act(async () => button('Retry pairing').click()) + expect(api.cluster.inviteNode).toHaveBeenCalledWith('overlay.example') + }) + + it('removes a manual entry and refreshes the list', async () => { + const onChanged = vi.fn() + await act(async () => + root.render( + createElement(ManualNodeItem, { + node: engineNode, + onChanged, + onError: vi.fn() + }) + ) + ) + await act(async () => button('Remove').click()) + expect(api.manualNodes.remove).toHaveBeenCalledWith(engineNode.id) + expect(onChanged).toHaveBeenCalledOnce() + expect(api.cluster.inviteNode).not.toHaveBeenCalled() + }) +}) diff --git a/docs/architecture.mdx b/docs/architecture.mdx index 6653aa857..bbd2e4030 100644 --- a/docs/architecture.mdx +++ b/docs/architecture.mdx @@ -851,18 +851,19 @@ Some networks block or filter multicast, so discovery is not the only path in. fixed interval. A PAIR peer enters the verified directory only after pinned cluster mTLS proves its current identity and service metadata; public HTTP or operator CA trust cannot substitute for that proof. Its configuration keeps an -independent manual record key until authentication succeeds. There is no +independent `preview:` record key until authentication succeeds. There is no plaintext downgrade for peer hardware, models, engine state or scheduling data. Explicitly user-added bare engines keep the separate manual routing bypass and -never establish a PAIR peer identity or provide scheduling telemetry. Recognizing -a PAIR endpoint prevents a bare-engine downgrade for that configuration's lifetime -in the manual-nodes worker, but this kind is not persisted. After that worker -restarts, a previously added PAIR address can become an unauthenticated plaintext -bare-engine route if no PAIR endpoint answers there, including if another machine -answers instead. Persisting each entry's PAIR-vs-bare-engine kind in -`manual-nodes.json` is a follow-up, alongside desktop handling of the additive -`peerRecordId` and `identityVerification` fields without keying previews by a -claimed `hostUuid`. +never establish a PAIR peer identity or provide scheduling telemetry. The broker +is the single persistence owner of `configs/manual-nodes.json`, including each +entry's explicitly chosen `pair` or `engine` kind. A `pair` entry never becomes an +engine bypass, even when offline or replayed after a worker/broker restart. +Historical entries without kind migrate on disk to `pair`; users of a former bare +engine must remove and re-add it as **Engine only**. A successful cluster removal +also forgets associated manual PAIR addresses. Verified associations retained in +the file help removal while a peer is offline; they are not routing proofs. +Desktop Add node and the TUI Manual tab register kind before PIN pairing, retaining +an unverified address if pairing fails. See [Routed networks](remote-networks.mdx). ## Observability and Distributed Tracing diff --git a/docs/remote-networks.mdx b/docs/remote-networks.mdx new file mode 100644 index 000000000..c4553356c --- /dev/null +++ b/docs/remote-networks.mdx @@ -0,0 +1,77 @@ +{/* +SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 +*/} + +# PAIR across routed networks + +mDNS is link-local. Tailscale, WireGuard and other routed links can carry PAIR +without multicast discovery, but PAIR does not enumerate their peers for you. + +## Add an address + +In **Add node**, enter a hostname (for example a MagicDNS name) or IP address, +without a scheme or port, and choose: + +- **PAIR machine**: saves a persistent PAIR entry, then starts the existing PIN + pairing exchange at that address. Read the PIN to the joining machine. Failed + or cancelled pairing leaves an unverified entry, not a routable engine. Retry + pairing from **Settings → Cluster → Manual entries**. Pairing by address keeps + the overlay address after trust is established. +- **Engine only**: explicitly enables a labelled manual bypass for a bare Ollama, + LM Studio or oMLX engine. PAIR trusts no PAIR identity or telemetry it reports. + A PAIR machine's engine is reachable this way only if its owner enabled LAN + engine binding. Do not use this as a substitute for pairing a PAIR machine. + +Hostnames are stored as entered, not replaced with the first resolved IP. Fresh +dials re-resolve names; live pinned connections may be reused. DNS changes can +recover without re-adding an entry. +An IP address cannot recover from renumbering by itself. Service callers can set +`tls_port` in `node/add` when pinned node-info is not on its default port. + +Manual entries can be listed/refreshed and removed in Cluster settings. Forgetting +an address alone does not revoke cluster membership. Successfully removing a peer +from the cluster also forgets its associated manual PAIR entries. Engine-only +entries are independent. The TUI **Manual** tab provides the same two choices +(`a` PAIR, `e` engine), `p` to retry pairing, and `r` to forget an address. + +## Persistence and migration + +The broker is the only writer/replayer of `configs/manual-nodes.json` under the +PAIR user-data directory, including for headless and TUI use. Each entry stores +`kind: "pair"` or `kind: "engine"`. Worker/broker restarts do not discard intent. + +Historical entries without kind are rewritten conservatively as `pair`. There +is no reliable historical evidence of consent to bypass peer authentication. +A previously configured bare engine therefore stops direct routing until the +user removes and re-adds it as **Engine only**. An offline PAIR entry never becomes +a bare-engine route, even if some other engine now answers at the same address. + +## Reachability and troubleshooting + +Allow only the participating machines through your overlay/network policy. The +usual service ports are: + +| Port | Purpose | +| --- | --- | +| `14321` | PIN pairing / cluster management | +| `14318` | Pinned mTLS node-info proof and telemetry | +| `14322` / `14323` | Pinned engine inventory / engine control | +| `14319` / `14320` | Cluster errors / workload relay | +| `11434` / `1234` / `1236` | Ollama / LM Studio / oMLX inference, as advertised | + +PAIR service ports may be moved by configuration/port negotiation. After proof, +use the peer's authenticated service map, not this table as a fixed promise. +Bare-engine entries support the fixed engine ports only. Both PAIR machines need +the authenticated service metadata contract; older peers remain unverified until +upgraded. Operator-CA trust is not a substitute for the cluster certificate pin. + +If an entry stays **Unverified**, check DNS resolution, overlay reachability, +node-info and pairing ports, the PIN exchange, and whether both machines are +upgraded. Healthy plaintext engine ports do not prove a PAIR peer. Verified +manual peers already use the same pinned-mTLS inventory and routing path as +verified discovered peers; no weaker overlay transport is introduced. + +Local unit tests are not a live multi-machine deployment check. Neither an +unverified preview nor an entry's stored prior peer association can authorize +routing, identity, scheduling inventory or telemetry. diff --git a/docs/terminal-interface.mdx b/docs/terminal-interface.mdx index 0358d5292..7906126e0 100644 --- a/docs/terminal-interface.mdx +++ b/docs/terminal-interface.mdx @@ -164,7 +164,7 @@ Tab switching and `q` do not work until you do. | 5 | **Workloads** | Live inference workloads: ID, model, engine, origin, destination, state, and age | | 6 | **Engines** | Selected node's engines: installed, running, healthy, port, and model inventory | | 7 | **Cluster** | This node's identity, cluster membership, and pairing | -| 8 | **Manual** | Nodes you added by address, with reachability | +| 8 | **Manual** | Persistent addresses, PAIR/engine kind, verification and reachability | | 9 | **Settings** | Node settings: force ports, cluster auto-sync, and cluster ID and name | | 10 | **Logs** | Service log output, with live log-level control | @@ -229,9 +229,16 @@ Press `d` instead to decline. After pairing, the peer appears under **Members**. | `i` | Invite by address | Cluster | | `a` | Accept an inbound invitation | Cluster | | `d` | Decline an inbound invitation | Cluster | -| `r` | Confirm removal of the selected member | Cluster | +| `r` | Confirm removal of the selected member; successful unpair also forgets its manual PAIR addresses | Cluster | | `L` | Confirm leaving the cluster (capital L) | Cluster | +On **Manual**, `a` adds a **PAIR machine** by hostname or IP and persists its +address before starting PIN pairing; `p` retries pairing the selected PAIR entry. +Unverified PAIR entries never route. `e` explicitly adds **Engine only**, a labelled +manual bypass that trusts no PAIR claims; a PAIR owner's engine needs LAN engine +binding enabled. `r` forgets a manual entry without removing cluster membership. +Old entries without kind migrate to PAIR; remove/re-add a bare engine with `e`. + Only pair when you trust both machines and the network. The PIN is a short-lived bootstrap code, not a durable credential. Refer to the [security policy](../SECURITY.md). diff --git a/services/lmstudio-proxy/manual_overlay_test.go b/services/lmstudio-proxy/manual_overlay_test.go new file mode 100644 index 000000000..c3b550ed0 --- /dev/null +++ b/services/lmstudio-proxy/manual_overlay_test.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" + "path/filepath" + "testing" +) + +// A broker-proven peer uses its manually configured overlay hostname, not a +// LAN address from discovery, and still requires a live exact certificate pin. +func TestManualOverlayHostnameKeepsPinnedTransport(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, dir, "cluster", "self", "peer") + n, ok := subscribedToNode(noderec.DirectoryNode{ + HostUUID: "peer", ClusterUUID: "peer", Name: "peer", IP: "Workstation.tail.example", + IPs: []string{"Workstation.tail.example"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceLMStudio: {Port: 1234}}, + }) + if !ok { + t.Fatal("manual address lost in relay projection") + } + disc := NewDiscovery() + disc.SetSubscribed([]Node{n}) + p := testProxy(disc, 1235) + p.mesh = clustertrust.Open(dir) + got := p.resolveCandidates("") + if len(got) != 1 || got[0].url.Scheme != "https" || got[0].url.Host != "Workstation.tail.example:1234" || got[0].peerUUID != "peer" { + t.Fatalf("overlay candidates = %+v", got) + } + clustertrusttest.RemovePeerPin(t, dir, "peer") + if got := p.resolveCandidates(""); len(got) != 0 { + t.Fatalf("removed peer still routable: %+v", got) + } +} diff --git a/services/nvpair-cluster-manager/README.md b/services/nvpair-cluster-manager/README.md index ef29dfa0a..f360a5f4c 100644 --- a/services/nvpair-cluster-manager/README.md +++ b/services/nvpair-cluster-manager/README.md @@ -181,6 +181,20 @@ set your per-request timeout above the pairing timeout for these two. No `id`; never carry an error; handle them off your read loop. +`cluster:pairing-completed` is broker-internal: after the inviter commits the +authenticated peer pin/member, and before publishing paired invite state, it +emits `{ address, peerUuid, clusterId, admissionEpoch }` inside the same +`rosterMu` transaction as the pin/member commit, ordering it before removal. This +is a fire-and-forget notification: it never waits for a broker reply. The broker's +independent read pump consumes it without a worker round trip or mutation-lock +wait, even while the request caller awaits removal. `address` is the original +dial name, not the +observed numeric source recorded in the roster. The broker persists its manual +PAIR aliases' cleanup-only association, allowing unpair to forget hostname +entries before any node-info inventory proof. Desktop and TUI never receive this +event; it grants no identity verification, inventory, telemetry or routing +authority. Failed/abandoned pairings emit no completion association. + ```jsonc // A pairing arrived and this node was invited. Prompt the user for the PIN, then respond-to-invite. {"jsonrpc":"2.0","method":"cluster:invite-received","params": } diff --git a/services/nvpair-cluster-manager/httpserver.go b/services/nvpair-cluster-manager/httpserver.go index 41ed0cf44..c916ae512 100644 --- a/services/nvpair-cluster-manager/httpserver.go +++ b/services/nvpair-cluster-manager/httpserver.go @@ -758,6 +758,21 @@ func (m *Manager) commitPairing(sess *pairingSession, pi *PairingInfo, cert *x50 log.Printf("pairing: clear superseded removal proof for %s: %v", admitted.NodeUUID, clearErr) } sess.peerPairing = &admitted + // This fire-and-forget write is ordered with the roster commit/removal. + // It never waits for an acknowledgement or a broker RPC: the broker's + // independent worker read pump consumes it while its caller may be + // waiting for nodes:remove, and the consumer takes no mutation lock or + // worker round trip. + if address := hostOnly(sess.joinerAddr); address != "" { + if notifyErr := m.codec.Notify("cluster:pairing-completed", struct { + Address string `json:"address"` + PeerUUID string `json:"peerUuid"` + ClusterID string `json:"clusterId"` + AdmissionEpoch uint64 `json:"admissionEpoch"` + }{address, admitted.NodeUUID, admitted.ClusterID, admitted.AdmissionEpoch}); notifyErr != nil { + log.Printf("pairing cleanup association notification failed: %v", notifyErr) + } + } committed = true }) return diff --git a/services/nvpair-cluster-manager/manual_association_test.go b/services/nvpair-cluster-manager/manual_association_test.go new file mode 100644 index 000000000..beba50890 --- /dev/null +++ b/services/nvpair-cluster-manager/manual_association_test.go @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "io" + "testing" +) + +func TestPairingCompletionPublishesOriginalHostnameForCleanup(t *testing.T) { + for _, commit := range []bool{true, false} { + t.Run(map[bool]string{true: "committed", false: "abandoned"}[commit], func(t *testing.T) { + m := newTestManager(t) + peer := newPinFixture(t, "peer") + pi, cert := pairingInfoFor(t, peer) + pi.Addr = "192.0.2.50:14321" // observed source, not the entered hostname + const inviteID = "manual-hostname" + m.putInvite(&Invite{InviteID: inviteID, ClusterID: "cluster-1", State: inviteStatePending}) + sess := putInviterSession(t, m, inviteID, "cluster-1") + sess.joinerAddr = "Workstation.tail.example:14321" + if !commit { + m.setClusterIdentity("", "") + } + var out bytes.Buffer + m.codec = NewCodec(&out) + err := m.finalizePairing(inviteID, sess, pi, cert) + if commit && err != nil { + t.Fatal(err) + } + if !commit && err == nil { + t.Fatal("abandoned pairing committed") + } + found := false + decoder := json.NewDecoder(&out) + for { + var event struct { + Method string `json:"method"` + Params struct { + Address string `json:"address"` + PeerUUID string `json:"peerUuid"` + ClusterID string `json:"clusterId"` + AdmissionEpoch uint64 `json:"admissionEpoch"` + } `json:"params"` + } + err := decoder.Decode(&event) + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if event.Method != "cluster:pairing-completed" { + continue + } + found = true + if event.Params.Address != "Workstation.tail.example" || event.Params.PeerUUID != peer.uuid || event.Params.ClusterID != "cluster-1" || event.Params.AdmissionEpoch != 1 { + t.Fatalf("cleanup association lost original address or authenticated peer: %+v", event.Params) + } + } + if found != commit { + t.Fatalf("cleanup association emitted=%v after committed=%v", found, commit) + } + }) + } +} diff --git a/services/nvpair-cluster-manager/pairing_publication_test.go b/services/nvpair-cluster-manager/pairing_publication_test.go new file mode 100644 index 000000000..94f50f103 --- /dev/null +++ b/services/nvpair-cluster-manager/pairing_publication_test.go @@ -0,0 +1,115 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "io" + "sync" + "testing" + "time" +) + +type pairingPublicationWriter struct { + manager *Manager + entered chan bool + release chan struct{} + mu sync.Mutex + frames []Message +} + +func (w *pairingPublicationWriter) Read([]byte) (int, error) { return 0, io.EOF } +func (w *pairingPublicationWriter) Write(p []byte) (int, error) { + var msg Message + if err := json.Unmarshal(p, &msg); err != nil { + return 0, err + } + if msg.Method == "cluster:pairing-completed" { + protected := !w.manager.rosterMu.TryLock() + if !protected { + w.manager.rosterMu.Unlock() + } + w.entered <- protected + <-w.release + } + w.mu.Lock() + w.frames = append(w.frames, msg) + w.mu.Unlock() + return len(p), nil +} + +// The real completion and removal execute concurrently. The publication seam +// observes the composition lock, and channels order the interleaving: no sleeps +// or assumption that the broker happened to receive the completion first. +func TestPairingCompletionPublicationOrdersConcurrentRemoval(t *testing.T) { + m := newTestManager(t) + peer := newPinFixture(t, "peer") + pi, cert := pairingInfoFor(t, peer) + pi.Addr = "" // no network removal leg is needed for the composition transaction + m.putInvite(&Invite{InviteID: "ordered", ClusterID: "cluster-1", State: inviteStatePending}) + sess := putInviterSession(t, m, "ordered", "cluster-1") + sess.joinerAddr = "overlay.example:14321" + writer := &pairingPublicationWriter{manager: m, entered: make(chan bool, 1), release: make(chan struct{})} + m.codec = NewCodec(writer) + var once sync.Once + unblock := func() { once.Do(func() { close(writer.release) }) } + t.Cleanup(unblock) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + completed := make(chan error, 1) + go func() { completed <- m.finalizePairing("ordered", sess, pi, cert) }() + select { + case protected := <-writer.entered: + if !protected { + t.Error("completion publication is outside the roster commit/removal boundary") + } + case <-ctx.Done(): + t.Fatal("completion never reached publication") + } + started, removed := make(chan struct{}), make(chan struct{}) + go func() { + close(started) + id := json.RawMessage("99") + params, _ := json.Marshal(map[string]string{"nodeUuid": peer.uuid}) + m.handleNodesRemove(&Message{JSONRPC: "2.0", ID: &id, Method: "nodes:remove", Params: params}) + close(removed) + }() + <-started + unblock() + select { + case err := <-completed: + if err != nil { + t.Fatal(err) + } + case <-ctx.Done(): + t.Fatal("completion stalled") + } + select { + case <-removed: + case <-ctx.Done(): + t.Fatal("removal stalled") + } + writer.mu.Lock() + defer writer.mu.Unlock() + completionIndex, removalIndex := -1, -1 + for i, msg := range writer.frames { + if msg.Method == "cluster:pairing-completed" { + completionIndex = i + } + if msg.ID != nil && string(*msg.ID) == "99" { + var result struct { + Removed bool `json:"removed"` + } + _ = json.Unmarshal(msg.Result, &result) + if msg.Error != nil || !result.Removed { + t.Fatalf("real removal failed: %+v", msg) + } + removalIndex = i + } + } + if completionIndex < 0 || removalIndex <= completionIndex { + t.Fatalf("completion/removal publication order: %d, %d", completionIndex, removalIndex) + } +} diff --git a/services/nvpair-manual-nodes/README.md b/services/nvpair-manual-nodes/README.md index 76d11709d..4358b7eec 100644 --- a/services/nvpair-manual-nodes/README.md +++ b/services/nvpair-manual-nodes/README.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # nvpair-manual-nodes -A Go service for managing manually configured nodes on networks where mDNS discovery is unavailable. Accepts node addresses via JSON-RPC, probes each for Ollama, LM Studio, and node-info, and emits status events. +A Go service for probing explicitly configured PAIR machines or bare engines on networks where mDNS is unavailable. The broker owns persistence; this worker owns live probe state. See [spec.md](spec.md) for the normative trust, intent and replay contract. Node-info JSON decoding reads at most 1 MiB, and model-list decoding at most 16 MiB. A JSON value that extends past its limit follows the existing probe or @@ -84,7 +84,7 @@ Emitted when a manually added node has been probed and its initial status determ PAIR peers must first prove identity through pinned cluster mTLS node-info on `:14318` (or the configured `tls_port`). Matching certificate, response and broker-owned `peerMetadata` identities are required. Only authenticated metadata supplies service ports; models and engine state are then read over pinned mTLS with no HTTP or operator-CA downgrade. Completed reads recheck the exact current pin and admission-bound `peerRecordId`. -Explicitly user-added bare engines retain independent manual routes: Ollama on `:11434`, LM Studio on `:1234`, and oMLX on `:1236`. They remain labelled `identityVerification: "manual"`, never acquire peer trust, and provide no hardware scheduling telemetry. Recognized PAIR configurations that cannot authenticate are labelled `unverified`, not silently downgraded into this bypass. The broker namespaces manual configurations separately from authenticated host UUIDs. Recognition lasts only for the configuration's lifetime in this worker: kind is not yet persisted in `manual-nodes.json`. After a worker restart, an address previously added for a PAIR peer can be routed as an unauthenticated plaintext bare engine if no PAIR endpoint answers there (including if another machine answers instead). Persisting PAIR-vs-bare-engine intent is follow-up work. +Explicitly user-added bare engines retain independent manual routes: Ollama on `:11434`, LM Studio on `:1234`, and oMLX on `:1236`. They remain labelled `identityVerification: "manual"`, never acquire peer trust, and provide no hardware scheduling telemetry. Each entry stores `kind: "pair"` or `kind: "engine"`. PAIR entries that cannot authenticate are `unverified` previews keyed `preview:`, never engine bypasses, including after worker/broker restarts or when no PAIR endpoint answers. Only an explicit `engine` choice creates a `manual:` bypass; it reads no PAIR identity, hardware, inventory or telemetry claims. The broker atomically persists kind in `configs/manual-nodes.json` and replays entries into replacement workers. Historical entries without kind are explicitly migrated on disk to `pair`: a former bare engine must be removed and re-added as **Engine only** to resume direct routing. Unknown kinds fail closed. `engineNative` is the remote engine's own report of itself — the counters, the rates and the active models the node's engine-manager publishes at `GET /v1/engine-stats`. It is read over the same transport as the model list beside it, because it is the same kind of data: a peer's engine counters are cluster data, so a clustered peer is read over cluster mTLS pinned to the principal the node advertises, and there is no weaker fallback for a peer. The field is absent when the node publishes no view — an engine with no admin API, an engine that could not be reached, or a node this one cannot read. A node whose view could not be re-read keeps the view it had (the last-good rule the node-scanner follows for a discovered node). It is read on a slower cadence than the probe (once a minute), because the engine's counters move slowly and every reading is a fresh observation. @@ -134,15 +134,16 @@ see [Worker control health](#worker-control-health) for its limited meaning. Add a node by address. The manager immediately probes it and emits a `node/discovered` event. -A hostname is preferred over an IP literal: probe clients disable keep-alives specifically so every probe re-resolves the name, which lets a node that gets a new address recover on its own. An IP-literal entry is dead once the device is renumbered. Supply the address on its own — a `host:port` string is not parsed, because ports are appended to it, so such an entry reads permanently down. +A hostname is preferred over an IP literal: fresh dials re-resolve the name, allowing recovery after renumbering. Direct engine probes disable keep-alives; authenticated clients may reuse a live pinned connection. An IP-literal entry cannot follow renumbering. The address is stored as entered. Supply it without a scheme or port; malformed addresses and `host:port` receive `-32602` with an actionable message. IPv6 literals are accepted without brackets. ```json -{"jsonrpc":"2.0","id":1,"method":"node/add","params":{"address":"10.0.1.50","name":"my-server"}} +{"jsonrpc":"2.0","id":1,"method":"node/add","params":{"address":"10.0.1.50","name":"my-server","kind":"pair"}} ``` | Param | Required | Description | |---|---|---| -| `address` | Yes | IP address or hostname of the node, with no port | +| `address` | Yes | IP address or hostname of the node, with no scheme or port; preserved as entered | +| `kind` | Choose explicitly | `pair` or `engine`; omission conservatively means `pair`, never an engine bypass | | `name` | No | Friendly name (used as node ID; defaults to `manual:
`) | | `tls_port` | No | Probe pinned cluster mTLS node-info on this port instead of the primary `14318`. Echoed back as `tls_enabled`; no plaintext downgrade | | `mtls` | No | Stored and echoed back as `mtls_required`. The probe transport itself is chosen by `tls_port` and live cluster membership, so this field records intent rather than driving it | @@ -159,7 +160,7 @@ Remove a previously added manual node. ### `nodes/list` -Returns all currently tracked manual nodes with their latest probe status. +Returns `{ nodes: ManualNodeStatus[] }` with each entry's `kind`, address, name, ID, `identityVerification` and probe status. Automatically tracked cluster addresses carry `from_cluster: true`; manual-entry management surfaces omit those rows. Removing a cluster peer via the broker also forgets its associated persisted PAIR addresses only after successful unpair. ```json {"jsonrpc":"2.0","id":3,"method":"nodes/list"} @@ -204,9 +205,9 @@ Each manual node is probed every 10 seconds, with a 3-second timeout per leg, fo - **LM Studio** on port 1234: `GET /v1/models`, which doubles as the liveness check and the model list - **Node Info** on port 14318, or `tls_port`: pinned cluster mTLS hardware inventory, matching identity and authenticated service metadata (`GET /v1/node-info`). Public responses cannot establish identity or scheduling measurements. -A node can have any combination of these, or none if the target is unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. +PAIR entries use only pinned cluster mTLS node-info and authenticated engine-manager inventory. Engine entries probe Ollama, LM Studio and oMLX (`:1236`) directly and never read node-info. A node can have any combination of eligible engines, or none if unreachable. Status changes trigger `node/updated` events. Because change detection compares CPU, memory, and GPU values, a node running node-info emits a `node/updated` on most probe cycles as utilization moves. -The three engine ports are compiled in: only the node-info leg's port can be moved, via `tls_port`. A remote engine on a non-default port is not discovered. +Bare-engine ports are fixed (`11434`, `1234`, `1236`). PAIR engine ports come only from authenticated broker-owned service metadata; `tls_port` can override its node-info port. Unverified peers cannot supply ports or model inventory. ## Shutdown diff --git a/services/nvpair-manual-nodes/entry.go b/services/nvpair-manual-nodes/entry.go new file mode 100644 index 000000000..19447dc9e --- /dev/null +++ b/services/nvpair-manual-nodes/entry.go @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "fmt" + "net" + "strings" +) + +// Missing historical intent is not consent to a plaintext engine bypass. +func validateManualEntry(entry *ManualEntry) error { + if entry.Kind == "" { + entry.Kind = "pair" + } + if entry.Kind != "pair" && entry.Kind != "engine" { + return fmt.Errorf("kind must be pair or engine") + } + if entry.Address == "" { + return fmt.Errorf("address is required") + } + if entry.TLSPort < 0 || entry.TLSPort > 65535 { + return fmt.Errorf("tls_port must be between 0 and 65535") + } + if net.ParseIP(entry.Address) != nil { + return nil + } + if strings.ContainsAny(entry.Address, ":/\\?#@[] \t\r\n") { + return fmt.Errorf("enter an IP address or hostname without a scheme or port; use tls_port for a PAIR node-info port") + } + host := strings.TrimSuffix(entry.Address, ".") + if len(host) == 0 || len(host) > 253 { + return fmt.Errorf("invalid hostname") + } + for _, label := range strings.Split(host, ".") { + if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return fmt.Errorf("invalid hostname") + } + for _, c := range label { + if !(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-') { + return fmt.Errorf("invalid hostname") + } + } + } + return nil +} diff --git a/services/nvpair-manual-nodes/kind_test.go b/services/nvpair-manual-nodes/kind_test.go new file mode 100644 index 000000000..3ca7800b8 --- /dev/null +++ b/services/nvpair-manual-nodes/kind_test.go @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net/http" + "testing" +) + +// Replaying the durable configuration into a fresh worker must not infer intent +// from whichever service happens to answer at that address after a restart. +func TestPairKindSurvivesWorkerRestartWithoutPairEndpoint(t *testing.T) { + configuration := []byte(`{"address":"overlay.example","kind":"pair"}`) + for restart := 0; restart < 2; restart++ { + m, rw, rt := newTestManager(t) + var entry ManualEntry + if err := json.Unmarshal(configuration, &entry); err != nil { + t.Fatal(err) + } + rt.set(http.MethodGet, "overlay.example:11434", "/", func(*http.Request) (*http.Response, error) { return httpJSON(200, `{}`) }) + rt.set(http.MethodGet, "overlay.example:11434", "/api/tags", func(*http.Request) (*http.Response, error) { + return httpJSON(200, `{"models":[{"name":"wrong-machine"}]}`) + }) + m.addNode(entry) + got := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/discovered"))) + if got.IdentityVerification != "unverified" || got.OllamaUp || got.LMStudioUp || got.OMLXUp || len(got.Models) != 0 || got.PeerRecordID != "" { + t.Fatalf("restart %d downgraded PAIR configuration to bare engine: %+v", restart, got) + } + } +} + +func TestPairHostnameResolvesWithoutChangingConfiguration(t *testing.T) { + m, rw, rt := newTestManager(t) + peer := newInventoryPeer(t, m, rt, "overlay", "peer") + configureHealthyNode(rt, "overlay", []string{"model"}, sampleInfo()) + peer.entry.Address = "localhost" + peer.entry.Kind = "pair" + m.addNode(peer.entry) + got := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/discovered"))) + if got.Address != "localhost" || got.IdentityVerification != "verified" || got.HostUUID != "peer" || len(got.Models) == 0 { + t.Fatalf("hostname did not resolve over pinned mTLS: %+v", got) + } +} + +func TestManualAddressValidationPreservesHostnames(t *testing.T) { + for _, address := range []string{"Workstation.tail.example", "100.64.1.2", "2001:db8::1", "node.local."} { + e := ManualEntry{Address: address, Kind: "pair"} + if err := validateManualEntry(&e); err != nil || e.Address != address { + t.Errorf("address %q: %+v %v", address, e, err) + } + } + for _, address := range []string{"host:14318", "https://host", "host/path", "bad host", "-bad.example", ""} { + e := ManualEntry{Address: address, Kind: "pair"} + if err := validateManualEntry(&e); err == nil { + t.Errorf("accepted malformed address %q", address) + } + } + e := ManualEntry{Address: "overlay.example"} + if err := validateManualEntry(&e); err != nil || e.Kind != "pair" { + t.Fatalf("missing intent must be pair: %+v %v", e, err) + } +} + +func TestEngineKindIgnoresPairClaims(t *testing.T) { + m, rw, rt := newTestManager(t) + var entry ManualEntry + if err := json.Unmarshal([]byte(`{"address":"engine.example","kind":"engine"}`), &entry); err != nil { + t.Fatal(err) + } + rt.set(http.MethodGet, "engine.example:14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { + t.Error("explicit engine must not read PAIR claims") + return httpJSON(200, `{"hostUuid":"claim"}`) + }) + rt.set(http.MethodGet, "engine.example:11434", "/", func(*http.Request) (*http.Response, error) { return httpJSON(200, `{}`) }) + rt.set(http.MethodGet, "engine.example:11434", "/api/tags", func(*http.Request) (*http.Response, error) { return httpJSON(200, `{"models":[{"name":"model"}]}`) }) + m.addNode(entry) + got := decodeParams[ManualNodeStatus](t, readCaptureUntil(t, rw, methodIs("node/discovered"))) + if got.IdentityVerification != "manual" || !got.OllamaUp || got.HostUUID != "" || got.ClusterUUID != "" || got.PeerRecordID != "" || got.NodeInfoUp || got.TelemetryValid { + t.Fatalf("engine intent not respected: %+v", got) + } +} diff --git a/services/nvpair-manual-nodes/manager.go b/services/nvpair-manual-nodes/manager.go index 0769ef042..e7f3c3bef 100644 --- a/services/nvpair-manual-nodes/manager.go +++ b/services/nvpair-manual-nodes/manager.go @@ -162,15 +162,11 @@ type NodeInfoResponse struct { ClusterUUID *string `json:"clusterUuid,omitempty"` } -// ManualEntry is the user-supplied identity of a manually added -// node. Address + Name are the historical fields. TLSPort is an -// optional hint that node-info on this host listens for HTTPS on -// the given port (typically 14319 with our default node-info -// flags); a zero TLSPort means "probe plain HTTP on 14318" — the -// historical behavior. MTLS is informational only on the entry -// (we don't probe differently), but it's persisted so the UI can -// show the operator the configuration they set. +// ManualEntry stores user intent, not endpoint-derived classification. Missing +// historical kind conservatively means pair. TLSPort overrides the pinned mTLS +// node-info port (default 14318); MTLS is retained as operator display metadata. type ManualEntry struct { + Kind string `json:"kind"` Address string `json:"address"` Name string `json:"name"` TLSPort int `json:"tls_port,omitempty"` @@ -185,6 +181,8 @@ type ManualEntry struct { // 14318 otherwise) so existing UI that displays the port stays // correct. type ManualNodeStatus struct { + Kind string `json:"kind"` + FromCluster bool `json:"from_cluster,omitempty"` IdentityVerification string `json:"identityVerification,omitempty"` PeerRecordID string `json:"peerRecordId,omitempty"` PeerMetadata *noderec.PeerMetadataParams `json:"peerMetadata,omitempty"` @@ -245,7 +243,6 @@ type trackedNode struct { entry ManualEntry status ManualNodeStatus fromCluster bool - peerSeen bool // consecutiveFails counts back-to-back probes where neither // service answered (OllamaUp && NodeInfoUp both false). Reset @@ -571,7 +568,7 @@ func (m *Manager) syncClusterMembers() { } m.mu.Lock() - var toProbe []ManualEntry + var toProbe []*trackedNode var toRemove []string for id, tn := range m.nodes { @@ -601,6 +598,9 @@ func (m *Manager) syncClusterMembers() { for _, member := range activeMembers { var existing *trackedNode for _, tn := range m.nodes { + if tn.entry.Kind == "engine" { + continue + } if (tn.status.HostUUID != "" && tn.status.HostUUID == member.NodeUUID) || tn.entry.Address == member.IPAddress { existing = tn @@ -616,18 +616,25 @@ func (m *Manager) syncClusterMembers() { } entry := ManualEntry{ + Kind: "pair", Address: member.IPAddress, Name: member.Name, } id := nodeID(entry) + if _, occupied := m.nodes[id]; occupied { + continue + } status := ManualNodeStatus{ - ID: id, - Name: entry.Name, - Address: entry.Address, - HostUUID: member.NodeUUID, - ClusterUUID: member.NodeUUID, - OllamaPort: 11434, - NodeInfoPort: 14318, + Kind: "pair", + FromCluster: true, + IdentityVerification: "unverified", + ID: id, + Name: entry.Name, + Address: entry.Address, + HostUUID: member.NodeUUID, + ClusterUUID: member.NodeUUID, + OllamaPort: 11434, + NodeInfoPort: 14318, } tn := &trackedNode{ entry: entry, @@ -636,29 +643,24 @@ func (m *Manager) syncClusterMembers() { } m.nodes[id] = tn m.startTelemetryLoopLocked(id, tn) - toProbe = append(toProbe, entry) + toProbe = append(toProbe, tn) } - m.mu.Unlock() for _, id := range toRemove { m.codec.Notify("node/removed", ManualNodeStatus{ID: id}) _ = m.codec.Notify("errors:clear", errors.ClearParams{ID: probeFailedID(id)}) } - for _, entry := range toProbe { - go func(e ManualEntry) { - m.probeNode(e) - id := nodeID(e) - m.mu.RLock() - tn, ok := m.nodes[id] - if !ok { - m.mu.RUnlock() - return + m.mu.Unlock() + for _, tracked := range toProbe { + go func(tn *trackedNode) { + m.probeNode(tn.entry) + m.mu.Lock() + defer m.mu.Unlock() + if current := m.nodes[nodeID(tn.entry)]; current == tn { + m.codec.Notify("node/discovered", tn.status) } - st := tn.status - m.mu.RUnlock() - m.codec.Notify("node/discovered", st) - }(entry) + }(tracked) } } @@ -689,24 +691,25 @@ func (m *Manager) probeNode(entry ManualEntry) { return } prior := tracked.status - peerSeen := tracked.peerSeen || tracked.fromCluster || prior.PeerRecordID != "" + fromCluster := tracked.fromCluster m.mu.RUnlock() // Use the request start as the age anchor so transport time cannot make a // remotely sampled observation look fresher than it is. nodeInfoReceived := m.clockNow() - info, nodeInfoPort, nodeInfoUp := m.authenticatedNodeInfo(m.ctx, entry, prior.ClusterUUID) + var info NodeInfoResponse + nodeInfoPort := 14318 + if entry.TLSPort > 0 { + nodeInfoPort = entry.TLSPort + } + nodeInfoUp := false clusterUUID := "" emPort := 0 + if entry.Kind != "engine" { + info, nodeInfoPort, nodeInfoUp = m.authenticatedNodeInfo(m.ctx, entry, prior.ClusterUUID) + } if nodeInfoUp { clusterUUID = *info.ClusterUUID emPort = info.PeerMetadata.Services[noderec.ServiceEngineManager] - peerSeen = true - } - if !peerSeen && entry.TLSPort == 0 && !entry.MTLS { - // Public PAIR presence can only deny the bare-engine bypass, never - // establish an identity or supply service inventory. - publicUp, _ := m.probeNodeInfo(m.client, "http", addr, 14318) - peerSeen = publicUp } emUp, emRes, engineNative := m.probePeerEngineManager(m.ctx, addr, clusterUUID, info.PeerRecordID, emPort, m.engineNativeDue(id)) @@ -714,10 +717,9 @@ func (m *Manager) probeNode(entry ManualEntry) { finalModelsByEngine := emRes.ModelsByEngine finalLoadedByEngine := emRes.LoadedByEngine - // Plaintext engine probes are only valid for unclustered/non-mTLS nodes; - // clustered peers reject non-loopback plaintext requests on their proxy - // ports and publish their authoritative inventory via Engine Manager (:14322). - canProbePlaintextEngines := !peerSeen && !entry.MTLS && entry.TLSPort == 0 + // Only an explicit engine choice enables the labelled manual bypass. + // PAIR intent never depends on endpoint availability, including after replay. + canProbePlaintextEngines := entry.Kind == "engine" var ( ollamaUp, lmStudioUp, omlxUp bool @@ -812,14 +814,16 @@ func (m *Manager) probeNode(entry ManualEntry) { } } - verification := "manual" - if peerSeen { - verification = "unverified" + verification := "unverified" + if entry.Kind == "engine" { + verification = "manual" } if nodeInfoUp { verification = "verified" } newStatus := ManualNodeStatus{ + Kind: entry.Kind, + FromCluster: fromCluster, IdentityVerification: verification, PeerRecordID: info.PeerRecordID, PeerMetadata: info.PeerMetadata, @@ -876,8 +880,10 @@ func (m *Manager) probeNode(entry ManualEntry) { return } } - tn.peerSeen = peerSeen + // State commit and publication form one transaction with replacement and + // removal. An older engine observation must never overtake a new PAIR preview. + defer m.mu.Unlock() prev := tn.status prevFails := tn.consecutiveFails // A failed node-info probe returns no HostUUID; preserve the last-learned one @@ -905,7 +911,6 @@ func (m *Manager) probeNode(entry ManualEntry) { tn.consecutiveFails++ } curFails := tn.consecutiveFails - m.mu.Unlock() changed := prev.IdentityVerification != newStatus.IdentityVerification || prev.PeerRecordID != newStatus.PeerRecordID || prev.OllamaUp != newStatus.OllamaUp || prev.LMStudioUp != newStatus.LMStudioUp || @@ -1280,7 +1285,7 @@ func (m *Manager) probeNodeTelemetry(ctx context.Context, id string, entry Manua trackedEntry := tn.entry prior := tn.status m.mu.RUnlock() - if trackedEntry != entry || prior.PeerRecordID == "" { + if entry.Kind == "engine" || trackedEntry != entry || prior.PeerRecordID == "" { return } nodeInfoReceived := m.clockNow() @@ -1297,8 +1302,8 @@ func (m *Manager) probeNodeTelemetry(ctx context.Context, id string, entry Manua m.mu.Unlock() return } + defer m.mu.Unlock() hostUUID := info.HostUUID - m.mu.Unlock() var msSince int64 if info.TelemetryValid { @@ -1339,26 +1344,35 @@ func (m *Manager) probeNodeTelemetry(ctx context.Context, id string, entry Manua } func (m *Manager) addNode(entry ManualEntry) ManualNodeStatus { + if entry.Kind == "" { + entry.Kind = "pair" + } id := nodeID(entry) nodeInfoPort := 14318 if entry.TLSPort > 0 { nodeInfoPort = entry.TLSPort } + verification := "unverified" + if entry.Kind == "engine" { + verification = "manual" + } status := ManualNodeStatus{ - ID: id, - Name: entry.Name, - Address: entry.Address, - OllamaPort: 11434, - NodeInfoPort: nodeInfoPort, - TLSEnabled: entry.TLSPort > 0, - MTLSRequired: entry.TLSPort > 0 && entry.MTLS, + Kind: entry.Kind, + IdentityVerification: verification, + ID: id, + Name: entry.Name, + Address: entry.Address, + OllamaPort: 11434, + NodeInfoPort: nodeInfoPort, + TLSEnabled: entry.TLSPort > 0, + MTLSRequired: entry.TLSPort > 0 && entry.MTLS, } var supersededClusterIDs []string m.mu.Lock() for existingID, tn := range m.nodes { - if existingID == id || !tn.fromCluster { + if existingID == id || !tn.fromCluster || entry.Kind == "engine" { continue } if tn.entry.Address == entry.Address { @@ -1375,30 +1389,31 @@ func (m *Manager) addNode(entry ManualEntry) ManualNodeStatus { supersededClusterIDs = append(supersededClusterIDs, existingID) } } - if existing, ok := m.nodes[id]; ok && existing.cancelTelemetry != nil { + existing, replacing := m.nodes[id] + if replacing && existing.cancelTelemetry != nil { existing.cancelTelemetry() } tn := &trackedNode{entry: entry, status: status} m.nodes[id] = tn m.startTelemetryLoopLocked(id, tn) - m.mu.Unlock() + if replacing { + // Withdraw the old route now, before the replacement's network probe. + m.codec.Notify("node/updated", status) + } for _, oldID := range supersededClusterIDs { m.codec.Notify("node/removed", ManualNodeStatus{ID: oldID}) _ = m.codec.Notify("errors:clear", errors.ClearParams{ID: probeFailedID(oldID)}) } + m.mu.Unlock() go func() { m.probeNode(entry) - m.mu.RLock() - tn, exists := m.nodes[id] - if !exists { - m.mu.RUnlock() - return + m.mu.Lock() + defer m.mu.Unlock() + if current := m.nodes[id]; current == tn { + m.codec.Notify("node/discovered", tn.status) } - discStatus := tn.status - m.mu.RUnlock() - m.codec.Notify("node/discovered", discStatus) }() return status @@ -1413,10 +1428,6 @@ func (m *Manager) removeNode(id string) bool { } delete(m.nodes, id) m.lastMembersMtime = time.Time{} - } - m.mu.Unlock() - - if exists { m.codec.Notify("node/removed", ManualNodeStatus{ID: id}) // Defensive clear: if this node had a probe-failed entry it // would outlive the node itself otherwise — the UI would @@ -1428,6 +1439,9 @@ func (m *Manager) removeNode(id string) bool { }); err != nil { log.Printf("failed to send errors:clear on remove for %s: %v", id, err) } + } + m.mu.Unlock() + if exists { m.syncClusterMembers() } return exists @@ -1518,8 +1532,8 @@ func (m *Manager) handleMessage(msg *Message) { m.codec.RespondError(msg.ID, -32602, "invalid params: expected {\"address\": \"...\"}") return } - if entry.Address == "" { - m.codec.RespondError(msg.ID, -32602, "address is required") + if err := validateManualEntry(&entry); err != nil { + m.codec.RespondError(msg.ID, -32602, err.Error()) return } status := m.addNode(entry) diff --git a/services/nvpair-manual-nodes/manager_test.go b/services/nvpair-manual-nodes/manager_test.go index c0bdeef38..bde327323 100644 --- a/services/nvpair-manual-nodes/manager_test.go +++ b/services/nvpair-manual-nodes/manager_test.go @@ -643,7 +643,7 @@ func TestAddThenRemoveBeforeInitialProbeDoesNotRediscover(t *testing.T) { return httpJSON(http.StatusOK, `{"models":[]}`) }) - status := m.addNode(ManualEntry{Address: "node.local", Name: "lab"}) + status := m.addNode(ManualEntry{Kind: "engine", Address: "node.local", Name: "lab"}) if status.ID != "lab" { t.Fatalf("add status = %+v", status) } diff --git a/services/nvpair-manual-nodes/peer_inventory.go b/services/nvpair-manual-nodes/peer_inventory.go index 6195796c7..ce40604fc 100644 --- a/services/nvpair-manual-nodes/peer_inventory.go +++ b/services/nvpair-manual-nodes/peer_inventory.go @@ -24,14 +24,14 @@ func (m *Manager) authenticatedNodeInfo(ctx context.Context, entry ManualEntry, if ctx == nil { ctx = context.Background() } - if m.mesh == nil { - return NodeInfoResponse{}, 0, false - } - m.mesh.Refresh() port := 14318 if entry.TLSPort > 0 { port = entry.TLSPort } + if m.mesh == nil { + return NodeInfoResponse{}, port, false + } + m.mesh.Refresh() url := "https://" + net.JoinHostPort(entry.Address, strconv.Itoa(port)) + "/v1/node-info" if expected == "" { cfg, ok := m.mesh.ClientTLSConfigAny() diff --git a/services/nvpair-manual-nodes/publication_test.go b/services/nvpair-manual-nodes/publication_test.go new file mode 100644 index 000000000..686a49cf8 --- /dev/null +++ b/services/nvpair-manual-nodes/publication_test.go @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "encoding/json" + "log/slog" + "net/http" + "sync" + "testing" +) + +// Pause exactly between probe state commit and wire publication, rather than +// relying on scheduler timing, a slow HTTP response, or sleeps. +type publicationLogGate struct { + entered, release chan struct{} + once sync.Once +} + +func (g *publicationLogGate) Enabled(context.Context, slog.Level) bool { return true } +func (g *publicationLogGate) Handle(_ context.Context, r slog.Record) error { + engineUp := false + r.Attrs(func(a slog.Attr) bool { + if a.Key == "ollama_up" { + engineUp = a.Value.Bool() + } + return true + }) + if r.Message == "manual node state changed" && engineUp { + g.once.Do(func() { close(g.entered); <-g.release }) + } + return nil +} +func (g *publicationLogGate) WithAttrs([]slog.Attr) slog.Handler { return g } +func (g *publicationLogGate) WithGroup(string) slog.Handler { return g } + +type publicationCapture struct { + *captureRW + mu sync.Mutex + statuses []ManualNodeStatus + pair chan struct{} + once sync.Once +} + +func (c *publicationCapture) Write(p []byte) (int, error) { + var frame struct { + Method string `json:"method"` + Params ManualNodeStatus `json:"params"` + } + if err := json.Unmarshal(p, &frame); err == nil && (frame.Method == "node/updated" || frame.Method == "node/discovered") { + c.mu.Lock() + c.statuses = append(c.statuses, frame.Params) + c.mu.Unlock() + if frame.Params.Kind == "pair" { + c.once.Do(func() { close(c.pair) }) + } + } + return c.captureRW.Write(p) +} + +func TestEnginePublicationCannotFollowPairReplacement(t *testing.T) { + m, _, rt := newTestManager(t) + capture := &publicationCapture{captureRW: newCaptureRW(), pair: make(chan struct{})} + m.codec = NewCodec(capture) + gate := &publicationLogGate{entered: make(chan struct{}), release: make(chan struct{})} + priorLogger := slog.Default() + slog.SetDefault(slog.New(gate)) + t.Cleanup(func() { slog.SetDefault(priorLogger) }) + var release sync.Once + unblock := func() { release.Do(func() { close(gate.release) }) } + t.Cleanup(unblock) + engine := ManualEntry{Kind: "engine", Name: "overlay", Address: "overlay.example"} + m.nodes[nodeID(engine)] = &trackedNode{entry: engine} + rt.set(http.MethodGet, "overlay.example:11434", "/", func(*http.Request) (*http.Response, error) { return httpJSON(200, `{}`) }) + rt.set(http.MethodGet, "overlay.example:11434", "/api/tags", func(*http.Request) (*http.Response, error) { return httpJSON(200, `{"models":[{"name":"bare"}]}`) }) + probed := make(chan struct{}) + go func() { m.probeNode(engine); close(probed) }() + select { + case <-gate.entered: + case <-m.ctx.Done(): + t.Fatal("probe never reached publication boundary") + } + pair := engine + pair.Kind = "pair" + replaced := make(chan struct{}) + // On the old implementation the state lock is already released: force the + // new preview onto the wire before allowing the old engine update through. + // With serialized publication, replacement must follow that old update. + if m.mu.TryLock() { + m.mu.Unlock() + m.addNode(pair) + close(replaced) + select { + case <-capture.pair: + case <-m.ctx.Done(): + t.Fatal("replacement preview missing") + } + unblock() + } else { + go func() { m.addNode(pair); close(replaced) }() + unblock() + } + for _, done := range []<-chan struct{}{probed, replaced, capture.pair} { + select { + case <-done: + case <-m.ctx.Done(): + t.Fatal("publication interleaving did not finish") + } + } + capture.mu.Lock() + defer capture.mu.Unlock() + seenPair := false + for _, s := range capture.statuses { + if s.Kind == "pair" { + seenPair = true + } + if seenPair && s.Kind == "engine" { + t.Fatalf("obsolete engine restored bypass after PAIR replacement: %+v", capture.statuses) + } + } + if !seenPair { + t.Fatal("PAIR replacement was not published") + } +} + +func TestKindReplacementWithdrawsBeforeNewProbe(t *testing.T) { + m, _, rt := newTestManager(t) + capture := &publicationCapture{captureRW: newCaptureRW(), pair: make(chan struct{})} + m.codec = NewCodec(capture) + peer := newInventoryPeer(t, m, rt, "overlay", "peer") + pair := peer.entry + pair.Kind = "pair" + pair.Name = "overlay" + engine := pair + engine.Kind = "engine" + m.nodes[nodeID(engine)] = &trackedNode{entry: engine, status: ManualNodeStatus{Kind: "engine", IdentityVerification: "manual", ID: nodeID(engine), Address: engine.Address, OllamaUp: true, OllamaModels: []string{"bare"}}} + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + rt.set(http.MethodGet, "overlay:14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { <-release; return httpJSON(503, `{}`) }) + m.addNode(pair) + select { + case <-capture.pair: + default: + t.Fatal("kind changed but the old engine route remains until a probe finishes") + } +} diff --git a/services/nvpair-manual-nodes/spec.md b/services/nvpair-manual-nodes/spec.md new file mode 100644 index 000000000..5614239b2 --- /dev/null +++ b/services/nvpair-manual-nodes/spec.md @@ -0,0 +1,147 @@ + + +# Manual nodes: normative service specification + +Adapted from NVIDIA/Personal-AI-Router@b3b25ae (NVIDIA/Personal-AI-Router#128) +for the fork's pinned-mTLS inventory contract. This specification describes the +current fork, not the upstream transport or persistence architecture. + +## Purpose and ownership + +`nvpair-manual-nodes` is the broker-supervised, client-only prober for addresses +mDNS cannot reach. It opens no HTTP listener. It owns only live configuration, +probe results and per-node telemetry loops. The broker is the sole durable owner +of `configs/manual-nodes.json` under the PAIR user-data root. Electron and TUI +are clients, not persistence owners. + +The broker already contained the persistence/replay work corresponding to +NVIDIA/Personal-AI-Router@aed93b6 (NVIDIA/Personal-AI-Router#60). Do not add a +second store or replay path. + +## Entry contract + +`node/add` accepts `{ address, kind?, name?, tls_port?, mtls? }`: + +- `address`: a bare IP literal or hostname. Hostnames, including case and a final + dot, are retained as entered and resolved when dialing. Schemes, paths, + whitespace and `host:port` are rejected with `-32602`; unbracketed IPv6 is valid. +- `kind`: `pair` (PAIR machine) or `engine` (explicit bare-engine bypass). Missing + historical intent means `pair`; unknown values are rejected. Endpoint responses + never choose or change kind. Re-adding with a different kind is a user action. +- `name`: optional configuration alias; the worker ID is name when supplied, + otherwise `manual:
`. This configuration ID is not a directory identity. +- `tls_port`: optional pinned-mTLS node-info port; zero uses `14318`. +- `mtls`: informational configuration metadata; never permits weaker peer reads. + +`node/add` returns initial `ManualNodeStatus` immediately and probes asynchronously. +Replacement/removal and all node-status/telemetry publications share the worker's +state lock. A probe cannot publish after its tracked entry has been replaced or +removed. Replacement publishes an empty initial status immediately, withdrawing +the old route before the new network probe completes. + +Status includes `kind`, `id`, `address`, optional name and configuration metadata, +`identityVerification`, availability, model inventory, and authenticated proof and +hardware fields when available. Initial PAIR status is `unverified`; initial engine +status is `manual`. `node/discovered` follows the first probe; `node/updated` follows +changed state. Removed/replaced configurations cannot publish late probe results. + +`nodes/list` returns `{ nodes: ManualNodeStatus[] }`. The worker also follows +cluster member addresses; those statuses carry `from_cluster: true` and kind +`pair`. Manual-entry management surfaces exclude these automatically tracked rows. +`node/remove` accepts `{ id }`, returns `{ removed }`, cancels telemetry and emits +`node/removed` for an existing entry. Removing only an address does not unpair. + +## Trust and routing invariants + +### PAIR machine + +1. Discover a currently pinned principal over cluster mTLS at the configured + address, then re-read node-info pinned to that exact principal. +2. Certificate principal, node-info identity, and broker-owned `peerMetadata` + identity must agree. Authenticated node-info must confirm its service port. +3. Only authenticated service metadata supplies engine-manager and proxy ports. + Models, engine state and optional engine-native counters are read over pinned + mTLS. Completed reads recheck the exact current pin and admission-bound + `peerRecordId` before publication. +4. The broker independently checks that proof before admitting service inventory. + A verified peer uses its proven UUID and the manual address in the relay + directory. Proxies use that address with pinned mTLS, not plaintext engine + probes. DNS names are dialed as names and may resolve again on subsequent dials. +5. Without proof, the broker projects only an unverified `preview:` record; + no routing, scheduling inventory, identity or telemetry authority is granted. + Failure, absence of node-info, an older peer, a worker/broker restart, or a new + machine answering at the same address cannot turn this into an engine entry. + There is no opt-in unclustered PAIR route. The operator must PIN-pair instead. + +### Engine only + +Only `kind: "engine"` enables the direct engine probes: Ollama `:11434`, LM Studio +`:1234`, oMLX `:1236`. The broker gives these `manual:` directory keys and +`identityVerification: "manual"`. They never read or trust PAIR node-info, +identity, hardware, telemetry, service metadata or engine-manager claims, even +if a PAIR endpoint exists at the same address. Their models are engine-local +inventory, not peer inventory. A PAIR machine's engine is reachable this way only +if its owner enabled LAN engine binding. Engine-only entries remain independent +of cluster membership/removal. + +## Persistence, migration and removal + +The canonical file is an array of `{ id, address, name?, kind, tls_port?, mtls?, +peer_uuid? }`. Writes use a private temporary file and rename. Successful broker +add/remove operations update this file; replacement workers receive `node/add` +replays. The worker itself writes no configuration file. + +On reading a historical entry without kind, the broker explicitly rewrites kind +as `pair` before replay. There is no safe way to infer historical engine consent +from address, optional TLS flags or a network response. Thus former bare engines +stop direct routing until the user removes/re-adds them as **Engine only**. Unknown +stored kinds or failed migration writes prevent replay rather than guessing. + +`peer_uuid` records a cleanup-only association from authenticated pairing +completion (or a later pinned inventory proof). Cluster-manager emits internal +`cluster:pairing-completed` with the original dial hostname, authenticated peer +UUID, cluster ID and admission epoch inside the roster commit/removal boundary, +before publishing paired state. It never waits for a broker acknowledgement. The broker +associates matching manual PAIR aliases durably, even though the roster records +an observed numeric source address and no node-info proof exists yet. Removed or +obsolete admissions cannot persist an association. Up to 16 transient address +hints can support an already in-flight unpair when the roster file has already +lost the peer; they expire on that request's success or failure and never enter +persistent configuration, inventory or routing. This field +is not an identity, inventory or transport proof; clients cannot supply it as an +authenticated association. Once non-empty, each entry's current `peer_uuid` +is authoritative for cleanup: only that peer matches. Address and cached-identity +fallbacks apply only when the entry has no association. Cached inventory and +ordinary re-adds cannot override a current association; authenticated pairing can +reassociate it, and deletion rechecks ownership atomically with the file write. +After successful `nodes:remove`, the broker forgets +all associated manual PAIR aliases and their live projections. Failed unpair keeps +them. A cleanup failure is returned to the client, not silently swallowed. This +adapts the behavior of NVIDIA/Personal-AI-Router@692950c +(NVIDIA/Personal-AI-Router#172) at the shared broker boundary for both clients. + +## Cadence, failure and process controls + +Topology probes run every 10 seconds with 3-second per-request deadlines. Pinned +telemetry refresh runs at roughly 2 seconds with bounded per-node jitter and a +1.5-second deadline. Engine-native counters refresh at a slower 60-second cadence. +Clients disable keep-alives for direct probes to permit fresh DNS resolution. +Three consecutive unreachable topology probes emit one `errors:report`; recovery +or removal clears it. Probe failures never delete the durable configuration. + +Node-info reads are bounded to 1 MiB, models to 16 MiB, and engine-native stats to +1 MiB. A malformed/oversized answer does not establish inventory. `worker/health` +is answered inline on the control loop and measures control responsiveness only. +`telemetry/set-scheduling-demand` controls poll intent, not probe cadence. `shutdown`, +stdin EOF or termination cancels work. `log/set-level` changes verbosity. Prompt, +completion, credential and key material must never enter logs. + +See [README.md](README.md) for CLI flags and examples; the broker README defines +relay envelopes and supervision. Unit tests cover intent replay without a PAIR +endpoint, explicit engines ignoring PAIR claims, hostname mTLS resolution, +conservative migration and successful-only unpair cleanup. Cross-process tests +exercise real-worker replacement in CI; local unit evidence is not deployment or +packaged-artifact certification. diff --git a/services/nvpair-manual-nodes/verified_inventory_test.go b/services/nvpair-manual-nodes/verified_inventory_test.go index 59a248718..b7bb9f897 100644 --- a/services/nvpair-manual-nodes/verified_inventory_test.go +++ b/services/nvpair-manual-nodes/verified_inventory_test.go @@ -36,7 +36,7 @@ func TestPublicManualInventoryCannotClaimPinnedPrincipal(t *testing.T) { // It may advertise models without acquiring a host identity or GPU telemetry. func TestBareEnginesUseExplicitManualInventory(t *testing.T) { m, rw, rt := newTestManager(t) - entry := ManualEntry{Name: "bare-engines", Address: "bare.local"} + entry := ManualEntry{Kind: "engine", Name: "bare-engines", Address: "bare.local"} rt.set(http.MethodGet, "bare.local:14318", "/v1/node-info", func(*http.Request) (*http.Response, error) { return httpJSON(http.StatusNotFound, `{}`) }) diff --git a/services/nvpair-tui/ui/manualnodes.go b/services/nvpair-tui/ui/manualnodes.go index b3d334cf0..13271bc75 100644 --- a/services/nvpair-tui/ui/manualnodes.go +++ b/services/nvpair-tui/ui/manualnodes.go @@ -23,11 +23,14 @@ const manualRefreshInterval = 10 * time.Second // manualNode is the subset of nvpair-manual-nodes' ManualNodeStatus shown. type manualNode struct { - ID string `json:"id"` - Name string `json:"name"` - Address string `json:"address"` - OllamaUp bool `json:"ollama_up"` - NodeInfoUp bool `json:"node_info_up"` + Kind string `json:"kind"` + IdentityVerification string `json:"identityVerification"` + FromCluster bool `json:"from_cluster"` + ID string `json:"id"` + Name string `json:"name"` + Address string `json:"address"` + OllamaUp bool `json:"ollama_up"` + NodeInfoUp bool `json:"node_info_up"` } // manualView manages user-added nodes that don't appear via mDNS: list, @@ -39,6 +42,7 @@ type manualView struct { nodes []manualNode input textinput.Model adding bool + kind string status string confirm *confirmPrompt width, height int @@ -56,8 +60,15 @@ type manualActionMsg struct { err error } +type manualPairSavedMsg struct { + address string + err error +} + var ( - manualAddKey = key.NewBinding(key.WithKeys("a"), key.WithHelp("a", "add node")) + manualAddKey = key.NewBinding(key.WithKeys("a"), key.WithHelp("a", "add PAIR machine")) + manualEngineKey = key.NewBinding(key.WithKeys("e"), key.WithHelp("e", "add engine only")) + manualPairKey = key.NewBinding(key.WithKeys("p"), key.WithHelp("p", "pair selected")) manualRemoveKey = key.NewBinding(key.WithKeys("r"), key.WithHelp("r", "remove node")) ) @@ -97,14 +108,14 @@ func (v *manualView) tickCmd() tea.Cmd { func (v *manualView) SetSize(w, h int) { v.width, v.height = w, h - const ollama, nodeinfo = 8, 9 + const ollama, nodeinfo = 8, 24 id := clampWidth((w-ollama-nodeinfo-2)/2, 8) addr := clampWidth(w-ollama-nodeinfo-id-2, 10) v.table.SetColumns([]table.Column{ {Title: "ID", Width: id}, {Title: "ADDRESS", Width: addr}, {Title: "OLLAMA", Width: ollama}, - {Title: "NODEINFO", Width: nodeinfo}, + {Title: "KIND / VERIFICATION", Width: nodeinfo}, }) v.table.SetWidth(w) v.table.SetHeight(clampWidth(h-2, 1)) @@ -119,6 +130,21 @@ func (v *manualView) Update(msg tea.Msg) tea.Cmd { v.setNodes(msg.nodes) } return nil + case manualPairSavedMsg: + if msg.err != nil { + v.status = "add " + msg.address + " failed: " + msg.err.Error() + return nil + } + return v.pairCmd(msg.address) + case clusterActionMsg: + if msg.err != nil { + v.status = "pairing failed (address kept): " + msg.err.Error() + } else if msg.rejected { + v.status = "pairing rejected: " + rejectReason(msg.reason) + } else { + v.status = "Pairing PIN: " + msg.pin + " (read it to the joining node)" + } + return v.listCmd() case manualTickMsg: return tea.Batch(v.listCmd(), v.tickCmd()) case manualActionMsg: @@ -152,7 +178,17 @@ func (v *manualView) handleKey(msg tea.KeyMsg) tea.Cmd { return cmd } switch { - case key.Matches(msg, manualAddKey): + case key.Matches(msg, manualPairKey): + idx := v.table.Cursor() + if idx >= 0 && idx < len(v.nodes) && v.nodes[idx].Kind == "pair" { + return v.pairCmd(v.nodes[idx].Address) + } + return nil + case key.Matches(msg, manualAddKey), key.Matches(msg, manualEngineKey): + v.kind = "pair" + if key.Matches(msg, manualEngineKey) { + v.kind = "engine" + } v.adding = true v.input.SetValue("") v.input.Focus() @@ -173,11 +209,28 @@ func (v *manualView) submitAdd() tea.Cmd { v.status = "address required" return nil } - return call(v.client, "node/add", map[string]string{"address": addr}, func(_ *rpc.Message, err error) tea.Msg { + kind := v.kind + if kind == "" { + kind = "pair" + } + return call(v.client, "node/add", map[string]string{"address": addr, "kind": kind}, func(_ *rpc.Message, err error) tea.Msg { + if kind == "pair" { + return manualPairSavedMsg{address: addr, err: err} + } return manualActionMsg{what: "add " + addr, err: err} }) } +func (v *manualView) pairCmd(address string) tea.Cmd { + return inviteNodeCmd(v.client, map[string]any{"address": address}, func(res inviteNodeResult, err error) tea.Msg { + pin := "" + if res.Pin != nil { + pin = *res.Pin + } + return clusterActionMsg{what: "pair", pin: pin, rejected: res.State == "rejected", reason: res.Reason, err: err} + }) +} + func (v *manualView) removeSelected() tea.Cmd { idx := v.table.Cursor() if idx < 0 || idx >= len(v.nodes) { @@ -192,14 +245,18 @@ func (v *manualView) removeSelected() tea.Cmd { } func (v *manualView) setNodes(nodes []manualNode) { - v.nodes = nodes + v.nodes = nil rows := make([]table.Row, 0, len(nodes)) for _, n := range nodes { + if n.FromCluster { + continue + } + v.nodes = append(v.nodes, n) rows = append(rows, table.Row{ truncate(n.ID, 16), n.Address, yesNo(n.OllamaUp), - yesNo(n.NodeInfoUp), + n.Kind + " / " + n.IdentityVerification, }) } v.table.SetRows(rows) @@ -208,12 +265,15 @@ func (v *manualView) setNodes(nodes []manualNode) { func (v *manualView) View() string { var b strings.Builder if len(v.nodes) == 0 { - b.WriteString(footerStyle.Render("No manual nodes. Press a to add one by address.")) + b.WriteString(footerStyle.Render("No manual entries. Press a for a PAIR machine or e for an engine.")) } else { b.WriteString(v.table.View()) } if v.adding { - b.WriteString("\nadd node: " + v.input.View()) + b.WriteString("\nadd " + v.kind + ": " + v.input.View()) + if v.kind == "engine" { + b.WriteString("\nManual bypass: trusts no PAIR claims. A PAIR owner's engine needs LAN engine binding enabled.") + } } if v.confirm.open() { b.WriteString("\n" + v.confirm.View()) @@ -228,5 +288,5 @@ func (v *manualView) Help() []key.Binding { if v.confirm.open() { return v.confirm.Help() } - return []key.Binding{manualAddKey, manualRemoveKey} + return []key.Binding{manualAddKey, manualEngineKey, manualPairKey, manualRemoveKey} } diff --git a/services/nvpair-tui/ui/manualnodes_test.go b/services/nvpair-tui/ui/manualnodes_test.go new file mode 100644 index 000000000..f7276f8fd --- /dev/null +++ b/services/nvpair-tui/ui/manualnodes_test.go @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package ui + +import ( + "strings" + "testing" +) + +func TestManualPairPersistsBeforeInviting(t *testing.T) { + transport, c := engineWire(t) + v := newManualView(transport.client) + v.SetSize(100, 24) + v.Update(engineKey("a")) + v.input.SetValue(" Workstation.tail.example ") + saved := engineRequest(t, v.submitAdd(), c, "node/add", `{"address":"Workstation.tail.example","kind":"pair"}`, `{"id":"manual:Workstation.tail.example","kind":"pair"}`) + invite := v.Update(saved) + result := engineRequest(t, invite, c, "cluster:invite-node", `{"address":"Workstation.tail.example"}`, `{"state":"pending","pin":"123456"}`) + v.Update(result) + if !strings.Contains(v.View(), "123456") { + t.Fatalf("PIN not shown: %s", v.View()) + } +} + +func TestManualEngineChoiceIsExplicitAndDoesNotPair(t *testing.T) { + transport, c := engineWire(t) + v := newManualView(transport.client) + v.SetSize(100, 24) + v.Update(engineKey("e")) + if !strings.Contains(v.View(), "trusts no PAIR claims") { + t.Fatal("engine bypass not explained") + } + v.input.SetValue("engine.example") + saved := engineRequest(t, v.submitAdd(), c, "node/add", `{"address":"engine.example","kind":"engine"}`, `{"id":"manual:engine.example","kind":"engine"}`) + if _, ok := saved.(manualActionMsg); !ok { + t.Fatalf("engine triggered pairing: %T", saved) + } +} + +func TestManualViewListsOnlyConfiguredEntriesWithIntent(t *testing.T) { + v := newManualView(nil) + v.SetSize(100, 24) + v.setNodes([]manualNode{{ID: "auto", FromCluster: true}, {ID: "overlay", Kind: "pair", Address: "overlay.example", IdentityVerification: "unverified"}, {ID: "engine", Kind: "engine", IdentityVerification: "manual"}}) + if len(v.nodes) != 2 || !strings.Contains(v.View(), "unverified") || !strings.Contains(v.View(), "engine / manual") { + t.Fatalf("manual listing = %s", v.View()) + } +} diff --git a/services/nvpair-ui-broker/README.md b/services/nvpair-ui-broker/README.md index ef958d7e5..5f7512914 100644 --- a/services/nvpair-ui-broker/README.md +++ b/services/nvpair-ui-broker/README.md @@ -821,14 +821,21 @@ report an error even when deletion succeeded. #### `node/add` / `node/remove` / `nodes/list` (manual nodes) -Relayed to `nvpair-manual-nodes`. `node/add` (`{ address, name?, tls_port?, mtls? }`) registers a user-added node and probes it; `node/remove` (`{ id }`) drops it; `nodes/list` returns the tracked manual nodes. Manually added nodes also surface in the shared `discovery:get-nodes` / `discovery:nodes-changed` snapshot — the broker merges `nvpair-manual-nodes`' `node/discovered|updated|removed` into the same store the scanner feeds. Separately from 10-second topology probes, `nvpair-manual-nodes` runs a 2 s ± 250 ms telemetry loop emitting internal `node/telemetry` frames (`id`, `hostUuid`, `gpuUtilizationPct`, `telemetryValid`, `msSince`, `sampleSeq`) for reachable nodes; the broker ingests these observations into its scheduling telemetry cache without modifying discovery state or proxy bindings. The broker persists successful `node/add` operations to `configs/manual-nodes.json` in the per-user data directory and replays them into a replacement `nvpair-manual-nodes` worker after startup; successful `node/remove` operations update the same durable list. Error `-32000 "manual-nodes not available"` when no manual-nodes worker is supervised. +Relayed to `nvpair-manual-nodes`. `node/add` (`{ address, kind?, name?, tls_port?, mtls? }`) registers a user-added node and probes it; `node/remove` (`{ id }`) drops it; `nodes/list` returns the tracked manual nodes. Manually added nodes also surface in the shared `discovery:get-nodes` / `discovery:nodes-changed` snapshot — the broker merges `nvpair-manual-nodes`' `node/discovered|updated|removed` into the same store the scanner feeds. Separately from 10-second topology probes, `nvpair-manual-nodes` runs a 2 s ± 250 ms telemetry loop emitting internal `node/telemetry` frames (`id`, `hostUuid`, `gpuUtilizationPct`, `telemetryValid`, `msSince`, `sampleSeq`) for reachable nodes; the broker ingests these observations into its scheduling telemetry cache without modifying discovery state or proxy bindings. The broker persists successful `node/add` operations to `configs/manual-nodes.json` in the per-user data directory and replays them into a replacement `nvpair-manual-nodes` worker after startup; successful `node/remove` operations update the same durable list. Error `-32000 "manual-nodes not available"` when no manual-nodes worker is supervised. -**Manual → proxy bridge.** When the broker supervises both `nvpair-manual-nodes` and a proxy, it also bridges a manual node whose engine is reachable into that proxy via `node/add-manual` (host/port from the node's per-engine status), so inference can route to it through `proxy:node/select` / `lmstudio-proxy:node/select` / `omlx-proxy:node/select` just like a relay-discovered node. This is per-engine: a node whose `ollama_*` status is up is bridged into `ollama-proxy` (host/port from `ollama_port`), one whose `lmstudio_*` status is up into `lmstudio-proxy` (from `lmstudio_port`), and one whose `omlx_*` status is up into `omlx-proxy` (from `omlx_port` or default `:1236`) — a node running multiple engines is bridged into each. Manual nodes are by definition the ones that never appear via the daemon's `_nvpair-node` discovery, so this explicit add is what makes them routable. The bridge tracks reachability: an engine that goes down (or a node that is removed, or whose prober crashes) is pulled back out with `node/remove-manual`. A proxy that isn't supervised → that leg is a no-op; manual nodes still appear in the discovery snapshot as before. +**Persistent intent.** `kind` is `pair` or `engine`; omission conservatively means `pair`. The broker persists kind and replays it on worker/broker restart. Historical entries without kind are rewritten as `pair` before replay, so former bare engines need explicit removal/re-addition as **Engine only**. Unknown kinds fail closed. Status includes kind and `identityVerification`; automatically tracked cluster addresses are marked `from_cluster: true`, not user-configured entries. A remembered `peer_uuid` associates an address for offline lifecycle removal, never for routing authorization. The broker consumes internal `cluster:pairing-completed` (`{ address, peerUuid, clusterId, admissionEpoch }`) only from its supervised cluster-manager after authenticated pairing commit. The original dial hostname associates matching manual PAIR aliases even before node-info proof and when the roster holds an observed numeric address. Only a current roster member with the same cluster/admission can persist an association; obsolete or removed completions are ignored. During an in-flight unpair, up to 16 original-address hints may be retained only until that request ends, to forget unassociated aliases even if the roster file already reflects removal when the ordered notification is consumed. Hints are never persisted or used for identity/routing. A non-empty persisted `peer_uuid` is authoritative for that entry: cached identities, reused addresses and hints cannot select it for a different peer's removal. Cached inventory can seed an unassociated entry but cannot replace an existing association, including on ordinary re-add. Deletion rechecks the association under the persistence lock so an in-flight reassociation is preserved. The event is never forwarded to desktop/TUI or admitted as peer inventory. + +**Manual → routing bridge.** Verified PAIR entries enter the authenticated relay with their manual address and proven service map, using pinned mTLS just like verified discovered peers. Unverified PAIR entries remain `preview:` and never route. Only explicitly chosen `engine` entries use `node/add-manual`, under `manual:` and verification `manual`, for reachable Ollama, LM Studio and oMLX engines. They trust no PAIR claims and provide no scheduling telemetry. Failed probes or worker retirement withdraw live routing without forgetting durable intent. A missing proxy makes that leg a no-op. **Manual-node engine view.** A manual node carries the same `engineNative` field a discovered node carries: the engine's own counters, rates and active models, as `nvpair-manual-nodes` read them from the node's engine-manager over the peer's pinned mTLS path (the same path the model list is read over). The broker projects the field onto the manual node's record in both the discovery store the `discovery:get-nodes` snapshot is taken from and the relay directory the `discovery:nodes-changed` stream is taken from, so a manual node and a discovered node present the engine's view identically. Without it, a manual node's record would carry no engine view at all: the node-scanner, which fetches the view for the nodes it discovers over mDNS, never sees a manual node. #### `cluster:` / `nodes:` (generic relay) +After successful `nodes:remove`, the broker also forgets every associated manual +PAIR address (including remembered associations when offline). Failed unpair keeps +them; cleanup errors are surfaced to the caller. This shared behavior covers both +desktop and TUI clients. Bare-engine entries remain independent. + Any request whose method starts with `cluster:` or `nodes:` is **forwarded to `nvpair-cluster-manager`** verbatim (no prefix stripping — these are the manager's own native namespaces), and the manager's result or JSON-RPC error is relayed straight back to the caller unchanged. As with the proxy relay this means the cluster-manager's whole surface is reachable through the broker without enumerating individual methods. Current methods include `cluster:get-node-id`, `cluster:set-identity`, `cluster:create`, `cluster:invite-node`, `cluster:respond-to-invite`, `cluster:cancel-invite`, `cluster:invite-status`, `nodes:get-initial`, and `nodes:remove` (see the [`nvpair-cluster-manager` README](../nvpair-cluster-manager/README.md) for their params/results and app-defined error codes). The cluster-manager also pushes notifications (`cluster:invite-received`, `cluster:invite-canceled`, `cluster:identity-changed`, `nodes:changed`, ...); the broker forwards them to the caller verbatim. Unlike `proxy:` / `workloads:*` these are **not opt-in** — there is no `cluster:subscribe`; the events always flow. @@ -903,7 +910,6 @@ Attach to a pre-existing endpoint: - **Engine-advertise control surface.** Engine registration is auto-driven only: the broker tracks local ollama / LM Studio on their fixed coordinates and registers `ol` / `lm` with the daemon while up. There's no manual-advertise RPC (custom service, port, name, or TXT), and no way to advertise anything other than the detected engines. - **node-info control surface.** The broker passes its cluster directory and local node ID, pushes demand and log-level controls, and publishes broker-owned service metadata via `nodeinfo:set-peer-metadata`. Public hardware previews remain available; only pinned mTLS inventory can feed service decisions. There is no public reconfiguration RPC or custom node-info port through the broker. -- **Manual-node persistence across restarts.** `nvpair-manual-nodes` remains an in-memory worker; the broker owns `configs/manual-nodes.json`, clears stale live projections when the worker exits, and replays the durable entries into its replacement. Trust-refresh snapshot/replay is serialized with worker callbacks and retirement; status snapshots from retired worker generations cannot republish inventory. - **Per-event push semantics.** `discovery:nodes-changed` always carries the full current snapshot, not a delta. For small N this is fine and lets the client treat the payload as authoritative without state reconciliation. `errors:update` is likewise a full snapshot. - **BYO-TLS peer authorization.** Operator CA trust and plaintext previews do not establish PAIR peer identity. Service inventory requires pinned cluster mTLS and matching authenticated metadata; older peers lacking that metadata remain previews until upgraded. - **Per-worker log-level granularity.** `log/set-level` fans the broker's single level out to every running child, but there's no way to set a different level per worker — it's one level for the whole process tree. diff --git a/services/nvpair-ui-broker/broker.go b/services/nvpair-ui-broker/broker.go index c8eea34d1..868abed05 100644 --- a/services/nvpair-ui-broker/broker.go +++ b/services/nvpair-ui-broker/broker.go @@ -405,11 +405,14 @@ type Broker struct { // alias is reprojected from manualNodeStatuses (its last-seen payload) so the // removed alias's address/models don't linger (the survivor's own periodic // probe emits no update while its status is unchanged). - manualMu sync.Mutex - manualNodeKeys map[string]string - manualNodeStatuses map[string]manualNodeStatusEntry - manualPersistMu sync.Mutex - manualMutationMu sync.Mutex + manualMu sync.Mutex + manualNodeKeys map[string]string + manualNodeStatuses map[string]manualNodeStatusEntry + manualPersistMu sync.Mutex + manualMutationMu sync.Mutex + manualCleanupMu sync.Mutex + manualPendingRemoval *pendingManualRemoval + manualRemovedAdmissions map[string]uint64 // manualMutationTail is closed when the most recently queued manual-node // mutation finishes. Each node/add or node/remove waits for the one before // it, so mutations apply in the order clients sent them without blocking @@ -1530,6 +1533,9 @@ func (b *Broker) upsertManualNode(s manualNodeStatus) { s = b.normalizeManualStatus(s) incoming := manualToEnriched(s) b.annotateManualTrust(&incoming, s) + if incoming.Trusted { + b.rememberManualPeer(s) + } key := incoming.storeKey() receivedAt := time.Now() @@ -2588,6 +2594,12 @@ func (b *Broker) forwardLogLevel(level string) { // errors:clear are routed into the nvpair-errors pipeline (the broker now // supervises it) rather than dropped. func (b *Broker) forwardClusterManagerNotification(method string, params json.RawMessage) { + if method == "cluster:pairing-completed" { + if err := b.associatePairedManualNodes(params); err != nil { + slog.Warn("failed to persist pairing cleanup association", "err", err) + } + return // broker-internal lifecycle metadata, not renderer peer identity + } if b.dispatchErrorsNotif("cluster-manager", method, params) { return } @@ -3739,7 +3751,67 @@ func (b *Broker) relayToClusterManager(msg *Message) { } return } + var manualAliases []string + removedPeerID := "" + if msg.Method == "nodes:remove" { + // Resolve and forget the same aliases even if another client has an + // in-flight manual add/remove when unpair begins. + b.manualMutationMu.Lock() + defer b.manualMutationMu.Unlock() + var params struct { + NodeUUID string `json:"nodeUuid"` + NodeID string `json:"nodeId"` + } + _ = json.Unmarshal(msg.Params, ¶ms) + id := params.NodeUUID + if id == "" { + id = params.NodeID + } + member, lookupErr := b.manualMemberForID(id) + if lookupErr != nil { + _ = b.codec.RespondError(msg.ID, -32000, fmt.Sprintf("manual-node lookup failed: %v", lookupErr)) + return + } + removedPeerID = member.NodeUUID + b.beginManualRemoval(member) + defer b.finishManualRemoval(false) + var err error + manualAliases, err = b.manualAliasesForPeer(id) + if err != nil { + _ = b.codec.RespondError(msg.ID, -32000, fmt.Sprintf("manual-node lookup failed: %v", err)) + return + } + } result, rpcErr, err := cm.Call(context.Background(), msg.Method, msg.Params) + if err == nil && rpcErr == nil && msg.Method == "nodes:remove" { + var removed struct { + Removed bool `json:"removed"` + } + if json.Unmarshal(result, &removed) == nil && removed.Removed { + // A pairing completion may have arrived while removal was in flight. + // Its cleanup-only association precedes this response on the worker + // stream, so include it even if no inventory proof was ever published. + hints := b.finishManualRemoval(true) + latest, lookupErr := b.manualAliasesForPeer(removedPeerID, hints...) + if lookupErr != nil { + _ = b.codec.RespondError(msg.ID, -32000, fmt.Sprintf("peer removed, but manual-node lookup failed: %v", lookupErr)) + return + } + seen := make(map[string]bool, len(manualAliases)) + for _, alias := range manualAliases { + seen[alias] = true + } + for _, alias := range latest { + if !seen[alias] { + manualAliases = append(manualAliases, alias) + } + } + if err := b.forgetManualAliases(manualAliases, removedPeerID); err != nil { + _ = b.codec.RespondError(msg.ID, -32000, fmt.Sprintf("peer removed, but forgetting manual entries failed: %v", err)) + return + } + } + } switch { case err != nil: if err := b.codec.RespondError(msg.ID, -32000, fmt.Sprintf("cluster-manager call failed: %v", err)); err != nil { diff --git a/services/nvpair-ui-broker/discovery_test.go b/services/nvpair-ui-broker/discovery_test.go index c173bea6b..6de23a4f6 100644 --- a/services/nvpair-ui-broker/discovery_test.go +++ b/services/nvpair-ui-broker/discovery_test.go @@ -57,13 +57,13 @@ func TestManualToEnrichedHostUUID(t *testing.T) { t.Fatalf("store key = %q, want the real hostUuid", withUUID.storeKey()) } claimedUUID := manualToEnriched(manualNodeStatus{ID: "manual:10.0.0.9", HostUUID: "real-uuid"}) - wantManualKey := noderec.ManualRecordID("manual:10.0.0.9") + wantManualKey := noderec.PreviewRecordID("manual:10.0.0.9", "") if claimedUUID.storeKey() != wantManualKey { - t.Fatalf("unproven hostUuid must use the manual config namespace: got %q, want %q", claimedUUID.storeKey(), wantManualKey) + t.Fatalf("unproven hostUuid must use the preview namespace: got %q, want %q", claimedUUID.storeKey(), wantManualKey) } noUUID := manualToEnriched(manualNodeStatus{ID: "manual:10.0.0.9"}) if noUUID.storeKey() != wantManualKey { - t.Fatalf("store key = %q, want namespaced manual config key %q", noUUID.storeKey(), wantManualKey) + t.Fatalf("store key = %q, want namespaced preview key %q", noUUID.storeKey(), wantManualKey) } } diff --git a/services/nvpair-ui-broker/manual_pairing.go b/services/nvpair-ui-broker/manual_pairing.go new file mode 100644 index 000000000..64e35c13e --- /dev/null +++ b/services/nvpair-ui-broker/manual_pairing.go @@ -0,0 +1,146 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" +) + +const manualRemovalHintLimit = 16 + +type manualClusterMember struct { + ID string `json:"id"` + NodeUUID string `json:"nodeUuid"` + Address string `json:"ipAddress"` + ClusterID string `json:"clusterId"` + AdmissionEpoch uint64 `json:"admissionEpoch"` + State string `json:"state"` +} + +type manualPairingCompletion struct { + Address string `json:"address"` + PeerUUID string `json:"peerUuid"` + ClusterID string `json:"clusterId"` + AdmissionEpoch uint64 `json:"admissionEpoch"` +} + +type pendingManualRemoval struct { + member manualClusterMember + addresses map[string]bool +} + +func (b *Broker) manualMemberForID(id string) (manualClusterMember, error) { + member := manualClusterMember{NodeUUID: id} + if id == "" || b.clusterDir == "" { + return member, nil + } + data, err := os.ReadFile(filepath.Join(b.clusterDir, "members.json")) + if errors.Is(err, os.ErrNotExist) { + return member, nil + } + if err != nil { + return member, err + } + var members []manualClusterMember + if err := json.Unmarshal(data, &members); err != nil { + return member, err + } + for _, candidate := range members { + if candidate.NodeUUID != "" && (candidate.NodeUUID == id || candidate.ID == id) { + return candidate, nil + } + } + return member, nil +} + +// manualMutationMu serializes these unpair transactions. The notification reader +// never takes that lock, so it can consume a completion while Call awaits removal. +func (b *Broker) beginManualRemoval(member manualClusterMember) { + b.manualCleanupMu.Lock() + defer b.manualCleanupMu.Unlock() + b.manualPendingRemoval = &pendingManualRemoval{member: member, addresses: make(map[string]bool)} +} + +// Hints live only until this unpair ends (success or failure), have a fixed size +// cap, are never persisted, and are consumed only for configuration deletion. +func (b *Broker) finishManualRemoval(removed bool) []string { + b.manualCleanupMu.Lock() + defer b.manualCleanupMu.Unlock() + pending := b.manualPendingRemoval + b.manualPendingRemoval = nil + if pending == nil { + return nil + } + if removed && pending.member.ClusterID != "" && pending.member.AdmissionEpoch > 0 { + if b.manualRemovedAdmissions == nil { + b.manualRemovedAdmissions = make(map[string]uint64) + } + key := pending.member.ClusterID + "\x00" + pending.member.NodeUUID + if pending.member.AdmissionEpoch > b.manualRemovedAdmissions[key] { + b.manualRemovedAdmissions[key] = pending.member.AdmissionEpoch + } + } + addresses := make([]string, 0, len(pending.addresses)) + for address := range pending.addresses { + addresses = append(addresses, address) + } + return addresses +} + +// Only the supervised cluster-manager's authenticated commit emits this event. +// Current roster/admission validation prevents obsolete completions from +// persisting an association after removal or readmission. No service authority +// is inferred, and there is no worker call or mutation-lock wait on this reader. +func (b *Broker) associatePairedManualNodes(params json.RawMessage) error { + var paired manualPairingCompletion + if err := json.Unmarshal(params, &paired); err != nil { + return err + } + if paired.Address == "" || paired.PeerUUID == "" || paired.ClusterID == "" || paired.AdmissionEpoch == 0 { + return fmt.Errorf("invalid pairing cleanup association") + } + b.manualCleanupMu.Lock() + defer b.manualCleanupMu.Unlock() + key := paired.ClusterID + "\x00" + paired.PeerUUID + if b.manualRemovedAdmissions[key] >= paired.AdmissionEpoch { + return nil + } + if pending := b.manualPendingRemoval; pending != nil && pending.member.NodeUUID == paired.PeerUUID && + (pending.member.ClusterID == "" || pending.member.ClusterID == paired.ClusterID) && + (pending.member.AdmissionEpoch == 0 || pending.member.AdmissionEpoch == paired.AdmissionEpoch) { + pending.member.ClusterID = paired.ClusterID + pending.member.AdmissionEpoch = paired.AdmissionEpoch + if len(pending.addresses) < manualRemovalHintLimit { + pending.addresses[paired.Address] = true + } + } + member, err := b.manualMemberForID(paired.PeerUUID) + if err != nil { + return err + } + if member.State != "member" || member.ClusterID != paired.ClusterID || member.AdmissionEpoch != paired.AdmissionEpoch { + return nil + } + b.manualPersistMu.Lock() + defer b.manualPersistMu.Unlock() + entries, err := loadPersistedManualNodes(b.manualNodesConfigPath) + if err != nil { + return err + } + changed := false + for i := range entries { + if entries[i].Kind == "pair" && entries[i].Address == paired.Address && entries[i].PeerUUID != paired.PeerUUID { + entries[i].PeerUUID = paired.PeerUUID + changed = true + } + } + if changed { + return writePersistedManualNodes(b.manualNodesConfigPath, entries) + } + return nil +} diff --git a/services/nvpair-ui-broker/manualnodes.go b/services/nvpair-ui-broker/manualnodes.go index d7c3bc668..651ba7299 100644 --- a/services/nvpair-ui-broker/manualnodes.go +++ b/services/nvpair-ui-broker/manualnodes.go @@ -7,6 +7,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "log/slog" "os" "path/filepath" @@ -26,6 +27,8 @@ import ( // (bridgeManualNode); the rest project into the discovery store via // manualToEnriched. type manualNodeStatus struct { + Kind string `json:"kind"` + FromCluster bool `json:"from_cluster,omitempty"` IdentityVerification string `json:"identityVerification,omitempty"` PeerRecordID string `json:"peerRecordId,omitempty"` PeerMetadata *noderec.PeerMetadataParams `json:"peerMetadata,omitempty"` @@ -82,11 +85,15 @@ type manualNodeTelemetryParams struct { } type persistedManualNode struct { - ID string `json:"id,omitempty"` - Address string `json:"address"` - Name string `json:"name,omitempty"` - TLSPort int `json:"tls_port,omitempty"` - MTLS bool `json:"mtls,omitempty"` + Kind string `json:"kind"` + // PeerUUID is a previously proven association used only to forget this + // address on unpair. It is never used as inventory or a routing proof. + PeerUUID string `json:"peer_uuid,omitempty"` + ID string `json:"id,omitempty"` + Address string `json:"address"` + Name string `json:"name,omitempty"` + TLSPort int `json:"tls_port,omitempty"` + MTLS bool `json:"mtls,omitempty"` } func (e persistedManualNode) key() string { @@ -122,9 +129,24 @@ func loadPersistedManualNodes(path string) ([]persistedManualNode, error) { return nil, err } valid := entries[:0] + migrated := false for _, entry := range entries { - if entry.Address != "" { - valid = append(valid, entry) + if entry.Address == "" { + continue + } + if entry.Kind == "" { + // Historical files cannot prove user consent to the engine bypass. + entry.Kind = "pair" + migrated = true + } + if entry.Kind != "pair" && entry.Kind != "engine" { + return nil, fmt.Errorf("invalid persisted manual node kind %q", entry.Kind) + } + valid = append(valid, entry) + } + if migrated { + if err := writePersistedManualNodes(path, valid); err != nil { + return nil, err } } return valid, nil @@ -152,6 +174,20 @@ func writePersistedManualNodes(path string, entries []persistedManualNode) error } func (b *Broker) persistManualNode(entry persistedManualNode) error { + if entry.Kind == "" { + entry.Kind = "pair" + } + if entry.Kind != "pair" && entry.Kind != "engine" { + return fmt.Errorf("invalid manual node kind") + } + // The client cannot choose a proven peer association. + entry.PeerUUID = "" + b.manualMu.Lock() + status := b.manualNodeStatuses[entry.key()].status + b.manualMu.Unlock() + if entry.Kind == "pair" && status.Address == entry.Address && status.IdentityVerification == "verified" { + entry.PeerUUID = status.HostUUID + } if entry.ID == "" { entry.ID = entry.key() } @@ -164,6 +200,9 @@ func (b *Broker) persistManualNode(entry persistedManualNode) error { replaced := false for i := range entries { if entries[i].key() == entry.key() { + if entry.Kind == "pair" && entries[i].Kind == "pair" && entries[i].Address == entry.Address && entries[i].PeerUUID != "" { + entry.PeerUUID = entries[i].PeerUUID + } entries[i] = entry replaced = true break @@ -222,7 +261,8 @@ func (b *Broker) manualAliasForKey(id string) string { } best := "" for alias, key := range b.manualNodeKeys { - if key == id && (best == "" || alias < best) { + status := b.manualNodeStatuses[alias].status + if (key == id || status.HostUUID == id || status.ClusterUUID == id) && (best == "" || alias < best) { best = alias } } @@ -232,6 +272,156 @@ func (b *Broker) manualAliasForKey(id string) string { return id } +// rememberManualPeer records only a current, authenticated association. The +// association helps lifecycle removal when the peer is later offline; it never +// authorizes transport or inventory. +func (b *Broker) rememberManualPeer(s manualNodeStatus) { + if b.manualNodesConfigPath == "" || s.Kind == "engine" || s.IdentityVerification != "verified" || s.HostUUID == "" { + return + } + b.manualPersistMu.Lock() + defer b.manualPersistMu.Unlock() + entries, err := loadPersistedManualNodes(b.manualNodesConfigPath) + if err != nil { + slog.Warn("failed to load manual peer association", "err", err) + return + } + for i := range entries { + if entries[i].key() == s.ID && entries[i].Address == s.Address && entries[i].Kind == "pair" && entries[i].PeerUUID == "" { + entries[i].PeerUUID = s.HostUUID + if err := writePersistedManualNodes(b.manualNodesConfigPath, entries); err != nil { + slog.Warn("failed to persist manual peer association", "err", err) + } + return + } + } +} + +func (b *Broker) manualAliasesForPeer(id string, addressHints ...string) ([]string, error) { + if id == "" { + return nil, nil + } + member, err := b.manualMemberForID(id) + if err != nil { + return nil, err + } + peerID := member.NodeUUID + addresses := make(map[string]bool, len(addressHints)+1) + if member.Address != "" { + addresses[member.Address] = true + } + for _, hint := range addressHints { + if hint != "" { + addresses[hint] = true + } + } + aliases := map[string]bool{} + b.manualMu.Lock() + statuses := make(map[string]manualNodeStatus, len(b.manualNodeStatuses)) + for alias, entry := range b.manualNodeStatuses { + statuses[alias] = entry.status + } + b.manualMu.Unlock() + b.manualPersistMu.Lock() + entries, err := loadPersistedManualNodes(b.manualNodesConfigPath) + b.manualPersistMu.Unlock() + if err != nil { + return nil, err + } + configured := make(map[string]bool, len(entries)) + for _, entry := range entries { + alias := entry.key() + configured[alias] = true + if entry.Kind != "pair" { + continue + } + // A current durable association wins over every cached identity, + // reused address, or transient hint. Only unassociated entries need + // the reachability/observation fallbacks. + if entry.PeerUUID != "" { + if entry.PeerUUID == peerID { + aliases[alias] = true + } + continue + } + s := statuses[alias] + if alias == id || addresses[entry.Address] || s.Address == entry.Address && s.Kind != "engine" && (s.HostUUID == peerID || s.ClusterUUID == peerID) { + aliases[alias] = true + } + } + // Automatically tracked cluster rows do not have durable configurations. + for alias, s := range statuses { + if configured[alias] || s.Kind == "engine" { + continue + } + if s.HostUUID == peerID || s.ClusterUUID == peerID || alias == id || addresses[s.Address] { + aliases[alias] = true + } + } + out := make([]string, 0, len(aliases)) + for alias := range aliases { + out = append(out, alias) + } + sort.Strings(out) + return out, nil +} + +// Recheck the current association atomically with the durable deletion: a +// pairing completion can reassociate an alias while the removal RPC is pending, +// after either snapshot was collected. +func (b *Broker) forgetPersistedManualPeerAlias(alias, peerID string) (bool, error) { + b.manualPersistMu.Lock() + defer b.manualPersistMu.Unlock() + entries, err := loadPersistedManualNodes(b.manualNodesConfigPath) + if err != nil { + return false, err + } + out := entries[:0] + found := false + for _, entry := range entries { + if entry.key() != alias { + out = append(out, entry) + continue + } + if entry.Kind != "pair" || entry.PeerUUID != "" && entry.PeerUUID != peerID { + return false, nil + } + found = true + } + if !found { + return true, nil + } // an automatically tracked, non-durable row + if err := writePersistedManualNodes(b.manualNodesConfigPath, out); err != nil { + return false, err + } + return true, nil +} + +// Caller serializes this transaction with manual-node mutations. +func (b *Broker) forgetManualAliases(aliases []string, peerID string) error { + for _, alias := range aliases { + removed, err := b.forgetPersistedManualPeerAlias(alias, peerID) + if err != nil { + return err + } + if !removed { + continue + } + if worker := b.getManualNodes(); worker != nil { + params, _ := json.Marshal(map[string]string{"id": alias}) + _, rpcErr, err := worker.Call(context.Background(), "node/remove", params) + if err != nil { + return err + } + if rpcErr != nil { + return fmt.Errorf("forget manual entry: %s", rpcErr.Message) + } + } + b.removeManualNode(alias) + } + return nil +} + func manualNodeTelemetry(status manualNodeStatus, hostUUID string) noderec.NodeTelemetry { var utilization uint32 for i := range status.GPUs { @@ -292,10 +482,11 @@ func manualToEnriched(s manualNodeStatus) EnrichedNode { recordID := s.PeerRecordID verification := "verified" if recordID == "" { - recordID = noderec.ManualRecordID(s.ID) + recordID = noderec.PreviewRecordID(s.ID, "") verification = "unverified" - if s.HostUUID == "" && s.ClusterUUID == "" && s.IdentityVerification != "unverified" { + if s.Kind == "engine" && s.IdentityVerification == "manual" { verification = "manual" + recordID = noderec.ManualRecordID(s.ID) } } models := s.Models @@ -349,7 +540,7 @@ func (b *Broker) annotateManualTrust(en *EnrichedNode, s manualNodeStatus) { en.Trusted = s.HostUUID == s.ClusterUUID && s.PeerMetadata != nil && s.PeerMetadata.HostUUID == s.HostUUID && b.currentPeerProof(s.ClusterUUID, s.PeerRecordID) if !en.Trusted && en.IdentityVerification == "verified" { en.IdentityVerification = "unverified" - en.PeerRecordID = noderec.ManualRecordID(s.ID) + en.PeerRecordID = noderec.PreviewRecordID(s.ID, "") } if en.Trusted { en.Clustered = true @@ -440,11 +631,13 @@ func (b *Broker) manualToDirectoryNode(s manualNodeStatus, key string) noderec.D } verification := "unverified" - if s.HostUUID == "" && s.ClusterUUID == "" && s.IdentityVerification != "unverified" { + recordID := noderec.PreviewRecordID(s.ID, "") + if s.Kind == "engine" && s.IdentityVerification == "manual" { verification = "manual" + recordID = noderec.ManualRecordID(s.ID) } return noderec.DirectoryNode{ - PeerRecordID: noderec.ManualRecordID(s.ID), + PeerRecordID: recordID, IdentityVerification: verification, HostUUID: hostUUID, Name: s.ID, @@ -608,7 +801,7 @@ func (b *Broker) bridgeToProxy(p *proxyProcess, engine string, s manualNodeStatu } // Proven PAIR peers travel exclusively through the authenticated relay. // Only independent, explicitly configured bare engines use this bypass. - if up && s.IdentityVerification != "unverified" && s.HostUUID == "" && s.ClusterUUID == "" && s.PeerRecordID == "" && s.Address != "" && port > 0 { + if up && s.Kind == "engine" && s.IdentityVerification == "manual" && s.HostUUID == "" && s.ClusterUUID == "" && s.PeerRecordID == "" && s.Address != "" && port > 0 { node := proxyManualNode{ ID: key, Host: s.Address, diff --git a/services/nvpair-ui-broker/manualnodes_completion_order_test.go b/services/nvpair-ui-broker/manualnodes_completion_order_test.go new file mode 100644 index 000000000..3f4bbebc0 --- /dev/null +++ b/services/nvpair-ui-broker/manualnodes_completion_order_test.go @@ -0,0 +1,87 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "testing" +) + +func TestManualRemovalHintsExpireWithoutGrantingAuthority(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manual-nodes.json") + if err := writePersistedManualNodes(path, []persistedManualNode{{ID: "alias", Kind: "pair", Address: "overlay-0.example"}}); err != nil { + t.Fatal(err) + } + b := &Broker{clusterDir: dir, manualNodesConfigPath: path, store: newDiscoveryStore()} + b.beginManualRemoval(manualClusterMember{NodeUUID: "peer", ClusterID: "cluster-1", AdmissionEpoch: 1}) + for i := 0; i < 2*manualRemovalHintLimit; i++ { + params, _ := json.Marshal(manualPairingCompletion{Address: fmt.Sprintf("overlay-%d.example", i), PeerUUID: "peer", ClusterID: "cluster-1", AdmissionEpoch: 1}) + if err := b.associatePairedManualNodes(params); err != nil { + t.Fatal(err) + } + } + if hints := b.finishManualRemoval(false); len(hints) != manualRemovalHintLimit { + t.Fatalf("unbounded cleanup hints: %d", len(hints)) + } + if b.manualPendingRemoval != nil { + t.Fatal("cleanup hints outlived the unpair") + } + stored, err := loadPersistedManualNodes(path) + if err != nil || len(stored) != 1 || stored[0].PeerUUID != "" || len(b.schedulerNodes()) != 0 { + t.Fatalf("transient hint became persistent association or authority: %+v %v", stored, err) + } +} + +func TestRemovedAdmissionRejectsCompletionDespiteStaleRoster(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manual-nodes.json") + if err := writePersistedManualNodes(path, []persistedManualNode{{ID: "alias", Kind: "pair", Address: "overlay.example"}}); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "members.json"), []byte(`[{"nodeUuid":"peer","clusterId":"cluster-1","admissionEpoch":1,"state":"member"}]`), 0600); err != nil { + t.Fatal(err) + } + b := &Broker{clusterDir: dir, manualNodesConfigPath: path} + b.beginManualRemoval(manualClusterMember{NodeUUID: "peer", ClusterID: "cluster-1", AdmissionEpoch: 1}) + b.finishManualRemoval(true) + if err := b.associatePairedManualNodes(json.RawMessage(`{"address":"overlay.example","peerUuid":"peer","clusterId":"cluster-1","admissionEpoch":1}`)); err != nil { + t.Fatal(err) + } + stored, err := loadPersistedManualNodes(path) + if err != nil || len(stored) != 1 || stored[0].PeerUUID != "" { + t.Fatalf("completion after successful removal was accepted: %+v %v", stored, err) + } +} + +func TestObsoletePairingCompletionCannotAssociateManualEntry(t *testing.T) { + for _, roster := range []string{ + `[]`, + `[{"nodeUuid":"peer","clusterId":"cluster-1","admissionEpoch":2,"state":"member"}]`, + `[{"nodeUuid":"peer","clusterId":"other-cluster","admissionEpoch":1,"state":"member"}]`, + `[{"nodeUuid":"peer","clusterId":"cluster-1","admissionEpoch":1,"state":"pending"}]`, + } { + t.Run(roster, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manual-nodes.json") + if err := writePersistedManualNodes(path, []persistedManualNode{{Kind: "pair", ID: "alias", Address: "overlay.example"}}); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "members.json"), []byte(roster), 0600); err != nil { + t.Fatal(err) + } + var out bytes.Buffer + b := &Broker{codec: NewCodec(&out), clusterDir: dir, manualNodesConfigPath: path} + b.forwardClusterManagerNotification("cluster:pairing-completed", json.RawMessage(`{"address":"overlay.example","peerUuid":"peer","clusterId":"cluster-1","admissionEpoch":1}`)) + stored, err := loadPersistedManualNodes(path) + if err != nil || len(stored) != 1 || stored[0].PeerUUID != "" { + t.Fatalf("obsolete completion persisted after removal/admission change: %+v %v", stored, err) + } + }) + } +} diff --git a/services/nvpair-ui-broker/manualnodes_pairing_test.go b/services/nvpair-ui-broker/manualnodes_pairing_test.go new file mode 100644 index 000000000..d15d18227 --- /dev/null +++ b/services/nvpair-ui-broker/manualnodes_pairing_test.go @@ -0,0 +1,122 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "bytes" + "encoding/json" + "net" + "os" + "path/filepath" + "testing" +) + +func TestPairingCompletionDuringUnpairForgetsHostname(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manual-nodes.json") + if err := writePersistedManualNodes(path, []persistedManualNode{{Kind: "pair", ID: "alias", Address: "Workstation.tail.example"}}); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "members.json"), []byte(`[{"nodeUuid":"peer","ipAddress":"192.0.2.50","clusterId":"cluster-1","admissionEpoch":1,"state":"member"}]`), 0600); err != nil { + t.Fatal(err) + } + clientBroker, client := net.Pipe() + defer clientBroker.Close() + defer client.Close() + clusterBroker, cluster := net.Pipe() + defer clusterBroker.Close() + defer cluster.Close() + b := &Broker{codec: NewCodec(clientBroker), clusterDir: dir, manualNodesConfigPath: path} + cm := &clusterManagerProcess{peer: NewPeer(NewCodec(clusterBroker)), onNotify: b.forwardClusterManagerNotification} + b.setClusterMgr(cm) + go cm.peer.Serve(nil, cm.handleNotify) + go func() { + codec := NewCodec(cluster) + msg, err := codec.Read() + if err != nil { + return + } + // Publication was ordered first, but the broker can consume it after + // the producer has already persisted the removed roster. + if err := os.WriteFile(filepath.Join(dir, "members.json"), []byte(`[]`), 0600); err != nil { + _ = codec.RespondError(msg.ID, -32000, err.Error()) + return + } + _ = codec.Notify("cluster:pairing-completed", manualPairingCompletion{Address: "Workstation.tail.example", PeerUUID: "peer", ClusterID: "cluster-1", AdmissionEpoch: 1}) + _ = codec.Respond(msg.ID, map[string]bool{"removed": true}) + }() + id := json.RawMessage("1") + go b.relayToClusterManager(&Message{JSONRPC: "2.0", ID: &id, Method: "nodes:remove", Params: json.RawMessage(`{"nodeUuid":"peer"}`)}) + response, err := NewCodec(client).Read() + if err != nil || response.Error != nil { + t.Fatalf("unpair response: %+v %v", response, err) + } + stored, err := loadPersistedManualNodes(path) + if err != nil || len(stored) != 0 { + t.Fatalf("in-flight association survived unpair: %+v %v", stored, err) + } +} + +func TestPairingCompletionForgetsHostnameBeforeInventoryProof(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manual-nodes.json") + if err := writePersistedManualNodes(path, []persistedManualNode{{Kind: "pair", ID: "original-alias", Address: "Workstation.tail.example"}, {Kind: "engine", ID: "independent-engine", Address: "Workstation.tail.example"}}); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "members.json"), []byte(`[{"id":"workstation","nodeUuid":"peer","ipAddress":"192.0.2.50","clusterId":"cluster-1","admissionEpoch":1,"state":"member"}]`), 0600); err != nil { + t.Fatal(err) + } + var frontend bytes.Buffer + b := &Broker{codec: NewCodec(&frontend), clusterDir: dir, manualNodesConfigPath: path, store: newDiscoveryStore()} + // This is a worker-owned post-authentication completion, not a public claim. + b.forwardClusterManagerNotification("cluster:pairing-completed", json.RawMessage(`{"address":"Workstation.tail.example","peerUuid":"peer","clusterId":"cluster-1","admissionEpoch":1}`)) + stored, err := loadPersistedManualNodes(path) + if err != nil { + t.Fatal(err) + } + for _, entry := range stored { + if entry.ID == "original-alias" && entry.PeerUUID != "peer" { + t.Fatalf("pairing did not persist hostname-to-peer cleanup association: %+v", entry) + } + if entry.Kind == "engine" && entry.PeerUUID != "" { + t.Fatal("pairing attached a bare engine to peer lifecycle") + } + } + if frontend.Len() != 0 { + t.Fatal("internal cleanup association leaked to renderer") + } + if len(b.schedulerNodes()) != 0 { + t.Fatal("cleanup association granted inventory or routing authority") + } + + // Restart the broker before any node-info proof. Only durable configuration + // and a numeric roster address remain; unpair must still forget the alias. + clientBroker, client := net.Pipe() + defer clientBroker.Close() + defer client.Close() + clusterBroker, cluster := net.Pipe() + defer clusterBroker.Close() + defer cluster.Close() + cm := &clusterManagerProcess{peer: NewPeer(NewCodec(clusterBroker))} + go cm.peer.Serve(nil, nil) + go func() { + codec := NewCodec(cluster) + msg, err := codec.Read() + if err == nil { + _ = codec.Respond(msg.ID, map[string]bool{"removed": true}) + } + }() + restarted := &Broker{codec: NewCodec(clientBroker), clusterDir: dir, manualNodesConfigPath: path} + restarted.setClusterMgr(cm) + id := json.RawMessage("1") + go restarted.relayToClusterManager(&Message{JSONRPC: "2.0", ID: &id, Method: "nodes:remove", Params: json.RawMessage(`{"nodeUuid":"peer"}`)}) + response, err := NewCodec(client).Read() + if err != nil || response.Error != nil { + t.Fatalf("unpair response: %+v, %v", response, err) + } + stored, err = loadPersistedManualNodes(path) + if err != nil || len(stored) != 1 || stored[0].Kind != "engine" { + t.Fatalf("hostname survived unpair and can replay: %+v, %v", stored, err) + } +} diff --git a/services/nvpair-ui-broker/manualnodes_persistence_test.go b/services/nvpair-ui-broker/manualnodes_persistence_test.go index 6903a87fd..a1522607a 100644 --- a/services/nvpair-ui-broker/manualnodes_persistence_test.go +++ b/services/nvpair-ui-broker/manualnodes_persistence_test.go @@ -6,23 +6,100 @@ package main import ( "encoding/json" "net" + "os" "path/filepath" "testing" "time" ) +func TestVerifiedAssociationSurvivesOfflineReplayWithoutGrantingTrust(t *testing.T) { + b := newManualTestBroker(t, "peer") + b.manualNodesConfigPath = filepath.Join(t.TempDir(), "manual-nodes.json") + entry := persistedManualNode{Kind: "pair", ID: "overlay", Address: "MagicDNS.example", Name: "overlay"} + if err := b.persistManualNode(entry); err != nil { + t.Fatal(err) + } + status := ownedManualStatus(t, b, manualNodeStatus{Kind: "pair", ID: "overlay", Address: entry.Address, HostUUID: "peer", IdentityVerification: "verified"}) + b.upsertManualNode(status) + stored, err := loadPersistedManualNodes(b.manualNodesConfigPath) + if err != nil || len(stored) != 1 || stored[0].PeerUUID != "peer" { + t.Fatalf("proven association: %+v %v", stored, err) + } + b.manualNodeStatuses = map[string]manualNodeStatusEntry{} + if err := b.persistManualNode(entry); err != nil { + t.Fatal(err) + } + aliases, err := b.manualAliasesForPeer("peer") + if err != nil || len(aliases) != 1 || aliases[0] != "overlay" { + t.Fatalf("offline aliases: %v %v", aliases, err) + } + b.upsertManualNode(manualNodeStatus{Kind: "pair", ID: "overlay", Address: entry.Address, IdentityVerification: "unverified"}) + if got := b.schedulerNodes(); len(got) != 0 { + t.Fatalf("stored association authorized routing: %+v", got) + } +} + +func TestUnpairMatchesRosterAddressBeforeInventoryProof(t *testing.T) { + dir := t.TempDir() + b := &Broker{clusterDir: dir, manualNodesConfigPath: filepath.Join(dir, "manual-nodes.json")} + if err := os.WriteFile(filepath.Join(dir, "members.json"), []byte(`[{"id":"workstation","nodeUuid":"peer","ipAddress":"MagicDNS.example"}]`), 0600); err != nil { + t.Fatal(err) + } + if err := writePersistedManualNodes(b.manualNodesConfigPath, []persistedManualNode{{ID: "overlay", Address: "MagicDNS.example", Kind: "pair"}, {ID: "bare", Address: "MagicDNS.example", Kind: "engine"}}); err != nil { + t.Fatal(err) + } + for _, id := range []string{"peer", "workstation"} { + aliases, err := b.manualAliasesForPeer(id) + if err != nil || len(aliases) != 1 || aliases[0] != "overlay" { + t.Fatalf("roster-only aliases for %q: %v %v", id, aliases, err) + } + } +} + +func TestLegacyManualIntentMigratesToPairOnDisk(t *testing.T) { + path := filepath.Join(t.TempDir(), "manual-nodes.json") + if err := os.WriteFile(path, []byte(`[{"id":"old","address":"MagicDNS.example","name":"old"}]`), 0600); err != nil { + t.Fatal(err) + } + entries, err := loadPersistedManualNodes(path) + if err != nil { + t.Fatal(err) + } + if len(entries) != 1 || entries[0].Kind != "pair" || entries[0].Address != "MagicDNS.example" { + t.Fatalf("migration = %+v", entries) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var stored []persistedManualNode + if err := json.Unmarshal(data, &stored); err != nil { + t.Fatal(err) + } + if len(stored) != 1 || stored[0].Kind != "pair" { + t.Fatalf("migration was not durable: %s", data) + } + if err := (&Broker{manualNodesConfigPath: path}).persistManualNode(persistedManualNode{ID: "engine", Address: "engine.example", Kind: "engine"}); err != nil { + t.Fatal(err) + } + entries, err = loadPersistedManualNodes(path) + if err != nil || len(entries) != 2 || entries[0].Kind != "engine" || entries[1].Kind != "pair" { + t.Fatalf("explicit engine persistence: %+v, %v", entries, err) + } +} + func TestPersistManualNodeRoundTripsAndRemoves(t *testing.T) { path := filepath.Join(t.TempDir(), "configs", "manual-nodes.json") b := &Broker{manualNodesConfigPath: path} - if err := b.persistManualNode(persistedManualNode{ID: "lab", Address: "node.local", Name: "lab", TLSPort: 14319, MTLS: true}); err != nil { + if err := b.persistManualNode(persistedManualNode{Kind: "pair", ID: "lab", Address: "node.local", Name: "lab", TLSPort: 14319, MTLS: true}); err != nil { t.Fatal(err) } entries, err := loadPersistedManualNodes(path) if err != nil { t.Fatal(err) } - if len(entries) != 1 || entries[0].ID != "lab" || entries[0].TLSPort != 14319 || !entries[0].MTLS { + if len(entries) != 1 || entries[0].Kind != "pair" || entries[0].ID != "lab" || entries[0].TLSPort != 14319 || !entries[0].MTLS { t.Fatalf("persisted entries = %+v", entries) } @@ -69,7 +146,7 @@ func TestReplayPersistedManualNodesIntoWorker(t *testing.T) { select { case entry := <-got: - if entry.Address != "node.local" || entry.Name != "lab" { + if entry.Kind != "pair" || entry.Address != "node.local" || entry.Name != "lab" { t.Fatalf("replayed entry = %+v", entry) } case <-time.After(2 * time.Second): diff --git a/services/nvpair-ui-broker/manualnodes_reassignment_test.go b/services/nvpair-ui-broker/manualnodes_reassignment_test.go new file mode 100644 index 000000000..82303cc0c --- /dev/null +++ b/services/nvpair-ui-broker/manualnodes_reassignment_test.go @@ -0,0 +1,107 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net" + "os" + "path/filepath" + "testing" +) + +func TestCurrentManualAssociationSurvivesStaleObservationAndReadd(t *testing.T) { + b := newManualTestBroker(t, "peer-a", "peer-b") + b.clusterDir = t.TempDir() + b.manualNodesConfigPath = filepath.Join(b.clusterDir, "manual-nodes.json") + entry := persistedManualNode{Kind: "pair", ID: "alias", Name: "alias", Address: "Workstation.tail.example", PeerUUID: "peer-a"} + if err := writePersistedManualNodes(b.manualNodesConfigPath, []persistedManualNode{entry}); err != nil { + t.Fatal(err) + } + roster := `[{"nodeUuid":"peer-a","clusterId":"fixture-cluster","admissionEpoch":1,"state":"member"},{"nodeUuid":"peer-b","clusterId":"fixture-cluster","admissionEpoch":1,"state":"member"}]` + if err := os.WriteFile(filepath.Join(b.clusterDir, "members.json"), []byte(roster), 0600); err != nil { + t.Fatal(err) + } + params, _ := json.Marshal(manualPairingCompletion{Address: entry.Address, PeerUUID: "peer-b", ClusterID: "fixture-cluster", AdmissionEpoch: 1}) + if err := b.associatePairedManualNodes(params); err != nil { + t.Fatal(err) + } + stale := ownedManualStatus(t, b, manualNodeStatus{Kind: "pair", IdentityVerification: "verified", ID: entry.ID, Address: entry.Address, HostUUID: "peer-a"}) + b.upsertManualNode(stale) + b.refreshManualNodesTrust() + entry.PeerUUID = "" // ordinary re-add cannot seed from the stale cached A proof + if err := b.persistManualNode(entry); err != nil { + t.Fatal(err) + } + stored, err := loadPersistedManualNodes(b.manualNodesConfigPath) + if err != nil || len(stored) != 1 || stored[0].PeerUUID != "peer-b" { + t.Fatalf("stale observation/re-add replaced current B association: %+v %v", stored, err) + } +} + +func TestUnpairAssociationOverridesCachedIdentityAndReusedAddress(t *testing.T) { + for _, address := range []string{"Workstation.tail.example", "192.0.2.50"} { + t.Run(address, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manual-nodes.json") + if err := writePersistedManualNodes(path, []persistedManualNode{{Kind: "pair", ID: "alias", Address: address, PeerUUID: "peer-b"}}); err != nil { + t.Fatal(err) + } + roster, _ := json.Marshal([]manualClusterMember{{NodeUUID: "peer-a", Address: address, State: "member"}, {NodeUUID: "peer-b", Address: "192.0.2.99", State: "member"}}) + if err := os.WriteFile(filepath.Join(dir, "members.json"), roster, 0600); err != nil { + t.Fatal(err) + } + b := &Broker{clusterDir: dir, manualNodesConfigPath: path, manualNodeStatuses: map[string]manualNodeStatusEntry{"alias": {status: manualNodeStatus{Kind: "pair", Address: address, HostUUID: "peer-a"}}}} + aliases, err := b.manualAliasesForPeer("peer-a") + if err != nil || len(aliases) != 0 { + t.Fatalf("removing A selected B's alias using stale identity/reused address: %v %v", aliases, err) + } + aliases, err = b.manualAliasesForPeer("peer-b") + if err != nil || len(aliases) != 1 || aliases[0] != "alias" { + t.Fatalf("current B association did not own its alias: %v %v", aliases, err) + } + }) + } +} + +func TestUnpairRechecksAssociationChangedDuringRequest(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "manual-nodes.json") + if err := writePersistedManualNodes(path, []persistedManualNode{{Kind: "pair", ID: "alias", Address: "overlay.example", PeerUUID: "peer-a"}}); err != nil { + t.Fatal(err) + } + roster := `[{"nodeUuid":"peer-a","clusterId":"cluster-1","admissionEpoch":1,"state":"member"},{"nodeUuid":"peer-b","clusterId":"cluster-1","admissionEpoch":1,"state":"member"}]` + if err := os.WriteFile(filepath.Join(dir, "members.json"), []byte(roster), 0600); err != nil { + t.Fatal(err) + } + clientBroker, client := net.Pipe() + defer clientBroker.Close() + defer client.Close() + clusterBroker, cluster := net.Pipe() + defer clusterBroker.Close() + defer cluster.Close() + b := &Broker{codec: NewCodec(clientBroker), clusterDir: dir, manualNodesConfigPath: path} + cm := &clusterManagerProcess{peer: NewPeer(NewCodec(clusterBroker)), onNotify: b.forwardClusterManagerNotification} + b.setClusterMgr(cm) + go cm.peer.Serve(nil, cm.handleNotify) + go func() { + codec := NewCodec(cluster) + msg, err := codec.Read() + if err != nil { + return + } + _ = codec.Notify("cluster:pairing-completed", manualPairingCompletion{Address: "overlay.example", PeerUUID: "peer-b", ClusterID: "cluster-1", AdmissionEpoch: 1}) + _ = codec.Respond(msg.ID, map[string]bool{"removed": true}) + }() + id := json.RawMessage("1") + go b.relayToClusterManager(&Message{JSONRPC: "2.0", ID: &id, Method: "nodes:remove", Params: json.RawMessage(`{"nodeUuid":"peer-a"}`)}) + response, err := NewCodec(client).Read() + if err != nil || response.Error != nil { + t.Fatalf("unpair response: %+v %v", response, err) + } + stored, err := loadPersistedManualNodes(path) + if err != nil || len(stored) != 1 || stored[0].PeerUUID != "peer-b" { + t.Fatalf("pre-removal snapshot deleted newly reassociated B entry: %+v %v", stored, err) + } +} diff --git a/services/nvpair-ui-broker/manualnodes_removal_test.go b/services/nvpair-ui-broker/manualnodes_removal_test.go new file mode 100644 index 000000000..087b81844 --- /dev/null +++ b/services/nvpair-ui-broker/manualnodes_removal_test.go @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/json" + "net" + "path/filepath" + "testing" +) + +func TestUnpairForgetsAllManualAddressesOnlyAfterSuccess(t *testing.T) { + for _, removed := range []bool{false, true} { + t.Run(map[bool]string{false: "not removed", true: "removed"}[removed], func(t *testing.T) { + path := filepath.Join(t.TempDir(), "manual-nodes.json") + entries := []persistedManualNode{{Kind: "pair", ID: "overlay", Address: "MagicDNS.example", PeerUUID: "peer"}, {Kind: "pair", ID: "wireguard", Address: "192.0.2.1", PeerUUID: "peer"}, {Kind: "engine", ID: "bare", Address: "bare.example"}} + if err := writePersistedManualNodes(path, entries); err != nil { + t.Fatal(err) + } + clientBroker, client := net.Pipe() + defer clientBroker.Close() + defer client.Close() + clusterBroker, cluster := net.Pipe() + defer clusterBroker.Close() + defer cluster.Close() + cm := &clusterManagerProcess{peer: NewPeer(NewCodec(clusterBroker))} + go cm.peer.Serve(nil, nil) + go func() { + codec := NewCodec(cluster) + msg, err := codec.Read() + if err == nil { + _ = codec.Respond(msg.ID, map[string]bool{"removed": removed}) + } + }() + b := &Broker{codec: NewCodec(clientBroker), manualNodesConfigPath: path} + b.setClusterMgr(cm) + id := json.RawMessage("1") + go b.relayToClusterManager(&Message{JSONRPC: "2.0", ID: &id, Method: "nodes:remove", Params: json.RawMessage(`{"nodeUuid":"peer"}`)}) + response, err := NewCodec(client).Read() + if err != nil || response.Error != nil { + t.Fatalf("remove response: %+v, %v", response, err) + } + got, err := loadPersistedManualNodes(path) + if err != nil { + t.Fatal(err) + } + want := 3 + if removed { + want = 1 + } + if len(got) != want { + t.Fatalf("entries after removed=%v: %+v", removed, got) + } + if removed && got[0].Kind != "engine" { + t.Fatalf("bare bypass entry removed by unpair: %+v", got) + } + }) + } +} diff --git a/services/nvpair-ui-broker/peer_inventory.go b/services/nvpair-ui-broker/peer_inventory.go index c77b35d34..9dd20d606 100644 --- a/services/nvpair-ui-broker/peer_inventory.go +++ b/services/nvpair-ui-broker/peer_inventory.go @@ -31,10 +31,10 @@ func (b *Broker) pushPeerMetadataToNodeInfo() { } func (b *Broker) normalizeManualStatus(s manualNodeStatus) manualNodeStatus { - if s.PeerRecordID == "" { + if s.Kind == "engine" && s.PeerRecordID == "" && s.HostUUID == "" && s.ClusterUUID == "" && s.IdentityVerification == "manual" { return s } - if s.HostUUID == s.ClusterUUID && s.PeerMetadata != nil && s.PeerMetadata.HostUUID == s.HostUUID && b.currentPeerProof(s.ClusterUUID, s.PeerRecordID) { + if s.Kind != "engine" && s.PeerRecordID != "" && s.HostUUID == s.ClusterUUID && s.PeerMetadata != nil && s.PeerMetadata.HostUUID == s.HostUUID && b.currentPeerProof(s.ClusterUUID, s.PeerRecordID) { return s } // A retired worker proof may remain as an unverified configuration preview, diff --git a/services/nvpair-ui-broker/verified_inventory_test.go b/services/nvpair-ui-broker/verified_inventory_test.go index 970eb147c..c2ceb817e 100644 --- a/services/nvpair-ui-broker/verified_inventory_test.go +++ b/services/nvpair-ui-broker/verified_inventory_test.go @@ -283,7 +283,7 @@ func TestScannerRestartWithdrawsProofButPreservesIndependentManualEngine(t *test node := verifiedDirectoryForTest(b) b.store.Upsert(directoryToEnriched(node), sourceScanner) b.relayDir.Apply(noderec.NotifyNodeDiscovered, node) - b.upsertManualNode(manualNodeStatus{ID: "manual:engine", Address: "192.0.2.3", OMLXUp: true, OMLXPort: 1236, OMLXModels: []string{"bare-model"}}) + b.upsertManualNode(manualNodeStatus{Kind: "engine", IdentityVerification: "manual", ID: "manual:engine", Address: "192.0.2.3", OMLXUp: true, OMLXPort: 1236, OMLXModels: []string{"bare-model"}}) b.clearScannerInventory() got := b.schedulerNodes() if len(got) != 1 || got[0].HostUUID != noderec.ManualRecordID("manual:engine") || len(got[0].Models) != 1 || got[0].Models[0] != "bare-model" { diff --git a/services/ollama-proxy/manual_overlay_test.go b/services/ollama-proxy/manual_overlay_test.go new file mode 100644 index 000000000..9333e45fc --- /dev/null +++ b/services/ollama-proxy/manual_overlay_test.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" + "path/filepath" + "testing" +) + +// A broker-proven peer uses its manually configured overlay hostname, not a +// LAN address from discovery, and still requires a live exact certificate pin. +func TestManualOverlayHostnameKeepsPinnedTransport(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, dir, "cluster", "self", "peer") + n, ok := subscribedToNode(noderec.DirectoryNode{ + HostUUID: "peer", ClusterUUID: "peer", Name: "peer", IP: "Workstation.tail.example", + IPs: []string{"Workstation.tail.example"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOllama: {Port: 11434}}, + }) + if !ok { + t.Fatal("manual address lost in relay projection") + } + disc := NewDiscovery() + disc.SetSubscribed([]Node{n}) + p := testProxy(disc, 11435) + p.mesh = clustertrust.Open(dir) + got := p.resolveCandidates("") + if len(got) != 1 || got[0].url.Scheme != "https" || got[0].url.Host != "Workstation.tail.example:11434" || got[0].peerUUID != "peer" { + t.Fatalf("overlay candidates = %+v", got) + } + clustertrusttest.RemovePeerPin(t, dir, "peer") + if got := p.resolveCandidates(""); len(got) != 0 { + t.Fatalf("removed peer still routable: %+v", got) + } +} diff --git a/services/omlx-proxy/manual_overlay_test.go b/services/omlx-proxy/manual_overlay_test.go new file mode 100644 index 000000000..495dbfaca --- /dev/null +++ b/services/omlx-proxy/manual_overlay_test.go @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "nvpair-shared/clustertrust" + "nvpair-shared/clustertrusttest" + "nvpair-shared/noderec" + "path/filepath" + "testing" +) + +// A broker-proven peer uses its manually configured overlay hostname, not a +// LAN address from discovery, and still requires a live exact certificate pin. +func TestManualOverlayHostnameKeepsPinnedTransport(t *testing.T) { + dir := filepath.Join(t.TempDir(), "cluster") + clustertrusttest.Join(t, dir, "cluster", "self", "peer") + n, ok := subscribedToNode(noderec.DirectoryNode{ + HostUUID: "peer", ClusterUUID: "peer", Name: "peer", IP: "Workstation.tail.example", + IPs: []string{"Workstation.tail.example"}, + Services: map[noderec.ServiceKey]noderec.ServiceStatus{noderec.ServiceOMLX: {Port: 1236}}, + }) + if !ok { + t.Fatal("manual address lost in relay projection") + } + disc := NewDiscovery() + disc.SetSubscribed([]Node{n}) + p := testProxy(disc, 1237) + p.mesh = clustertrust.Open(dir) + got := p.resolveCandidates("") + if len(got) != 1 || got[0].url.Scheme != "https" || got[0].url.Host != "Workstation.tail.example:1236" || got[0].peerUUID != "peer" { + t.Fatalf("overlay candidates = %+v", got) + } + clustertrusttest.RemovePeerPin(t, dir, "peer") + if got := p.resolveCandidates(""); len(got) != 0 { + t.Fatalf("removed peer still routable: %+v", got) + } +} diff --git a/services/tests/broker_management_test.go b/services/tests/broker_management_test.go index 7544fbdc1..84e9cba5d 100644 --- a/services/tests/broker_management_test.go +++ b/services/tests/broker_management_test.go @@ -207,7 +207,7 @@ func TestBrokerBridgesBareManualOllamaIntoProxy(t *testing.T) { waitForMethod(t, msgs, "app:ready", 10*time.Second) const nodeName = "xproc-manual-proxy-bridge" - addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":900,"method":"node/add","params":{"address":"127.0.0.1","name":%q}}`, nodeName) + "\n" + addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":900,"method":"node/add","params":{"kind":"engine","address":"127.0.0.1","name":%q}}`, nodeName) + "\n" if _, err := stdin.Write([]byte(addReq)); err != nil { t.Fatalf("write node/add: %v", err) } @@ -289,7 +289,7 @@ func TestBrokerKeepsUnclusteredManualPAIRAsUnverifiedPreview(t *testing.T) { for { n, ok := manualInventory(t, stdin, msgs, nodeName) if ok && n.IdentityVerification == "unverified" { - if n.PeerRecordID != noderec.ManualRecordID(nodeName) || n.Trusted || len(n.Models) != 0 { + if n.PeerRecordID != noderec.PreviewRecordID(nodeName, "") || n.Trusted || len(n.Models) != 0 { t.Fatalf("PAIR preview acquired peer trust or routing inventory: %+v", n) } if tc.hostUUID != "" && (n.HostUUID == tc.hostUUID || n.PeerRecordID == tc.hostUUID) { @@ -325,6 +325,72 @@ func TestBrokerKeepsUnclusteredManualPAIRAsUnverifiedPreview(t *testing.T) { } } +// A live engine at a replayed PAIR address is not consent to a manual bypass. +func TestManualPairIntentSurvivesWorkerRestartWithoutPairEndpoint(t *testing.T) { + t.Cleanup(fakeOllama(t)) + t.Cleanup(fakeNodeInfo(t, nil)) + stdin, msgs, _, cleanup := startBrokerWith(t, "--manual-nodes-path", manualNodesBin) + t.Cleanup(cleanup) + waitForMethod(t, msgs, "app:ready", 10*time.Second) + initialWorker := waitWorkerRunning(t, stdin, msgs, "manual-nodes") + const name = "pair-restart-intent" + added := callRPC(t, stdin, msgs, "node/add", map[string]string{"address": "127.0.0.1", "name": name, "kind": "pair"}, 5*time.Second) + if added.Error != nil { + t.Fatalf("add: %+v", added.Error) + } + for generation := 0; generation < 2; generation++ { + deadline := time.Now().Add(15 * time.Second) + seen := false + for time.Now().Before(deadline) { + n, ok := manualInventory(t, stdin, msgs, name) + if ok { + seen = true + if n.IdentityVerification != "unverified" || n.PeerRecordID != noderec.PreviewRecordID(name, "") || n.Trusted || len(n.Models) != 0 { + t.Fatalf("generation %d downgraded PAIR: %+v", generation, n) + } + break + } + time.Sleep(100 * time.Millisecond) + } + if !seen { + t.Fatalf("generation %d did not replay PAIR entry", generation) + } + listed := callRPC(t, stdin, msgs, "nodes/list", nil, 5*time.Second) + var list struct { + Nodes []struct { + ID string `json:"id"` + Kind string `json:"kind"` + OllamaUp bool `json:"ollama_up"` + } `json:"nodes"` + } + if err := json.Unmarshal(listed.Result, &list); err != nil { + t.Fatal(err) + } + for _, n := range list.Nodes { + if n.ID == name && (n.Kind != "pair" || n.OllamaUp) { + t.Fatalf("replayed configuration: %+v", n) + } + } + if generation == 0 { + restarted := callRPC(t, stdin, msgs, "workers:restart", map[string]string{"name": "manual-nodes"}, 10*time.Second) + if restarted.Error != nil { + t.Fatalf("restart: %+v", restarted.Error) + } + deadline := time.Now().Add(15 * time.Second) + for { + worker := waitWorkerRunning(t, stdin, msgs, "manual-nodes") + if worker.PID != initialWorker.PID { + break + } + if time.Now().After(deadline) { + t.Fatal("manual worker did not change process") + } + time.Sleep(100 * time.Millisecond) + } + } + } +} + // Bare LM Studio has the same explicit manual bypass as bare Ollama. func TestBrokerBridgesBareManualLMStudioIntoProxy(t *testing.T) { configDir := persistLMStudioProxyPort(t, freePort(t)) @@ -341,7 +407,7 @@ func TestBrokerBridgesBareManualLMStudioIntoProxy(t *testing.T) { waitForMethod(t, msgs, "app:ready", 10*time.Second) const nodeName = "xproc-manual-lmstudio-bridge" - addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":920,"method":"node/add","params":{"address":"127.0.0.1","name":%q}}`, nodeName) + "\n" + addReq := fmt.Sprintf(`{"jsonrpc":"2.0","id":920,"method":"node/add","params":{"kind":"engine","address":"127.0.0.1","name":%q}}`, nodeName) + "\n" if _, err := stdin.Write([]byte(addReq)); err != nil { t.Fatalf("write node/add: %v", err) } diff --git a/services/versions.json b/services/versions.json index 44677892d..0cc2e7abd 100644 --- a/services/versions.json +++ b/services/versions.json @@ -13,9 +13,9 @@ "nvpair-node-settings": "3.2.0", "nvpair-ui-broker": "2.5.0", "nvpair-engine-manager": "1.4.0", - "nvpair-cluster-manager": "2.1.1", + "nvpair-cluster-manager": "2.2.0", "nvpair-job-scheduler": "1.1.0", - "nvpair-tui": "1.2.1", + "nvpair-tui": "1.3.0", "omlx-proxy": "1.4.2" } }