From 15a4e980d1abcc2be7b3171a563b40b149ef8dd2 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:56:13 -0700 Subject: [PATCH 1/6] feat(byo): record seat-registered on an agent's first authenticated call The browser can see page, command and listen steps, but not whether the registered agent ever calls the backend when the page is closed. The first token use in agentRuntimeAuth now writes one byo_telemetry row, step seat-registered, for the installing user. The browser cannot post it. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: tern Agent-Model: claude/claude-haiku-5-5/default --- .../unit/middleware/agentRuntimeAuth.test.js | 9 +++++ .../unit/routes/stats.byo-step.test.js | 7 ++++ .../unit/services/byoTelemetry.test.js | 35 +++++++++++++++++++ backend/middleware/agentRuntimeAuth.ts | 4 +++ backend/routes/stats.ts | 18 +++------- backend/services/byoTelemetry.ts | 25 +++++++++++++ docs/development/byo-funnel-telemetry.md | 2 ++ 7 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 backend/__tests__/unit/services/byoTelemetry.test.js create mode 100644 backend/services/byoTelemetry.ts diff --git a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js index ee7be69a2b..cada5c0c5b 100644 --- a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js +++ b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js @@ -23,6 +23,7 @@ const mockInstallationFind = jest.fn(); const mockInstallationUpdateOne = jest.fn(); const mockCompleteStarterTask = jest.fn(); const mockMarkByoConnectionAttempt = jest.fn(); +const mockRecordByoEvent = jest.fn(); jest.mock('../../../services/starterTaskService', () => ({ completeConnectAgentStarterTask: (...args) => mockCompleteStarterTask(...args), @@ -30,6 +31,9 @@ jest.mock('../../../services/starterTaskService', () => ({ jest.mock('../../../services/onboardingMarks', () => ({ markByoConnectionAttempt: (...args) => mockMarkByoConnectionAttempt(...args), })); +jest.mock('../../../services/byoTelemetry', () => ({ + recordByoEvent: (...args) => mockRecordByoEvent(...args), +})); jest.mock('../../../models/User', () => { function User() {} @@ -133,6 +137,9 @@ describe('agentRuntimeAuth path 2 (install-bound token) — #66 fix', () => { agentLabel: 'codex', }); expect(mockMarkByoConnectionAttempt).toHaveBeenCalledWith('human-a', matchedInstall); + // Backend-side seat-registered row, written once for the first use of the token. + expect(mockRecordByoEvent).toHaveBeenCalledTimes(1); + expect(mockRecordByoEvent).toHaveBeenCalledWith('human-a', 'seat-registered'); }); test('does not fire the starter hook when the token was used before (#916)', async () => { @@ -150,6 +157,7 @@ describe('agentRuntimeAuth path 2 (install-bound token) — #66 fix', () => { await agentRuntimeAuth(buildReq('cm_agent_test'), buildRes(), jest.fn()); expect(mockCompleteStarterTask).not.toHaveBeenCalled(); + expect(mockRecordByoEvent).not.toHaveBeenCalled(); }); }); @@ -188,6 +196,7 @@ describe('agentRuntimeAuth path 1 (User-row token) — first-use starter hook (# 'human-1', expect.objectContaining({ podId: expect.anything(), installedBy: 'human-1' }), ); + expect(mockRecordByoEvent).toHaveBeenCalledWith('human-1', 'seat-registered'); }); test('stays silent on every subsequent authentication', async () => { diff --git a/backend/__tests__/unit/routes/stats.byo-step.test.js b/backend/__tests__/unit/routes/stats.byo-step.test.js index a0952ebd97..fdc067b10c 100644 --- a/backend/__tests__/unit/routes/stats.byo-step.test.js +++ b/backend/__tests__/unit/routes/stats.byo-step.test.js @@ -85,6 +85,13 @@ describe('POST /api/stats/byo-step', () => { expect(mockMarkSurface).not.toHaveBeenCalled(); }); + it('rejects seat-registered from the browser: the backend alone writes that row', async () => { + const res = await request(app).post('/api/stats/byo-step').send({ step: 'seat-registered' }); + + expect(res.status).toBe(400); + expect(mockInsertOne).not.toHaveBeenCalled(); + }); + it('rejects unknown steps without writing a mark or telemetry event', async () => { const res = await request(app) .post('/api/stats/byo-step') diff --git a/backend/__tests__/unit/services/byoTelemetry.test.js b/backend/__tests__/unit/services/byoTelemetry.test.js new file mode 100644 index 0000000000..f24080d3a3 --- /dev/null +++ b/backend/__tests__/unit/services/byoTelemetry.test.js @@ -0,0 +1,35 @@ +const mockInsertOne = jest.fn(); +const mockWarn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + +jest.mock('mongoose', () => ({ + connection: { db: { collection: () => ({ insertOne: (...args) => mockInsertOne(...args) }) } }, +})); + +const { recordByoEvent } = require('../../../services/byoTelemetry'); + +describe('recordByoEvent', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockInsertOne.mockResolvedValue({ acknowledged: true }); + }); + + afterAll(() => mockWarn.mockRestore()); + + it('writes the fixed step with the user, the extra fields and server time', async () => { + await recordByoEvent('human-1', 'seat-registered', { commandType: 'cli' }); + + expect(mockInsertOne).toHaveBeenCalledWith({ + userId: 'human-1', + step: 'seat-registered', + commandType: 'cli', + at: expect.any(Date), + }); + }); + + it('never throws when the write fails, so the caller flow continues', async () => { + mockInsertOne.mockRejectedValue(new Error('db down')); + + await expect(recordByoEvent('human-1', 'seat-registered')).resolves.toBeUndefined(); + expect(mockWarn).toHaveBeenCalledWith('[byo-telemetry] write failed:', 'db down'); + }); +}); diff --git a/backend/middleware/agentRuntimeAuth.ts b/backend/middleware/agentRuntimeAuth.ts index e73d4d1544..29963c2395 100644 --- a/backend/middleware/agentRuntimeAuth.ts +++ b/backend/middleware/agentRuntimeAuth.ts @@ -9,6 +9,7 @@ import { touchLastActive } from './auth'; import Pod from '../models/Pod'; import AgentCredential from '../models/AgentCredential'; import { markByoConnectionAttempt } from '../services/onboardingMarks'; +import { recordByoEvent } from '../services/byoTelemetry'; import { isSpawnCredential } from '../services/spawnCredentialService'; import { embeddedRuntimeTokenExpired, runtimeCredentialRefusalReason } from '../services/runtimeCredentialGate'; @@ -148,6 +149,8 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next for (const installationRow of installations) { markByoConnectionAttempt(installationRow.installedBy, installationRow); } + // One row per token, not per installation: the seat is the token. + void recordByoEvent(installations[0]?.installedBy, 'seat-registered'); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { completeConnectAgentStarterTask } = require('../services/starterTaskService'); void completeConnectAgentStarterTask({ @@ -205,6 +208,7 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next if (isFirstLegacyTokenUse) { markByoConnectionAttempt(installation.installedBy, installation); + void recordByoEvent(installation.installedBy, 'seat-registered'); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { completeConnectAgentStarterTask } = require('../services/starterTaskService'); void completeConnectAgentStarterTask({ diff --git a/backend/routes/stats.ts b/backend/routes/stats.ts index f68ab8b587..647420269b 100644 --- a/backend/routes/stats.ts +++ b/backend/routes/stats.ts @@ -84,7 +84,7 @@ const { rateLimit } = require('express-rate-limit'); // eslint-disable-next-line global-require const { cloudflareIpRateLimitKeyGenerator } = require('../middleware/ipRateLimit'); // eslint-disable-next-line global-require -const mongoose = require('mongoose'); +const { recordByoEvent } = require('../services/byoTelemetry'); const BYO_STEPS = new Set([ 'page-viewed', 'seat-created', 'mcp-command-copied', 'cli-command-copied', @@ -115,18 +115,10 @@ router.post('/byo-step', rateLimit({ markByoCommandCopied(req.userId, step === 'mcp-command-copied' ? 'mcp' : 'cli'); } if (SURFACE_STEPS[step]) markSurface(req.userId, SURFACE_STEPS[step]); - try { - await mongoose.connection.db.collection('byo_telemetry').insertOne({ - userId: req.userId, - step, - ...(step === 'mcp-command-copied' ? { commandType: 'mcp' } : {}), - ...(step === 'cli-command-copied' ? { commandType: 'cli' } : {}), - at: new Date(), - }); - } catch (err: any) { - // Telemetry must never fail the caller's flow. - console.warn('[byo-telemetry] write failed:', err?.message); - } + await recordByoEvent(req.userId, step as any, { + ...(step === 'mcp-command-copied' ? { commandType: 'mcp' } : {}), + ...(step === 'cli-command-copied' ? { commandType: 'cli' } : {}), + }); return res.json({ ok: true }); }); diff --git a/backend/services/byoTelemetry.ts b/backend/services/byoTelemetry.ts new file mode 100644 index 0000000000..39c41c2510 --- /dev/null +++ b/backend/services/byoTelemetry.ts @@ -0,0 +1,25 @@ +import mongoose from 'mongoose'; + +// Server-side BYO funnel rows. The browser posts its steps through +// POST /api/stats/byo-step; events only the backend can see (an agent's first +// authenticated call) are written here directly, so they never depend on the +// user keeping the page open. Fixed step names only: never a command, key or name. +export type ServerByoStep = 'seat-registered'; + +export async function recordByoEvent( + userId: unknown, + step: ServerByoStep, + extra: Record = {}, +): Promise { + try { + await mongoose.connection.db.collection('byo_telemetry').insertOne({ + userId, + step, + ...extra, + at: new Date(), + }); + } catch (err: any) { + // Telemetry must never fail the caller's flow. + console.warn('[byo-telemetry] write failed:', err?.message); + } +} diff --git a/docs/development/byo-funnel-telemetry.md b/docs/development/byo-funnel-telemetry.md index 41da62872f..f3bafdca77 100644 --- a/docs/development/byo-funnel-telemetry.md +++ b/docs/development/byo-funnel-telemetry.md @@ -18,6 +18,8 @@ As of TASK-483 and TASK-493 (2026-10-24), the browser sends: | `hosted-offer-clicked` | The user activated the offer to start hosted setup. Recorded before name lookup, including failed attempts; concurrent duplicate activations are suppressed. A retry records another activation. This is separate from choosing the initial hosted mode. | | `listen-timeout` | The page's connection polling window expired. | +The backend writes one more row, `seat-registered`, itself; the browser cannot post it (the route answers 400). It is written once per runtime token, on the token's first authenticated call (`middleware/agentRuntimeAuth.ts`), with the installing user as `userId`. It fires whether or not the user is still on the page, so it measures registration when `listen-confirmed` is missing. It is not limited to BYO: a hosted seat's first call also writes it. + Before TASK-483, the client never emitted `page-viewed` or `seat-created`. Historical counts of those events cannot measure page or command exposure. A selected-text copy from a command or key block dismisses the hosted offer but does not claim that a complete From bd6ed6b1422cf89a266572e21bd5a3073083a1f1 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:59:12 -0700 Subject: [PATCH 2/6] fix(byo): the daemon copy button records daemon-command-copied The Add a computer command copy (V2AgentBYO copy key daemon-setup) wrote the clipboard but posted no telemetry step, so a user who copied the daemon command was invisible to the copy count. Prod shows seats created and zero copy rows. The step is telemetry only and sets no onboarding copy mark. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: tern Agent-Model: claude/claude-haiku-5-5/default --- .../unit/routes/stats.byo-step.test.js | 14 +++ backend/routes/stats.ts | 2 +- docs/development/byo-funnel-telemetry.md | 1 + .../V2AgentBYOCopyTelemetry.test.tsx | 101 ++++++++++++++++++ frontend/src/v2/components/V2AgentBYO.tsx | 1 + frontend/src/v2/utils/byoTelemetry.ts | 3 +- 6 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 frontend/src/v2/__tests__/V2AgentBYOCopyTelemetry.test.tsx diff --git a/backend/__tests__/unit/routes/stats.byo-step.test.js b/backend/__tests__/unit/routes/stats.byo-step.test.js index fdc067b10c..153ab258f9 100644 --- a/backend/__tests__/unit/routes/stats.byo-step.test.js +++ b/backend/__tests__/unit/routes/stats.byo-step.test.js @@ -60,6 +60,20 @@ describe('POST /api/stats/byo-step', () => { expect(JSON.stringify(mockInsertOne.mock.calls[0][0])).not.toContain('cm_agent_secret'); }); + it('records the daemon copy as telemetry only, with no onboarding copy mark', async () => { + const res = await request(app) + .post('/api/stats/byo-step') + .send({ step: 'daemon-command-copied', command: 'commonly daemon register' }); + + expect(res.status).toBe(200); + expect(mockMarkByoCommandCopied).not.toHaveBeenCalled(); + expect(mockInsertOne).toHaveBeenCalledWith({ + userId: 'human-1', + step: 'daemon-command-copied', + at: expect.any(Date), + }); + }); + it.each(['hosted-offer-shown', 'hosted-offer-clicked'])('accepts %s without copied-command marks or extra payload', async (step) => { const res = await request(app).post('/api/stats/byo-step') .send({ step, command: 'fixture-only-secret', agentName: 'private-agent' }); diff --git a/backend/routes/stats.ts b/backend/routes/stats.ts index 647420269b..192b188955 100644 --- a/backend/routes/stats.ts +++ b/backend/routes/stats.ts @@ -87,7 +87,7 @@ const { cloudflareIpRateLimitKeyGenerator } = require('../middleware/ipRateLimit const { recordByoEvent } = require('../services/byoTelemetry'); const BYO_STEPS = new Set([ - 'page-viewed', 'seat-created', 'mcp-command-copied', 'cli-command-copied', + 'page-viewed', 'seat-created', 'mcp-command-copied', 'cli-command-copied', 'daemon-command-copied', 'token-copied', 'listen-confirmed', 'listen-timeout', 'hosted-offer-shown', 'hosted-offer-clicked', 'invite-card-shown', 'invite-card-dismissed', 'byo-offer-shown', ]); diff --git a/docs/development/byo-funnel-telemetry.md b/docs/development/byo-funnel-telemetry.md index f3bafdca77..88f8b28cf4 100644 --- a/docs/development/byo-funnel-telemetry.md +++ b/docs/development/byo-funnel-telemetry.md @@ -12,6 +12,7 @@ As of TASK-483 and TASK-493 (2026-10-24), the browser sends: | `seat-created` | A runtime key was issued and its command rendered. Once per issued command. This is the command-exposure denominator. | | `mcp-command-copied` | The Claude or Cursor copy button successfully wrote the command. | | `cli-command-copied` | The listener copy button successfully wrote the command. | +| `daemon-command-copied` | The "Add a computer" daemon command copy button successfully wrote the command. Telemetry only: it sets no onboarding copy mark, so stalled-connect messaging is unchanged. Before TASK-653 this copy posted nothing, so a user who used this path was invisible to the copy count. | | `token-copied` | The token copy button successfully wrote the key. | | `listen-confirmed` | The connection poll observed this agent using its key after issuance (with existing clock-skew allowance). | | `hosted-offer-shown` | The timer or desktop exit-intent path displayed the hosted offer. The shared session guard records it once, including when both triggers run. | diff --git a/frontend/src/v2/__tests__/V2AgentBYOCopyTelemetry.test.tsx b/frontend/src/v2/__tests__/V2AgentBYOCopyTelemetry.test.tsx new file mode 100644 index 0000000000..10315d65f0 --- /dev/null +++ b/frontend/src/v2/__tests__/V2AgentBYOCopyTelemetry.test.tsx @@ -0,0 +1,101 @@ +// @ts-nocheck +// BYO copy telemetry: every copy surface that writes a command must post its +// step. The "Add a computer" daemon command was the one that did not (TASK-653: +// 6 seats were created in a week and no copy step was ever recorded). +import React from 'react'; +import { render, screen, waitFor, fireEvent, act } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import V2AgentBYO from '../components/V2AgentBYO'; +import { AuthContext } from '../../context/AuthContext'; +import { recordByoStep } from '../utils/byoTelemetry'; + +jest.mock('../utils/byoTelemetry', () => ({ + recordByoStep: jest.fn(), + recordByoStepOnce: jest.fn(), + resetByoStepsForTest: jest.fn(), +})); + +jest.mock('axios', () => { + const mock = { + get: jest.fn(), + post: jest.fn(), + patch: jest.fn(), + delete: jest.fn(), + defaults: { baseURL: '', headers: { common: {} } }, + interceptors: { + request: { use: jest.fn(), eject: jest.fn() }, + response: { use: jest.fn(), eject: jest.fn() }, + }, + }; + return { __esModule: true, default: mock, ...mock }; +}); + +const axios = jest.requireMock('axios').default; + +const authValue = { + currentUser: { _id: 'u1', username: 'sam' }, + user: { _id: 'u1', username: 'sam' }, + token: 'user-jwt', + loading: false, + error: null, + isAuthenticated: true, + register: jest.fn(), + login: jest.fn(), + logout: jest.fn(), + updateProfile: jest.fn(), +}; + +const mockGet = () => { + axios.get.mockImplementation((url) => { + if (url.startsWith('/api/registry/agents/next-free-name?')) return Promise.resolve({ data: { name: 'agent' } }); + if (url.startsWith('/api/hosted/availability')) { + return Promise.resolve({ data: { configured: false, caps: { agentsPerUser: 1, turnsPerDay: 200 } } }); + } + if (url === '/api/pods') return Promise.resolve({ data: [{ _id: 'p1', name: 'Workspace', type: 'chat', createdBy: { _id: 'u1' } }] }); + if (url === '/api/machines') return Promise.resolve({ data: { machines: [], offlineAfterMs: 90000 } }); + return Promise.resolve({ data: {} }); + }); +}; + +const renderPage = () => render( + + + + + , +); + +beforeEach(() => { + window.history.pushState({}, '', '/v2/agents/byo'); + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText: jest.fn().mockResolvedValue(undefined) } }); + mockGet(); +}); + +afterEach(() => { + jest.clearAllMocks(); +}); + +const stepsPosted = () => recordByoStep.mock.calls.map(([, step]) => step); + +test('copying the add-a-computer daemon command posts daemon-command-copied', async () => { + renderPage(); + await waitFor(() => expect(axios.get).toHaveBeenCalledWith('/api/machines', expect.anything())); + const block = await screen.findByTestId('byo-add-computer'); + await act(async () => { + fireEvent.click(block.querySelector('button.v2-byo__copy')); + }); + + expect(navigator.clipboard.writeText).toHaveBeenCalledWith(expect.stringContaining('commonly daemon')); + expect(stepsPosted()).toContain('daemon-command-copied'); +}); + +test('a failed daemon copy posts no copy step', async () => { + navigator.clipboard.writeText.mockRejectedValueOnce(new Error('Denied')); + renderPage(); + const block = await screen.findByTestId('byo-add-computer'); + await act(async () => { + fireEvent.click(block.querySelector('button.v2-byo__copy')); + }); + + expect(stepsPosted()).not.toContain('daemon-command-copied'); +}); diff --git a/frontend/src/v2/components/V2AgentBYO.tsx b/frontend/src/v2/components/V2AgentBYO.tsx index e324763a2e..bd7daad929 100644 --- a/frontend/src/v2/components/V2AgentBYO.tsx +++ b/frontend/src/v2/components/V2AgentBYO.tsx @@ -990,6 +990,7 @@ const V2AgentBYO: React.FC = () => { setTimeout(() => setCopied((c) => (c === key ? null : c)), 1500); if (key === 'claude' || key === 'cursor') byoStep('mcp-command-copied'); else if (key === 'listen') byoStep('cli-command-copied'); + else if (key === 'daemon-setup') byoStep('daemon-command-copied'); else if (key === 'tok') byoStep('token-copied'); } catch { // Clipboard may be unavailable (non-HTTPS, sandbox); user can select manually. diff --git a/frontend/src/v2/utils/byoTelemetry.ts b/frontend/src/v2/utils/byoTelemetry.ts index 5f13a2a2a4..66d074b7b2 100644 --- a/frontend/src/v2/utils/byoTelemetry.ts +++ b/frontend/src/v2/utils/byoTelemetry.ts @@ -1,7 +1,8 @@ import type { V2ApiClient } from '../hooks/useV2Api'; export type ByoStep = 'page-viewed' | 'seat-created' | 'mcp-command-copied' | 'cli-command-copied' - | 'token-copied' | 'listen-confirmed' | 'listen-timeout' | 'hosted-offer-shown' | 'hosted-offer-clicked' + | 'daemon-command-copied' | 'token-copied' | 'listen-confirmed' | 'listen-timeout' + | 'hosted-offer-shown' | 'hosted-offer-clicked' | 'invite-card-shown' | 'invite-card-dismissed' | 'byo-offer-shown'; // Fixed event names only. Commands and credentials must never enter telemetry. From be20ad89963b49b330f5d65453bd348b8ee519ce Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:05:35 -0700 Subject: [PATCH 3/6] test(byo): create the telemetry warn spy per test restoreMocks restores module-level spies before the first test, so the failure case asserted on an unwrapped console.warn. Create it in beforeEach. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: tern Agent-Model: claude/claude-haiku-5-5/default --- backend/__tests__/unit/services/byoTelemetry.test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/__tests__/unit/services/byoTelemetry.test.js b/backend/__tests__/unit/services/byoTelemetry.test.js index f24080d3a3..de3df0e41d 100644 --- a/backend/__tests__/unit/services/byoTelemetry.test.js +++ b/backend/__tests__/unit/services/byoTelemetry.test.js @@ -1,5 +1,5 @@ const mockInsertOne = jest.fn(); -const mockWarn = jest.spyOn(console, 'warn').mockImplementation(() => {}); +let mockWarn; jest.mock('mongoose', () => ({ connection: { db: { collection: () => ({ insertOne: (...args) => mockInsertOne(...args) }) } }, @@ -10,11 +10,10 @@ const { recordByoEvent } = require('../../../services/byoTelemetry'); describe('recordByoEvent', () => { beforeEach(() => { jest.clearAllMocks(); + mockWarn = jest.spyOn(console, 'warn').mockImplementation(() => {}); mockInsertOne.mockResolvedValue({ acknowledged: true }); }); - afterAll(() => mockWarn.mockRestore()); - it('writes the fixed step with the user, the extra fields and server time', async () => { await recordByoEvent('human-1', 'seat-registered', { commandType: 'cli' }); From 594fcedea8bf6588e4ea39b29c5d4c021c15961a Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:49:13 -0700 Subject: [PATCH 4/6] fix(byo): write seat-registered once per token under concurrent first calls Two first calls with one token could both read no lastUsedAt and both insert the seat-registered row. The row now takes _id from a sha256 of the token record hash, so the unique _id index makes a racing second insert a no-op. The hash is one-way and is never the credential. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: tern Agent-Model: claude/claude-haiku-5-5/default --- .../unit/middleware/agentRuntimeAuth.test.js | 5 ++-- .../unit/services/byoTelemetry.test.js | 23 ++++++++++++++++++- backend/middleware/agentRuntimeAuth.ts | 6 ++--- backend/services/byoTelemetry.ts | 18 +++++++++++++-- 4 files changed, 44 insertions(+), 8 deletions(-) diff --git a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js index cada5c0c5b..e50a3b3c3a 100644 --- a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js +++ b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js @@ -33,6 +33,7 @@ jest.mock('../../../services/onboardingMarks', () => ({ })); jest.mock('../../../services/byoTelemetry', () => ({ recordByoEvent: (...args) => mockRecordByoEvent(...args), + seatRegisteredKey: () => `seat-registered:${'a'.repeat(64)}`, })); jest.mock('../../../models/User', () => { @@ -139,7 +140,7 @@ describe('agentRuntimeAuth path 2 (install-bound token) — #66 fix', () => { expect(mockMarkByoConnectionAttempt).toHaveBeenCalledWith('human-a', matchedInstall); // Backend-side seat-registered row, written once for the first use of the token. expect(mockRecordByoEvent).toHaveBeenCalledTimes(1); - expect(mockRecordByoEvent).toHaveBeenCalledWith('human-a', 'seat-registered'); + expect(mockRecordByoEvent).toHaveBeenCalledWith('human-a', 'seat-registered', {}, expect.stringMatching(/^seat-registered:[0-9a-f]{64}$/)); }); test('does not fire the starter hook when the token was used before (#916)', async () => { @@ -196,7 +197,7 @@ describe('agentRuntimeAuth path 1 (User-row token) — first-use starter hook (# 'human-1', expect.objectContaining({ podId: expect.anything(), installedBy: 'human-1' }), ); - expect(mockRecordByoEvent).toHaveBeenCalledWith('human-1', 'seat-registered'); + expect(mockRecordByoEvent).toHaveBeenCalledWith('human-1', 'seat-registered', {}, expect.stringMatching(/^seat-registered:[0-9a-f]{64}$/)); }); test('stays silent on every subsequent authentication', async () => { diff --git a/backend/__tests__/unit/services/byoTelemetry.test.js b/backend/__tests__/unit/services/byoTelemetry.test.js index de3df0e41d..1f85796dd6 100644 --- a/backend/__tests__/unit/services/byoTelemetry.test.js +++ b/backend/__tests__/unit/services/byoTelemetry.test.js @@ -5,7 +5,7 @@ jest.mock('mongoose', () => ({ connection: { db: { collection: () => ({ insertOne: (...args) => mockInsertOne(...args) }) } }, })); -const { recordByoEvent } = require('../../../services/byoTelemetry'); +const { recordByoEvent, seatRegisteredKey } = require('../../../services/byoTelemetry'); describe('recordByoEvent', () => { beforeEach(() => { @@ -31,4 +31,25 @@ describe('recordByoEvent', () => { await expect(recordByoEvent('human-1', 'seat-registered')).resolves.toBeUndefined(); expect(mockWarn).toHaveBeenCalledWith('[byo-telemetry] write failed:', 'db down'); }); + + it('keys a once-per-seat row by a hash of the token, never the token itself', () => { + const key = seatRegisteredKey('token-record-hash'); + + expect(key).toMatch(/^seat-registered:[0-9a-f]{64}$/); + expect(key).not.toContain('token-record-hash'); + expect(seatRegisteredKey('token-record-hash')).toBe(key); + }); + + it('writes the keyed row with _id set, so a racing second write is a no-op', async () => { + const dup = Object.assign(new Error('E11000 duplicate key'), { code: 11000 }); + mockInsertOne.mockResolvedValueOnce({ acknowledged: true }).mockRejectedValueOnce(dup); + const key = seatRegisteredKey('token-record-hash'); + + await recordByoEvent('human-1', 'seat-registered', {}, key); + await expect(recordByoEvent('human-1', 'seat-registered', {}, key)).resolves.toBeUndefined(); + + expect(mockInsertOne).toHaveBeenNthCalledWith(1, expect.objectContaining({ _id: key, step: 'seat-registered' })); + expect(mockInsertOne).toHaveBeenNthCalledWith(2, expect.objectContaining({ _id: key, step: 'seat-registered' })); + expect(mockWarn).not.toHaveBeenCalled(); + }); }); diff --git a/backend/middleware/agentRuntimeAuth.ts b/backend/middleware/agentRuntimeAuth.ts index 29963c2395..479dee4697 100644 --- a/backend/middleware/agentRuntimeAuth.ts +++ b/backend/middleware/agentRuntimeAuth.ts @@ -9,7 +9,7 @@ import { touchLastActive } from './auth'; import Pod from '../models/Pod'; import AgentCredential from '../models/AgentCredential'; import { markByoConnectionAttempt } from '../services/onboardingMarks'; -import { recordByoEvent } from '../services/byoTelemetry'; +import { recordByoEvent, seatRegisteredKey } from '../services/byoTelemetry'; import { isSpawnCredential } from '../services/spawnCredentialService'; import { embeddedRuntimeTokenExpired, runtimeCredentialRefusalReason } from '../services/runtimeCredentialGate'; @@ -150,7 +150,7 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next markByoConnectionAttempt(installationRow.installedBy, installationRow); } // One row per token, not per installation: the seat is the token. - void recordByoEvent(installations[0]?.installedBy, 'seat-registered'); + void recordByoEvent(installations[0]?.installedBy, 'seat-registered', {}, seatRegisteredKey(tokenHash)); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { completeConnectAgentStarterTask } = require('../services/starterTaskService'); void completeConnectAgentStarterTask({ @@ -208,7 +208,7 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next if (isFirstLegacyTokenUse) { markByoConnectionAttempt(installation.installedBy, installation); - void recordByoEvent(installation.installedBy, 'seat-registered'); + void recordByoEvent(installation.installedBy, 'seat-registered', {}, seatRegisteredKey(tokenHash)); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { completeConnectAgentStarterTask } = require('../services/starterTaskService'); void completeConnectAgentStarterTask({ diff --git a/backend/services/byoTelemetry.ts b/backend/services/byoTelemetry.ts index 39c41c2510..962e1630dd 100644 --- a/backend/services/byoTelemetry.ts +++ b/backend/services/byoTelemetry.ts @@ -1,3 +1,4 @@ +import { createHash } from 'crypto'; import mongoose from 'mongoose'; // Server-side BYO funnel rows. The browser posts its steps through @@ -6,19 +7,32 @@ import mongoose from 'mongoose'; // user keeping the page open. Fixed step names only: never a command, key or name. export type ServerByoStep = 'seat-registered'; +// A once-per-seat event is keyed by a hash of the token record's hash, so the +// unique _id index makes concurrent first calls write one row. The hash is one-way +// and is never the credential itself. +export function seatRegisteredKey(tokenHash: string): string { + return `seat-registered:${createHash('sha256').update(tokenHash).digest('hex')}`; +} + export async function recordByoEvent( userId: unknown, step: ServerByoStep, extra: Record = {}, + dedupeKey?: string, ): Promise { try { - await mongoose.connection.db.collection('byo_telemetry').insertOne({ + // The collection is untyped here; a string _id is only set for the keyed seat row. + const doc: Record = { + ...(dedupeKey ? { _id: dedupeKey } : {}), userId, step, ...extra, at: new Date(), - }); + }; + await mongoose.connection.db.collection('byo_telemetry').insertOne(doc as any); } catch (err: any) { + // A duplicate key means this seat's row already exists: the expected outcome of a race. + if (err?.code === 11000) return; // Telemetry must never fail the caller's flow. console.warn('[byo-telemetry] write failed:', err?.message); } From 629fb5001453a927cd2e14816590fe157730e67a Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:55:32 -0700 Subject: [PATCH 5/6] test(byo): prove the seat-registered race against real Mongo; name read-back field at The duplicate-key case was only a mock. The service-tier test runs two concurrent first calls with one key on real Mongo and asserts one row. Also names the read-back field at in the telemetry doc. Co-Authored-By: Claude Haiku 5.5 Co-authored-by: tern Agent-Model: claude/claude-haiku-5-5/default --- .../service/byoTelemetry.service.test.js | 56 +++++++++++++++++++ docs/development/byo-funnel-telemetry.md | 4 +- 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 backend/__tests__/service/byoTelemetry.service.test.js diff --git a/backend/__tests__/service/byoTelemetry.service.test.js b/backend/__tests__/service/byoTelemetry.service.test.js new file mode 100644 index 0000000000..ce0e978733 --- /dev/null +++ b/backend/__tests__/service/byoTelemetry.service.test.js @@ -0,0 +1,56 @@ +// Real Mongo: the once-per-seat guarantee is the unique _id index, so a mock +// cannot prove it. Two concurrent first calls with one token must leave one row. +const mongoose = require('mongoose'); +const { setupMongoDb, closeMongoDb, clearMongoDb } = require('../utils/testUtils'); +const { recordByoEvent, seatRegisteredKey } = require('../../services/byoTelemetry'); + +const rows = () => mongoose.connection.db.collection('byo_telemetry').find({}).toArray(); + +describe('byo_telemetry seat-registered on real Mongo', () => { + beforeAll(async () => { + await setupMongoDb(); + }); + + afterAll(async () => { + await closeMongoDb(); + }); + + beforeEach(async () => { + await clearMongoDb(); + // clearMongoDb only resets model collections; byo_telemetry is a raw collection. + await mongoose.connection.db.collection('byo_telemetry').deleteMany({}); + }); + + it('two concurrent first calls with one token leave exactly one row', async () => { + const key = seatRegisteredKey('token-record-hash'); + + await Promise.all([ + recordByoEvent('human-1', 'seat-registered', {}, key), + recordByoEvent('human-1', 'seat-registered', {}, key), + ]); + + const stored = await rows(); + expect(stored).toHaveLength(1); + expect(stored[0]._id).toBe(key); + expect(stored[0]).toEqual(expect.objectContaining({ userId: 'human-1', step: 'seat-registered' })); + expect(stored[0].at).toBeInstanceOf(Date); + }); + + it('a different token for the same installer is a separate seat row', async () => { + await Promise.all([ + recordByoEvent('human-1', 'seat-registered', {}, seatRegisteredKey('token-a')), + recordByoEvent('human-1', 'seat-registered', {}, seatRegisteredKey('token-b')), + ]); + + expect(await rows()).toHaveLength(2); + }); + + it('keyless browser rows keep a Mongo-generated _id', async () => { + await recordByoEvent('human-1', 'seat-created'); + await recordByoEvent('human-1', 'seat-created'); + + const stored = await rows(); + expect(stored).toHaveLength(2); + stored.forEach((row) => expect(row._id).toBeInstanceOf(mongoose.Types.ObjectId)); + }); +}); diff --git a/docs/development/byo-funnel-telemetry.md b/docs/development/byo-funnel-telemetry.md index 88f8b28cf4..7966f17b39 100644 --- a/docs/development/byo-funnel-telemetry.md +++ b/docs/development/byo-funnel-telemetry.md @@ -19,7 +19,9 @@ As of TASK-483 and TASK-493 (2026-10-24), the browser sends: | `hosted-offer-clicked` | The user activated the offer to start hosted setup. Recorded before name lookup, including failed attempts; concurrent duplicate activations are suppressed. A retry records another activation. This is separate from choosing the initial hosted mode. | | `listen-timeout` | The page's connection polling window expired. | -The backend writes one more row, `seat-registered`, itself; the browser cannot post it (the route answers 400). It is written once per runtime token, on the token's first authenticated call (`middleware/agentRuntimeAuth.ts`), with the installing user as `userId`. It fires whether or not the user is still on the page, so it measures registration when `listen-confirmed` is missing. It is not limited to BYO: a hosted seat's first call also writes it. +The backend writes one more row, `seat-registered`, itself; the browser cannot post it (the route answers 400). It is written once per runtime token, on the token's first authenticated call (`middleware/agentRuntimeAuth.ts`), with the installing user as `userId`. It fires whether or not the user is still on the page, so it measures registration when `listen-confirmed` is missing. It is not limited to BYO: a hosted seat's first call also writes it. Its `_id` is `seat-registered:`, so a racing second first call is a duplicate-key no-op and the row is never written twice. + +Read back `step`, `userId` and `at` (server time). The field is `at`, not `createdAt`; a count on `createdAt` sees nothing. Rows from the browser have a Mongo-generated `_id`. Before TASK-483, the client never emitted `page-viewed` or `seat-created`. Historical counts of those events cannot measure page or command exposure. From 10f185e25c3f028afe031a9bf9233f81809fb231 Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:03:10 -0700 Subject: [PATCH 6/6] test(byo): pin that a used User-row token records no seat-registered row Co-Authored-By: Claude Haiku 5.5 Co-authored-by: tern Agent-Model: claude/claude-haiku-5-5/default --- backend/__tests__/unit/middleware/agentRuntimeAuth.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js index e50a3b3c3a..a6a5696c0f 100644 --- a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js +++ b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js @@ -211,6 +211,7 @@ describe('agentRuntimeAuth path 1 (User-row token) — first-use starter hook (# await agentRuntimeAuth(buildReq('cm_agent_test'), buildRes(), jest.fn()); expect(mockCompleteStarterTask).not.toHaveBeenCalled(); + expect(mockRecordByoEvent).not.toHaveBeenCalled(); }); test('returns 401 when token doesnt match any install OR user', async () => {