diff --git a/backend/__tests__/service/byoTelemetry.service.test.js b/backend/__tests__/service/byoTelemetry.service.test.js new file mode 100644 index 0000000000..ce0e978733 --- /dev/null +++ b/backend/__tests__/service/byoTelemetry.service.test.js @@ -0,0 +1,56 @@ +// Real Mongo: the once-per-seat guarantee is the unique _id index, so a mock +// cannot prove it. Two concurrent first calls with one token must leave one row. +const mongoose = require('mongoose'); +const { setupMongoDb, closeMongoDb, clearMongoDb } = require('../utils/testUtils'); +const { recordByoEvent, seatRegisteredKey } = require('../../services/byoTelemetry'); + +const rows = () => mongoose.connection.db.collection('byo_telemetry').find({}).toArray(); + +describe('byo_telemetry seat-registered on real Mongo', () => { + beforeAll(async () => { + await setupMongoDb(); + }); + + afterAll(async () => { + await closeMongoDb(); + }); + + beforeEach(async () => { + await clearMongoDb(); + // clearMongoDb only resets model collections; byo_telemetry is a raw collection. + await mongoose.connection.db.collection('byo_telemetry').deleteMany({}); + }); + + it('two concurrent first calls with one token leave exactly one row', async () => { + const key = seatRegisteredKey('token-record-hash'); + + await Promise.all([ + recordByoEvent('human-1', 'seat-registered', {}, key), + recordByoEvent('human-1', 'seat-registered', {}, key), + ]); + + const stored = await rows(); + expect(stored).toHaveLength(1); + expect(stored[0]._id).toBe(key); + expect(stored[0]).toEqual(expect.objectContaining({ userId: 'human-1', step: 'seat-registered' })); + expect(stored[0].at).toBeInstanceOf(Date); + }); + + it('a different token for the same installer is a separate seat row', async () => { + await Promise.all([ + recordByoEvent('human-1', 'seat-registered', {}, seatRegisteredKey('token-a')), + recordByoEvent('human-1', 'seat-registered', {}, seatRegisteredKey('token-b')), + ]); + + expect(await rows()).toHaveLength(2); + }); + + it('keyless browser rows keep a Mongo-generated _id', async () => { + await recordByoEvent('human-1', 'seat-created'); + await recordByoEvent('human-1', 'seat-created'); + + const stored = await rows(); + expect(stored).toHaveLength(2); + stored.forEach((row) => expect(row._id).toBeInstanceOf(mongoose.Types.ObjectId)); + }); +}); diff --git a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js index ee7be69a2b..a6a5696c0f 100644 --- a/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js +++ b/backend/__tests__/unit/middleware/agentRuntimeAuth.test.js @@ -23,6 +23,7 @@ const mockInstallationFind = jest.fn(); const mockInstallationUpdateOne = jest.fn(); const mockCompleteStarterTask = jest.fn(); const mockMarkByoConnectionAttempt = jest.fn(); +const mockRecordByoEvent = jest.fn(); jest.mock('../../../services/starterTaskService', () => ({ completeConnectAgentStarterTask: (...args) => mockCompleteStarterTask(...args), @@ -30,6 +31,10 @@ jest.mock('../../../services/starterTaskService', () => ({ jest.mock('../../../services/onboardingMarks', () => ({ markByoConnectionAttempt: (...args) => mockMarkByoConnectionAttempt(...args), })); +jest.mock('../../../services/byoTelemetry', () => ({ + recordByoEvent: (...args) => mockRecordByoEvent(...args), + seatRegisteredKey: () => `seat-registered:${'a'.repeat(64)}`, +})); jest.mock('../../../models/User', () => { function User() {} @@ -133,6 +138,9 @@ describe('agentRuntimeAuth path 2 (install-bound token) — #66 fix', () => { agentLabel: 'codex', }); expect(mockMarkByoConnectionAttempt).toHaveBeenCalledWith('human-a', matchedInstall); + // Backend-side seat-registered row, written once for the first use of the token. + expect(mockRecordByoEvent).toHaveBeenCalledTimes(1); + expect(mockRecordByoEvent).toHaveBeenCalledWith('human-a', 'seat-registered', {}, expect.stringMatching(/^seat-registered:[0-9a-f]{64}$/)); }); test('does not fire the starter hook when the token was used before (#916)', async () => { @@ -150,6 +158,7 @@ describe('agentRuntimeAuth path 2 (install-bound token) — #66 fix', () => { await agentRuntimeAuth(buildReq('cm_agent_test'), buildRes(), jest.fn()); expect(mockCompleteStarterTask).not.toHaveBeenCalled(); + expect(mockRecordByoEvent).not.toHaveBeenCalled(); }); }); @@ -188,6 +197,7 @@ describe('agentRuntimeAuth path 1 (User-row token) — first-use starter hook (# 'human-1', expect.objectContaining({ podId: expect.anything(), installedBy: 'human-1' }), ); + expect(mockRecordByoEvent).toHaveBeenCalledWith('human-1', 'seat-registered', {}, expect.stringMatching(/^seat-registered:[0-9a-f]{64}$/)); }); test('stays silent on every subsequent authentication', async () => { @@ -201,6 +211,7 @@ describe('agentRuntimeAuth path 1 (User-row token) — first-use starter hook (# await agentRuntimeAuth(buildReq('cm_agent_test'), buildRes(), jest.fn()); expect(mockCompleteStarterTask).not.toHaveBeenCalled(); + expect(mockRecordByoEvent).not.toHaveBeenCalled(); }); test('returns 401 when token doesnt match any install OR user', async () => { diff --git a/backend/__tests__/unit/routes/stats.byo-step.test.js b/backend/__tests__/unit/routes/stats.byo-step.test.js index a0952ebd97..153ab258f9 100644 --- a/backend/__tests__/unit/routes/stats.byo-step.test.js +++ b/backend/__tests__/unit/routes/stats.byo-step.test.js @@ -60,6 +60,20 @@ describe('POST /api/stats/byo-step', () => { expect(JSON.stringify(mockInsertOne.mock.calls[0][0])).not.toContain('cm_agent_secret'); }); + it('records the daemon copy as telemetry only, with no onboarding copy mark', async () => { + const res = await request(app) + .post('/api/stats/byo-step') + .send({ step: 'daemon-command-copied', command: 'commonly daemon register' }); + + expect(res.status).toBe(200); + expect(mockMarkByoCommandCopied).not.toHaveBeenCalled(); + expect(mockInsertOne).toHaveBeenCalledWith({ + userId: 'human-1', + step: 'daemon-command-copied', + at: expect.any(Date), + }); + }); + it.each(['hosted-offer-shown', 'hosted-offer-clicked'])('accepts %s without copied-command marks or extra payload', async (step) => { const res = await request(app).post('/api/stats/byo-step') .send({ step, command: 'fixture-only-secret', agentName: 'private-agent' }); @@ -85,6 +99,13 @@ describe('POST /api/stats/byo-step', () => { expect(mockMarkSurface).not.toHaveBeenCalled(); }); + it('rejects seat-registered from the browser: the backend alone writes that row', async () => { + const res = await request(app).post('/api/stats/byo-step').send({ step: 'seat-registered' }); + + expect(res.status).toBe(400); + expect(mockInsertOne).not.toHaveBeenCalled(); + }); + it('rejects unknown steps without writing a mark or telemetry event', async () => { const res = await request(app) .post('/api/stats/byo-step') diff --git a/backend/__tests__/unit/services/byoTelemetry.test.js b/backend/__tests__/unit/services/byoTelemetry.test.js new file mode 100644 index 0000000000..1f85796dd6 --- /dev/null +++ b/backend/__tests__/unit/services/byoTelemetry.test.js @@ -0,0 +1,55 @@ +const mockInsertOne = jest.fn(); +let mockWarn; + +jest.mock('mongoose', () => ({ + connection: { db: { collection: () => ({ insertOne: (...args) => mockInsertOne(...args) }) } }, +})); + +const { recordByoEvent, seatRegisteredKey } = require('../../../services/byoTelemetry'); + +describe('recordByoEvent', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockWarn = jest.spyOn(console, 'warn').mockImplementation(() => {}); + mockInsertOne.mockResolvedValue({ acknowledged: true }); + }); + + it('writes the fixed step with the user, the extra fields and server time', async () => { + await recordByoEvent('human-1', 'seat-registered', { commandType: 'cli' }); + + expect(mockInsertOne).toHaveBeenCalledWith({ + userId: 'human-1', + step: 'seat-registered', + commandType: 'cli', + at: expect.any(Date), + }); + }); + + it('never throws when the write fails, so the caller flow continues', async () => { + mockInsertOne.mockRejectedValue(new Error('db down')); + + await expect(recordByoEvent('human-1', 'seat-registered')).resolves.toBeUndefined(); + expect(mockWarn).toHaveBeenCalledWith('[byo-telemetry] write failed:', 'db down'); + }); + + it('keys a once-per-seat row by a hash of the token, never the token itself', () => { + const key = seatRegisteredKey('token-record-hash'); + + expect(key).toMatch(/^seat-registered:[0-9a-f]{64}$/); + expect(key).not.toContain('token-record-hash'); + expect(seatRegisteredKey('token-record-hash')).toBe(key); + }); + + it('writes the keyed row with _id set, so a racing second write is a no-op', async () => { + const dup = Object.assign(new Error('E11000 duplicate key'), { code: 11000 }); + mockInsertOne.mockResolvedValueOnce({ acknowledged: true }).mockRejectedValueOnce(dup); + const key = seatRegisteredKey('token-record-hash'); + + await recordByoEvent('human-1', 'seat-registered', {}, key); + await expect(recordByoEvent('human-1', 'seat-registered', {}, key)).resolves.toBeUndefined(); + + expect(mockInsertOne).toHaveBeenNthCalledWith(1, expect.objectContaining({ _id: key, step: 'seat-registered' })); + expect(mockInsertOne).toHaveBeenNthCalledWith(2, expect.objectContaining({ _id: key, step: 'seat-registered' })); + expect(mockWarn).not.toHaveBeenCalled(); + }); +}); diff --git a/backend/middleware/agentRuntimeAuth.ts b/backend/middleware/agentRuntimeAuth.ts index e73d4d1544..479dee4697 100644 --- a/backend/middleware/agentRuntimeAuth.ts +++ b/backend/middleware/agentRuntimeAuth.ts @@ -9,6 +9,7 @@ import { touchLastActive } from './auth'; import Pod from '../models/Pod'; import AgentCredential from '../models/AgentCredential'; import { markByoConnectionAttempt } from '../services/onboardingMarks'; +import { recordByoEvent, seatRegisteredKey } from '../services/byoTelemetry'; import { isSpawnCredential } from '../services/spawnCredentialService'; import { embeddedRuntimeTokenExpired, runtimeCredentialRefusalReason } from '../services/runtimeCredentialGate'; @@ -148,6 +149,8 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next for (const installationRow of installations) { markByoConnectionAttempt(installationRow.installedBy, installationRow); } + // One row per token, not per installation: the seat is the token. + void recordByoEvent(installations[0]?.installedBy, 'seat-registered', {}, seatRegisteredKey(tokenHash)); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { completeConnectAgentStarterTask } = require('../services/starterTaskService'); void completeConnectAgentStarterTask({ @@ -205,6 +208,7 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next if (isFirstLegacyTokenUse) { markByoConnectionAttempt(installation.installedBy, installation); + void recordByoEvent(installation.installedBy, 'seat-registered', {}, seatRegisteredKey(tokenHash)); // eslint-disable-next-line global-require, @typescript-eslint/no-require-imports const { completeConnectAgentStarterTask } = require('../services/starterTaskService'); void completeConnectAgentStarterTask({ diff --git a/backend/routes/stats.ts b/backend/routes/stats.ts index f68ab8b587..192b188955 100644 --- a/backend/routes/stats.ts +++ b/backend/routes/stats.ts @@ -84,10 +84,10 @@ const { rateLimit } = require('express-rate-limit'); // eslint-disable-next-line global-require const { cloudflareIpRateLimitKeyGenerator } = require('../middleware/ipRateLimit'); // eslint-disable-next-line global-require -const mongoose = require('mongoose'); +const { recordByoEvent } = require('../services/byoTelemetry'); const BYO_STEPS = new Set([ - 'page-viewed', 'seat-created', 'mcp-command-copied', 'cli-command-copied', + 'page-viewed', 'seat-created', 'mcp-command-copied', 'cli-command-copied', 'daemon-command-copied', 'token-copied', 'listen-confirmed', 'listen-timeout', 'hosted-offer-shown', 'hosted-offer-clicked', 'invite-card-shown', 'invite-card-dismissed', 'byo-offer-shown', ]); @@ -115,18 +115,10 @@ router.post('/byo-step', rateLimit({ markByoCommandCopied(req.userId, step === 'mcp-command-copied' ? 'mcp' : 'cli'); } if (SURFACE_STEPS[step]) markSurface(req.userId, SURFACE_STEPS[step]); - try { - await mongoose.connection.db.collection('byo_telemetry').insertOne({ - userId: req.userId, - step, - ...(step === 'mcp-command-copied' ? { commandType: 'mcp' } : {}), - ...(step === 'cli-command-copied' ? { commandType: 'cli' } : {}), - at: new Date(), - }); - } catch (err: any) { - // Telemetry must never fail the caller's flow. - console.warn('[byo-telemetry] write failed:', err?.message); - } + await recordByoEvent(req.userId, step as any, { + ...(step === 'mcp-command-copied' ? { commandType: 'mcp' } : {}), + ...(step === 'cli-command-copied' ? { commandType: 'cli' } : {}), + }); return res.json({ ok: true }); }); diff --git a/backend/services/byoTelemetry.ts b/backend/services/byoTelemetry.ts new file mode 100644 index 0000000000..962e1630dd --- /dev/null +++ b/backend/services/byoTelemetry.ts @@ -0,0 +1,39 @@ +import { createHash } from 'crypto'; +import mongoose from 'mongoose'; + +// Server-side BYO funnel rows. The browser posts its steps through +// POST /api/stats/byo-step; events only the backend can see (an agent's first +// authenticated call) are written here directly, so they never depend on the +// user keeping the page open. Fixed step names only: never a command, key or name. +export type ServerByoStep = 'seat-registered'; + +// A once-per-seat event is keyed by a hash of the token record's hash, so the +// unique _id index makes concurrent first calls write one row. The hash is one-way +// and is never the credential itself. +export function seatRegisteredKey(tokenHash: string): string { + return `seat-registered:${createHash('sha256').update(tokenHash).digest('hex')}`; +} + +export async function recordByoEvent( + userId: unknown, + step: ServerByoStep, + extra: Record = {}, + dedupeKey?: string, +): Promise { + try { + // The collection is untyped here; a string _id is only set for the keyed seat row. + const doc: Record = { + ...(dedupeKey ? { _id: dedupeKey } : {}), + userId, + step, + ...extra, + at: new Date(), + }; + await mongoose.connection.db.collection('byo_telemetry').insertOne(doc as any); + } catch (err: any) { + // A duplicate key means this seat's row already exists: the expected outcome of a race. + if (err?.code === 11000) return; + // Telemetry must never fail the caller's flow. + console.warn('[byo-telemetry] write failed:', err?.message); + } +} diff --git a/docs/development/byo-funnel-telemetry.md b/docs/development/byo-funnel-telemetry.md index 41da62872f..7966f17b39 100644 --- a/docs/development/byo-funnel-telemetry.md +++ b/docs/development/byo-funnel-telemetry.md @@ -12,12 +12,17 @@ As of TASK-483 and TASK-493 (2026-10-24), the browser sends: | `seat-created` | A runtime key was issued and its command rendered. Once per issued command. This is the command-exposure denominator. | | `mcp-command-copied` | The Claude or Cursor copy button successfully wrote the command. | | `cli-command-copied` | The listener copy button successfully wrote the command. | +| `daemon-command-copied` | The "Add a computer" daemon command copy button successfully wrote the command. Telemetry only: it sets no onboarding copy mark, so stalled-connect messaging is unchanged. Before TASK-653 this copy posted nothing, so a user who used this path was invisible to the copy count. | | `token-copied` | The token copy button successfully wrote the key. | | `listen-confirmed` | The connection poll observed this agent using its key after issuance (with existing clock-skew allowance). | | `hosted-offer-shown` | The timer or desktop exit-intent path displayed the hosted offer. The shared session guard records it once, including when both triggers run. | | `hosted-offer-clicked` | The user activated the offer to start hosted setup. Recorded before name lookup, including failed attempts; concurrent duplicate activations are suppressed. A retry records another activation. This is separate from choosing the initial hosted mode. | | `listen-timeout` | The page's connection polling window expired. | +The backend writes one more row, `seat-registered`, itself; the browser cannot post it (the route answers 400). It is written once per runtime token, on the token's first authenticated call (`middleware/agentRuntimeAuth.ts`), with the installing user as `userId`. It fires whether or not the user is still on the page, so it measures registration when `listen-confirmed` is missing. It is not limited to BYO: a hosted seat's first call also writes it. Its `_id` is `seat-registered:`, so a racing second first call is a duplicate-key no-op and the row is never written twice. + +Read back `step`, `userId` and `at` (server time). The field is `at`, not `createdAt`; a count on `createdAt` sees nothing. Rows from the browser have a Mongo-generated `_id`. + Before TASK-483, the client never emitted `page-viewed` or `seat-created`. Historical counts of those events cannot measure page or command exposure. A selected-text copy from a command or key block dismisses the hosted offer but does not claim that a complete diff --git a/frontend/src/v2/__tests__/V2AgentBYOCopyTelemetry.test.tsx b/frontend/src/v2/__tests__/V2AgentBYOCopyTelemetry.test.tsx new file mode 100644 index 0000000000..10315d65f0 --- /dev/null +++ b/frontend/src/v2/__tests__/V2AgentBYOCopyTelemetry.test.tsx @@ -0,0 +1,101 @@ +// @ts-nocheck +// BYO copy telemetry: every copy surface that writes a command must post its +// step. The "Add a computer" daemon command was the one that did not (TASK-653: +// 6 seats were created in a week and no copy step was ever recorded). +import React from 'react'; +import { render, screen, waitFor, fireEvent, act } from '@testing-library/react'; +import { MemoryRouter } from 'react-router-dom'; +import V2AgentBYO from '../components/V2AgentBYO'; +import { AuthContext } from '../../context/AuthContext'; +import { recordByoStep } from '../utils/byoTelemetry'; + +jest.mock('../utils/byoTelemetry', () => ({ + recordByoStep: jest.fn(), + recordByoStepOnce: jest.fn(), + resetByoStepsForTest: jest.fn(), +})); + +jest.mock('axios', () => { + const mock = { + get: jest.fn(), + post: jest.fn(), + patch: jest.fn(), + delete: jest.fn(), + defaults: { baseURL: '', headers: { common: {} } }, + interceptors: { + request: { use: jest.fn(), eject: jest.fn() }, + response: { use: jest.fn(), eject: jest.fn() }, + }, + }; + return { __esModule: true, default: mock, ...mock }; +}); + +const axios = jest.requireMock('axios').default; + +const authValue = { + currentUser: { _id: 'u1', username: 'sam' }, + user: { _id: 'u1', username: 'sam' }, + token: 'user-jwt', + loading: false, + error: null, + isAuthenticated: true, + register: jest.fn(), + login: jest.fn(), + logout: jest.fn(), + updateProfile: jest.fn(), +}; + +const mockGet = () => { + axios.get.mockImplementation((url) => { + if (url.startsWith('/api/registry/agents/next-free-name?')) return Promise.resolve({ data: { name: 'agent' } }); + if (url.startsWith('/api/hosted/availability')) { + return Promise.resolve({ data: { configured: false, caps: { agentsPerUser: 1, turnsPerDay: 200 } } }); + } + if (url === '/api/pods') return Promise.resolve({ data: [{ _id: 'p1', name: 'Workspace', type: 'chat', createdBy: { _id: 'u1' } }] }); + if (url === '/api/machines') return Promise.resolve({ data: { machines: [], offlineAfterMs: 90000 } }); + return Promise.resolve({ data: {} }); + }); +}; + +const renderPage = () => render( + + + + + , +); + +beforeEach(() => { + window.history.pushState({}, '', '/v2/agents/byo'); + Object.defineProperty(navigator, 'clipboard', { configurable: true, value: { writeText: jest.fn().mockResolvedValue(undefined) } }); + mockGet(); +}); + +afterEach(() => { + jest.clearAllMocks(); +}); + +const stepsPosted = () => recordByoStep.mock.calls.map(([, step]) => step); + +test('copying the add-a-computer daemon command posts daemon-command-copied', async () => { + renderPage(); + await waitFor(() => expect(axios.get).toHaveBeenCalledWith('/api/machines', expect.anything())); + const block = await screen.findByTestId('byo-add-computer'); + await act(async () => { + fireEvent.click(block.querySelector('button.v2-byo__copy')); + }); + + expect(navigator.clipboard.writeText).toHaveBeenCalledWith(expect.stringContaining('commonly daemon')); + expect(stepsPosted()).toContain('daemon-command-copied'); +}); + +test('a failed daemon copy posts no copy step', async () => { + navigator.clipboard.writeText.mockRejectedValueOnce(new Error('Denied')); + renderPage(); + const block = await screen.findByTestId('byo-add-computer'); + await act(async () => { + fireEvent.click(block.querySelector('button.v2-byo__copy')); + }); + + expect(stepsPosted()).not.toContain('daemon-command-copied'); +}); diff --git a/frontend/src/v2/components/V2AgentBYO.tsx b/frontend/src/v2/components/V2AgentBYO.tsx index e324763a2e..bd7daad929 100644 --- a/frontend/src/v2/components/V2AgentBYO.tsx +++ b/frontend/src/v2/components/V2AgentBYO.tsx @@ -990,6 +990,7 @@ const V2AgentBYO: React.FC = () => { setTimeout(() => setCopied((c) => (c === key ? null : c)), 1500); if (key === 'claude' || key === 'cursor') byoStep('mcp-command-copied'); else if (key === 'listen') byoStep('cli-command-copied'); + else if (key === 'daemon-setup') byoStep('daemon-command-copied'); else if (key === 'tok') byoStep('token-copied'); } catch { // Clipboard may be unavailable (non-HTTPS, sandbox); user can select manually. diff --git a/frontend/src/v2/utils/byoTelemetry.ts b/frontend/src/v2/utils/byoTelemetry.ts index 5f13a2a2a4..66d074b7b2 100644 --- a/frontend/src/v2/utils/byoTelemetry.ts +++ b/frontend/src/v2/utils/byoTelemetry.ts @@ -1,7 +1,8 @@ import type { V2ApiClient } from '../hooks/useV2Api'; export type ByoStep = 'page-viewed' | 'seat-created' | 'mcp-command-copied' | 'cli-command-copied' - | 'token-copied' | 'listen-confirmed' | 'listen-timeout' | 'hosted-offer-shown' | 'hosted-offer-clicked' + | 'daemon-command-copied' | 'token-copied' | 'listen-confirmed' | 'listen-timeout' + | 'hosted-offer-shown' | 'hosted-offer-clicked' | 'invite-card-shown' | 'invite-card-dismissed' | 'byo-offer-shown'; // Fixed event names only. Commands and credentials must never enter telemetry.