Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions backend/__tests__/service/byoTelemetry.service.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Real Mongo: the once-per-seat guarantee is the unique _id index, so a mock
// cannot prove it. Two concurrent first calls with one token must leave one row.
const mongoose = require('mongoose');
const { setupMongoDb, closeMongoDb, clearMongoDb } = require('../utils/testUtils');
const { recordByoEvent, seatRegisteredKey } = require('../../services/byoTelemetry');

const rows = () => mongoose.connection.db.collection('byo_telemetry').find({}).toArray();

describe('byo_telemetry seat-registered on real Mongo', () => {
beforeAll(async () => {
await setupMongoDb();
});

afterAll(async () => {
await closeMongoDb();
});

beforeEach(async () => {
await clearMongoDb();
// clearMongoDb only resets model collections; byo_telemetry is a raw collection.
await mongoose.connection.db.collection('byo_telemetry').deleteMany({});
});

it('two concurrent first calls with one token leave exactly one row', async () => {
const key = seatRegisteredKey('token-record-hash');

await Promise.all([
recordByoEvent('human-1', 'seat-registered', {}, key),
recordByoEvent('human-1', 'seat-registered', {}, key),
]);

const stored = await rows();
expect(stored).toHaveLength(1);
expect(stored[0]._id).toBe(key);
expect(stored[0]).toEqual(expect.objectContaining({ userId: 'human-1', step: 'seat-registered' }));
expect(stored[0].at).toBeInstanceOf(Date);
});

it('a different token for the same installer is a separate seat row', async () => {
await Promise.all([
recordByoEvent('human-1', 'seat-registered', {}, seatRegisteredKey('token-a')),
recordByoEvent('human-1', 'seat-registered', {}, seatRegisteredKey('token-b')),
]);

expect(await rows()).toHaveLength(2);
});

it('keyless browser rows keep a Mongo-generated _id', async () => {
await recordByoEvent('human-1', 'seat-created');
await recordByoEvent('human-1', 'seat-created');

const stored = await rows();
expect(stored).toHaveLength(2);
stored.forEach((row) => expect(row._id).toBeInstanceOf(mongoose.Types.ObjectId));
});
});
11 changes: 11 additions & 0 deletions backend/__tests__/unit/middleware/agentRuntimeAuth.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,18 @@ const mockInstallationFind = jest.fn();
const mockInstallationUpdateOne = jest.fn();
const mockCompleteStarterTask = jest.fn();
const mockMarkByoConnectionAttempt = jest.fn();
const mockRecordByoEvent = jest.fn();

jest.mock('../../../services/starterTaskService', () => ({
completeConnectAgentStarterTask: (...args) => mockCompleteStarterTask(...args),
}));
jest.mock('../../../services/onboardingMarks', () => ({
markByoConnectionAttempt: (...args) => mockMarkByoConnectionAttempt(...args),
}));
jest.mock('../../../services/byoTelemetry', () => ({
recordByoEvent: (...args) => mockRecordByoEvent(...args),
seatRegisteredKey: () => `seat-registered:${'a'.repeat(64)}`,
}));

jest.mock('../../../models/User', () => {
function User() {}
Expand Down Expand Up @@ -133,6 +138,9 @@ describe('agentRuntimeAuth path 2 (install-bound token) — #66 fix', () => {
agentLabel: 'codex',
});
expect(mockMarkByoConnectionAttempt).toHaveBeenCalledWith('human-a', matchedInstall);
// Backend-side seat-registered row, written once for the first use of the token.
expect(mockRecordByoEvent).toHaveBeenCalledTimes(1);
expect(mockRecordByoEvent).toHaveBeenCalledWith('human-a', 'seat-registered', {}, expect.stringMatching(/^seat-registered:[0-9a-f]{64}$/));
});

test('does not fire the starter hook when the token was used before (#916)', async () => {
Expand All @@ -150,6 +158,7 @@ describe('agentRuntimeAuth path 2 (install-bound token) — #66 fix', () => {
await agentRuntimeAuth(buildReq('cm_agent_test'), buildRes(), jest.fn());

expect(mockCompleteStarterTask).not.toHaveBeenCalled();
expect(mockRecordByoEvent).not.toHaveBeenCalled();
});
});

Expand Down Expand Up @@ -188,6 +197,7 @@ describe('agentRuntimeAuth path 1 (User-row token) — first-use starter hook (#
'human-1',
expect.objectContaining({ podId: expect.anything(), installedBy: 'human-1' }),
);
expect(mockRecordByoEvent).toHaveBeenCalledWith('human-1', 'seat-registered', {}, expect.stringMatching(/^seat-registered:[0-9a-f]{64}$/));
});

test('stays silent on every subsequent authentication', async () => {
Expand All @@ -201,6 +211,7 @@ describe('agentRuntimeAuth path 1 (User-row token) — first-use starter hook (#
await agentRuntimeAuth(buildReq('cm_agent_test'), buildRes(), jest.fn());

expect(mockCompleteStarterTask).not.toHaveBeenCalled();
expect(mockRecordByoEvent).not.toHaveBeenCalled();
});

test('returns 401 when token doesnt match any install OR user', async () => {
Expand Down
21 changes: 21 additions & 0 deletions backend/__tests__/unit/routes/stats.byo-step.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,20 @@ describe('POST /api/stats/byo-step', () => {
expect(JSON.stringify(mockInsertOne.mock.calls[0][0])).not.toContain('cm_agent_secret');
});

it('records the daemon copy as telemetry only, with no onboarding copy mark', async () => {
const res = await request(app)
.post('/api/stats/byo-step')
.send({ step: 'daemon-command-copied', command: 'commonly daemon register' });

expect(res.status).toBe(200);
expect(mockMarkByoCommandCopied).not.toHaveBeenCalled();
expect(mockInsertOne).toHaveBeenCalledWith({
userId: 'human-1',
step: 'daemon-command-copied',
at: expect.any(Date),
});
});

it.each(['hosted-offer-shown', 'hosted-offer-clicked'])('accepts %s without copied-command marks or extra payload', async (step) => {
const res = await request(app).post('/api/stats/byo-step')
.send({ step, command: 'fixture-only-secret', agentName: 'private-agent' });
Expand All @@ -85,6 +99,13 @@ describe('POST /api/stats/byo-step', () => {
expect(mockMarkSurface).not.toHaveBeenCalled();
});

it('rejects seat-registered from the browser: the backend alone writes that row', async () => {
const res = await request(app).post('/api/stats/byo-step').send({ step: 'seat-registered' });

expect(res.status).toBe(400);
expect(mockInsertOne).not.toHaveBeenCalled();
});

it('rejects unknown steps without writing a mark or telemetry event', async () => {
const res = await request(app)
.post('/api/stats/byo-step')
Expand Down
55 changes: 55 additions & 0 deletions backend/__tests__/unit/services/byoTelemetry.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
const mockInsertOne = jest.fn();
let mockWarn;

jest.mock('mongoose', () => ({
connection: { db: { collection: () => ({ insertOne: (...args) => mockInsertOne(...args) }) } },
}));

const { recordByoEvent, seatRegisteredKey } = require('../../../services/byoTelemetry');

describe('recordByoEvent', () => {
beforeEach(() => {
jest.clearAllMocks();
mockWarn = jest.spyOn(console, 'warn').mockImplementation(() => {});
mockInsertOne.mockResolvedValue({ acknowledged: true });
});

it('writes the fixed step with the user, the extra fields and server time', async () => {
await recordByoEvent('human-1', 'seat-registered', { commandType: 'cli' });

expect(mockInsertOne).toHaveBeenCalledWith({
userId: 'human-1',
step: 'seat-registered',
commandType: 'cli',
at: expect.any(Date),
});
});

it('never throws when the write fails, so the caller flow continues', async () => {
mockInsertOne.mockRejectedValue(new Error('db down'));

await expect(recordByoEvent('human-1', 'seat-registered')).resolves.toBeUndefined();
expect(mockWarn).toHaveBeenCalledWith('[byo-telemetry] write failed:', 'db down');
});

it('keys a once-per-seat row by a hash of the token, never the token itself', () => {
const key = seatRegisteredKey('token-record-hash');

expect(key).toMatch(/^seat-registered:[0-9a-f]{64}$/);
expect(key).not.toContain('token-record-hash');
expect(seatRegisteredKey('token-record-hash')).toBe(key);
});

it('writes the keyed row with _id set, so a racing second write is a no-op', async () => {
const dup = Object.assign(new Error('E11000 duplicate key'), { code: 11000 });
mockInsertOne.mockResolvedValueOnce({ acknowledged: true }).mockRejectedValueOnce(dup);
const key = seatRegisteredKey('token-record-hash');

await recordByoEvent('human-1', 'seat-registered', {}, key);
await expect(recordByoEvent('human-1', 'seat-registered', {}, key)).resolves.toBeUndefined();

expect(mockInsertOne).toHaveBeenNthCalledWith(1, expect.objectContaining({ _id: key, step: 'seat-registered' }));
expect(mockInsertOne).toHaveBeenNthCalledWith(2, expect.objectContaining({ _id: key, step: 'seat-registered' }));
expect(mockWarn).not.toHaveBeenCalled();
});
});
4 changes: 4 additions & 0 deletions backend/middleware/agentRuntimeAuth.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import { touchLastActive } from './auth';
import Pod from '../models/Pod';
import AgentCredential from '../models/AgentCredential';
import { markByoConnectionAttempt } from '../services/onboardingMarks';
import { recordByoEvent, seatRegisteredKey } from '../services/byoTelemetry';
import { isSpawnCredential } from '../services/spawnCredentialService';
import { embeddedRuntimeTokenExpired, runtimeCredentialRefusalReason } from '../services/runtimeCredentialGate';

Expand Down Expand Up @@ -148,6 +149,8 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next
for (const installationRow of installations) {
markByoConnectionAttempt(installationRow.installedBy, installationRow);
}
// One row per token, not per installation: the seat is the token.
void recordByoEvent(installations[0]?.installedBy, 'seat-registered', {}, seatRegisteredKey(tokenHash));
// eslint-disable-next-line global-require, @typescript-eslint/no-require-imports
const { completeConnectAgentStarterTask } = require('../services/starterTaskService');
void completeConnectAgentStarterTask({
Expand Down Expand Up @@ -205,6 +208,7 @@ export default async function agentRuntimeAuth(req: Request, res: Response, next

if (isFirstLegacyTokenUse) {
markByoConnectionAttempt(installation.installedBy, installation);
void recordByoEvent(installation.installedBy, 'seat-registered', {}, seatRegisteredKey(tokenHash));
// eslint-disable-next-line global-require, @typescript-eslint/no-require-imports
const { completeConnectAgentStarterTask } = require('../services/starterTaskService');
void completeConnectAgentStarterTask({
Expand Down
20 changes: 6 additions & 14 deletions backend/routes/stats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,10 +84,10 @@ const { rateLimit } = require('express-rate-limit');
// eslint-disable-next-line global-require
const { cloudflareIpRateLimitKeyGenerator } = require('../middleware/ipRateLimit');
// eslint-disable-next-line global-require
const mongoose = require('mongoose');
const { recordByoEvent } = require('../services/byoTelemetry');

const BYO_STEPS = new Set([
'page-viewed', 'seat-created', 'mcp-command-copied', 'cli-command-copied',
'page-viewed', 'seat-created', 'mcp-command-copied', 'cli-command-copied', 'daemon-command-copied',
'token-copied', 'listen-confirmed', 'listen-timeout', 'hosted-offer-shown', 'hosted-offer-clicked',
'invite-card-shown', 'invite-card-dismissed', 'byo-offer-shown',
]);
Expand Down Expand Up @@ -115,18 +115,10 @@ router.post('/byo-step', rateLimit({
markByoCommandCopied(req.userId, step === 'mcp-command-copied' ? 'mcp' : 'cli');
}
if (SURFACE_STEPS[step]) markSurface(req.userId, SURFACE_STEPS[step]);
try {
await mongoose.connection.db.collection('byo_telemetry').insertOne({
userId: req.userId,
step,
...(step === 'mcp-command-copied' ? { commandType: 'mcp' } : {}),
...(step === 'cli-command-copied' ? { commandType: 'cli' } : {}),
at: new Date(),
});
} catch (err: any) {
// Telemetry must never fail the caller's flow.
console.warn('[byo-telemetry] write failed:', err?.message);
}
await recordByoEvent(req.userId, step as any, {
...(step === 'mcp-command-copied' ? { commandType: 'mcp' } : {}),
...(step === 'cli-command-copied' ? { commandType: 'cli' } : {}),
});
return res.json({ ok: true });
});

Expand Down
39 changes: 39 additions & 0 deletions backend/services/byoTelemetry.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { createHash } from 'crypto';
import mongoose from 'mongoose';

// Server-side BYO funnel rows. The browser posts its steps through
// POST /api/stats/byo-step; events only the backend can see (an agent's first
// authenticated call) are written here directly, so they never depend on the
// user keeping the page open. Fixed step names only: never a command, key or name.
export type ServerByoStep = 'seat-registered';

// A once-per-seat event is keyed by a hash of the token record's hash, so the
// unique _id index makes concurrent first calls write one row. The hash is one-way
// and is never the credential itself.
export function seatRegisteredKey(tokenHash: string): string {
return `seat-registered:${createHash('sha256').update(tokenHash).digest('hex')}`;
}

export async function recordByoEvent(
userId: unknown,
step: ServerByoStep,
extra: Record<string, unknown> = {},
dedupeKey?: string,
): Promise<void> {
try {
// The collection is untyped here; a string _id is only set for the keyed seat row.
const doc: Record<string, unknown> = {
...(dedupeKey ? { _id: dedupeKey } : {}),
userId,
step,
...extra,
at: new Date(),
};
await mongoose.connection.db.collection('byo_telemetry').insertOne(doc as any);
} catch (err: any) {
// A duplicate key means this seat's row already exists: the expected outcome of a race.
if (err?.code === 11000) return;
// Telemetry must never fail the caller's flow.
console.warn('[byo-telemetry] write failed:', err?.message);
}
}
5 changes: 5 additions & 0 deletions docs/development/byo-funnel-telemetry.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,17 @@ As of TASK-483 and TASK-493 (2026-10-24), the browser sends:
| `seat-created` | A runtime key was issued and its command rendered. Once per issued command. This is the command-exposure denominator. |
| `mcp-command-copied` | The Claude or Cursor copy button successfully wrote the command. |
| `cli-command-copied` | The listener copy button successfully wrote the command. |
| `daemon-command-copied` | The "Add a computer" daemon command copy button successfully wrote the command. Telemetry only: it sets no onboarding copy mark, so stalled-connect messaging is unchanged. Before TASK-653 this copy posted nothing, so a user who used this path was invisible to the copy count. |
| `token-copied` | The token copy button successfully wrote the key. |
| `listen-confirmed` | The connection poll observed this agent using its key after issuance (with existing clock-skew allowance). |
| `hosted-offer-shown` | The timer or desktop exit-intent path displayed the hosted offer. The shared session guard records it once, including when both triggers run. |
| `hosted-offer-clicked` | The user activated the offer to start hosted setup. Recorded before name lookup, including failed attempts; concurrent duplicate activations are suppressed. A retry records another activation. This is separate from choosing the initial hosted mode. |
| `listen-timeout` | The page's connection polling window expired. |

The backend writes one more row, `seat-registered`, itself; the browser cannot post it (the route answers 400). It is written once per runtime token, on the token's first authenticated call (`middleware/agentRuntimeAuth.ts`), with the installing user as `userId`. It fires whether or not the user is still on the page, so it measures registration when `listen-confirmed` is missing. It is not limited to BYO: a hosted seat's first call also writes it. Its `_id` is `seat-registered:<sha256 of the token record hash>`, so a racing second first call is a duplicate-key no-op and the row is never written twice.

Read back `step`, `userId` and `at` (server time). The field is `at`, not `createdAt`; a count on `createdAt` sees nothing. Rows from the browser have a Mongo-generated `_id`.

Before TASK-483, the client never emitted `page-viewed` or `seat-created`.
Historical counts of those events cannot measure page or command exposure.
A selected-text copy from a command or key block dismisses the hosted offer but does not claim that a complete
Expand Down
Loading
Loading