From f72a7280db13a46cc99ed6e2242e6049911211ca Mon Sep 17 00:00:00 2001 From: Mark IJbema Date: Thu, 21 May 2026 12:49:00 +0200 Subject: [PATCH] feat(admin): show experiment request log --- .../app/admin/api/model-experiments/hooks.ts | 5 + apps/web/src/app/admin/gateway/page.tsx | 15 +- .../ModelExperimentRequestsContent.tsx | 144 ++++++++++++++++++ .../admin/model-experiments-router.test.ts | 64 ++++++++ .../routers/admin/model-experiments-router.ts | 49 ++++++ 5 files changed, 276 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/app/admin/model-experiments/ModelExperimentRequestsContent.tsx diff --git a/apps/web/src/app/admin/api/model-experiments/hooks.ts b/apps/web/src/app/admin/api/model-experiments/hooks.ts index f3b132bf65a..984bbf77f6f 100644 --- a/apps/web/src/app/admin/api/model-experiments/hooks.ts +++ b/apps/web/src/app/admin/api/model-experiments/hooks.ts @@ -16,6 +16,11 @@ export function useModelExperiment(id: string | null) { }); } +export function useModelExperimentRequests() { + const trpc = useTRPC(); + return useQuery(trpc.admin.modelExperiments.listRequests.queryOptions()); +} + function useInvalidate() { const trpc = useTRPC(); const qc = useQueryClient(); diff --git a/apps/web/src/app/admin/gateway/page.tsx b/apps/web/src/app/admin/gateway/page.tsx index e3b6d49d792..622d1eff0fe 100644 --- a/apps/web/src/app/admin/gateway/page.tsx +++ b/apps/web/src/app/admin/gateway/page.tsx @@ -9,14 +9,21 @@ import { SyncProvidersContent } from '@/app/admin/sync-providers/SyncProvidersCo import { CustomLlmsContent } from '@/app/admin/custom-llms/CustomLlmsContent'; import { RoutingContent } from '@/app/admin/gateway/RoutingContent'; import { ModelExperimentsContent } from '@/app/admin/model-experiments/ModelExperimentsContent'; +import { ModelExperimentRequestsContent } from '@/app/admin/model-experiments/ModelExperimentRequestsContent'; const VALID_TABS: readonly string[] = [ 'sync-providers', 'custom-llms', 'routing', 'model-experiments', + 'experiment-requests', ]; -type Tab = 'sync-providers' | 'custom-llms' | 'routing' | 'model-experiments'; +type Tab = + | 'sync-providers' + | 'custom-llms' + | 'routing' + | 'model-experiments' + | 'experiment-requests'; const isValidTab = (value: string | null): value is Tab => value !== null && VALID_TABS.includes(value); @@ -76,6 +83,9 @@ export default function AdminGatewayPage() { Model Experiments + + Experiment Requests + @@ -89,6 +99,9 @@ export default function AdminGatewayPage() { + + + diff --git a/apps/web/src/app/admin/model-experiments/ModelExperimentRequestsContent.tsx b/apps/web/src/app/admin/model-experiments/ModelExperimentRequestsContent.tsx new file mode 100644 index 00000000000..f7f573f1ac5 --- /dev/null +++ b/apps/web/src/app/admin/model-experiments/ModelExperimentRequestsContent.tsx @@ -0,0 +1,144 @@ +'use client'; + +import { Badge } from '@/components/ui/badge'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { useModelExperimentRequests } from '@/app/admin/api/model-experiments/hooks'; + +function formatDate(value: string): string { + return new Date(value).toLocaleString(); +} + +function formatNullable(value: string | null): string { + return value && value.length > 0 ? value : '—'; +} + +function formatTokens(value: number): string { + return value.toLocaleString(); +} + +function formatMicrodollars(value: number | null): string { + if (value === null) return '—'; + return value.toLocaleString(); +} + +function HashCell({ value }: { value: string }) { + const display = value.length > 18 ? `${value.slice(0, 18)}…` : value; + return ( + + {display} + + ); +} + +export function ModelExperimentRequestsContent() { + const { data, isLoading } = useModelExperimentRequests(); + const rows = data?.items ?? []; + + return ( +
+
+

Experiment Requests

+

+ Last 100 requests attributed to model experiments. Prompt bodies stay in R2; this table + shows attribution, routing, and usage metadata only. +

+
+ + {isLoading ? ( +
Loading…
+ ) : rows.length === 0 ? ( +
+ No experiment requests recorded yet. +
+ ) : ( +
+ + + + Created + Experiment + Variant + Request + Usage + Prompt hash + User / org + + + + {rows.map(row => ( + + + {formatDate(row.createdAt)} + + +
{row.experimentName}
+
+ {row.publicModelId} +
+
+ {row.experimentId} +
+ + +
{row.variantLabel}
+
+ {row.variantVersionId} +
+ + +
+ {row.requestKind} + {row.allocationSubject} + {row.wasTruncated && truncated} +
+
+ Client: {formatNullable(row.clientRequestId)} +
+
{row.usageId}
+ + +
+ Tokens: {formatTokens(row.inputTokens)} in / {formatTokens(row.outputTokens)}{' '} + out +
+
+ Cache: {formatTokens(row.cacheWriteTokens)} write /{' '} + {formatTokens(row.cacheHitTokens)} hit +
+
+ Cost: {formatMicrodollars(row.costMicrodollars)} µ$ · discount:{' '} + {formatMicrodollars(row.cacheDiscountMicrodollars)} µ$ +
+ {row.hasError && error} + + + + + +
{row.userId}
+
+ {formatNullable(row.organizationId)} +
+
+ {formatNullable(row.provider)} / {formatNullable(row.inferenceProvider)} +
+
+ {formatNullable(row.requestedModel)} → {formatNullable(row.upstreamModel)} +
+ + + ))} + +
+
+ )} +
+ ); +} diff --git a/apps/web/src/routers/admin/model-experiments-router.test.ts b/apps/web/src/routers/admin/model-experiments-router.test.ts index 960cfe0a793..14367c9213f 100644 --- a/apps/web/src/routers/admin/model-experiments-router.test.ts +++ b/apps/web/src/routers/admin/model-experiments-router.test.ts @@ -3,7 +3,9 @@ import { cleanupDbForTest, db } from '@/lib/drizzle'; import { insertTestUser } from '@/tests/helpers/user.helper'; import { createCallerForUser } from '@/routers/test-utils'; import { + microdollar_usage, model_experiment, + model_experiment_request, model_experiment_variant, model_experiment_variant_version, } from '@kilocode/db/schema'; @@ -11,6 +13,7 @@ import { decryptApiKey } from '@/lib/ai-gateway/byok/encryption'; import { BYOK_ENCRYPTION_KEY } from '@/lib/config.server'; import { eq } from 'drizzle-orm'; import type { User } from '@kilocode/db/schema'; +import { randomUUID } from 'crypto'; let admin: User; @@ -93,6 +96,67 @@ describe('admin.modelExperiments — basic CRUD', () => { expect(withArchived.items.map(i => i.id).sort()).toEqual([a.id, b.id].sort()); }); + it('lists the last experiment attribution requests with usage and variant metadata', async () => { + const { caller, experimentId, variantA } = + await makeDraftWithTwoVariants('kilo/preview-requests'); + await caller.admin.modelExperiments.activate({ id: experimentId }); + const [version] = await db + .select({ id: model_experiment_variant_version.id }) + .from(model_experiment_variant_version) + .where(eq(model_experiment_variant_version.variant_id, variantA.id)) + .limit(1); + expect(version).toBeDefined(); + + const usageId = randomUUID(); + await db.insert(microdollar_usage).values({ + id: usageId, + kilo_user_id: 'request-user', + cost: 0, + input_tokens: 123, + output_tokens: 45, + cache_write_tokens: 6, + cache_hit_tokens: 7, + provider: 'custom', + model: 'partner-checkpoint-rc1', + requested_model: 'kilo/preview-requests', + inference_provider: 'partner', + organization_id: null, + }); + await db.insert(model_experiment_request).values({ + usage_id: usageId, + variant_version_id: version.id, + allocation_subject: 'user', + client_request_id: 'client-request-123', + request_kind: 'chat_completions', + request_body_sha256: '__failed__', + was_truncated: true, + }); + + const requests = await caller.admin.modelExperiments.listRequests(); + + expect(requests.items[0]).toEqual( + expect.objectContaining({ + usageId, + experimentId, + experimentName: 'exp-kilo/preview-requests', + publicModelId: 'kilo/preview-requests', + variantId: variantA.id, + variantLabel: 'control', + variantVersionId: version.id, + allocationSubject: 'user', + clientRequestId: 'client-request-123', + requestKind: 'chat_completions', + requestBodySha256: '__failed__', + wasTruncated: true, + userId: 'request-user', + requestedModel: 'kilo/preview-requests', + upstreamModel: 'partner-checkpoint-rc1', + inputTokens: 123, + outputTokens: 45, + }) + ); + }); + it('rejects delete unless status is draft', async () => { const { caller, experimentId } = await makeDraftWithTwoVariants('kilo/preview-delete'); await caller.admin.modelExperiments.activate({ id: experimentId }); diff --git a/apps/web/src/routers/admin/model-experiments-router.ts b/apps/web/src/routers/admin/model-experiments-router.ts index 56e51c577ae..6fba3c40a8d 100644 --- a/apps/web/src/routers/admin/model-experiments-router.ts +++ b/apps/web/src/routers/admin/model-experiments-router.ts @@ -1,7 +1,9 @@ import { adminProcedure, createTRPCRouter } from '@/lib/trpc/init'; import { db } from '@/lib/drizzle'; import { + microdollar_usage, model_experiment, + model_experiment_request, model_experiment_variant, model_experiment_variant_version, } from '@kilocode/db/schema'; @@ -287,6 +289,53 @@ export const adminModelExperimentsRouter = createTRPCRouter({ return { items: rows }; }), + listRequests: adminProcedure.query(async () => { + const rows = await db + .select({ + usageId: model_experiment_request.usage_id, + createdAt: model_experiment_request.created_at, + experimentId: model_experiment.id, + experimentName: model_experiment.name, + publicModelId: model_experiment.public_model_id, + variantId: model_experiment_variant.id, + variantLabel: model_experiment_variant.label, + variantVersionId: model_experiment_variant_version.id, + allocationSubject: model_experiment_request.allocation_subject, + clientRequestId: model_experiment_request.client_request_id, + requestKind: model_experiment_request.request_kind, + requestBodySha256: model_experiment_request.request_body_sha256, + wasTruncated: model_experiment_request.was_truncated, + userId: microdollar_usage.kilo_user_id, + organizationId: microdollar_usage.organization_id, + requestedModel: microdollar_usage.requested_model, + upstreamModel: microdollar_usage.model, + provider: microdollar_usage.provider, + inferenceProvider: microdollar_usage.inference_provider, + inputTokens: microdollar_usage.input_tokens, + outputTokens: microdollar_usage.output_tokens, + cacheWriteTokens: microdollar_usage.cache_write_tokens, + cacheHitTokens: microdollar_usage.cache_hit_tokens, + costMicrodollars: microdollar_usage.cost, + cacheDiscountMicrodollars: microdollar_usage.cache_discount, + hasError: microdollar_usage.has_error, + }) + .from(model_experiment_request) + .innerJoin(microdollar_usage, eq(model_experiment_request.usage_id, microdollar_usage.id)) + .innerJoin( + model_experiment_variant_version, + eq(model_experiment_request.variant_version_id, model_experiment_variant_version.id) + ) + .innerJoin( + model_experiment_variant, + eq(model_experiment_variant_version.variant_id, model_experiment_variant.id) + ) + .innerJoin(model_experiment, eq(model_experiment_variant.experiment_id, model_experiment.id)) + .orderBy(desc(model_experiment_request.created_at)) + .limit(100); + + return { items: rows }; + }), + get: adminProcedure.input(idSchema).query(async ({ input }) => { const experiment = await loadExperimentOrThrow(input.id); const variants = await listVariantsWithCurrentVersion(input.id);