From eb26a755d2c50907d9278f49ed47b1db78505450 Mon Sep 17 00:00:00 2001 From: Mark IJbema Date: Thu, 21 May 2026 12:09:47 +0200 Subject: [PATCH 1/2] feat(gateway): expose active experiment models --- .../src/app/api/openrouter/models/route.ts | 6 +- .../list-available-experiment-models.test.ts | 68 +++++++++++++++++++ .../list-available-experiment-models.ts | 52 ++++++++++++++ .../lib/organizations/organization-models.ts | 2 + 4 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.test.ts create mode 100644 apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.ts diff --git a/apps/web/src/app/api/openrouter/models/route.ts b/apps/web/src/app/api/openrouter/models/route.ts index c9af1304bba..e09714ddd54 100644 --- a/apps/web/src/app/api/openrouter/models/route.ts +++ b/apps/web/src/app/api/openrouter/models/route.ts @@ -8,6 +8,7 @@ import { getDirectByokModelsForUser } from '@/lib/ai-gateway/providers/direct-by import { getAvailableModelsForOrganization } from '@/lib/organizations/organization-models'; import { FEATURE_HEADER, validateFeatureHeader } from '@/lib/feature-detection'; import { filterByFeature } from '@/lib/ai-gateway/models'; +import { listAvailableExperimentModels } from '@/lib/ai-gateway/experiments/list-available-experiment-models'; async function tryGetUserFromAuth() { try { @@ -43,7 +44,10 @@ export async function GET( return NextResponse.json(data); } const byokModels = auth?.user ? await getDirectByokModelsForUser(auth.user.id) : []; - return NextResponse.json({ data: filterByFeature(data.data.concat(byokModels), feature) }); + const experimentModels = await listAvailableExperimentModels(); + return NextResponse.json({ + data: filterByFeature(data.data.concat(byokModels, experimentModels), feature), + }); } catch (error) { captureException(error, { tags: { endpoint: 'openrouter/models' }, diff --git a/apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.test.ts b/apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.test.ts new file mode 100644 index 00000000000..6c39570dfbd --- /dev/null +++ b/apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it } from '@jest/globals'; +import { db } from '@/lib/drizzle'; +import { model_experiment } from '@kilocode/db/schema'; +import { listAvailableExperimentModels } from './list-available-experiment-models'; +import { inArray } from 'drizzle-orm'; + +const testPublicIds = [ + 'kilo/preview-active-a', + 'kilo/preview-active-b', + 'kilo/preview-paused', + 'kilo/preview-draft', +]; + +describe('listAvailableExperimentModels', () => { + afterEach(async () => { + await db + .delete(model_experiment) + .where(inArray(model_experiment.public_model_id, testPublicIds)); + }); + + it('returns only active experiment public ids as selectable models', async () => { + await db + .delete(model_experiment) + .where(inArray(model_experiment.public_model_id, testPublicIds)); + await db.insert(model_experiment).values([ + { + public_model_id: 'kilo/preview-active-a', + name: 'Preview Active A', + description: 'First active preview', + status: 'active', + }, + { + public_model_id: 'kilo/preview-active-b', + name: 'Preview Active B', + description: null, + status: 'active', + }, + { + public_model_id: 'kilo/preview-paused', + name: 'Preview Paused', + description: 'Paused preview', + status: 'paused', + }, + { + public_model_id: 'kilo/preview-draft', + name: 'Preview Draft', + description: 'Draft preview', + status: 'draft', + }, + ]); + + const models = await listAvailableExperimentModels(); + + expect(models.map(model => model.id)).toEqual([ + 'kilo/preview-active-a', + 'kilo/preview-active-b', + ]); + expect(models[0]).toEqual( + expect.objectContaining({ + id: 'kilo/preview-active-a', + name: 'Preview Active A', + description: 'First active preview', + context_length: 200000, + }) + ); + expect(models[1].description).toBe('Preview Active B'); + }); +}); diff --git a/apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.ts b/apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.ts new file mode 100644 index 00000000000..00f11530268 --- /dev/null +++ b/apps/web/src/lib/ai-gateway/experiments/list-available-experiment-models.ts @@ -0,0 +1,52 @@ +import { asc, eq } from 'drizzle-orm'; +import { model_experiment } from '@kilocode/db/schema'; +import { readDb } from '@/lib/drizzle'; +import type { OpenRouterModel } from '@/lib/organizations/organization-types'; + +const DEFAULT_CONTEXT_LENGTH = 200_000; + +function toCreatedSeconds(createdAt: string): number { + const ms = new Date(createdAt).getTime(); + return Number.isFinite(ms) ? Math.floor(ms / 1000) : 0; +} + +function convertExperimentModel(row: typeof model_experiment.$inferSelect): OpenRouterModel { + return { + id: row.public_model_id, + name: row.name, + created: toCreatedSeconds(row.created_at), + description: row.description ?? row.name, + architecture: { + input_modalities: ['text'], + output_modalities: ['text'], + tokenizer: 'Other', + }, + top_provider: { + is_moderated: false, + context_length: DEFAULT_CONTEXT_LENGTH, + max_completion_tokens: null, + }, + pricing: { + prompt: '0.0000000', + completion: '0.0000000', + request: '0', + image: '0', + web_search: '0', + internal_reasoning: '0', + input_cache_read: '0.00000000', + input_cache_write: '0.00000000', + }, + context_length: DEFAULT_CONTEXT_LENGTH, + per_request_limits: null, + supported_parameters: ['max_tokens', 'temperature', 'tools', 'reasoning', 'include_reasoning'], + }; +} + +export async function listAvailableExperimentModels(): Promise { + const rows = await readDb + .select() + .from(model_experiment) + .where(eq(model_experiment.status, 'active')) + .orderBy(asc(model_experiment.public_model_id)); + return rows.map(convertExperimentModel); +} diff --git a/apps/web/src/lib/organizations/organization-models.ts b/apps/web/src/lib/organizations/organization-models.ts index 749bc6fe27f..6e04f65cfae 100644 --- a/apps/web/src/lib/organizations/organization-models.ts +++ b/apps/web/src/lib/organizations/organization-models.ts @@ -9,6 +9,7 @@ import { listAvailableCustomLlms } from '@/lib/ai-gateway/custom-llm/listAvailab import { getDirectByokModelsForOrganization } from '@/lib/ai-gateway/providers/direct-byok'; import { getOrganizationById } from '@/lib/organizations/organizations'; import { getEffectiveModelRestrictions } from '@/lib/organizations/model-restrictions'; +import { listAvailableExperimentModels } from '@/lib/ai-gateway/experiments/list-available-experiment-models'; export async function getAvailableModelsForOrganization( organizationId: string @@ -40,6 +41,7 @@ export async function getAvailableModelsForOrganization( filteredModels.push(...(await getDirectByokModelsForOrganization(organizationId))); filteredModels.push(...(await listAvailableCustomLlms(organizationId))); + filteredModels.push(...(await listAvailableExperimentModels())); return { ...responseData, From 051a000abb437d214588f8b43daf9391a3140e95 Mon Sep 17 00:00:00 2001 From: Mark IJbema Date: Thu, 21 May 2026 13:02:05 +0200 Subject: [PATCH 2/2] fix(gateway): apply org restrictions to experiment models --- .../lib/organizations/organization-models.ts | 7 ++-- .../organization-settings-router.test.ts | 34 ++++++++++++++++--- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/apps/web/src/lib/organizations/organization-models.ts b/apps/web/src/lib/organizations/organization-models.ts index 6e04f65cfae..b41711c30ec 100644 --- a/apps/web/src/lib/organizations/organization-models.ts +++ b/apps/web/src/lib/organizations/organization-models.ts @@ -26,12 +26,14 @@ export async function getAvailableModelsForOrganization( } const responseData = await getEnhancedOpenRouterModels(); + const experimentModels = await listAvailableExperimentModels(); + const restrictionCandidates = responseData.data.concat(experimentModels); - let filteredModels = responseData.data; + let filteredModels = restrictionCandidates; if (hasActiveModelRestrictions(restrictions)) { const isAllowed = createAllowPredicateFromRestrictions(restrictions); const models = []; - for (const model of responseData.data) { + for (const model of restrictionCandidates) { if (await isAllowed(model.id)) { models.push(model); } @@ -41,7 +43,6 @@ export async function getAvailableModelsForOrganization( filteredModels.push(...(await getDirectByokModelsForOrganization(organizationId))); filteredModels.push(...(await listAvailableCustomLlms(organizationId))); - filteredModels.push(...(await listAvailableExperimentModels())); return { ...responseData, diff --git a/apps/web/src/routers/organizations/organization-settings-router.test.ts b/apps/web/src/routers/organizations/organization-settings-router.test.ts index 133cfe2e8ae..ccd33126e5c 100644 --- a/apps/web/src/routers/organizations/organization-settings-router.test.ts +++ b/apps/web/src/routers/organizations/organization-settings-router.test.ts @@ -11,8 +11,8 @@ import type { OpenRouterModelsResponse, } from '@/lib/organizations/organization-types'; import type { User, Organization } from '@kilocode/db/schema'; -import { organizations } from '@kilocode/db/schema'; -import { eq } from 'drizzle-orm'; +import { model_experiment, organizations } from '@kilocode/db/schema'; +import { eq, inArray } from 'drizzle-orm'; import { randomUUID } from 'crypto'; import { db } from '@/lib/drizzle'; @@ -250,6 +250,17 @@ describe('organizations settings trpc router', () => { }); describe('listAvailableModels procedure', () => { + const experimentPublicIds = ['kilo/preview-allowed-by-policy', 'kilo/preview-denied-by-policy']; + + async function deleteExperimentModels() { + await db + .delete(model_experiment) + .where(inArray(model_experiment.public_model_id, experimentPublicIds)); + } + + beforeEach(deleteExperimentModels); + afterEach(deleteExperimentModels); + function makeOpenRouterModel(id: string): OpenRouterModel { return { id, @@ -282,12 +293,24 @@ describe('organizations settings trpc router', () => { } satisfies OpenRouterModelsResponse; mockedGetEnhancedOpenRouterModels.mockResolvedValue(openRouterModelsResponse); + await db.insert(model_experiment).values([ + { + public_model_id: 'kilo/preview-allowed-by-policy', + name: 'Allowed experiment', + status: 'active', + }, + { + public_model_id: 'kilo/preview-denied-by-policy', + name: 'Denied experiment', + status: 'active', + }, + ]); const orgWithDenyList = await createTestOrganization( 'Model Deny List', owner.id, 0, - { model_deny_list: ['openai/gpt-4o'] }, + { model_deny_list: ['openai/gpt-4o', 'kilo/preview-denied-by-policy'] }, false ); await addUserToOrganization(orgWithDenyList.id, member.id, 'member'); @@ -297,7 +320,10 @@ describe('organizations settings trpc router', () => { organizationId: orgWithDenyList.id, }); - expect(result.data.map(model => model.id)).toEqual(['anthropic/claude-3-opus']); + expect(result.data.map(model => model.id)).toEqual([ + 'anthropic/claude-3-opus', + 'kilo/preview-allowed-by-policy', + ]); }); it('should include new models from allowed providers when they are not denied', async () => {