Skip to content

test(webapp): replace slow metadata replica guard with unit test #4312

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
carderne merged 1 commit into from
Jul 20, 2026
Merged

test(webapp): replace slow metadata replica guard with unit test #4312

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
293 changes: 75 additions & 218 deletions apps/webapp/test/metadataRouteReplicaLag.guard.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,17 @@
// Property: the metadata GET loader resolves a live run on a replica+buffer double-miss via its primary
// fallback, returning 200 + the run's metadata. Drives the REAL exported `loader` end-to-end:
// 1. runStore.findRun(..., $replica) → REPLICA (lagging) → null
// 2. findRunByIdWithMollifierFallback(...) → buffer MISS (null)
// 3. runStore.findRunOnPrimary(...) → owning PRIMARY → HIT
// Store is REAL: a split RoutingRunStore over two testcontainer Postgres DBs, the owning
// (legacy/control-plane) REPLICA frozen behind the shared laggingReplica. Only the loader's webapp
// singletons (auth, $replica brand, mollifier buffer, route builder, logging) are stubbed.

import { describe, expect, vi } from "vitest";
import { heteroRunOpsPostgresTest, laggingReplica } from "@internal/testcontainers";
import type { PrismaClient } from "@trigger.dev/database";
import type { RunOpsPrismaClient } from "@internal/run-ops-database";
import { PostgresRunStore, RoutingRunStore } from "@internal/run-store";
import type { CreateRunInput } from "@internal/run-store";

// vi.mock factories are hoisted above the imports, so anything they reference must be created inside
// vi.hoisted. `holder.store` is filled in per-test with the REAL split router; the mocked
// `~/v3/runStore.server` export is a stable Proxy that forwards every property access to it. The
// branded `$replica` marker uses the global-registry symbol the run-store brands replicas with
// (readReplicaClient.ts) — a branded client makes the routing store keep the read on the owning
// REPLICA (no primary escalation), which under lag is exactly the miss the primary fallback recovers.
const { holder } = vi.hoisted(() => {
const REPLICA_BRAND = Symbol.for("trigger.dev/run-store/read-replica");
return {
holder: {
store: undefined as unknown,
brandedReplica: { [REPLICA_BRAND]: true } as object,
environment: undefined as unknown,
bufferResult: null as unknown,
},
};
});
import { beforeEach, describe, expect, it, vi } from "vitest";

const { mocks } = vi.hoisted(() => ({
mocks: {
replica: {},
environment: { id: "env_meta", organizationId: "org_meta" },
authenticateApiRequest: vi.fn(),
findRun: vi.fn(),
findRunOnPrimary: vi.fn(),
findRunByIdWithMollifierFallback: vi.fn(),
},
}));
Comment thread
carderne marked this conversation as resolved.

vi.mock("~/db.server", () => ({ prisma: {}, $replica: holder.brandedReplica }));
vi.mock("~/db.server", () => ({ prisma: {}, $replica: mocks.replica }));
vi.mock("~/env.server", () => ({
env: {
TASK_RUN_METADATA_MAXIMUM_SIZE: 256 * 1024,
Expand All @@ -41,42 +20,30 @@ vi.mock("~/env.server", () => ({
TRIGGER_MOLLIFIER_METADATA_BACKOFF_STEP_MS: 10,
},
}));
// The route module builds its action at import time via createActionApiRoute; stub the builder so the
// heavy platform/auth middleware graph never evaluates. We drive the exported `loader` directly.
vi.mock("~/services/routeBuilders/apiBuilder.server", () => ({
createActionApiRoute: () => ({ action: vi.fn() }),
}));
vi.mock("~/services/apiAuth.server", () => ({
authenticateApiRequest: vi.fn(async () => ({ environment: holder.environment })),
authenticateApiRequest: mocks.authenticateApiRequest,
}));
// Inject the REAL split router. A stable Proxy keeps the named import binding constant while
// forwarding every method to the per-test router set in holder.store.
vi.mock("~/v3/runStore.server", () => ({
runStore: new Proxy(
{},
{
get(_target, prop) {
const store = holder.store as Record<string | symbol, unknown>;
if (!store) throw new Error("test bug: holder.store not initialised before loader ran");
const value = store[prop];
return typeof value === "function"
? (value as (...a: unknown[]) => unknown).bind(store)
: value;
},
}
),
runStore: {
findRun: mocks.findRun,
findRunOnPrimary: mocks.findRunOnPrimary,
},
}));
// Buffer fallback returns whatever the test set (null = buffer miss, the double-miss scenario).
vi.mock("~/v3/mollifier/readFallback.server", () => ({
findRunByIdWithMollifierFallback: vi.fn(async () => holder.bufferResult),
findRunByIdWithMollifierFallback: mocks.findRunByIdWithMollifierFallback,
}));
// Action-only leaf; stub to keep the import graph light.
vi.mock("~/v3/mollifier/applyMetadataMutation.server", () => ({
applyMetadataMutationToBufferedRun: vi.fn(),
}));
vi.mock("~/services/metadata/updateMetadataInstance.server", () => ({
updateMetadataService: { call: vi.fn(async () => undefined) },
}));
vi.mock("~/services/realtime/runChangeNotifierInstance.server", () => ({
publishChangeRecord: vi.fn(),
}));
vi.mock("~/v3/services/common.server", () => ({
ServiceValidationError: class extends Error {},
}));
Expand All @@ -86,176 +53,66 @@ vi.mock("~/services/logger.server", () => ({

import { loader } from "~/routes/api.v1.runs.$runId.metadata";

// A cuid (25 chars after `run_`) classifies LEGACY, so both the create and the friendlyId-keyed reads
// route to the legacy (control-plane) store — the store that owns this run.
const CUID_25 = "c".repeat(25);
const friendlyId = "run_meta_live";
const runWhere = { friendlyId, runtimeEnvironmentId: mocks.environment.id };
const metadataSelect = { select: { metadata: true, metadataType: true } };

async function seedEnvironmentLegacy(prisma: PrismaClient, suffix: string) {
const organization = await prisma.organization.create({
data: { title: `Org ${suffix}`, slug: `org-${suffix}` },
});
const project = await prisma.project.create({
data: {
name: `Project ${suffix}`,
slug: `project-${suffix}`,
externalRef: `proj_${suffix}`,
organizationId: organization.id,
},
});
const environment = await prisma.runtimeEnvironment.create({
data: {
type: "DEVELOPMENT",
slug: "dev",
projectId: project.id,
organizationId: organization.id,
apiKey: `tr_dev_${suffix}`,
pkApiKey: `pk_dev_${suffix}`,
shortcode: `short_${suffix}`,
},
});
return { organization, project, environment };
}

function buildCreateRunInput(params: {
runId: string;
friendlyId: string;
organizationId: string;
projectId: string;
runtimeEnvironmentId: string;
metadata?: string;
metadataType?: string;
}): CreateRunInput {
return {
data: {
id: params.runId,
engine: "V2",
status: "PENDING",
friendlyId: params.friendlyId,
runtimeEnvironmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
organizationId: params.organizationId,
projectId: params.projectId,
taskIdentifier: "my-task",
payload: '{"hello":"world"}',
payloadType: "application/json",
metadata: params.metadata,
metadataType: params.metadataType,
context: { foo: "bar" },
traceContext: { trace: "ctx" },
traceId: "trace_1",
spanId: "span_1",
runTags: ["alpha"],
queue: "task/my-task",
isTest: false,
taskEventStore: "taskEvent",
depth: 0,
createdAt: new Date("2024-01-01T00:00:00.000Z"),
},
snapshot: {
engine: "V2",
executionStatus: "RUN_CREATED",
description: "Run was created",
runStatus: "PENDING",
environmentId: params.runtimeEnvironmentId,
environmentType: "DEVELOPMENT",
projectId: params.projectId,
organizationId: params.organizationId,
},
};
}

function buildRouterWithLaggingLegacyReplica(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) {
const legacyReplica = laggingReplica(prisma14, [{ model: "taskRun", mode: "missing" }]);
const legacyStore = new PostgresRunStore({
prisma: prisma14,
readOnlyPrisma: legacyReplica.client,
schemaVariant: "legacy",
});
const newStore = new PostgresRunStore({
prisma: prisma17 as never,
readOnlyPrisma: prisma17 as never,
schemaVariant: "dedicated",
});
const router = new RoutingRunStore({ new: newStore, legacy: legacyStore });
return { router, legacyStore, legacyReplica };
}

async function callLoader(friendlyId: string) {
const response = await loader({
request: new Request("https://api.trigger.dev/api/v1/runs/" + friendlyId + "/metadata", {
async function callLoader(runId = friendlyId) {
return (await loader({
request: new Request(`https://example.com/api/v1/runs/${runId}/metadata`, {
headers: { Authorization: "Bearer tr_dev_meta" },
}),
params: { runId: friendlyId },
params: { runId },
context: {} as never,
});
return response as Response;
})) as Response;
}

describe("metadata GET loader under replica lag", () => {
heteroRunOpsPostgresTest(
"metadata GET loader resolves a live run on a replica and buffer double-miss",
async ({ prisma14, prisma17 }) => {
const { router, legacyStore, legacyReplica } = buildRouterWithLaggingLegacyReplica(
prisma14,
prisma17
);

const seed = await seedEnvironmentLegacy(prisma14, "meta");
const runId = `run_${CUID_25}`; // cuid → LEGACY-owned
const friendlyId = "run_meta_live";
const metadata = '{"phase":"one"}';
await legacyStore.createRun(
buildCreateRunInput({
runId,
friendlyId,
organizationId: seed.organization.id,
projectId: seed.project.id,
runtimeEnvironmentId: seed.environment.id,
metadata,
metadataType: "application/json",
})
);

// Wire the loader's module singletons: the real split router, the authenticated env, and a
// buffer that MISSES (the run has drained from the buffer to the primary but the replica has
// not caught up — the exact double-miss under test).
holder.store = router;
holder.environment = { id: seed.environment.id, organizationId: seed.organization.id };
holder.bufferResult = null;

const response = await callLoader(friendlyId);
const body = (await response.json()) as {
metadata?: unknown;
metadataType?: unknown;
error?: string;
};

// The replica WAS consulted (and, being frozen, missed) — proving the read genuinely went
// through the lagging replica and the recovery is the primary fallback, not a lucky replica hit.
expect(legacyReplica.wasHit()).toBe(true);

// The property: the loader re-reads the owning primary and returns the live run's metadata.
expect(response.status).toBe(200);
expect(body.metadata).toBe(metadata);
expect(body.metadataType).toBe("application/json");
}
);
beforeEach(() => {
vi.clearAllMocks();
mocks.authenticateApiRequest.mockResolvedValue({ environment: mocks.environment });
mocks.findRun.mockResolvedValue(null);
mocks.findRunByIdWithMollifierFallback.mockResolvedValue(null);
mocks.findRunOnPrimary.mockResolvedValue(null);
});

// Negative control: when the run truly does not exist anywhere (replica miss + buffer miss +
// primary miss), the loader must still 404. This pins the behavior to "recover a LIVE run" rather
// than "never 404".
heteroRunOpsPostgresTest(
"metadata GET loader 404s when the run is absent on the primary too",
async ({ prisma14, prisma17 }) => {
const { router } = buildRouterWithLaggingLegacyReplica(prisma14, prisma17);
const seed = await seedEnvironmentLegacy(prisma14, "meta_absent");
describe("metadata GET loader under replica lag", () => {
it("resolves a live run after a replica and buffer double-miss", async () => {
const metadata = '{"phase":"one"}';
mocks.findRunOnPrimary.mockResolvedValue({
metadata,
metadataType: "application/json",
});

const response = await callLoader();

expect(mocks.findRun).toHaveBeenCalledWith(runWhere, metadataSelect, mocks.replica);
expect(mocks.findRunByIdWithMollifierFallback).toHaveBeenCalledWith({
runId: friendlyId,
environmentId: mocks.environment.id,
organizationId: mocks.environment.organizationId,
});
expect(mocks.findRunOnPrimary).toHaveBeenCalledWith(runWhere, metadataSelect);
expect(mocks.findRun.mock.invocationCallOrder[0]).toBeLessThan(
mocks.findRunByIdWithMollifierFallback.mock.invocationCallOrder[0]
);
expect(mocks.findRunByIdWithMollifierFallback.mock.invocationCallOrder[0]).toBeLessThan(
mocks.findRunOnPrimary.mock.invocationCallOrder[0]
);
await expect(response.json()).resolves.toEqual({
metadata,
metadataType: "application/json",
});
expect(response.status).toBe(200);
});

holder.store = router;
holder.environment = { id: seed.environment.id, organizationId: seed.organization.id };
holder.bufferResult = null;
it("returns 404 when the run is absent from the primary too", async () => {
const response = await callLoader("run_does_not_exist");

const response = await callLoader("run_does_not_exist");
expect(response.status).toBe(404);
}
);
expect(mocks.findRunOnPrimary).toHaveBeenCalledWith(
{ friendlyId: "run_does_not_exist", runtimeEnvironmentId: mocks.environment.id },
metadataSelect
);
await expect(response.json()).resolves.toEqual({ error: "Run not found" });
expect(response.status).toBe(404);
});
});