From fd13088f8953396214b6fcee4e8590eafa151fef Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Fri, 10 Jul 2026 10:36:33 +0100 Subject: [PATCH 01/15] fix(supervisor): authenticate compute snapshot callbacks (#60) * fix(supervisor): authenticate compute snapshot callbacks * fix(supervisor): harden compute snapshot callback token derivation Derive a domain-separated key for callback HMACs instead of using the worker secret directly, reject an empty callback secret at construction (covering the file-based secret path), tighten the worker token env to be non-empty, and document the token's binding scope and the callback channel assumptions it relies on. --- .../secure-compute-snapshot-callbacks.md | 6 + apps/supervisor/src/env.ts | 2 +- apps/supervisor/src/index.ts | 5 +- .../services/computeSnapshotService.test.ts | 120 +++++++++++++++++- .../src/services/computeSnapshotService.ts | 92 +++++++++++++- apps/supervisor/src/workloadServer/index.ts | 2 + 6 files changed, 219 insertions(+), 8 deletions(-) create mode 100644 .server-changes/secure-compute-snapshot-callbacks.md diff --git a/.server-changes/secure-compute-snapshot-callbacks.md b/.server-changes/secure-compute-snapshot-callbacks.md new file mode 100644 index 00000000000..5f362f3fa41 --- /dev/null +++ b/.server-changes/secure-compute-snapshot-callbacks.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: fix +--- + +Reject compute snapshot callbacks that do not match the snapshot request that created them. diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 32e669b5914..c352f7b1b2a 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -14,7 +14,7 @@ export const Env = z // Required settings TRIGGER_API_URL: z.string().url(), - TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file + TRIGGER_WORKER_TOKEN: z.string().min(1), // accepts file:// path to read from a file MANAGED_WORKER_SECRET: z.string(), OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index aeda154c355..28d0d1bba7c 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -289,8 +289,10 @@ class ManagedSupervisor { }); } + const workerToken = getWorkerToken(); + this.workerSession = new SupervisorSession({ - workerToken: getWorkerToken(), + workerToken, apiUrl: env.TRIGGER_API_URL, instanceName: env.TRIGGER_WORKER_INSTANCE_NAME, managedWorkerSecret: env.MANAGED_WORKER_SECRET, @@ -568,6 +570,7 @@ class ManagedSupervisor { checkpointClient: this.checkpointClient, computeManager: this.computeManager, tracing: this.tracing, + snapshotCallbackSecret: workerToken, wideEventOpts: this.wideEventOpts, wideEventsNoisyRoutes: this.wideEventsNoisyRoutes, }); diff --git a/apps/supervisor/src/services/computeSnapshotService.test.ts b/apps/supervisor/src/services/computeSnapshotService.test.ts index 44a13a4bdd3..0877f5e9d7a 100644 --- a/apps/supervisor/src/services/computeSnapshotService.test.ts +++ b/apps/supervisor/src/services/computeSnapshotService.test.ts @@ -20,13 +20,26 @@ function createService() { snapshot, } as unknown as ComputeWorkloadManager; + const submitSuspendCompletion = vi.fn(async () => ({ success: true })); + const service = new ComputeSnapshotService({ computeManager, - workerClient: {} as SupervisorHttpClient, + workerClient: { submitSuspendCompletion } as unknown as SupervisorHttpClient, wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "test-secret", }); - return { service, snapshot }; + return { service, snapshot, submitSuspendCompletion }; +} + +function dispatchedMetadata(snapshot: { + mock: { calls: Array }>> }; +}) { + const metadata = snapshot.mock.calls[0]?.[0]?.metadata; + if (!metadata) { + throw new Error("Snapshot was not dispatched"); + } + return metadata; } function delayedSnapshot(runnerId = "runner-1") { @@ -38,6 +51,24 @@ function delayedSnapshot(runnerId = "runner-1") { } describe("ComputeSnapshotService", () => { + it("refuses to construct with an empty callback secret", () => { + const computeManager = { + snapshotDelayMs: DELAY_MS, + snapshotDispatchLimit: 1, + snapshot: vi.fn(async () => true), + } as unknown as ComputeWorkloadManager; + + expect( + () => + new ComputeSnapshotService({ + computeManager, + workerClient: {} as SupervisorHttpClient, + wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "", + }) + ).toThrow(); + }); + it("dispatches a scheduled snapshot after the delay", async () => { const { service, snapshot } = createService(); try { @@ -46,7 +77,12 @@ describe("ComputeSnapshotService", () => { await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); expect(snapshot).toHaveBeenCalledWith({ runnerId: "runner-1", - metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" }, + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_1", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), }); } finally { service.stop(); @@ -121,8 +157,84 @@ describe("ComputeSnapshotService", () => { expect(snapshot).toHaveBeenCalledTimes(1); expect(snapshot).toHaveBeenCalledWith({ runnerId: "runner-1", - metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_2" }, + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_2", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), + }); + } finally { + service.stop(); + } + }); + + it("accepts a snapshot callback with the dispatched token", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata, + }); + + expect(result).toEqual({ ok: true, status: 200 }); + expect(submitSuspendCompletion).toHaveBeenCalledWith({ + runId: "run_1", + snapshotId: "snapshot_1", + body: { + success: true, + checkpoint: { + type: "COMPUTE", + location: "compute_snapshot_1", + }, + }, + }); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback without a valid token", async () => { + const { service, submitSuspendCompletion } = createService(); + try { + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" }, + }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback whose token is for a different snapshot", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { ...metadata, snapshotFriendlyId: "snapshot_2" }, }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); } finally { service.stop(); } diff --git a/apps/supervisor/src/services/computeSnapshotService.ts b/apps/supervisor/src/services/computeSnapshotService.ts index 216753fc12d..6af307d0e30 100644 --- a/apps/supervisor/src/services/computeSnapshotService.ts +++ b/apps/supervisor/src/services/computeSnapshotService.ts @@ -1,3 +1,4 @@ +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; import pLimit from "p-limit"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic"; @@ -16,6 +17,13 @@ import { type WideEventOptions, } from "../wideEvents/index.js"; +const SNAPSHOT_CALLBACK_NONCE_METADATA_KEY = "snapshotCallbackNonce"; +const SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY = "snapshotCallbackToken"; + +// Domain-separation label so the callback-signing key is derived from, rather +// than equal to, the secret used for other protocols. Bump the suffix to rotate. +const SNAPSHOT_CALLBACK_KEY_INFO = "compute-snapshot-callback-v1"; + type DelayedSnapshot = { runnerId: string; runFriendlyId: string; @@ -34,6 +42,7 @@ export type ComputeSnapshotServiceOptions = { workerClient: SupervisorHttpClient; tracing?: OtlpTraceService; wideEventOpts: WideEventOptions; + snapshotCallbackSecret: string; }; export class ComputeSnapshotService { @@ -48,6 +57,7 @@ export class ComputeSnapshotService { private readonly workerClient: SupervisorHttpClient; private readonly tracing?: OtlpTraceService; private readonly wideEventOpts: WideEventOptions; + private readonly snapshotCallbackKey: Buffer; constructor(opts: ComputeSnapshotServiceOptions) { this.computeManager = opts.computeManager; @@ -55,6 +65,18 @@ export class ComputeSnapshotService { this.tracing = opts.tracing; this.wideEventOpts = opts.wideEventOpts; + // Reject an empty secret up front: an empty HMAC key would make callback + // tokens forgeable by anyone. Guarding here (rather than only at env parse) + // also covers the case where the secret is read from an empty file. + if (!opts.snapshotCallbackSecret) { + throw new Error("snapshotCallbackSecret must not be empty"); + } + // Derive a dedicated key by domain separation so the raw secret is never + // used directly as a MAC key for this protocol. + this.snapshotCallbackKey = createHmac("sha256", opts.snapshotCallbackSecret) + .update(SNAPSHOT_CALLBACK_KEY_INFO) + .digest(); + this.dispatchLimit = pLimit(this.computeManager.snapshotDispatchLimit); this.timerWheel = new TimerWheel({ delayMs: this.computeManager.snapshotDelayMs, @@ -146,15 +168,29 @@ export class ComputeSnapshotService { instanceId: body.instance_id, status: body.status, error: body.status === "failed" ? body.error : undefined, - metadata: body.metadata, + runId, + snapshotFriendlyId, durationMs: body.duration_ms, }); if (!runId || !snapshotFriendlyId) { - this.logger.error("Snapshot callback missing metadata", { body }); + this.logger.error("Snapshot callback missing metadata", { + status: body.status, + instanceId: body.instance_id, + metadataKeys: Object.keys(body.metadata ?? {}), + }); return { ok: false as const, status: 400 }; } + if (!this.#verifyCallbackToken(body.metadata, runId, snapshotFriendlyId)) { + this.logger.error("Snapshot callback failed token verification", { + runId, + snapshotFriendlyId, + instanceId: body.instance_id, + }); + return { ok: false as const, status: 401 }; + } + this.#emitSnapshotSpan(runId, body.duration_ms, snapshotId); if (body.status === "completed") { @@ -266,11 +302,18 @@ export class ComputeSnapshotService { }, }, async () => { + const callbackNonce = randomBytes(16).toString("hex"); const result = await this.computeManager.snapshot({ runnerId: snapshot.runnerId, metadata: { runId: snapshot.runFriendlyId, snapshotFriendlyId: snapshot.snapshotFriendlyId, + [SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]: callbackNonce, + [SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]: this.#createCallbackToken( + callbackNonce, + snapshot.runFriendlyId, + snapshot.snapshotFriendlyId + ), }, }); @@ -281,6 +324,51 @@ export class ComputeSnapshotService { ); } + #createCallbackToken(nonce: string, runFriendlyId: string, snapshotFriendlyId: string): string { + return createHmac("sha256", this.snapshotCallbackKey) + .update(nonce) + .update("\0") + .update(runFriendlyId) + .update("\0") + .update(snapshotFriendlyId) + .digest("hex"); + } + + /** + * Verify that a callback carries a token this supervisor issued for the given + * run and snapshot. The token binds only the identifiers known at dispatch + * time (nonce, run, snapshot); it intentionally does not cover result fields + * such as the snapshot location or status/error, which are produced by the + * gateway after the snapshot and so cannot be signed in advance. Verification + * is also stateless, so a token is not single-use. + * + * This closes the primary risk (a caller that can merely reach the endpoint + * cannot mint a valid token, so cannot forge a result for an arbitrary run). + * It does not defend against an attacker who can observe a genuine callback + * and then replay it or alter its unsigned result fields - that relies on the + * gateway->supervisor callback channel being authenticated and encrypted. + */ + #verifyCallbackToken( + metadata: Record | undefined, + runFriendlyId: string, + snapshotFriendlyId: string + ): boolean { + const nonce = metadata?.[SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]; + const token = metadata?.[SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]; + + if (!nonce || !token) { + return false; + } + + const expected = this.#createCallbackToken(nonce, runFriendlyId, snapshotFriendlyId); + const expectedBuffer = Buffer.from(expected, "hex"); + const tokenBuffer = Buffer.from(token, "hex"); + + return ( + expectedBuffer.length === tokenBuffer.length && timingSafeEqual(expectedBuffer, tokenBuffer) + ); + } + #emitSnapshotSpan(runFriendlyId: string, durationMs?: number, snapshotId?: string) { if (!this.tracing) return; diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index bacdbbe06d9..66ca047f3ef 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -86,6 +86,7 @@ type WorkloadServerOptions = { checkpointClient?: CheckpointClient; computeManager?: ComputeWorkloadManager; tracing?: OtlpTraceService; + snapshotCallbackSecret: string; wideEventOpts: WideEventOptions; /** When true, high-frequency HTTP routes also emit wide events. */ wideEventsNoisyRoutes: boolean; @@ -136,6 +137,7 @@ export class WorkloadServer extends EventEmitter { workerClient: opts.workerClient, tracing: opts.tracing, wideEventOpts: this.wideEventOpts, + snapshotCallbackSecret: opts.snapshotCallbackSecret, }); } From 3d19aaadb1c48e12fe91decb517bb830bf72a981 Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:14:27 +0100 Subject: [PATCH 02/15] feat: deployment-scoped runner boundary auth (#77) * feat(core,supervisor,webapp): authenticate the runner boundary with a deployment-scoped token Supervisor mints a signed, deployment-scoped token at pod creation and injects it as the TRIGGER_DEPLOYMENT_ID value; old runners forward it unchanged. The supervisor verifies it on the workload API (HTTP routes + WS handshake), adds a runConnected ownership guard, and forwards the verified background worker id upstream. The platform checks the run belongs to that worker on start/complete/continue/snapshots-since/snapshots-latest, gated by a created-at cutoff (disabled/log/enforce). Also adds mint/verify helpers to core and an OTLP worker_id unwrap so the metric stays a short, stable id. refs TRI-11844 * refactor(core,supervisor,webapp): scope worker actions by environment Fold the tenant check into the snapshot read each worker action already performs (scope by the token's environment id) instead of a separate per-run version lookup, so enforcement adds no extra database reads on the hot path. The supervisor forwards the verified environment id only when enforcing; the platform scopes the snapshot read by it when present. The no-token created-at suppression is now a separate, default-off gate that only reads a run row when enabled and no header is present, so feature-off is identical to prior behaviour. Drop the now-unused background_worker_id claim from the token. Refs TRI-11844 * refactor(core,supervisor): mint the deployment token deterministically per deployment Drop the wall-clock issued-at and use a caller-supplied absolute expiry when minting, so the same deployment mints byte-identical tokens on every pod and every supervisor node. The runner stamps TRIGGER_DEPLOYMENT_ID verbatim as the OpenTelemetry worker.id, so a per-pod-unique token would explode that attribute's cardinality (and ride to any external exporter); a per-deployment token keeps it at one value per deployment, matching the prior friendlyId. The supervisor drives the absolute expiry via WORKLOAD_TOKEN_EXP. Refs TRI-11844 * fix(webapp): decode deployment-token worker.id in span and log telemetry ingest The worker.id a runner stamps into span/log metadata can be a deployment token; only the metrics ingest path decoded it back to the friendlyId, so spans and logs stored the raw token in customer-facing metadata. Unwrap it on the span/log path too, at the storage boundary, so the credential never lands in ClickHouse and the value stays a stable deployment id. Non-token values (bare friendlyIds, dev ids) take a zero-cost fast path. Refs TRI-11851 * chore: apply oxfmt formatting * refactor(supervisor): drop speculative start_type from token mint The deployment token is injected once at pod creation and frozen across warm starts and restores (nothing re-injects TRIGGER_DEPLOYMENT_ID), so mint only ever happens at cold start. The start_type label was always cold - a cardinality-1 dimension - and the warm/restore variants were unreachable. Drop them; add the dimension back if a future token refreshes on warm start. Refs TRI-11844 * fix(supervisor): don't leak WORKLOAD_TOKEN_SECRET in debug log; allow same-runner reconnect to rebind run * test(core): fix flaky retry-delay boundary assertion (>= minDelay) * refactor(webapp): use lru-cache for the worker_id unwrap memo Replace the plain-Map memo with an LRU. The active-deployment working set is unbounded, so an insertion-order memo can thrash to worse-than-no-cache once churn exceeds the cap; an LRU keeps the hot set and degrades gracefully. Uses lru-cache directly (the lib @internal/cache wraps; its async SWR cache is the wrong shape for this synchronous ingest hot path) - same version already in the lockfile, no new package version. Refs TRI-11851 * fix(webapp): route created-at gate run lookup through the run-store * fix(supervisor): redact TRIGGER_DEPLOYMENT_ID from relayed debug logs * fix(supervisor): import deployment token minting --------- Co-authored-by: Chris Arderne --- .changeset/workload-deployment-token.md | 5 + .server-changes/workload-token-supervisor.md | 6 + .server-changes/workload-token-webapp.md | 6 + apps/supervisor/src/env.ts | 17 ++ apps/supervisor/src/index.ts | 12 ++ .../supervisor/src/workloadManager/compute.ts | 2 +- apps/supervisor/src/workloadManager/docker.ts | 2 +- .../src/workloadManager/kubernetes.ts | 2 +- apps/supervisor/src/workloadManager/types.ts | 2 + apps/supervisor/src/workloadServer/index.ts | 162 +++++++++++++++++- .../workloadServer.auth.test.ts | 107 ++++++++++++ .../workloadServer.authLog.test.ts | 87 ++++++++++ apps/supervisor/src/workloadToken.ts | 85 +++++++++ apps/webapp/app/env.server.ts | 7 + ...s.$snapshotFriendlyId.attempts.complete.ts | 2 + ...hots.$snapshotFriendlyId.attempts.start.ts | 2 + ....snapshots.$snapshotFriendlyId.continue.ts | 8 + ...ns.runs.$runFriendlyId.snapshots.latest.ts | 2 + ...nFriendlyId.snapshots.since.$snapshotId.ts | 2 + .../routeBuilders/apiBuilder.server.ts | 8 + apps/webapp/app/v3/otlpTransform.server.ts | 7 +- .../worker/workerGroupTokenService.server.ts | 141 ++++++++++++++- .../workloadTokenAuthorization.server.ts | 27 +++ apps/webapp/app/v3/workerIdUnwrap.server.ts | 75 ++++++++ apps/webapp/package.json | 3 + apps/webapp/test/workerIdUnwrap.test.ts | 71 ++++++++ .../test/workloadTokenAuthorization.test.ts | 26 +++ .../workloadTokenGate.integration.test.ts | 88 ++++++++++ .../run-engine/src/engine/index.ts | 23 ++- .../src/engine/systems/checkpointSystem.ts | 9 +- .../engine/systems/executionSnapshotSystem.ts | 31 +++- .../src/engine/systems/runAttemptSystem.ts | 31 +++- .../run-store/src/PostgresRunStore.ts | 8 +- .../run-store/src/runOpsStore.ts | 6 +- internal-packages/run-store/src/types.ts | 5 +- packages/core/src/v3/index.ts | 1 + packages/core/src/v3/jwt.ts | 16 +- .../core/src/v3/runEngineWorker/consts.ts | 3 + .../src/v3/runEngineWorker/supervisor/http.ts | 33 +++- packages/core/src/v3/runEngineWorker/types.ts | 2 + .../src/v3/workloadDeploymentToken.test.ts | 127 ++++++++++++++ .../core/src/v3/workloadDeploymentToken.ts | 99 +++++++++++ packages/core/test/taskExecutor.test.ts | 6 +- pnpm-lock.yaml | 15 ++ 44 files changed, 1329 insertions(+), 50 deletions(-) create mode 100644 .changeset/workload-deployment-token.md create mode 100644 .server-changes/workload-token-supervisor.md create mode 100644 .server-changes/workload-token-webapp.md create mode 100644 apps/supervisor/src/workloadServer/workloadServer.auth.test.ts create mode 100644 apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts create mode 100644 apps/supervisor/src/workloadToken.ts create mode 100644 apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts create mode 100644 apps/webapp/app/v3/workerIdUnwrap.server.ts create mode 100644 apps/webapp/test/workerIdUnwrap.test.ts create mode 100644 apps/webapp/test/workloadTokenAuthorization.test.ts create mode 100644 apps/webapp/test/workloadTokenGate.integration.test.ts create mode 100644 packages/core/src/v3/workloadDeploymentToken.test.ts create mode 100644 packages/core/src/v3/workloadDeploymentToken.ts diff --git a/.changeset/workload-deployment-token.md b/.changeset/workload-deployment-token.md new file mode 100644 index 00000000000..6a3c08cfc77 --- /dev/null +++ b/.changeset/workload-deployment-token.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Add helpers to mint and verify the deployment-scoped token used to authenticate run controllers to the platform. diff --git a/.server-changes/workload-token-supervisor.md b/.server-changes/workload-token-supervisor.md new file mode 100644 index 00000000000..b7c82c88881 --- /dev/null +++ b/.server-changes/workload-token-supervisor.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: fix +--- + +Authenticate run controllers to the platform with a signed, deployment-scoped token. diff --git a/.server-changes/workload-token-webapp.md b/.server-changes/workload-token-webapp.md new file mode 100644 index 00000000000..b743c446308 --- /dev/null +++ b/.server-changes/workload-token-webapp.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Verify that worker actions (starting, completing, and continuing a run, and reading its snapshots) target a run belonging to the caller's environment. diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index c352f7b1b2a..6c954b4e76a 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -16,6 +16,15 @@ export const Env = z TRIGGER_API_URL: z.string().url(), TRIGGER_WORKER_TOKEN: z.string().min(1), // accepts file:// path to read from a file MANAGED_WORKER_SECRET: z.string(), + + // Deployment token: sign a token into TRIGGER_DEPLOYMENT_ID at pod creation and verify it on + // inbound workload calls. "disabled" = off; "log" = mint + verify + metrics only; "enforce" = + // also reject invalid tokens. + WORKLOAD_TOKEN_SECRET: z.string().optional(), + WORKLOAD_TOKEN_ENFORCEMENT: z.enum(["disabled", "log", "enforce"]).default("disabled"), + // Absolute expiry for minted deployment tokens. Deterministic (no wall-clock issued-at) so every + // pod of a deployment carries an identical token; bump before this date. Must outlive any run. + WORKLOAD_TOKEN_EXP: z.string().datetime().default("2032-01-01T00:00:00.000Z"), OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners // Workload API settings (coordinator mode) - the workload API is what the run controller connects to @@ -365,6 +374,14 @@ export const Env = z path: ["TRIGGER_WORKLOAD_API_DOMAIN"], }); } + if (data.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled" && !data.WORKLOAD_TOKEN_SECRET) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "WORKLOAD_TOKEN_SECRET is required when WORKLOAD_TOKEN_ENFORCEMENT is not disabled", + path: ["WORKLOAD_TOKEN_SECRET"], + }); + } if ( data.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED && !data.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 0df315317bf..cf73be90eea 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -27,6 +27,7 @@ import { register } from "./metrics.js"; import { PodCleaner } from "./services/podCleaner.js"; import { FailedPodHandler } from "./services/failedPodHandler.js"; import { getWorkerToken } from "./workerToken.js"; +import { mintDeploymentToken } from "./workloadToken.js"; import { OtlpTraceService } from "./services/otlpTraceService.js"; import { WarmStartVerificationService, @@ -96,6 +97,7 @@ class ManagedSupervisor { COMPUTE_GATEWAY_AUTH_TOKEN, DOCKER_REGISTRY_PASSWORD, TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD, + WORKLOAD_TOKEN_SECRET, ...envWithoutSecrets } = env; @@ -606,6 +608,15 @@ class ManagedSupervisor { throw new Error("Image is missing"); } + const deploymentToken = await mintDeploymentToken({ + deployment: message.deployment.friendlyId, + deployment_version: message.backgroundWorker.version, + environment_id: message.environment.id, + environment_type: message.environment.type, + org_id: message.organization.id, + project_id: message.project.id, + }); + await this.workloadManager.create({ dequeuedAt: message.dequeuedAt, dequeueResponseMs: timings.dequeueResponseMs, @@ -620,6 +631,7 @@ class ManagedSupervisor { deploymentFriendlyId: message.deployment.friendlyId, deploymentVersion: message.backgroundWorker.version, runtime: message.backgroundWorker.runtime, + deploymentToken, runId: message.run.id, runFriendlyId: message.run.friendlyId, version: message.version, diff --git a/apps/supervisor/src/workloadManager/compute.ts b/apps/supervisor/src/workloadManager/compute.ts index 857e129a83f..66e4a482ad4 100644 --- a/apps/supervisor/src/workloadManager/compute.ts +++ b/apps/supervisor/src/workloadManager/compute.ts @@ -151,7 +151,7 @@ export class ComputeWorkloadManager implements WorkloadManager { TRIGGER_DEQUEUED_AT_MS: String(opts.dequeuedAt.getTime()), TRIGGER_POD_SCHEDULED_AT_MS: String(Date.now()), TRIGGER_ENV_ID: opts.envId, - TRIGGER_DEPLOYMENT_ID: opts.deploymentFriendlyId, + TRIGGER_DEPLOYMENT_ID: opts.deploymentToken ?? opts.deploymentFriendlyId, TRIGGER_DEPLOYMENT_VERSION: opts.deploymentVersion, TRIGGER_RUN_ID: opts.runFriendlyId, TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId, diff --git a/apps/supervisor/src/workloadManager/docker.ts b/apps/supervisor/src/workloadManager/docker.ts index 1d8fec1df78..70ca02654f6 100644 --- a/apps/supervisor/src/workloadManager/docker.ts +++ b/apps/supervisor/src/workloadManager/docker.ts @@ -72,7 +72,7 @@ export class DockerWorkloadManager implements WorkloadManager { `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`, `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`, `TRIGGER_ENV_ID=${opts.envId}`, - `TRIGGER_DEPLOYMENT_ID=${opts.deploymentFriendlyId}`, + `TRIGGER_DEPLOYMENT_ID=${opts.deploymentToken ?? opts.deploymentFriendlyId}`, `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`, `TRIGGER_RUN_ID=${opts.runFriendlyId}`, `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`, diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index 282ac80b4fd..5e9940c0e3a 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -158,7 +158,7 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, { name: "TRIGGER_DEPLOYMENT_ID", - value: opts.deploymentFriendlyId, + value: opts.deploymentToken ?? opts.deploymentFriendlyId, }, { name: "TRIGGER_DEPLOYMENT_VERSION", diff --git a/apps/supervisor/src/workloadManager/types.ts b/apps/supervisor/src/workloadManager/types.ts index 4945b1382d9..eda9162c7d6 100644 --- a/apps/supervisor/src/workloadManager/types.ts +++ b/apps/supervisor/src/workloadManager/types.ts @@ -44,6 +44,8 @@ export interface WorkloadManagerCreateOptions { deploymentVersion: string; // Canonical runtime identifier (e.g. "node", "node-22", "node-24") runtime?: string; + // When set, overrides the TRIGGER_DEPLOYMENT_ID value the runner forwards as its identity header. + deploymentToken?: string; runId: string; runFriendlyId: string; snapshotId: string; diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index 66ca047f3ef..86717438c72 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -25,6 +25,11 @@ import { type Namespace, Server, type Socket } from "socket.io"; import { z } from "zod"; import { env } from "../env.js"; import { register } from "../metrics.js"; +import { + verifyDeploymentIdHeader, + workloadTokenEnforced, + workloadTokensEnabled, +} from "../workloadToken.js"; import { ComputeSnapshotService, type RunTraceContext, @@ -171,6 +176,34 @@ export class WorkloadServer extends EventEmitter { return this.headerValueFromRequest(req, WORKLOAD_HEADERS.PROJECT_REF); } + /** + * Verify the deployment token from the workload deployment-id header and return the verified + * environment_id to forward upstream. The env id is only forwarded in enforce mode: in log mode + * we still verify + record metrics but attach no header (so the platform never scopes). Only + * enforce fails a request, and only for a present-but-invalid token; absent and legacy ids pass. + */ + private async authorizeWorkloadRequest( + req: IncomingMessage + ): Promise<{ ok: true; environmentId?: string } | { ok: false }> { + if (!workloadTokensEnabled) { + return { ok: true }; + } + + const result = await verifyDeploymentIdHeader(this.deploymentIdFromRequest(req), "http"); + + if (result.outcome === "jwt_invalid" && workloadTokenEnforced) { + return { ok: false }; + } + + return { + ok: true, + environmentId: + workloadTokenEnforced && result.outcome === "jwt_valid" + ? result.claims.environment_id + : undefined, + }; + } + /** * Sets common route meta on the wide-event state from URL params. */ @@ -252,11 +285,17 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const startResponse = await this.workerClient.startRunAttempt( params.runFriendlyId, params.snapshotFriendlyId, body, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!startResponse.success) { @@ -288,6 +327,11 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const runnerId = this.runnerIdFromRequest(req); // A completion attempt invalidates any pending delayed snapshot @@ -306,7 +350,8 @@ export class WorkloadServer extends EventEmitter { params.runFriendlyId, params.snapshotFriendlyId, body, - runnerId + runnerId, + auth.environmentId ); if (!completeResponse.success) { @@ -338,6 +383,11 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const heartbeatResponse = await this.workerClient.heartbeatRun( params.runFriendlyId, params.snapshotFriendlyId, @@ -375,6 +425,11 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { reply, params, req } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const runnerId = this.runnerIdFromRequest(req); const deploymentVersion = this.deploymentVersionFromRequest(req); const projectRef = this.projectRefFromRequest(req); @@ -471,6 +526,11 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { req, reply, params } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } this.logger.debug("Run continuation request", { params }); // Cancel any pending delayed snapshot for this run @@ -479,7 +539,8 @@ export class WorkloadServer extends EventEmitter { const continuationResult = await this.workerClient.continueRunExecution( params.runFriendlyId, params.snapshotFriendlyId, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!continuationResult.success) { @@ -513,10 +574,16 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { req, reply, params } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const sinceSnapshotResponse = await this.workerClient.getSnapshotsSince( params.runFriendlyId, params.snapshotFriendlyId, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!sinceSnapshotResponse.success) { @@ -587,9 +654,18 @@ export class WorkloadServer extends EventEmitter { const { req, reply, params, body } = ctx; reply.empty(204); + // Redact TRIGGER_DEPLOYMENT_ID before relaying to the platform. + const sanitizedBody = + body.properties && "TRIGGER_DEPLOYMENT_ID" in body.properties + ? { + ...body, + properties: { ...body.properties, TRIGGER_DEPLOYMENT_ID: "[redacted]" }, + } + : body; + await this.workerClient.sendDebugLog( params.runFriendlyId, - body, + sanitizedBody, this.runnerIdFromRequest(req) ); }, @@ -683,7 +759,31 @@ export class WorkloadServer extends EventEmitter { return; } - this.logger.debug("[WS] auth success", socket.data); + if (workloadTokensEnabled) { + const result = await verifyDeploymentIdHeader(socket.data.deploymentId, "ws"); + + if (result.outcome === "jwt_invalid" && workloadTokenEnforced) { + this.logger.error("[WS] deployment token verification failed", { + runnerId: socket.data.runnerId, + }); + socket.disconnect(true); + return; + } + + // Re-source the deployment id from the verified claim; the raw header may be an opaque token. + // A legacy bare id is itself the friendlyId, so it's safe to keep. + socket.data.deploymentFriendlyId = + result.outcome === "jwt_valid" + ? result.claims.deployment + : result.outcome === "legacy_bare" + ? socket.data.deploymentId + : undefined; + } + + this.logger.debug("[WS] handshake complete", { + runnerId: socket.data.runnerId, + deploymentFriendlyId: socket.data.deploymentFriendlyId, + }); next(); }); @@ -695,7 +795,7 @@ export class WorkloadServer extends EventEmitter { const getSocketMetadata = () => { return { - deploymentId: socket.data.deploymentId, + deploymentId: socket.data.deploymentFriendlyId ?? socket.data.deploymentId, runId: socket.data.runFriendlyId, snapshotId: socket.data.snapshotId, runnerId: socket.data.runnerId, @@ -714,8 +814,9 @@ export class WorkloadServer extends EventEmitter { populate: (state) => { state.extras.event = event; setMeta(state, "run_id", friendlyId); - if (socket.data.deploymentId) { - setMeta(state, "deployment_id", socket.data.deploymentId); + const deploymentId = socket.data.deploymentFriendlyId ?? socket.data.deploymentId; + if (deploymentId) { + setMeta(state, "deployment_id", deploymentId); } if (socket.data.runnerId) setMeta(state, "runner_id", socket.data.runnerId); state.extras.socket_id = socket.id; @@ -727,6 +828,33 @@ export class WorkloadServer extends EventEmitter { const runConnected = (friendlyId: string) => { socketLogger.debug("runConnected", { ...getSocketMetadata() }); + // Only the owning runner may (re)bind a run. A live socket from a *different* + // runner keeps its binding so an unrelated connection can't hijack the run. But + // the newest socket for the *same* runner is a legitimate reconnection/handoff and + // is allowed to take over even while the stale socket still reports connected - + // otherwise, during a reconnect race the fresh socket would silently stay unbound + // (missing continue/cancel/suspend notifications) until the dead socket times out. + const existing = this.runSockets.get(friendlyId); + if (existing && existing.id !== socket.id && existing.connected) { + const sameRunner = + !!socket.data.runnerId && existing.data.runnerId === socket.data.runnerId; + + if (!sameRunner) { + socketLogger.warn("runConnected: run already bound to another socket", { + ...getSocketMetadata(), + friendlyId, + existingSocketId: existing.id, + }); + return; + } + + socketLogger.debug("runConnected: replacing stale socket for same runner", { + ...getSocketMetadata(), + friendlyId, + existingSocketId: existing.id, + }); + } + // If there's already a run ID set, we should "disconnect" it from this socket if (socket.data.runFriendlyId && socket.data.runFriendlyId !== friendlyId) { socketLogger.debug("runConnected: disconnecting existing run", { @@ -746,6 +874,22 @@ export class WorkloadServer extends EventEmitter { const runDisconnected = (friendlyId: string, reason: string) => { socketLogger.debug("runDisconnected", { ...getSocketMetadata() }); + // A newer socket may have taken over this run (same-runner reconnect race). If the + // run is now bound to a different socket, this stale socket must not clear the fresh + // binding or emit a spurious disconnect - just drop its own reference and bail. + const bound = this.runSockets.get(friendlyId); + if (bound && bound.id !== socket.id) { + socketLogger.debug("runDisconnected: run rebound to another socket, skipping", { + ...getSocketMetadata(), + friendlyId, + boundSocketId: bound.id, + }); + if (socket.data.runFriendlyId === friendlyId) { + socket.data.runFriendlyId = undefined; + } + return; + } + // The run is gone from this runner (crash, exit, or replaced by a new // run), so a pending delayed snapshot for it is stale. Genuine // waitpoint suspensions keep the socket connected, so this doesn't diff --git a/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts b/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts new file mode 100644 index 00000000000..3e8fb6e9de7 --- /dev/null +++ b/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts @@ -0,0 +1,107 @@ +import { mintWorkloadDeploymentToken } from "@trigger.dev/core/v3"; +import { WORKLOAD_HEADERS } from "@trigger.dev/core/v3/workers"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Set enforce mode + secret before env.ts parses (vi.mock is hoisted above imports, so the secret +// must be a literal here). SECRET below mirrors it for use in the test body. +vi.mock("std-env", () => ({ + env: { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + WORKLOAD_TOKEN_SECRET: "integration-test-secret", + WORKLOAD_TOKEN_ENFORCEMENT: "enforce", + }, +})); + +const SECRET = "integration-test-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const { WorkloadServer } = await import("./index.js"); + +const PORT = 18732; +const BASE = `http://127.0.0.1:${PORT}`; + +function claims(environmentId = "env_test_123") { + return { + deployment: "deployment_test", + deployment_version: "20260710.1", + environment_id: environmentId, + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", + }; +} + +// Records the args each relay method is called with so we can assert the forwarded claim. +const calls: { getSnapshotsSince: any[][] } = { getSnapshotsSince: [] }; + +const workerClient = { + getSnapshotsSince: vi.fn(async (...args: any[]) => { + calls.getSnapshotsSince.push(args); + return { success: true as const, data: { snapshots: [] } }; + }), +} as any; + +let server: InstanceType; + +beforeAll(async () => { + server = new WorkloadServer({ + port: PORT, + workerClient, + snapshotCallbackSecret: "snapshot-callback-secret", + wideEventOpts: { service: "supervisor", env: { nodeId: "test" }, enabled: false }, + wideEventsNoisyRoutes: false, + }); + await server.start(); +}); + +afterAll(async () => { + await server.stop(); +}); + +function snapshotsSince(deploymentIdHeader?: string) { + const headers: Record = { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + }; + if (deploymentIdHeader !== undefined) { + headers[WORKLOAD_HEADERS.DEPLOYMENT_ID] = deploymentIdHeader; + } + return fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { headers }); +} + +describe("WorkloadServer auth (enforce mode)", () => { + it("allows a valid token and forwards the verified environment_id", async () => { + const token = await mintWorkloadDeploymentToken(claims("env_forwarded_42"), SECRET, EXP); + const res = await snapshotsSince(token); + + expect(res.status).toBe(200); + const lastCall = calls.getSnapshotsSince.at(-1)!; + // getSnapshotsSince(runId, snapshotId, runnerId, environmentId) + expect(lastCall[3]).toBe("env_forwarded_42"); + }); + + it("rejects a token signed with the wrong secret (401) and does not relay", async () => { + const before = calls.getSnapshotsSince.length; + const badToken = await mintWorkloadDeploymentToken(claims(), "wrong-secret", EXP); + const res = await snapshotsSince(badToken); + + expect(res.status).toBe(401); + expect(calls.getSnapshotsSince.length).toBe(before); + }); + + it("allows a legacy bare friendlyId and forwards no environment_id", async () => { + const res = await snapshotsSince("deployment_legacy_bare"); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); + + it("allows an absent token and forwards no environment_id", async () => { + const res = await snapshotsSince(undefined); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); +}); diff --git a/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts b/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts new file mode 100644 index 00000000000..8e66f738f64 --- /dev/null +++ b/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts @@ -0,0 +1,87 @@ +import { mintWorkloadDeploymentToken } from "@trigger.dev/core/v3"; +import { WORKLOAD_HEADERS } from "@trigger.dev/core/v3/workers"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Log mode: mint + verify + metrics, but the platform must NOT be scoped, so no environment_id is +// forwarded even for a valid token. (vi.mock is hoisted; secret literal here, mirrored below.) +vi.mock("std-env", () => ({ + env: { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + WORKLOAD_TOKEN_SECRET: "integration-test-secret", + WORKLOAD_TOKEN_ENFORCEMENT: "log", + }, +})); + +const SECRET = "integration-test-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const { WorkloadServer } = await import("./index.js"); + +const PORT = 18733; +const BASE = `http://127.0.0.1:${PORT}`; + +const claims = { + deployment: "deployment_test", + deployment_version: "20260710.1", + environment_id: "env_should_not_forward", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", +}; + +const calls: { getSnapshotsSince: any[][] } = { getSnapshotsSince: [] }; + +const workerClient = { + getSnapshotsSince: vi.fn(async (...args: any[]) => { + calls.getSnapshotsSince.push(args); + return { success: true as const, data: { snapshots: [] } }; + }), +} as any; + +let server: InstanceType; + +beforeAll(async () => { + server = new WorkloadServer({ + port: PORT, + workerClient, + snapshotCallbackSecret: "snapshot-callback-secret", + wideEventOpts: { service: "supervisor", env: { nodeId: "test" }, enabled: false }, + wideEventsNoisyRoutes: false, + }); + await server.start(); +}); + +afterAll(async () => { + await server.stop(); +}); + +describe("WorkloadServer auth (log mode)", () => { + it("allows a valid token but forwards no environment_id", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const res = await fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { + headers: { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + [WORKLOAD_HEADERS.DEPLOYMENT_ID]: token, + }, + }); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); + + it("does not reject an invalid token in log mode", async () => { + const badToken = await mintWorkloadDeploymentToken(claims, "wrong-secret", EXP); + const res = await fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { + headers: { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + [WORKLOAD_HEADERS.DEPLOYMENT_ID]: badToken, + }, + }); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); +}); diff --git a/apps/supervisor/src/workloadToken.ts b/apps/supervisor/src/workloadToken.ts new file mode 100644 index 00000000000..914411dd735 --- /dev/null +++ b/apps/supervisor/src/workloadToken.ts @@ -0,0 +1,85 @@ +import { + classifyDeploymentIdHeader, + mintWorkloadDeploymentToken, + type WorkloadDeploymentTokenClaims, + type WorkloadDeploymentTokenInput, +} from "@trigger.dev/core/v3"; +import { Counter } from "prom-client"; +import { env } from "./env.js"; +import { register } from "./metrics.js"; + +const secret = env.WORKLOAD_TOKEN_SECRET; + +// Absolute expiry (epoch seconds) shared by every mint, so tokens stay byte-deterministic per +// deployment regardless of when/where a pod is created. +const tokenExpSeconds = Math.floor(new Date(env.WORKLOAD_TOKEN_EXP).getTime() / 1000); + +/** Mint + verify run in "log" (dry-run) and "enforce"; the env superRefine guarantees a secret then. */ +export const workloadTokensEnabled = env.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled"; + +/** Only "enforce" rejects a present-but-invalid token; "log" observes and always allows. */ +export const workloadTokenEnforced = env.WORKLOAD_TOKEN_ENFORCEMENT === "enforce"; + +const mintCounter = new Counter({ + name: "workload_token_minted_total", + help: "Deployment tokens minted and injected into TRIGGER_DEPLOYMENT_ID at pod creation", + labelNames: ["env_type"] as const, + registers: [register], +}); + +export type WorkloadAuthTransport = "http" | "ws"; +export type WorkloadAuthOutcome = "jwt_valid" | "jwt_invalid" | "legacy_bare" | "token_absent"; + +const verifyCounter = new Counter({ + name: "workload_auth_verify_total", + help: "Runner-boundary token verification outcomes at the supervisor workload server", + labelNames: ["outcome", "transport", "env_type"] as const, + registers: [register], +}); + +export async function mintDeploymentToken( + claims: WorkloadDeploymentTokenInput +): Promise { + if (!workloadTokensEnabled || !secret) { + return undefined; + } + + const token = await mintWorkloadDeploymentToken(claims, secret, tokenExpSeconds); + mintCounter.inc({ env_type: claims.environment_type }); + return token; +} + +export type VerifiedDeploymentHeader = + | { outcome: "jwt_valid"; claims: WorkloadDeploymentTokenClaims } + | { outcome: "jwt_invalid" | "legacy_bare" | "token_absent"; claims?: undefined }; + +/** + * Verify the deployment-id header value and record the outcome. "jwt_valid" returns the claims so the + * caller can forward the verified environment_id upstream; other outcomes carry no trusted data. + */ +export async function verifyDeploymentIdHeader( + value: string | undefined, + transport: WorkloadAuthTransport +): Promise { + const result = await classify(value); + verifyCounter.inc({ + outcome: result.outcome, + transport, + env_type: result.outcome === "jwt_valid" ? result.claims.environment_type : "unknown", + }); + return result; +} + +async function classify(value: string | undefined): Promise { + if (!value || !secret) { + return { outcome: "token_absent" }; + } + + const result = await classifyDeploymentIdHeader(value, secret); + + if (result.outcome === "jwt_valid" && result.claims) { + return { outcome: "jwt_valid", claims: result.claims }; + } + + return { outcome: result.outcome === "jwt_valid" ? "jwt_invalid" : result.outcome }; +} diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 973560ca07f..3aa040f0fde 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -670,6 +670,13 @@ const EnvironmentSchema = z MANAGED_WORKER_SECRET: z.string().default("managed-secret"), + // Tenant scoping on worker actions is header-driven (folded into the engine snapshot read) and + // needs no flag. This is only the no-header fallback: when "1", a worker action on a run created + // after WORKLOAD_TOKEN_CUTOFF without a verified env header is rejected; runs on or before the + // cutoff pass (grandfathered). Default off = no run-row read, byte-for-byte today's behavior. + WORKLOAD_CREATED_AT_GATE_ENABLED: z.string().default("0"), + WORKLOAD_TOKEN_CUTOFF: z.string().datetime().optional(), + // Development OTEL environment variables DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(), DEV_OTEL_METRICS_ENDPOINT: z.string().optional(), diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts index 051b6d4d04f..342ca3f2822 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts @@ -18,6 +18,7 @@ export const action = createActionWorkerApiRoute( body, params, runnerId, + environmentId, }): Promise> => { const { completion } = body; const { runFriendlyId, snapshotFriendlyId } = params; @@ -27,6 +28,7 @@ export const action = createActionWorkerApiRoute( snapshotFriendlyId, completion, runnerId, + environmentId, }); return json({ result: completeResult }); diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts index a28a362ed93..558fa0a0c01 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts @@ -18,6 +18,7 @@ export const action = createActionWorkerApiRoute( body, params, runnerId, + environmentId, }): Promise> => { const { runFriendlyId, snapshotFriendlyId } = params; @@ -26,6 +27,7 @@ export const action = createActionWorkerApiRoute( snapshotFriendlyId, isWarmStart: body.isWarmStart, runnerId, + environmentId, }); return json(runExecutionData); diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts index 3c264adcd23..1aef207aac4 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts @@ -17,6 +17,7 @@ export const loader = createLoaderWorkerApiRoute( authenticatedWorker, params, runnerId, + environmentId, }): Promise> => { const { runFriendlyId, snapshotFriendlyId } = params; @@ -27,10 +28,17 @@ export const loader = createLoaderWorkerApiRoute( runFriendlyId, snapshotFriendlyId, runnerId, + environmentId, }); return json(continuationResult); } catch (error) { + // An authorization rejection is thrown as a Response; propagate it as-is rather than + // masking it as a generic 422. + if (error instanceof Response) { + throw error; + } + logger.warn("Failed to continue run execution", { runFriendlyId, snapshotFriendlyId, diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts index 73df07748b9..46d84741061 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts @@ -13,11 +13,13 @@ export const loader = createLoaderWorkerApiRoute( async ({ authenticatedWorker, params, + environmentId, }): Promise> => { const { runFriendlyId } = params; const executionData = await authenticatedWorker.getLatestSnapshot({ runFriendlyId, + environmentId, }); if (!executionData) { diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts index 87e1ed5eedf..7b91dc476c7 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts @@ -14,12 +14,14 @@ export const loader = createLoaderWorkerApiRoute( async ({ authenticatedWorker, params, + environmentId, }): Promise> => { const { runFriendlyId, snapshotId } = params; const snapshots = await authenticatedWorker.getSnapshotsSince({ runFriendlyId, snapshotId, + environmentId, }); if (!snapshots) { diff --git a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts index a74d49d2290..94c2929f5f7 100644 --- a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts @@ -1537,6 +1537,7 @@ type WorkerLoaderHandlerFunction< ? z.infer : undefined; runnerId?: string; + environmentId?: string; }) => Promise; export function createLoaderWorkerApiRoute< @@ -1601,6 +1602,8 @@ export function createLoaderWorkerApiRoute< } const runnerId = request.headers.get(WORKER_HEADERS.RUNNER_ID) ?? undefined; + // `|| undefined` so a blank header can't become a zero-match snapshot filter (→ false reject). + const environmentId = request.headers.get(WORKER_HEADERS.ENVIRONMENT_ID) || undefined; const result = await handler({ params: parsedParams, @@ -1609,6 +1612,7 @@ export function createLoaderWorkerApiRoute< request, headers: parsedHeaders, runnerId, + environmentId, }); return result; } catch (error) { @@ -1660,6 +1664,7 @@ type WorkerActionHandlerFunction< ? z.infer : undefined; runnerId?: string; + environmentId?: string; }) => Promise; export function createActionWorkerApiRoute< @@ -1758,6 +1763,8 @@ export function createActionWorkerApiRoute< } const runnerId = request.headers.get(WORKER_HEADERS.RUNNER_ID) ?? undefined; + // `|| undefined` so a blank header can't become a zero-match snapshot filter (→ false reject). + const environmentId = request.headers.get(WORKER_HEADERS.ENVIRONMENT_ID) || undefined; const result = await handler({ params: parsedParams, @@ -1767,6 +1774,7 @@ export function createActionWorkerApiRoute< body: parsedBody, headers: parsedHeaders, runnerId, + environmentId, }); return result; } catch (error) { diff --git a/apps/webapp/app/v3/otlpTransform.server.ts b/apps/webapp/app/v3/otlpTransform.server.ts index d66f1a8b821..4de989b0a45 100644 --- a/apps/webapp/app/v3/otlpTransform.server.ts +++ b/apps/webapp/app/v3/otlpTransform.server.ts @@ -12,6 +12,7 @@ import type { import { SeverityNumber, Span_SpanKind, Status_StatusCode } from "@trigger.dev/otlp-importer"; import type { MetricsV1Input } from "@internal/clickhouse"; import { generateSpanId } from "./eventRepository/common.server"; +import { unwrapWorkerIdInMetadata } from "./workerIdUnwrap.server"; import type { CreatableEventKind, CreatableEventStatus, @@ -468,7 +469,11 @@ function resolveDataPointContext( function extractEventProperties(attributes: KeyValue[], prefix?: string) { return { - metadata: convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]), + // Decode a deployment-token worker.id back to its friendlyId so the credential never lands in + // stored span/log metadata (the metrics path unwraps its own worker_id separately). + metadata: unwrapWorkerIdInMetadata( + convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]) + ), environmentId: extractStringAttribute(attributes, [ prefix, SemanticInternalAttributes.ENVIRONMENT_ID, diff --git a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts index b1660065646..23d20e108fb 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts @@ -18,11 +18,14 @@ import { fromFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import { WORKER_HEADERS, type WorkerQueueClass } from "@trigger.dev/core/v3/workers"; import type { RuntimeEnvironment, WorkerInstanceGroup } from "@trigger.dev/database"; import { Prisma, WorkerInstanceGroupType } from "@trigger.dev/database"; -import { SENSITIVE_WORKER_HEADERS, sanitizeWorkerHeaders } from "./sanitizeWorkerHeaders"; +import { json } from "@remix-run/server-runtime"; import { createHash, timingSafeEqual } from "crypto"; import { customAlphabet } from "nanoid"; +import { Counter } from "prom-client"; import { z } from "zod"; import { env } from "~/env.server"; +import { metricsRegister } from "~/metrics.server"; +import { evaluateCreatedAtGate } from "./workloadTokenAuthorization.server"; import { isWorkerQueueDequeueDisabled, recordBlockedDequeue, @@ -42,6 +45,30 @@ const authenticatedWorkerInstanceCache = singleton( createAuthenticatedWorkerInstanceCache ); +// Opt-in suppression of untokened worker actions on runs created after the cutoff. Only the +// no-header path ever reads a run row, and only when this is on - default off means feature-off is +// byte-for-byte today's behavior (no extra reads). Tenant scoping itself is header-driven (folded +// into the engine snapshot read) and needs no platform flag. +const workloadCreatedAtGateEnabled = env.WORKLOAD_CREATED_AT_GATE_ENABLED === "1"; +const workloadTokenCutoff = env.WORKLOAD_TOKEN_CUTOFF + ? new Date(env.WORKLOAD_TOKEN_CUTOFF) + : undefined; + +if (workloadCreatedAtGateEnabled && !workloadTokenCutoff) { + logger.warn( + "WORKLOAD_CREATED_AT_GATE_ENABLED is set but WORKLOAD_TOKEN_CUTOFF is missing; the created-at gate stays off until a cutoff is configured" + ); +} + +type WorkloadGateAction = "start" | "complete" | "continue" | "snapshots_since"; + +const workloadAuthGateCounter = new Counter({ + name: "workload_auth_gate_total", + help: "Deployment token authorization outcomes on worker actions", + labelNames: ["outcome", "action"] as const, + registers: [metricsRegister], +}); + function createAuthenticatedWorkerInstanceCache() { return createCache({ authenticatedWorkerInstance: new Namespace( @@ -188,6 +215,7 @@ export class WorkerGroupTokenService extends WithRunEngine { if (a.byteLength !== b.byteLength) { logger.error("[WorkerGroupTokenService] Managed secret length mismatch", { + managedWorkerSecret, headers: this.sanitizeHeaders(request), }); return; @@ -195,6 +223,7 @@ export class WorkerGroupTokenService extends WithRunEngine { if (!timingSafeEqual(a, b)) { logger.error("[WorkerGroupTokenService] Managed secret mismatch", { + managedWorkerSecret, headers: this.sanitizeHeaders(request), }); return; @@ -316,10 +345,16 @@ export class WorkerGroupTokenService extends WithRunEngine { } } - // Strip sensitive headers before logging request headers — see - // `sanitizeWorkerHeaders`. - private sanitizeHeaders(request: Request, denylist = SENSITIVE_WORKER_HEADERS) { - return sanitizeWorkerHeaders(request.headers, denylist); + private sanitizeHeaders(request: Request, skipHeaders = ["authorization"]) { + const sanitizedHeaders: Partial> = {}; + + for (const [key, value] of request.headers.entries()) { + if (!skipHeaders.includes(key.toLowerCase())) { + sanitizedHeaders[key] = value; + } + } + + return sanitizedHeaders; } } @@ -398,6 +433,55 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { }); } + /** + * The no-header fallback. When the env header is present the engine scopes the snapshot read by it + * (nothing to do here). When it's absent, this optionally suppresses runs created after the cutoff - + * a run that new enough should have carried a token, so a missing one is treated as out-of-scope. + * Only runs when the gate is enabled AND a cutoff is set; that's the ONLY path that reads a run row. + */ + private async assertCreatedAtGate({ + runId, + environmentId, + action, + }: { + runId: string; + environmentId?: string; + action: WorkloadGateAction; + }): Promise { + if (environmentId) { + // Scoping is delegated to the engine snapshot read; no run-row read here. Recorded so the + // platform can see how much traffic is env-scoped as enforcement rolls out. + workloadAuthGateCounter.inc({ outcome: "env_scoped", action }); + return; + } + + if (!workloadCreatedAtGateEnabled || !workloadTokenCutoff) { + return; + } + + const run = await this._engine.runStore.findRun({ id: runId }, { select: { createdAt: true } }); + + if (!run) { + // Let the engine method surface the canonical not-found error. + return; + } + + const { allow, outcome } = evaluateCreatedAtGate({ + runCreatedAt: run.createdAt, + cutoff: workloadTokenCutoff, + }); + + workloadAuthGateCounter.inc({ outcome, action }); + + if (!allow) { + logger.warn("[workload-auth] rejecting untokened worker action created after cutoff", { + action, + runId, + }); + throw json({ error: "Run does not belong to this worker" }, { status: 403 }); + } + } + async heartbeatRun({ runFriendlyId, snapshotFriendlyId, @@ -420,22 +504,31 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { snapshotFriendlyId, isWarmStart, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; isWarmStart?: boolean; runnerId?: string; + environmentId?: string; }): Promise< StartRunAttemptResult & { envVars: Record; } > { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "start", + }); + const engineResult = await this._engine.startRunAttempt({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), isWarmStart, workerId: this.workerInstanceId, runnerId, + environmentId, }); const defaultMachinePreset = machinePresetFromName(defaultMachine); @@ -470,24 +563,42 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { snapshotFriendlyId, completion, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; completion: TaskRunExecutionResult; runnerId?: string; + environmentId?: string; }): Promise { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "complete", + }); + return await this._engine.completeRunAttempt({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), completion, workerId: this.workerInstanceId, runnerId, + environmentId, }); } - async getLatestSnapshot({ runFriendlyId }: { runFriendlyId: string }) { + async getLatestSnapshot({ + runFriendlyId, + environmentId, + }: { + runFriendlyId: string; + environmentId?: string; + }) { + // No created-at gate: the only untokened caller is an internal warm-start poll that legitimately + // has no token, so an absent header must not reject. When a header is present the engine scopes. return await this._engine.getRunExecutionData({ runId: fromFriendlyId(runFriendlyId), + environmentId, }); } @@ -515,29 +626,47 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { runFriendlyId, snapshotFriendlyId, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; runnerId?: string; + environmentId?: string; }) { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "continue", + }); + return await this._engine.continueRunExecution({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), workerId: this.workerInstanceId, runnerId, + environmentId, }); } async getSnapshotsSince({ runFriendlyId, snapshotId, + environmentId, }: { runFriendlyId: string; snapshotId: string; + environmentId?: string; }) { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "snapshots_since", + }); + return await this._engine.getSnapshotsSince({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotId), + environmentId, }); } diff --git a/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts new file mode 100644 index 00000000000..a38097b62b2 --- /dev/null +++ b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts @@ -0,0 +1,27 @@ +/** + * The no-header created-at gate. Tenant scoping is handled by the env-scoped snapshot read in the + * engine; this only covers the fallback where no verified env header was forwarded (caller predates + * tokens, or the supervisor isn't enforcing): + * + * run created on/before cutoff -> allow (grandfather legacy untokened runs) + * run created after cutoff -> reject (a new-enough run should have carried a token) + * + * Pure and env-import-free so it stays trivially testable. + */ + +export type CreatedAtGateOutcome = "grandfathered" | "suppressed"; + +export type CreatedAtGateEvaluation = { + outcome: CreatedAtGateOutcome; + allow: boolean; +}; + +export function evaluateCreatedAtGate(params: { + runCreatedAt: Date; + cutoff: Date; +}): CreatedAtGateEvaluation { + const createdAfterCutoff = params.runCreatedAt.getTime() > params.cutoff.getTime(); + return createdAfterCutoff + ? { outcome: "suppressed", allow: false } + : { outcome: "grandfathered", allow: true }; +} diff --git a/apps/webapp/app/v3/workerIdUnwrap.server.ts b/apps/webapp/app/v3/workerIdUnwrap.server.ts new file mode 100644 index 00000000000..7300b740690 --- /dev/null +++ b/apps/webapp/app/v3/workerIdUnwrap.server.ts @@ -0,0 +1,75 @@ +// Decodes a `worker_id` that may be a deployment token back to its friendlyId, so the stored/queried +// value stays a short, stable id. Runs on the OTEL ingest hot path: base64url + JSON decode only (no +// verify), LRU-memoized. Fails safe — a non-token value passes through unchanged. + +import { LRUCache } from "lru-cache"; +import { SemanticInternalAttributes } from "@trigger.dev/core/v3"; + +// The runner flattens `worker.id` (the raw TRIGGER_DEPLOYMENT_ID) into span/log metadata under this +// key, mirroring how the runner composes it, so it can't drift. +const WORKER_ID_METADATA_KEY = `${SemanticInternalAttributes.METADATA}.${SemanticInternalAttributes.WORKER_ID}`; + +/** + * Unwrap, in place, the `worker.id` a runner stamps into span/log metadata: a deployment-token value + * becomes its friendlyId, everything else is left untouched. Keeps the credential out of stored + * telemetry and the value a stable deployment id (the metrics path unwraps its own worker_id). + */ +export function unwrapWorkerIdInMetadata< + T extends Record, +>(metadata: T | undefined): T | undefined { + if (metadata) { + const value = metadata[WORKER_ID_METADATA_KEY]; + if (typeof value === "string") { + (metadata as Record)[WORKER_ID_METADATA_KEY] = + unwrapWorkerId(value); + } + } + return metadata; +} + +// LRU, not FIFO: the active-deployment working set is unbounded, so an insertion-order memo thrashes +// once it exceeds the cap. Raw lru-cache (not @internal/cache's async wrapper) since this is sync. +const memo = new LRUCache({ max: 32_768 }); + +export function unwrapWorkerId(value: string | undefined): string | undefined { + if (!value) { + return value; + } + + // Fast path: a minted token is a JWT, whose base64url header always begins "eyJ". Anything else — + // a bare deployment friendlyId, "unmanaged", a dev id — returns immediately with no decode and no + // memo entry, so the non-token case costs nothing (and stays free once telemetry no longer carries + // tokens at all). + if (!value.startsWith("eyJ")) { + return value; + } + + const cached = memo.get(value); + if (cached !== undefined) { + return cached; + } + + const unwrapped = decodeDeploymentFriendlyId(value) ?? value; + memo.set(value, unwrapped); + + return unwrapped; +} + +function decodeDeploymentFriendlyId(value: string): string | undefined { + const parts = value.split("."); + // Not a JWT (three non-empty segments) → a legacy bare friendlyId; leave it as-is. + if (parts.length !== 3 || !parts[1]) { + return undefined; + } + + try { + const payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as { + deployment?: unknown; + }; + return typeof payload.deployment === "string" && payload.deployment.length > 0 + ? payload.deployment + : undefined; + } catch { + return undefined; + } +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 31b2bb9f831..60495a47156 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -154,6 +154,9 @@ "isbot": "^3.6.5", "jose": "^5.4.0", "json-stable-stringify": "^1.3.0", + "jsonpointer": "^5.0.1", + "lodash.omit": "^4.5.0", + "lru-cache": "^11.2.4", "lucide-react": "^0.229.0", "marked": "^4.0.18", "match-sorter": "^6.3.4", diff --git a/apps/webapp/test/workerIdUnwrap.test.ts b/apps/webapp/test/workerIdUnwrap.test.ts new file mode 100644 index 00000000000..ea787a81108 --- /dev/null +++ b/apps/webapp/test/workerIdUnwrap.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { mintWorkloadDeploymentToken, SemanticInternalAttributes } from "@trigger.dev/core/v3"; +import { unwrapWorkerId, unwrapWorkerIdInMetadata } from "~/v3/workerIdUnwrap.server"; + +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); +const WORKER_ID_KEY = `${SemanticInternalAttributes.METADATA}.${SemanticInternalAttributes.WORKER_ID}`; + +function mint(deployment: string) { + return mintWorkloadDeploymentToken( + { + deployment, + deployment_version: "20260709.1", + environment_id: "env_1", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", + }, + "any-secret", + EXP + ); +} + +describe("unwrapWorkerId", () => { + it("unwraps a real minted token to its deployment friendlyId", async () => { + // Decode is signature-independent (display-only), so the secret is irrelevant here. + expect(unwrapWorkerId(await mint("deployment_abc123"))).toBe("deployment_abc123"); + }); + + it("passes a legacy bare friendlyId through unchanged", () => { + expect(unwrapWorkerId("deployment_legacy")).toBe("deployment_legacy"); + }); + + it("passes undefined and garbage through unchanged (fail-safe)", () => { + expect(unwrapWorkerId(undefined)).toBeUndefined(); + expect(unwrapWorkerId("")).toBe(""); + expect(unwrapWorkerId("a.b.c")).toBe("a.b.c"); + expect(unwrapWorkerId("not-a-token")).toBe("not-a-token"); + }); + + it("is stable across repeated calls (memoized)", async () => { + const token = await mint("deployment_memo"); + expect(unwrapWorkerId(token)).toBe("deployment_memo"); + expect(unwrapWorkerId(token)).toBe("deployment_memo"); + }); +}); + +describe("unwrapWorkerIdInMetadata", () => { + it("unwraps a token worker.id in the metadata bag, leaving other keys untouched", async () => { + const metadata = { + [WORKER_ID_KEY]: await mint("deployment_span"), + "$metadata.custom": "keep-me", + "$metadata.worker.version": "20260709.1", + }; + + const result = unwrapWorkerIdInMetadata(metadata); + + expect(result?.[WORKER_ID_KEY]).toBe("deployment_span"); + expect(result?.["$metadata.custom"]).toBe("keep-me"); + expect(result?.["$metadata.worker.version"]).toBe("20260709.1"); + }); + + it("leaves a legacy bare friendlyId worker.id unchanged", () => { + const metadata = { [WORKER_ID_KEY]: "deployment_legacy" }; + expect(unwrapWorkerIdInMetadata(metadata)?.[WORKER_ID_KEY]).toBe("deployment_legacy"); + }); + + it("handles absent worker.id and undefined metadata", () => { + expect(unwrapWorkerIdInMetadata(undefined)).toBeUndefined(); + expect(unwrapWorkerIdInMetadata({ "$metadata.other": "x" })?.["$metadata.other"]).toBe("x"); + }); +}); diff --git a/apps/webapp/test/workloadTokenAuthorization.test.ts b/apps/webapp/test/workloadTokenAuthorization.test.ts new file mode 100644 index 00000000000..205db263255 --- /dev/null +++ b/apps/webapp/test/workloadTokenAuthorization.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { evaluateCreatedAtGate } from "~/v3/services/worker/workloadTokenAuthorization.server"; + +const cutoff = new Date("2026-07-09T00:00:00.000Z"); +const before = new Date("2026-07-01T00:00:00.000Z"); +const after = new Date("2026-07-10T00:00:00.000Z"); + +describe("evaluateCreatedAtGate", () => { + it("grandfathers a run created before the cutoff", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: before, cutoff }); + expect(result.outcome).toBe("grandfathered"); + expect(result.allow).toBe(true); + }); + + it("suppresses a run created after the cutoff", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: after, cutoff }); + expect(result.outcome).toBe("suppressed"); + expect(result.allow).toBe(false); + }); + + it("treats a run created exactly at the cutoff as grandfathered (not after)", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: cutoff, cutoff }); + expect(result.outcome).toBe("grandfathered"); + expect(result.allow).toBe(true); + }); +}); diff --git a/apps/webapp/test/workloadTokenGate.integration.test.ts b/apps/webapp/test/workloadTokenGate.integration.test.ts new file mode 100644 index 00000000000..ea6458b5760 --- /dev/null +++ b/apps/webapp/test/workloadTokenGate.integration.test.ts @@ -0,0 +1,88 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; + +// The env-scoped snapshot read the platform now performs for worker actions. Mirrors the where clause +// in getLatestExecutionSnapshot so this exercises the real select against the schema. +async function readLatestSnapshot(prisma: PrismaClient, runId: string, environmentId?: string) { + return prisma.taskRunExecutionSnapshot.findFirst({ + where: { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }, + orderBy: { createdAt: "desc" }, + }); +} + +async function seed(prisma: PrismaClient) { + const org = await prisma.organization.create({ + data: { title: "Org", slug: `org-${Date.now()}` }, + }); + const project = await prisma.project.create({ + data: { + name: "Project", + slug: `proj-${Date.now()}`, + externalRef: `proj_${Date.now()}`, + organizationId: org.id, + }, + }); + const env = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: "prod", + projectId: project.id, + organizationId: org.id, + apiKey: "api_key", + pkApiKey: "pk_api_key", + shortcode: "short", + }, + }); + + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${Date.now()}`, + taskIdentifier: "test-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_1", + spanId: "span_1", + queue: "task/test-task", + runtimeEnvironmentId: env.id, + projectId: project.id, + organizationId: org.id, + }, + }); + + await prisma.taskRunExecutionSnapshot.create({ + data: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "seed", + runId: run.id, + runStatus: "PENDING", + environmentId: env.id, + environmentType: "PRODUCTION", + projectId: project.id, + organizationId: org.id, + }, + }); + + return { env, run }; +} + +describe("env-scoped snapshot read against a real DB row", () => { + postgresTest( + "returns the snapshot for the matching env and nothing for another", + async ({ prisma }) => { + const { env, run } = await seed(prisma as PrismaClient); + + // No env scoping -> found (internal callers) + expect(await readLatestSnapshot(prisma as PrismaClient, run.id)).not.toBeNull(); + + // Matching env -> found + expect(await readLatestSnapshot(prisma as PrismaClient, run.id, env.id)).not.toBeNull(); + + // Different env -> not found (rejected as a tenant boundary) + expect( + await readLatestSnapshot(prisma as PrismaClient, run.id, "clenvdoesnotexist000000000") + ).toBeNull(); + } + ); +}); diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index bf9612ba198..3c1f4330a0f 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1497,6 +1497,7 @@ export class RunEngine { workerId, runnerId, isWarmStart, + environmentId, tx, }: { runId: string; @@ -1504,6 +1505,7 @@ export class RunEngine { workerId?: string; runnerId?: string; isWarmStart?: boolean; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { return this.runAttemptSystem.startRunAttempt({ @@ -1512,6 +1514,7 @@ export class RunEngine { workerId, runnerId, isWarmStart, + environmentId, tx, }); } @@ -1523,12 +1526,14 @@ export class RunEngine { completion, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; completion: TaskRunExecutionResult; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { return this.runAttemptSystem.completeRunAttempt({ runId, @@ -1536,6 +1541,7 @@ export class RunEngine { completion, workerId, runnerId, + environmentId, }); } @@ -2014,12 +2020,14 @@ export class RunEngine { snapshotId, workerId, runnerId, + environmentId, tx, }: { runId: string; snapshotId: string; workerId?: string; runnerId?: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { return this.checkpointSystem.continueRunExecution({ @@ -2027,6 +2035,7 @@ export class RunEngine { snapshotId, workerId, runnerId, + environmentId, tx, }); } @@ -2062,14 +2071,21 @@ export class RunEngine { /** Get required data to execute the run */ async getRunExecutionData({ runId, + environmentId, tx, }: { runId: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.prisma; try { - const snapshot = await getLatestExecutionSnapshot(prisma, runId, this.runStore); + const snapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.runStore, + environmentId + ); return executionDataFromSnapshot(snapshot); } catch (e) { this.logger.error("Failed to getRunExecutionData", { @@ -2086,10 +2102,12 @@ export class RunEngine { async getSnapshotsSince({ runId, snapshotId, + environmentId, tx, }: { runId: string; snapshotId: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const useReplica = @@ -2107,7 +2125,8 @@ export class RunEngine { runId, snapshotId, this.runStore, - repairClient + repairClient, + environmentId ); return snapshots.map(executionDataFromSnapshot); }; diff --git a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts index 7bac40838f8..a8f4eab1fdd 100644 --- a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts @@ -271,18 +271,25 @@ export class CheckpointSystem { snapshotId, workerId, runnerId, + environmentId, tx, }: { runId: string; snapshotId: string; workerId?: string; runnerId?: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.$.prisma; return await this.$.runLock.lock("continueRunExecution", [runId], async () => { - const snapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const snapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (snapshot.id !== snapshotId) { throw new ServiceValidationError( diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts index b98f2920b39..dca4c66b2e7 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts @@ -11,7 +11,7 @@ import type { Waitpoint, } from "@trigger.dev/database"; import type { RunStore } from "@internal/run-store"; -import { ExecutionSnapshotNotFoundError } from "../errors.js"; +import { ExecutionSnapshotNotFoundError, ServiceValidationError } from "../errors.js"; import type { HeartbeatTimeouts } from "../types.js"; import type { SystemResources } from "./systems.js"; @@ -195,16 +195,21 @@ async function fetchWaitpointsInChunks( return allWaitpoints; } -/* Gets the most recent valid snapshot for a run */ +/** + * Gets the most recent valid snapshot for a run. When `environmentId` is provided the read is scoped + * to that environment (tenant boundary): a run in another environment reads as not-found and rejects + * with a 404 rather than leaking existence. Internal callers omit it to read regardless of env. + */ export async function getLatestExecutionSnapshot( prisma: PrismaClientOrTransaction, runId: string, - runStore?: RunStore + runStore?: RunStore, + environmentId?: string ): Promise { const snapshot = runStore - ? await runStore.findLatestExecutionSnapshot(runId, prisma) + ? await runStore.findLatestExecutionSnapshot(runId, prisma, environmentId) : await prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where: { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }, include: { completedWaitpoints: true, checkpoint: true, @@ -213,6 +218,9 @@ export async function getLatestExecutionSnapshot( }); if (!snapshot) { + if (environmentId) { + throw new ServiceValidationError(`No execution snapshot found for TaskRun ${runId}`, 404); + } throw new Error(`No execution snapshot found for TaskRun ${runId}`); } @@ -295,19 +303,24 @@ export async function getExecutionSnapshotsSince( runStore?: RunStore, // The primary, for read-repair when `prisma` is a lagging read replica (see Step 3). Omit when // `prisma` is already the primary. - repairClient?: PrismaClientOrTransaction + repairClient?: PrismaClientOrTransaction, + // When set, scopes both reads to this environment (tenant boundary): a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string ): Promise { + const envScope = environmentId ? { environmentId } : {}; + // Step 1: Find the createdAt of the sinceSnapshotId const sinceSnapshot = runStore ? await runStore.findExecutionSnapshot( { - where: { id: sinceSnapshotId, runId }, + where: { id: sinceSnapshotId, runId, ...envScope }, select: { createdAt: true }, }, prisma ) : await prisma.taskRunExecutionSnapshot.findFirst({ - where: { id: sinceSnapshotId, runId }, + where: { id: sinceSnapshotId, runId, ...envScope }, select: { createdAt: true }, }); @@ -323,6 +336,7 @@ export async function getExecutionSnapshotsSince( runId, isValid: true, createdAt: { gt: sinceSnapshot.createdAt }, + ...envScope, }, include: { checkpoint: true, @@ -338,6 +352,7 @@ export async function getExecutionSnapshotsSince( runId, isValid: true, createdAt: { gt: sinceSnapshot.createdAt }, + ...envScope, }, include: { checkpoint: true, diff --git a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts index 922b41cd225..c7a331fe9de 100644 --- a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts @@ -300,6 +300,7 @@ export class RunAttemptSystem { workerId, runnerId, isWarmStart, + environmentId, tx, }: { runId: string; @@ -307,6 +308,7 @@ export class RunAttemptSystem { workerId?: string; runnerId?: string; isWarmStart?: boolean; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.$.prisma; @@ -316,7 +318,12 @@ export class RunAttemptSystem { "startRunAttempt", async (span) => { return this.$.runLock.lock("startRunAttempt", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { //if there is a big delay between the snapshot and the attempt, the snapshot might have changed @@ -643,12 +650,14 @@ export class RunAttemptSystem { completion, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; completion: TaskRunExecutionResult; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { await this.#notifyMetadataUpdated(runId, completion); @@ -661,6 +670,7 @@ export class RunAttemptSystem { tx: this.$.prisma, workerId, runnerId, + environmentId, }); } case false: { @@ -671,6 +681,7 @@ export class RunAttemptSystem { tx: this.$.prisma, workerId, runnerId, + environmentId, }); } } @@ -683,6 +694,7 @@ export class RunAttemptSystem { tx, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; @@ -690,6 +702,7 @@ export class RunAttemptSystem { tx: PrismaClientOrTransaction; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { const prisma = tx ?? this.$.prisma; @@ -698,7 +711,12 @@ export class RunAttemptSystem { "#completeRunAttemptSuccess", async (span) => { return this.$.runLock.lock("attemptSucceeded", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); @@ -868,6 +886,7 @@ export class RunAttemptSystem { runnerId, completion, forceRequeue, + environmentId, tx, }: { runId: string; @@ -876,6 +895,7 @@ export class RunAttemptSystem { runnerId?: string; completion: TaskRunFailedExecutionResult; forceRequeue?: boolean; + environmentId?: string; tx: PrismaClientOrTransaction; }): Promise { const prisma = this.$.prisma; @@ -885,7 +905,12 @@ export class RunAttemptSystem { "completeRunAttemptFailure", async (span) => { return this.$.runLock.lock("attemptFailed", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 5f636ada987..1b0d2f89cee 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -1708,15 +1708,17 @@ export class PostgresRunStore implements RunStore { async findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + environmentId?: string ): Promise | null> { const prisma = client ?? this.readOnlyPrisma; + const where = { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }; if (this.schemaVariant === "dedicated") { const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where, include: { checkpoint: true }, orderBy: { createdAt: "desc" }, }); @@ -1728,7 +1730,7 @@ export class PostgresRunStore implements RunStore { } return prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where, include: { completedWaitpoints: true, checkpoint: true, diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index d14ae525224..ba496980768 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -841,14 +841,16 @@ export class RoutingRunStore implements RunStore { // OUTPUT is silently missing from the resume payload — re-resolve them across BOTH DBs. async findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + environmentId?: string ): Promise | null> { const owningStore = this.#routeOrNew(runId); const snapshot = await owningStore.findLatestExecutionSnapshot( runId, - RoutingRunStore.#ownPrimary(owningStore, client) + RoutingRunStore.#ownPrimary(owningStore, client), + environmentId ); if (snapshot) { await this.#reresolveCompletedWaitpointsCrossDb( diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index b55e8f968c2..374a550e419 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -673,7 +673,10 @@ export interface RunStore { // Snapshot group findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string ): Promise | null>; diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index 9052884bbd6..35dbfa287f9 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -30,6 +30,7 @@ export * from "./resource-catalog-api.js"; export * from "./types/index.js"; export { links } from "./links.js"; export * from "./jwt.js"; +export * from "./workloadDeploymentToken.js"; export * from "./idempotencyKeys.js"; export * from "./streams/asyncIterableStream.js"; export * from "./utils/getEnv.js"; diff --git a/packages/core/src/v3/jwt.ts b/packages/core/src/v3/jwt.ts index d71f1e7f1e8..6845225be8a 100644 --- a/packages/core/src/v3/jwt.ts +++ b/packages/core/src/v3/jwt.ts @@ -4,6 +4,10 @@ export type GenerateJWTOptions = { secretKey: string; payload: Record; expirationTime?: number | Date | string; + // Skip the `iat` claim. Combined with an absolute `expirationTime`, this makes the signed token a + // pure function of its payload — the same claims mint byte-identical tokens. Off by default so + // ordinary short-lived tokens keep their issued-at. + omitIssuedAt?: boolean; }; export const JWT_ALGORITHM = "HS256"; @@ -15,13 +19,17 @@ export async function generateJWT(options: GenerateJWTOptions): Promise const secret = new TextEncoder().encode(options.secretKey); - return new SignJWT(options.payload) + const jwt = new SignJWT(options.payload) .setIssuer(JWT_ISSUER) .setAudience(JWT_AUDIENCE) .setProtectedHeader({ alg: JWT_ALGORITHM }) - .setIssuedAt() - .setExpirationTime(options.expirationTime ?? "15m") - .sign(secret); + .setExpirationTime(options.expirationTime ?? "15m"); + + if (!options.omitIssuedAt) { + jwt.setIssuedAt(); + } + + return jwt.sign(secret); } export type ValidationResult = diff --git a/packages/core/src/v3/runEngineWorker/consts.ts b/packages/core/src/v3/runEngineWorker/consts.ts index 29dc76b5358..e04e83875b2 100644 --- a/packages/core/src/v3/runEngineWorker/consts.ts +++ b/packages/core/src/v3/runEngineWorker/consts.ts @@ -3,6 +3,9 @@ export const WORKER_HEADERS = { DEPLOYMENT_ID: "x-trigger-worker-deployment-id", MANAGED_SECRET: "x-trigger-worker-managed-secret", RUNNER_ID: "x-trigger-worker-runner-id", + // Verified environment id recovered from the deployment token. Set by the supervisor only when + // enforcing; the platform scopes the worker-action's snapshot read by it. + ENVIRONMENT_ID: "x-trigger-worker-environment-id", }; export const WORKLOAD_HEADERS = { diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts index bbb1fe227de..d6d51b1c2b2 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/http.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/http.ts @@ -143,7 +143,8 @@ export class SupervisorHttpClient { runId: string, snapshotId: string, body: WorkerApiRunAttemptStartRequestBody, - runnerId?: string + runnerId?: string, + environmentId?: string ) { return wrapZodFetch( WorkerApiRunAttemptStartResponseBody, @@ -153,6 +154,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, body: JSON.stringify(body), } @@ -163,7 +165,8 @@ export class SupervisorHttpClient { runId: string, snapshotId: string, body: WorkerApiRunAttemptCompleteRequestBody, - runnerId?: string + runnerId?: string, + environmentId?: string ) { return wrapZodFetch( WorkerApiRunAttemptCompleteResponseBody, @@ -173,13 +176,14 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, body: JSON.stringify(body), } ); } - async getLatestSnapshot(runId: string, runnerId?: string) { + async getLatestSnapshot(runId: string, runnerId?: string, environmentId?: string) { return wrapZodFetch( WorkerApiRunLatestSnapshotResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/latest`, @@ -188,12 +192,18 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, } ); } - async getSnapshotsSince(runId: string, snapshotId: string, runnerId?: string) { + async getSnapshotsSince( + runId: string, + snapshotId: string, + runnerId?: string, + environmentId?: string + ) { return wrapZodFetch( WorkerApiRunSnapshotsSinceResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/since/${snapshotId}`, @@ -202,6 +212,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, } ); @@ -235,7 +246,12 @@ export class SupervisorHttpClient { } } - async continueRunExecution(runId: string, snapshotId: string, runnerId?: string) { + async continueRunExecution( + runId: string, + snapshotId: string, + runnerId?: string, + environmentId?: string + ) { return wrapZodFetch( WorkerApiContinueRunExecutionRequestBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/continue`, @@ -244,6 +260,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, }, { @@ -295,4 +312,10 @@ export class SupervisorHttpClient { [WORKER_HEADERS.RUNNER_ID]: runnerId, }); } + + private environmentIdHeader(environmentId?: string): Record { + return createHeaders({ + [WORKER_HEADERS.ENVIRONMENT_ID]: environmentId, + }); + } } diff --git a/packages/core/src/v3/runEngineWorker/types.ts b/packages/core/src/v3/runEngineWorker/types.ts index b54d23ccde7..7a76af3f4cb 100644 --- a/packages/core/src/v3/runEngineWorker/types.ts +++ b/packages/core/src/v3/runEngineWorker/types.ts @@ -29,4 +29,6 @@ export type WorkloadClientSocketData = { runnerId: string; runFriendlyId?: string; snapshotId?: string; + // Deployment friendlyId resolved from the verified token; prefer over the raw deploymentId. + deploymentFriendlyId?: string; }; diff --git a/packages/core/src/v3/workloadDeploymentToken.test.ts b/packages/core/src/v3/workloadDeploymentToken.test.ts new file mode 100644 index 00000000000..4a807b002ce --- /dev/null +++ b/packages/core/src/v3/workloadDeploymentToken.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; +import { generateJWT } from "./jwt.js"; +import { + classifyDeploymentIdHeader, + looksLikeWorkloadDeploymentToken, + mintWorkloadDeploymentToken, + verifyWorkloadDeploymentToken, + WORKLOAD_DEPLOYMENT_TOKEN_VERSION, + type WorkloadDeploymentTokenInput, +} from "./workloadDeploymentToken.js"; + +const SECRET = "test-workload-token-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const claims: WorkloadDeploymentTokenInput = { + deployment: "deployment_abc123", + deployment_version: "20260709.1", + environment_id: "env_1", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", +}; + +describe("workloadDeploymentToken", () => { + it("mints a token that verifies and round-trips every claim", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.claims.ver).toBe(WORKLOAD_DEPLOYMENT_TOKEN_VERSION); + expect(result.claims.deployment).toBe(claims.deployment); + expect(result.claims.deployment_version).toBe(claims.deployment_version); + expect(result.claims.environment_id).toBe(claims.environment_id); + expect(result.claims.environment_type).toBe(claims.environment_type); + expect(result.claims.org_id).toBe(claims.org_id); + expect(result.claims.project_id).toBe(claims.project_id); + }); + + it("sets the caller-supplied absolute exp", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.claims.exp).toBe(EXP); + }); + + it("mints byte-identical tokens for identical claims (deterministic, no iat)", async () => { + const a = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const b = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + expect(a).toBe(b); + + const result = await verifyWorkloadDeploymentToken(a, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect((result.claims as Record).iat).toBeUndefined(); + }); + + it("rejects a token signed with a different secret", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await verifyWorkloadDeploymentToken(token, "wrong-secret"); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("invalid_signature"); + }); + + it("rejects a tampered token", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const [header, payload, signature] = token.split("."); + const tampered = `${header}.${payload}x.${signature}`; + const result = await verifyWorkloadDeploymentToken(tampered, SECRET); + expect(result.ok).toBe(false); + }); + + it("rejects a valid JWT that is missing required claims", async () => { + // Signed with the right secret, but not a workload deployment token. + const token = await generateJWT({ + secretKey: SECRET, + payload: { foo: "bar" }, + expirationTime: "1825d", + }); + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("malformed_claims"); + }); + + describe("looksLikeWorkloadDeploymentToken", () => { + it("is true for a minted token", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + expect(looksLikeWorkloadDeploymentToken(token)).toBe(true); + }); + + it("is false for a legacy bare friendlyId", () => { + expect(looksLikeWorkloadDeploymentToken("deployment_abc123")).toBe(false); + }); + + it("is false for empty and malformed values", () => { + expect(looksLikeWorkloadDeploymentToken("")).toBe(false); + expect(looksLikeWorkloadDeploymentToken("a.b")).toBe(false); + expect(looksLikeWorkloadDeploymentToken("a..c")).toBe(false); + }); + }); + + describe("classifyDeploymentIdHeader", () => { + it("classifies a minted token as jwt_valid and returns claims", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await classifyDeploymentIdHeader(token, SECRET); + expect(result.outcome).toBe("jwt_valid"); + expect(result.claims?.deployment).toBe(claims.deployment); + }); + + it("classifies a JWT with a bad signature as jwt_invalid", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await classifyDeploymentIdHeader(token, "wrong-secret"); + expect(result.outcome).toBe("jwt_invalid"); + expect(result.claims).toBeUndefined(); + }); + + it("classifies a bare friendlyId as legacy_bare", async () => { + const result = await classifyDeploymentIdHeader("deployment_abc123", SECRET); + expect(result.outcome).toBe("legacy_bare"); + }); + }); +}); diff --git a/packages/core/src/v3/workloadDeploymentToken.ts b/packages/core/src/v3/workloadDeploymentToken.ts new file mode 100644 index 00000000000..4f0d367038f --- /dev/null +++ b/packages/core/src/v3/workloadDeploymentToken.ts @@ -0,0 +1,99 @@ +import { z } from "zod"; +import { generateJWT, validateJWT } from "./jwt.js"; + +/** + * Signed, deployment-scoped token carried in the TRIGGER_DEPLOYMENT_ID env var to identify a run + * controller. Long-lived identity token, not a rotating credential; the `ver` claim allows a future + * format revision. + */ +export const WORKLOAD_DEPLOYMENT_TOKEN_VERSION = 1 as const; + +export const WorkloadDeploymentTokenClaims = z.object({ + ver: z.literal(WORKLOAD_DEPLOYMENT_TOKEN_VERSION), + /** Deployment friendlyId (deployment_xxxx). */ + deployment: z.string(), + deployment_version: z.string(), + environment_id: z.string(), + environment_type: z.string(), + org_id: z.string(), + project_id: z.string(), + exp: z.number(), +}); + +export type WorkloadDeploymentTokenClaims = z.infer; + +/** Claims the caller supplies; `ver` and `exp` are set by the minter. */ +export type WorkloadDeploymentTokenInput = Omit; + +/** + * `expiresAtSeconds` is an absolute epoch (caller-supplied — the supervisor drives it). Combined with + * the omitted `iat`, the token is byte-deterministic per deployment: identical claims mint identical + * bytes, so the OTel `worker.id` the runner derives from it stays one value per deployment. + */ +export async function mintWorkloadDeploymentToken( + claims: WorkloadDeploymentTokenInput, + secret: string, + expiresAtSeconds: number +): Promise { + return generateJWT({ + secretKey: secret, + payload: { ver: WORKLOAD_DEPLOYMENT_TOKEN_VERSION, ...claims }, + expirationTime: expiresAtSeconds, + omitIssuedAt: true, + }); +} + +export type WorkloadDeploymentTokenVerification = + | { ok: true; claims: WorkloadDeploymentTokenClaims } + | { ok: false; reason: "invalid_signature" | "malformed_claims"; error: string }; + +export async function verifyWorkloadDeploymentToken( + token: string, + secret: string +): Promise { + const result = await validateJWT(token, secret); + + if (!result.ok) { + return { ok: false, reason: "invalid_signature", error: result.error }; + } + + const parsed = WorkloadDeploymentTokenClaims.safeParse(result.payload); + + if (!parsed.success) { + return { ok: false, reason: "malformed_claims", error: parsed.error.message }; + } + + return { ok: true, claims: parsed.data }; +} + +/** + * A legacy TRIGGER_DEPLOYMENT_ID is a bare friendlyId (deployment_xxxx); a minted token is a JWT of + * three non-empty base64url segments. Used by the dry-run to tell a pre-upgrade runner from a token. + */ +export function looksLikeWorkloadDeploymentToken(value: string): boolean { + const parts = value.split("."); + return parts.length === 3 && parts.every((part) => part.length > 0); +} + +export type DeploymentIdHeaderOutcome = "jwt_valid" | "jwt_invalid" | "legacy_bare"; + +/** + * Classify the value carried in the deployment-id header for the rollout metric, verifying the + * signature when it is JWT-shaped. Returns the claims on a valid token so callers avoid a second verify. + */ +export async function classifyDeploymentIdHeader( + value: string, + secret: string +): Promise<{ outcome: DeploymentIdHeaderOutcome; claims?: WorkloadDeploymentTokenClaims }> { + if (!looksLikeWorkloadDeploymentToken(value)) { + return { outcome: "legacy_bare" }; + } + + const result = await verifyWorkloadDeploymentToken(value, secret); + + if (result.ok) { + return { outcome: "jwt_valid", claims: result.claims }; + } + + return { outcome: "jwt_invalid" }; +} diff --git a/packages/core/test/taskExecutor.test.ts b/packages/core/test/taskExecutor.test.ts index 9bbda935f6d..d46a3fbe67e 100644 --- a/packages/core/test/taskExecutor.test.ts +++ b/packages/core/test/taskExecutor.test.ts @@ -1802,8 +1802,10 @@ describe("TaskExecutor", () => { // Rate limit errors should use the rate limit retry delay expect((result.result as any).retry.delay).toBeGreaterThan(0); } else { - // Other retryable errors should use the exponential backoff - expect((result.result as any).retry.delay).toBeGreaterThan(1000); + // Other retryable errors should use the exponential backoff. The first retry uses + // minDelay (1000ms) as its base, and jitter can land exactly on the floor, so this + // is >= rather than > to avoid a flaky boundary failure. + expect((result.result as any).retry.delay).toBeGreaterThanOrEqual(1000); expect((result.result as any).retry.delay).toBeLessThan(5000); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0e328c76a4..29450d4b162 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -585,6 +585,15 @@ importers: json-stable-stringify: specifier: ^1.3.0 version: 1.3.0 + jsonpointer: + specifier: ^5.0.1 + version: 5.0.1 + lodash.omit: + specifier: ^4.5.0 + version: 4.5.0 + lru-cache: + specifier: ^11.2.4 + version: 11.2.4 lucide-react: specifier: ^0.229.0 version: 0.229.0(react@18.3.1) @@ -11391,6 +11400,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + jsonpointer@5.0.1: + resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} + engines: {node: '>=0.10.0'} + jsonwebtoken@9.0.2: resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} engines: {node: '>=12', npm: '>=6'} @@ -26634,6 +26647,8 @@ snapshots: '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) jsep: 1.4.0 + jsonpointer@5.0.1: {} + jsonwebtoken@9.0.2: dependencies: jws: 3.2.3 From 05a3ec10c1bfcb960bdca7fa2e793d74d6abdd7a Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:14:33 +0100 Subject: [PATCH 03/15] fix(webapp): require same-origin navigation for GET /logout to block CSRF logout (#72) * fix(webapp): require same-origin navigation for GET /logout to block CSRF logout The /logout route exposes a `loader` that runs on GET and calls `authenticator.logout(...)`. A cross-site top-level GET navigation (e.g. a victim clicking a link to https://cloud.trigger.dev/logout on an attacker's page) therefore logs the victim out. Because the `__session` cookie is `SameSite=Lax`, the browser still sends it on cross-site top-level GET navigations, so SameSite does not mitigate this (CWE-352, forced logout). Gate the loader with a same-origin check (`isSameOriginNavigation`, deny-by- default on missing headers), redirecting to `/` otherwise. Server-initiated `throw redirect("/logout")` flows (auto-logout, SSO expiry) and the in-app SideMenu logout link are same-origin and unaffected; cross-site clicks are rejected. Deliberately not converted to POST-only so the existing `throw redirect("/logout")` flows keep working. Advisory: GHSA-2cqj-rq4j-8835 * chore(webapp): shorten logout navigation comments --- apps/webapp/app/routes/logout.tsx | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/logout.tsx b/apps/webapp/app/routes/logout.tsx index 20b4705f352..fe6368ee3be 100644 --- a/apps/webapp/app/routes/logout.tsx +++ b/apps/webapp/app/routes/logout.tsx @@ -1,6 +1,8 @@ -import { type ActionFunction, type LoaderFunction } from "@remix-run/node"; +import { redirect, type ActionFunction, type LoaderFunction } from "@remix-run/node"; +import { env } from "~/env.server"; import { authenticator } from "~/services/auth.server"; import { sanitizeRedirectPath } from "~/utils"; +import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; import { SSO_SESSION_EXPIRED_REASON } from "~/utils/ssoSession"; function logoutRedirectTo(request: Request): string { @@ -18,5 +20,10 @@ export const action: ActionFunction = async ({ request }) => { }; export const loader: LoaderFunction = async ({ request }) => { + // GET /logout is state-changing, so reject cross-site navigations. + if (!isSameOriginNavigation(request, env.LOGIN_ORIGIN)) { + throw redirect("/"); + } + return await authenticator.logout(request, { redirectTo: logoutRedirectTo(request) }); }; From 845cd21e7892e80c0a84a0e9d54ea8212e281904 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:14:42 +0100 Subject: [PATCH 04/15] fix(tsql): validate window-function names against the allowlist (#68) --- .server-changes/tsql-window-function-allowlist.md | 6 ++++++ internal-packages/tsql/src/query/printer.ts | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 .server-changes/tsql-window-function-allowlist.md diff --git a/.server-changes/tsql-window-function-allowlist.md b/.server-changes/tsql-window-function-allowlist.md new file mode 100644 index 00000000000..a56a881ae73 --- /dev/null +++ b/.server-changes/tsql-window-function-allowlist.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Window-function names in the query compiler are now validated against the allowlist, matching how other function calls are handled. diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index 6f1af07d71c..00ff8eabdba 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -3127,8 +3127,16 @@ export class ClickHousePrinter { } private visitWindowFunction(node: WindowFunction): string { + // Validate the function name against the allowlist and resolve it to its + // safe ClickHouse name. Without this, an attacker-controlled name would be + // emitted raw into the query (mirrors the validation in visitCall). + const funcMeta = findTSQLFunction(node.name) ?? findTSQLAggregation(node.name); + if (!funcMeta) { + throw new QueryError(`Unknown function: ${node.name}`); + } + const args = node.args ? node.args.map((a) => this.visit(a)) : []; - const funcCall = `${node.name}(${args.join(", ")})`; + const funcCall = `${funcMeta.clickhouseName}(${args.join(", ")})`; if (node.over_identifier) { return `${funcCall} OVER ${this.printIdentifier(node.over_identifier)}`; From 2d195ba873bb63eba6432b0eca6902a19310cb1b Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:14:50 +0100 Subject: [PATCH 05/15] fix(webapp): scope deployment background-worker lookup to the authenticated environment (#66) --- .../scope-deployment-background-worker-lookup.md | 6 ++++++ .../services/createDeploymentBackgroundWorkerV4.server.ts | 1 + 2 files changed, 7 insertions(+) create mode 100644 .server-changes/scope-deployment-background-worker-lookup.md diff --git a/.server-changes/scope-deployment-background-worker-lookup.md b/.server-changes/scope-deployment-background-worker-lookup.md new file mode 100644 index 00000000000..74c2aa7ee6d --- /dev/null +++ b/.server-changes/scope-deployment-background-worker-lookup.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Background-worker deployment lookups are now scoped to the authenticated environment. diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index a53dee42353..66a17bb9f65 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -51,6 +51,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { const deployment = await this._prisma.workerDeployment.findFirst({ where: { friendlyId: deploymentId, + environmentId: environment.id, }, }); From d2121555a219f37dc7691de25a1a0408db694b33 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:14:58 +0100 Subject: [PATCH 06/15] fix(webapp): rate-limit unauthenticated OTLP ingestion endpoints (#61) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(webapp): rate-limit unauthenticated OTLP ingestion endpoints Add a per-IP rate limiter to /otel/* to bound unauthenticated bulk-injection and DoS against the OTLP ingestion path. Fails open on limiter errors and when the source IP is unknown, with generous, configurable defaults so legitimate telemetry is not throttled. This is a minimal hardening step only — it does NOT authenticate the endpoints (see PR description). * fix(webapp): make OTLP per-IP rate limiter opt-in and document IP-keying caveats The per-IP limiter on /otel/* was enabled by default and keys on the source IP. Where legitimate clients share a single egress IP (NAT or a shared proxy) their telemetry collapses into one bucket and could be throttled, so a default-on limiter risks dropping telemetry that flows fine today (fail-open only covers limiter errors, not over-limit). Default OTLP_RATE_LIMIT_ENABLED to 0 so it is opt-in, and document the preconditions for enabling it safely: each client must present a distinct IP through a proxy that appends the real client IP to X-Forwarded-For, and the limit must be sized for aggregate shared-source volume. Without such a proxy the X-Forwarded-For value is client-controlled and the per-IP bound is bypassable. * fix(webapp): harden optional OTLP request rate limiting --- .server-changes/otlp-ingestion-rate-limit.md | 6 ++ apps/webapp/app/entry.server.tsx | 1 + apps/webapp/app/env.server.ts | 16 ++++ .../app/services/otlpRateLimit.server.ts | 91 +++++++++++++++++++ apps/webapp/server.ts | 2 + 5 files changed, 116 insertions(+) create mode 100644 .server-changes/otlp-ingestion-rate-limit.md create mode 100644 apps/webapp/app/services/otlpRateLimit.server.ts diff --git a/.server-changes/otlp-ingestion-rate-limit.md b/.server-changes/otlp-ingestion-rate-limit.md new file mode 100644 index 00000000000..acb9c623122 --- /dev/null +++ b/.server-changes/otlp-ingestion-rate-limit.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Added optional request rate limiting for telemetry ingestion endpoints. diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 672916d8103..da1ab5a433a 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -300,6 +300,7 @@ singleton("SentryTenantContextProcessor", () => { export { apiRateLimiter } from "./services/apiRateLimit.server"; export { engineRateLimiter } from "./services/engineRateLimit.server"; +export { otlpRateLimiter } from "./services/otlpRateLimit.server"; export { runWithHttpContext } from "./services/httpAsyncStorage.server"; export { tenantContextMiddleware } from "./services/tenantContextResolver.server"; export { socketIo } from "./v3/handleSocketIo.server"; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 3aa040f0fde..de36b225398 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -547,6 +547,22 @@ const EnvironmentSchema = z API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"), API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60), + // Per-IP rate limit for the unauthenticated OTLP ingestion endpoints + // (/otel/*). Bounds unauthenticated request rates. Opt-in + // (disabled by default): because it keys on the source IP, it is only + // safe to enable when each client presents a distinct IP through a proxy + // that appends the real client IP to X-Forwarded-For. Enabling it where + // many clients share one egress IP (e.g. behind NAT or a shared proxy) + // would collapse that traffic into a single bucket and could throttle + // legitimate telemetry. Set OTLP_RATE_LIMIT_ENABLED=1 to enable, then tune + // OTLP_RATE_LIMIT_MAX / OTLP_RATE_LIMIT_WINDOW for expected volume. + OTLP_RATE_LIMIT_ENABLED: z.string().default("0"), + OTLP_RATE_LIMIT_WINDOW: z + .string() + .regex(/^\d+ ?(?:ms|s|m|h|d)$/) + .default("1m"), + OTLP_RATE_LIMIT_MAX: z.coerce.number().int().positive().default(3000), + DEPOT_TOKEN: z.string().optional(), DEPOT_ORG_ID: z.string().optional(), DEPOT_REGION: z.string().default("us-east-1"), diff --git a/apps/webapp/app/services/otlpRateLimit.server.ts b/apps/webapp/app/services/otlpRateLimit.server.ts new file mode 100644 index 00000000000..eef07990ecd --- /dev/null +++ b/apps/webapp/app/services/otlpRateLimit.server.ts @@ -0,0 +1,91 @@ +import { Ratelimit } from "@upstash/ratelimit"; +import type { NextFunction, Request, Response } from "express"; +import { env } from "~/env.server"; +import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server"; +import { extractClientIp } from "~/utils/extractClientIp.server"; +import { singleton } from "~/utils/singleton"; +import { logger } from "./logger.server"; + +const OTLP_PATH = /^\/otel\//i; + +function getOtlpIpRateLimiter() { + return singleton( + "otlpIpRateLimiter", + () => + new RateLimiter({ + redisClient: createRedisRateLimitClient({ + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }), + keyPrefix: "otlp:ip", + limiter: Ratelimit.slidingWindow( + env.OTLP_RATE_LIMIT_MAX, + env.OTLP_RATE_LIMIT_WINDOW as Parameters[1] + ), + logSuccess: false, + logFailure: true, + }) + ); +} + +/** + * Per-IP rate limiter for the OTLP ingestion endpoints (`/otel/*`). + * + * These endpoints are currently unauthenticated (see SEC-98), so the source IP + * is the only identity available to key on. This bounds unauthenticated + * request rates and is NOT a substitute for authenticating the + * endpoints. It fails open (allows the request) whenever the source cannot be + * identified or the limiter backend errors, so a limiter outage never drops + * legitimate telemetry. + * + * Opt-in (disabled unless `OTLP_RATE_LIMIT_ENABLED=1`). Because it keys on the + * source IP, two preconditions must hold before enabling it, or it can drop + * legitimate telemetry: + * + * 1. Each client must present a distinct IP. Where many clients share one + * egress IP (e.g. behind NAT or a shared proxy) their traffic collapses + * into a single bucket and can be throttled together. Size + * `OTLP_RATE_LIMIT_MAX` for the aggregate volume of a shared source, not a + * single client. + * 2. The IP must be trustworthy. `extractClientIp` takes the last + * `X-Forwarded-For` hop, which is only spoof-resistant behind a proxy that + * appends the real client IP. Without such a proxy the value is + * client-controlled and the per-IP bound is bypassable. + */ +export async function otlpRateLimiter(req: Request, res: Response, next: NextFunction) { + if (env.OTLP_RATE_LIMIT_ENABLED !== "1") { + return next(); + } + + if (req.method.toUpperCase() === "OPTIONS" || !OTLP_PATH.test(req.path)) { + return next(); + } + + const xff = req.headers["x-forwarded-for"]; + const ip = extractClientIp(Array.isArray(xff) ? xff.join(",") : (xff ?? null)) ?? req.ip; + + if (!ip) { + // Fail open: without a source we cannot fairly rate limit. + return next(); + } + + try { + const { success, reset } = await getOtlpIpRateLimiter().limit(ip); + + if (!success) { + const retryAfterSeconds = Math.max(1, Math.ceil((reset - Date.now()) / 1000)); + res.setHeader("Retry-After", retryAfterSeconds.toString()); + res.status(429).send("Too Many Requests"); + return; + } + } catch (error) { + // Fail open: a rate-limiter backend outage must not drop telemetry. + logger.warn("otlpRateLimiter: limiter error, allowing request", { error }); + } + + return next(); +} diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index 3794441abbd..fb4e2035947 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -147,6 +147,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { const wss: WebSocketServer | undefined = build.entry.module.wss; const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter; const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter; + const otlpRateLimiter: RequestHandler = build.entry.module.otlpRateLimiter; const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext; const tenantContextMiddleware: RequestHandler = build.entry.module.tenantContextMiddleware; @@ -198,6 +199,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { app.use(apiRateLimiter); app.use(engineRateLimiter); + app.use(otlpRateLimiter); app.use(tenantContextMiddleware); From 3cbbaed1fc6c5cc49115cf0ea34186a5a7e3e7f5 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:15:07 +0100 Subject: [PATCH 07/15] fix(webapp): scope GitHub App installation update to the caller's org (#58) * fix(webapp): scope GitHub App installation update to the caller's org * fix(webapp): make GitHub App install session single-use Invalidate the GitHub App installation state cookie once a callback consumes an installation_id, so a single install initiation cannot be replayed against other installation_ids. --- .../scope-github-app-update-to-org.md | 6 ++++ .../app/routes/_app.github.callback/route.tsx | 34 +++++++++++++++---- apps/webapp/app/services/gitHub.server.ts | 22 +++++++----- 3 files changed, 48 insertions(+), 14 deletions(-) create mode 100644 .server-changes/scope-github-app-update-to-org.md diff --git a/.server-changes/scope-github-app-update-to-org.md b/.server-changes/scope-github-app-update-to-org.md new file mode 100644 index 00000000000..df417e52dd4 --- /dev/null +++ b/.server-changes/scope-github-app-update-to-org.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Updating a GitHub App installation from the callback flow is now scoped to your own organization, so an installation ID belonging to another organization can no longer be used to refresh that organization's installation record. The GitHub App installation session is also now single-use, so completing an installation callback invalidates its state and it can no longer be replayed. diff --git a/apps/webapp/app/routes/_app.github.callback/route.tsx b/apps/webapp/app/routes/_app.github.callback/route.tsx index cd67f27f54c..94e13ae1854 100644 --- a/apps/webapp/app/routes/_app.github.callback/route.tsx +++ b/apps/webapp/app/routes/_app.github.callback/route.tsx @@ -1,6 +1,9 @@ import { type LoaderFunctionArgs } from "@remix-run/node"; import { z } from "zod"; -import { validateGitHubAppInstallSession } from "~/services/gitHubSession.server"; +import { + destroyGitHubAppInstallSession, + validateGitHubAppInstallSession, +} from "~/services/gitHubSession.server"; import { linkGitHubAppInstallation, updateGitHubAppInstallation } from "~/services/gitHub.server"; import { logger } from "~/services/logger.server"; import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; @@ -75,6 +78,15 @@ export async function loader({ request }: LoaderFunctionArgs) { return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app"); } + // The install session is single-use: once a callback consumes an + // installation_id, invalidate the state cookie so the same initiation + // cannot be replayed against other installation_ids. + const clearInstallSession = await destroyGitHubAppInstallSession(cookieHeader); + const consumingSession = (response: Response) => { + response.headers.append("Set-Cookie", clearInstallSession); + return response; + }; + switch (callbackData.setup_action) { case "install": { const [error] = await tryCatch( @@ -85,23 +97,33 @@ export async function loader({ request }: LoaderFunctionArgs) { logger.error("Failed to link GitHub App installation", { error, }); - return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app"); + return consumingSession( + await redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app") + ); } - return redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully"); + return consumingSession( + await redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully") + ); } case "update": { - const [error] = await tryCatch(updateGitHubAppInstallation(callbackData.installation_id)); + const [error] = await tryCatch( + updateGitHubAppInstallation(callbackData.installation_id, organizationId) + ); if (error) { logger.error("Failed to update GitHub App installation", { error, }); - return redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App"); + return consumingSession( + await redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App") + ); } - return redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully"); + return consumingSession( + await redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully") + ); } case "request": { diff --git a/apps/webapp/app/services/gitHub.server.ts b/apps/webapp/app/services/gitHub.server.ts index e67895fe066..00ec2087f04 100644 --- a/apps/webapp/app/services/gitHub.server.ts +++ b/apps/webapp/app/services/gitHub.server.ts @@ -58,26 +58,32 @@ export async function linkGitHubAppInstallation( } /** - * Links a GitHub App installation to a Trigger organization + * Updates a GitHub App installation owned by the given Trigger organization */ -export async function updateGitHubAppInstallation(installationId: number): Promise { +export async function updateGitHubAppInstallation( + installationId: number, + organizationId: string +): Promise { if (!githubApp) { throw new Error("GitHub App is not enabled"); } - const octokit = await githubApp.getInstallationOctokit(installationId); - const { data: installation } = await octokit.rest.apps.getInstallation({ - installation_id: installationId, - }); - + // Scope the lookup to the caller's organization so a cross-tenant + // installation_id cannot update another org's record. Resolve ownership + // before calling GitHub to avoid burning the victim's API rate limit. const existingInstallation = await prisma.githubAppInstallation.findFirst({ - where: { appInstallationId: installationId }, + where: { appInstallationId: installationId, organizationId }, }); if (!existingInstallation) { throw new Error("GitHub App installation not found"); } + const octokit = await githubApp.getInstallationOctokit(installationId); + const { data: installation } = await octokit.rest.apps.getInstallation({ + installation_id: installationId, + }); + const repositorySelection = installation.repository_selection === "all" ? "ALL" : "SELECTED"; // repos are updated asynchronously via webhook events From 7cb35f1249f36ad60c13b2362a742900629052e9 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:15:15 +0100 Subject: [PATCH 08/15] fix(webapp,helm): require control-plane auth secrets and remove weak bundled-datastore defaults (#57) * fix(webapp): require control-plane auth secrets * fix(helm): remove weak default credentials for bundled datastores Empty the shipped postgres/clickhouse/minio/registry credential defaults so an unconfigured install fails closed instead of booting with publicly-known passwords. Add fail-closed validation guards for the bundled (deploy=true) datastores and feed CI throwaway render-time values so chart linting stays green. * fix(webapp): reject known-insecure control-plane secret defaults Add a blocklist refine to PROVIDER_SECRET, COORDINATOR_SECRET and MANAGED_WORKER_SECRET so the webapp rejects the publicly-known placeholder strings at boot, matching the value the coordinator already refuses at startup. Add the three now-required vars to the root .env.example and align the supervisor MANAGED_WORKER_SECRET example with the Docker example so local setups authenticate consistently. * fix(helm): fail closed on missing control-plane secrets and fix datastore guard guidance Blank the bundled provider/coordinator/managed-worker secret defaults and add a render-time guard so an unconfigured install fails closed instead of shipping repo-published tokens, matching the datastore fail-closed pattern. Require both postgres.auth.postgresPassword and postgres.auth.password, and point the Postgres/ClickHouse guard messages at the key paths the webapp actually consumes so operators are not sent to a crash loop. Add placeholder credentials to values-production-example.yaml and the CI lint values so the documented install paths still render, and update the README to reflect that a values file with credentials is now required. * test(webapp): provide required control-plane secrets in e2e harness The e2e webapp harness (startTestServer) spawns the production webapp bundle, which now fails closed when PROVIDER_SECRET / COORDINATOR_SECRET / MANAGED_WORKER_SECRET are unset. Supply strong test values (not the blocklisted known-insecure defaults) so the webapp boots and the E2E Tests: Webapp check passes. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(webapp,hosting): fail closed on empty app secrets and stop shipping working ones Extend the fail-closed convention to the two remaining app secrets and the Docker Compose self-host path: - env.server: SESSION_SECRET and MAGIC_LINK_SECRET now require .min(1), so an empty value fails the schema parse at boot rather than running with an empty secret (SESSION_SECRET is the JWT signing key) or falling through to the emailAuth runtime guard. - hosting/docker/.env.example and apps/supervisor/.env.example: ship the crypto/auth secrets empty with a short 'generate with openssl rand -hex 16' comment instead of the previous working public values. A bare 'cp .env.example .env && docker compose up' now fails closed on the first missing secret; the documented setup already generates all six. - Tests: cover empty and unset rejection for all five required secrets. Addresses SEC-387 / GHSA-pqxw-g93w-hj9x (the Docker path the earlier commits in this PR did not cover). * fix(webapp,helm): reject published application secret defaults * fix(helm): reject unsupported bundled datastore secret references * format * .env * feat(helm): auto-generate app and control-plane secrets instead of failing closed Leave secrets.* empty to have the chart generate a strong value on first install, retained across upgrades via lookup so ENCRYPTION_KEY/SESSION_SECRET are never rotated. Explicit values and existingSecret still win. Removes the now-redundant fail-closed guards for these secrets (the webapp still rejects known-insecure values at startup). * feat(hosting): add docker generate-secrets.sh to fill self-hosting secrets Fills empty required secrets in .env with openssl-random values. Safe to re-run: never overwrites an already-set value, so restarts and re-runs don't rotate ENCRYPTION_KEY/SESSION_SECRET. Docs updated to run it during setup. * ci(helm): lint and render release-helm with ci lint-values The bundled-datastore passwords are now fail-closed, so a values.yaml-only lint/template render fails. Supply ci/lint-values.yaml like the prerelease workflow already does, keeping the chart release pipeline green. * feat(webapp): allow opting out of the insecure-default secret blocklist ALLOW_INSECURE_DEFAULT_SECRETS lets a deployment boot while still using a known-published default it cannot safely rotate yet (e.g. ENCRYPTION_KEY protects existing data, SESSION_SECRET rotation logs everyone out). It only bypasses the known-insecure blocklist; the min-length and 32-byte ENCRYPTION_KEY checks still apply. A loud warning is logged at boot naming any secret still on a published default. * feat(hosting): auto-generate bundled-datastore passwords for docker self-hosting Remove the shipped weak defaults for postgres/clickhouse/minio/registry. generate-secrets.sh now fills each datastore password (openssl rand), and for the registry writes a matching bcrypt htpasswd via docker. Connection URLs are derived from the single password var by compose interpolation, so server and client always match; datastore services fail closed with a helpful message if a password is unset. Never clobbers an existing value - use --force to rotate. * docs(self-hosting): document secret auto-generation, rotation gotchas, and the insecure-default opt-out Note that generate-secrets.sh also fills the bundled datastore passwords; add ALLOW_INSECURE_DEFAULT_SECRETS to the webapp env list; add a Helm secret generation/rotation section covering upgrade retention, the GitOps regeneration caveat, and the lack of a clean ENCRYPTION_KEY migration. * feat(helm): auto-generate bundled-datastore passwords Generate the postgres/clickhouse/minio passwords once into a chart-managed datastore Secret (retained across upgrades via lookup) and point each bundled subchart at it via auth.existingSecret; the webapp reads them back through secretKeyRef and $(VAR) URL interpolation, so server and client always match. The registry password is generated and retained inside secrets.yaml (consumed at render time by htpasswd + dockerconfigjson). Drops the deploy=true datastore password guards - a bare helm install now renders and deploys with strong, unique credentials, and explicit values or an existingSecret still win. * fix(helm): start the s2 container via args, not command The s2 image ENTRYPOINT is ["./s2"] with the subcommand/flags as CMD. Setting Kubernetes command overrode the entrypoint and tried to exec "lite" directly, so the pod failed with StartError and realtime streams v2 (the default) never came up. Passing them as args preserves the entrypoint (./s2 lite ...). Verified live in kind: the s2 pod now reaches Running. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: nicktrn <55853254+nicktrn@users.noreply.github.com> --- .env.example | 1 + .github/workflows/helm-prerelease.yml | 4 +- .github/workflows/release-helm.yml | 4 +- .../require-webapp-auth-secrets.md | 6 + apps/supervisor/.env.example | 4 +- apps/webapp/app/env.server.ts | 57 ++++++++- apps/webapp/test/env.server.test.ts | 84 +++++++++++++ apps/webapp/test/registryConfig.test.ts | 3 + apps/webapp/test/setup.ts | 3 + docs/self-hosting/docker.mdx | 20 +++- docs/self-hosting/env/webapp.mdx | 3 +- docs/self-hosting/kubernetes.mdx | 26 ++++- hosting/docker/.env.example | 56 +++++---- hosting/docker/generate-secrets.sh | 110 ++++++++++++++++++ hosting/docker/registry/auth.htpasswd | 1 - hosting/docker/webapp/docker-compose.yml | 22 ++-- hosting/k8s/helm/README.md | 27 +++-- hosting/k8s/helm/ci/lint-values.yaml | 26 +++++ hosting/k8s/helm/templates/NOTES.txt | 20 +--- hosting/k8s/helm/templates/_helpers.tpl | 35 +++++- .../k8s/helm/templates/datastore-secret.yaml | 39 +++++++ hosting/k8s/helm/templates/electric.yaml | 7 ++ hosting/k8s/helm/templates/s2.yaml | 5 +- hosting/k8s/helm/templates/secrets.yaml | 30 +++-- .../templates/validate-external-config.yaml | 17 ++- hosting/k8s/helm/templates/webapp.yaml | 23 ++++ .../k8s/helm/values-production-example.yaml | 9 ++ hosting/k8s/helm/values.yaml | 66 +++++++---- .../testcontainers/src/webapp.ts | 5 + 29 files changed, 602 insertions(+), 111 deletions(-) create mode 100644 .server-changes/require-webapp-auth-secrets.md create mode 100644 apps/webapp/test/env.server.test.ts create mode 100755 hosting/docker/generate-secrets.sh create mode 100644 hosting/k8s/helm/ci/lint-values.yaml create mode 100644 hosting/k8s/helm/templates/datastore-secret.yaml diff --git a/.env.example b/.env.example index f37a1a5194c..901fbd7b3e0 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ SESSION_SECRET=abcdef1234 MAGIC_LINK_SECRET=abcdef1234 ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal +MANAGED_WORKER_SECRET=abcdef1234 # Must match the supervisor's MANAGED_WORKER_SECRET LOGIN_ORIGIN=http://localhost:3030 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public # This sets the URL used for direct connections to the database and should only be needed in limited circumstances diff --git a/.github/workflows/helm-prerelease.yml b/.github/workflows/helm-prerelease.yml index 37b909e415e..7d49868eeaf 100644 --- a/.github/workflows/helm-prerelease.yml +++ b/.github/workflows/helm-prerelease.yml @@ -52,12 +52,14 @@ jobs: - name: Lint Helm Chart run: | - helm lint ./hosting/k8s/helm/ + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml - name: Render templates run: | helm template test-release ./hosting/k8s/helm/ \ --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ --output-dir ./helm-output - name: Validate manifests diff --git a/.github/workflows/release-helm.yml b/.github/workflows/release-helm.yml index a5afd8b1b24..0ce968ebc47 100644 --- a/.github/workflows/release-helm.yml +++ b/.github/workflows/release-helm.yml @@ -47,12 +47,14 @@ jobs: - name: Lint Helm Chart run: | - helm lint ./hosting/k8s/helm/ + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml - name: Render templates run: | helm template test-release ./hosting/k8s/helm/ \ --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ --output-dir ./helm-output - name: Validate manifests diff --git a/.server-changes/require-webapp-auth-secrets.md b/.server-changes/require-webapp-auth-secrets.md new file mode 100644 index 00000000000..52cb61d871a --- /dev/null +++ b/.server-changes/require-webapp-auth-secrets.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: breaking +--- + +Self-hosted deployments no longer ship shared default credentials; fresh installs generate their own. If yours still uses a previously published default, set a unique value before upgrading, or set `ALLOW_INSECURE_DEFAULT_SECRETS=true` to keep booting while you migrate. diff --git a/apps/supervisor/.env.example b/apps/supervisor/.env.example index 4355a4eea30..f71cb88f3eb 100644 --- a/apps/supervisor/.env.example +++ b/apps/supervisor/.env.example @@ -1,8 +1,8 @@ # This needs to match the token of the worker group you want to connect to TRIGGER_WORKER_TOKEN= -# This needs to match the MANAGED_WORKER_SECRET env var on the webapp -MANAGED_WORKER_SECRET=managed-secret +# Must match the webapp's MANAGED_WORKER_SECRET. Generate with: openssl rand -hex 16 +MANAGED_WORKER_SECRET= # Point this at the webapp in prod TRIGGER_API_URL=http://localhost:3030 diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index de36b225398..53b5edf0a46 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -85,6 +85,29 @@ const S2EnvSchema = z.preprocess( ]) ); +// Previously published secret values must never be accepted, including when +// an existing deployment or external secret manager still supplies one. +const INSECURE_SECRET_VALUES = [ + "managed-secret", + "2818143646516f6fffd707b36f334bbb", + "44da78b7bbb0dfe709cf38931d25dcdd", + "f686147ab967943ebbe9ed3b496e465a", + "447c29678f9eaf289e9c4b70d3dd8a7f", +]; + +// Escape hatch for deployments that can't rotate a published default yet (e.g. +// ENCRYPTION_KEY protects existing data). Read raw: a refine can't see the +// sibling parsed flag. +const allowInsecureDefaultSecrets = ["true", "1"].includes( + (process.env.ALLOW_INSECURE_DEFAULT_SECRETS ?? "").toLowerCase().trim() +); + +const isNotInsecureSecret = (value: string) => + allowInsecureDefaultSecrets || !INSECURE_SECRET_VALUES.includes(value); + +const INSECURE_SECRET_MESSAGE = + "must not be a known-insecure published default; set a strong, unique value. If you cannot rotate it yet (e.g. it protects existing encrypted data or active sessions), set ALLOW_INSECURE_DEFAULT_SECRETS=1 to boot while you migrate."; + const EnvironmentSchema = z .object({ NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]), @@ -188,14 +211,15 @@ const EnvironmentSchema = z // Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES). CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(), CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(), - SESSION_SECRET: z.string(), - MAGIC_LINK_SECRET: z.string(), + SESSION_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), + MAGIC_LINK_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), ENCRYPTION_KEY: z .string() .refine( (val) => Buffer.from(val, "utf8").length === 32, "ENCRYPTION_KEY must be exactly 32 bytes" - ), + ) + .refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), WHITELISTED_EMAILS: z .string() .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") @@ -684,7 +708,11 @@ const EnvironmentSchema = z EVENTS_LOAD_SHEDDING_THRESHOLD: z.coerce.number().int().default(100000), EVENTS_LOAD_SHEDDING_ENABLED: z.string().default("1"), - MANAGED_WORKER_SECRET: z.string().default("managed-secret"), + MANAGED_WORKER_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), + + // Allow booting with a known-insecure published default secret. Temporary + // bridge for deployments that can't rotate yet; rotate as soon as possible. + ALLOW_INSECURE_DEFAULT_SECRETS: BoolEnv.default(false), // Tenant scoping on worker actions is header-driven (folded into the engine snapshot read) and // needs no flag. This is only the no-header fallback: when "1", a worker action on a run created @@ -2113,3 +2141,24 @@ const EnvironmentSchema = z export type Environment = z.infer; export const env = EnvironmentSchema.parse(process.env); + +if (env.ALLOW_INSECURE_DEFAULT_SECRETS) { + const insecure = ( + [ + ["SESSION_SECRET", env.SESSION_SECRET], + ["MAGIC_LINK_SECRET", env.MAGIC_LINK_SECRET], + ["ENCRYPTION_KEY", env.ENCRYPTION_KEY], + ["MANAGED_WORKER_SECRET", env.MANAGED_WORKER_SECRET], + ] as const + ) + .filter(([, value]) => INSECURE_SECRET_VALUES.includes(value)) + .map(([name]) => name); + + if (insecure.length > 0) { + console.warn( + `⚠️ ALLOW_INSECURE_DEFAULT_SECRETS is enabled and these secrets still use a known-insecure published default: ${insecure.join( + ", " + )}. This is insecure - rotate them as soon as you can.` + ); + } +} diff --git a/apps/webapp/test/env.server.test.ts b/apps/webapp/test/env.server.test.ts new file mode 100644 index 00000000000..0fe13efbe26 --- /dev/null +++ b/apps/webapp/test/env.server.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const originalEnv = process.env; + +const requiredEnv = { + NODE_ENV: "test", + DATABASE_URL: "postgresql://test:test@localhost:5432/test", + DIRECT_URL: "postgresql://test:test@localhost:5432/test", + SESSION_SECRET: "test-session-secret", + MAGIC_LINK_SECRET: "test-magic-link-secret", + ENCRYPTION_KEY: "test-encryption-keeeeey-32-bytes", + CLICKHOUSE_URL: "http://localhost:8123", + DEPLOY_REGISTRY_HOST: "registry.example.com", + MANAGED_WORKER_SECRET: "test-managed-worker-secret", +}; + +describe("webapp environment secrets", () => { + afterEach(() => { + process.env = originalEnv; + vi.resetModules(); + }); + + it.each(["SESSION_SECRET", "MAGIC_LINK_SECRET", "ENCRYPTION_KEY", "MANAGED_WORKER_SECRET"])( + "requires %s to be explicitly set", + async (key) => { + process.env = { ...requiredEnv }; + delete process.env[key]; + + await expect(import("../app/env.server")).rejects.toThrow(key); + } + ); + + it.each(["SESSION_SECRET", "MAGIC_LINK_SECRET", "ENCRYPTION_KEY", "MANAGED_WORKER_SECRET"])( + "rejects an empty %s", + async (key) => { + process.env = { ...requiredEnv, [key]: "" }; + + await expect(import("../app/env.server")).rejects.toThrow(key); + } + ); + + it.each([ + ["SESSION_SECRET", "2818143646516f6fffd707b36f334bbb"], + ["MAGIC_LINK_SECRET", "44da78b7bbb0dfe709cf38931d25dcdd"], + ["ENCRYPTION_KEY", "f686147ab967943ebbe9ed3b496e465a"], + ["MANAGED_WORKER_SECRET", "managed-secret"], + ["MANAGED_WORKER_SECRET", "447c29678f9eaf289e9c4b70d3dd8a7f"], + ])("rejects the known-insecure default value for %s", async (key, insecureValue) => { + process.env = { ...requiredEnv, [key]: insecureValue }; + + await expect(import("../app/env.server")).rejects.toThrow(key); + }); + + it("accepts explicitly configured secrets", async () => { + process.env = { ...requiredEnv }; + + const { env } = await import("../app/env.server"); + + expect(env.SESSION_SECRET).toBe(requiredEnv.SESSION_SECRET); + expect(env.MAGIC_LINK_SECRET).toBe(requiredEnv.MAGIC_LINK_SECRET); + expect(env.ENCRYPTION_KEY).toBe(requiredEnv.ENCRYPTION_KEY); + expect(env.MANAGED_WORKER_SECRET).toBe(requiredEnv.MANAGED_WORKER_SECRET); + }); + + it("allows a known-insecure default when ALLOW_INSECURE_DEFAULT_SECRETS is set", async () => { + process.env = { + ...requiredEnv, + ALLOW_INSECURE_DEFAULT_SECRETS: "1", + ENCRYPTION_KEY: "f686147ab967943ebbe9ed3b496e465a", + MANAGED_WORKER_SECRET: "managed-secret", + }; + + const { env } = await import("../app/env.server"); + + expect(env.ENCRYPTION_KEY).toBe("f686147ab967943ebbe9ed3b496e465a"); + expect(env.MANAGED_WORKER_SECRET).toBe("managed-secret"); + }); + + it("still rejects an empty secret even with ALLOW_INSECURE_DEFAULT_SECRETS", async () => { + process.env = { ...requiredEnv, ALLOW_INSECURE_DEFAULT_SECRETS: "1", SESSION_SECRET: "" }; + + await expect(import("../app/env.server")).rejects.toThrow("SESSION_SECRET"); + }); +}); diff --git a/apps/webapp/test/registryConfig.test.ts b/apps/webapp/test/registryConfig.test.ts index 341761d0115..b9a8c966123 100644 --- a/apps/webapp/test/registryConfig.test.ts +++ b/apps/webapp/test/registryConfig.test.ts @@ -10,6 +10,9 @@ describe("getRegistryConfig", () => { MAGIC_LINK_SECRET: "test-magic-link-secret", ENCRYPTION_KEY: "test-encryption-keeeeey-32-bytes", CLICKHOUSE_URL: "http://localhost:8123", + PROVIDER_SECRET: "test-provider-secret", + COORDINATOR_SECRET: "test-coordinator-secret", + MANAGED_WORKER_SECRET: "test-managed-worker-secret", }; beforeEach(() => { diff --git a/apps/webapp/test/setup.ts b/apps/webapp/test/setup.ts index db07cb1402a..521a9624b7d 100644 --- a/apps/webapp/test/setup.ts +++ b/apps/webapp/test/setup.ts @@ -13,6 +13,9 @@ config({ path: path.resolve(__dirname, "../.env") }); // the pair — the ioredis mock below forces lazyConnect, so nothing ever dials. process.env.REDIS_HOST ??= "localhost"; process.env.REDIS_PORT ??= "6379"; +process.env.PROVIDER_SECRET ??= "test-provider-secret"; +process.env.COORDINATOR_SECRET ??= "test-coordinator-secret"; +process.env.MANAGED_WORKER_SECRET ??= "test-managed-worker-secret"; // Worker singletons construct a RedisWorker at import time whose ioredis client // connects eagerly, so any test importing the service graph opens real Redis diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx index ead4281a6df..6c8de292f17 100644 --- a/docs/self-hosting/docker.mdx +++ b/docs/self-hosting/docker.mdx @@ -75,12 +75,23 @@ git clone --depth=1 https://github.com/triggerdotdev/trigger.dev cd trigger.dev/hosting/docker ``` -2. Create a `.env` file +2. Create a `.env` file and generate secrets ```bash cp .env.example .env + +# Fills the required secrets in .env with strong, unique values. +# Safe to re-run - it never overwrites a secret you've already set. +./generate-secrets.sh ``` + + The stack ships no working default credentials. `generate-secrets.sh` fills the + application secrets and the bundled datastore passwords with strong, unique values. + Keep them safe - rotating the encryption key or session secret later will invalidate + existing sessions and encrypted data. + + 3. Start the webapp ```bash @@ -130,6 +141,13 @@ docker compose up -d 4. Configure the supervisor using the [environment variables](/self-hosting/env/supervisor) in your `.env` file, including the [worker token](#worker-token). + + For a split webapp/worker setup, set `MANAGED_WORKER_SECRET` on the worker to + the **same** value as the webapp's `MANAGED_WORKER_SECRET`. Don't run + `generate-secrets.sh` on the worker host - it would create a mismatched value + and the worker would fail to authenticate. + + 5. Apply the changes: ```bash diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 1f629718b52..09416f0f38f 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -11,7 +11,8 @@ mode: "wide" | `SESSION_SECRET` | Yes | — | Session encryption secret. Run: `openssl rand -hex 16` | | `MAGIC_LINK_SECRET` | Yes | — | Magic link encryption secret. Run: `openssl rand -hex 16` | | `ENCRYPTION_KEY` | Yes | — | Secret store encryption key. Run: `openssl rand -hex 16` | -| `MANAGED_WORKER_SECRET` | No | managed-secret | Managed worker secret. Should be changed and match supervisor. | +| `MANAGED_WORKER_SECRET` | Yes | — | Managed worker secret. Must be set and match supervisor. Run: `openssl rand -hex 32` | +| `ALLOW_INSECURE_DEFAULT_SECRETS` | No | false | Boot even if a secret is still a known-insecure published default. Temporary escape hatch for values you can't safely rotate yet (see [Secret generation and rotation](/self-hosting/kubernetes#secret-generation-and-rotation)). | | **Domains & ports** | | | | | `REMIX_APP_PORT` | No | 3030 | Remix app port. | | `APP_ORIGIN` | Yes | http://localhost:3030 | App origin URL. | diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx index 5e24b675f4a..33d06f3086c 100644 --- a/docs/self-hosting/kubernetes.mdx +++ b/docs/self-hosting/kubernetes.mdx @@ -121,8 +121,10 @@ The default values are insecure and are only suitable for testing. You will need Create a `values-custom.yaml` file to override the defaults. For example: ```yaml -# Generate new secrets with `openssl rand -hex 16` -# WARNING: You should probably use an existingSecret instead +# Leave these unset to have the chart auto-generate strong values on first +# install (retained across upgrades). Set them explicitly only if you need to +# control the value - e.g. sharing MANAGED_WORKER_SECRET with an external +# supervisor - or use an existingSecret. secrets: enabled: true sessionSecret: "your-32-char-hex-secret-1" @@ -133,6 +135,8 @@ secrets: # - SESSION_SECRET # - MAGIC_LINK_SECRET # - ENCRYPTION_KEY +# - PROVIDER_SECRET +# - COORDINATOR_SECRET # - MANAGED_WORKER_SECRET # - OBJECT_STORE_ACCESS_KEY_ID # - OBJECT_STORE_SECRET_ACCESS_KEY @@ -176,6 +180,24 @@ helm upgrade -n trigger --install trigger \ -f values-custom.yaml ``` +### Secret generation and rotation + +Application, control-plane, and bundled-datastore secrets left unset are generated on +first install and **retained across `helm upgrade`** - they are never rotated +automatically, so sessions, encrypted data, and datastore volumes survive upgrades. + + + GitOps tools that render with `helm template` (e.g. Argo CD) cannot read the existing + secret, so they regenerate these values on every sync - which rotates them. If you + deploy via GitOps, always supply your own `secrets.existingSecret` (and datastore + credentials) so nothing is generated in-cluster. + + +There is no clean migration for a compromised `ENCRYPTION_KEY`: changing it makes +existing encrypted data unreadable. If a deployment is still running a previously +published default and cannot rotate yet, set `ALLOW_INSECURE_DEFAULT_SECRETS=true` on +the webapp to keep booting while you plan a migration. + ### Extra env You can set extra environment variables on all services. For example: diff --git a/hosting/docker/.env.example b/hosting/docker/.env.example index 383b6f79dfe..4c7cf11bc70 100644 --- a/hosting/docker/.env.example +++ b/hosting/docker/.env.example @@ -3,13 +3,16 @@ # - You should change them to suit your needs, especially the secrets # - See the docs for more information: https://trigger.dev/docs/self-hosting/overview -# Secrets -# - Do NOT use these defaults in production -# - Generate your own by running `openssl rand -hex 16` for each secret -SESSION_SECRET=2818143646516f6fffd707b36f334bbb -MAGIC_LINK_SECRET=44da78b7bbb0dfe709cf38931d25dcdd -ENCRYPTION_KEY=f686147ab967943ebbe9ed3b496e465a -MANAGED_WORKER_SECRET=447c29678f9eaf289e9c4b70d3dd8a7f +# Secrets — REQUIRED, no defaults. The stack will not boot until each is set to a unique value. +# Generate each with: openssl rand -hex 16 +SESSION_SECRET= +MAGIC_LINK_SECRET= +ENCRYPTION_KEY= +# These authenticate the internal control-plane connections. Generate each with: openssl rand -hex 16 +# COORDINATOR_SECRET must match the coordinator's PLATFORM_SECRET; MANAGED_WORKER_SECRET the supervisor's. +PROVIDER_SECRET= +COORDINATOR_SECRET= +MANAGED_WORKER_SECRET= # Worker token # - This is the token for the worker to connect to the webapp @@ -24,13 +27,14 @@ MANAGED_WORKER_SECRET=447c29678f9eaf289e9c4b70d3dd8a7f # OTEL_EXPORTER_OTLP_ENDPOINT=https://trigger.example.com/otel # Postgres -# - Do NOT use these defaults in production -# - Especially if you decide to expose the database to the internet +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it (or openssl rand -hex 16). +# - DATABASE_URL / DIRECT_URL are derived from POSTGRES_PASSWORD automatically - only set them +# below to point at an external Postgres (POSTGRES_PASSWORD is then unused). # POSTGRES_USER=postgres -POSTGRES_PASSWORD=unsafe-postgres-pw +POSTGRES_PASSWORD= # POSTGRES_DB=postgres -DATABASE_URL=postgresql://postgres:unsafe-postgres-pw@postgres:5432/main?schema=public&sslmode=disable -DIRECT_URL=postgresql://postgres:unsafe-postgres-pw@postgres:5432/main?schema=public&sslmode=disable +# DATABASE_URL=postgresql://user:password@host:5432/main?schema=public&sslmode=disable +# DIRECT_URL=postgresql://user:password@host:5432/main?schema=public&sslmode=disable # Trigger image tag # - This is the version of the webapp and worker images to use, they should be locked to a specific version in production @@ -59,19 +63,21 @@ DEV_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8030/otel # NODE_MAX_OLD_SPACE_SIZE=8192 # ClickHouse -# - Do NOT use these defaults in production +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it. +# - CLICKHOUSE_URL / RUN_REPLICATION_CLICKHOUSE_URL are derived from CLICKHOUSE_PASSWORD +# automatically - only set them below to point at an external ClickHouse. CLICKHOUSE_USER=default -CLICKHOUSE_PASSWORD=password -CLICKHOUSE_URL=http://default:password@clickhouse:8123?secure=false -RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@clickhouse:8123 +CLICKHOUSE_PASSWORD= +# CLICKHOUSE_URL=http://user:password@host:8123?secure=false +# RUN_REPLICATION_CLICKHOUSE_URL=http://user:password@host:8123 # Docker Registry -# - When testing locally, the default values should be fine -# - When deploying to production, you will have to change these, especially the password and URL +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it - it also writes +# the matching registry/auth.htpasswd (bcrypt) the bundled registry authenticates against. # - See the docs for more information: https://trigger.dev/docs/self-hosting/docker#registry-setup DOCKER_REGISTRY_URL=localhost:5000 DOCKER_REGISTRY_USERNAME=registry-user -DOCKER_REGISTRY_PASSWORD=very-secure-indeed +DOCKER_REGISTRY_PASSWORD= # When using an external registry you will have to change this # On Docker Hub it should generally be the same as your username DOCKER_REGISTRY_NAMESPACE=trigger @@ -80,8 +86,10 @@ DOCKER_REGISTRY_NAMESPACE=trigger # - You need to log into the Minio dashboard and create a bucket called "packets" # - See the docs for more information: https://trigger.dev/docs/self-hosting/docker#object-storage # Default provider (backward compatible - no protocol prefix) +# - Secret access key is REQUIRED, no default. Run ./generate-secrets.sh to fill it. +# - For the bundled MinIO, these ARE its root credentials (MINIO_ROOT_USER/PASSWORD derive from them). OBJECT_STORE_ACCESS_KEY_ID=admin -OBJECT_STORE_SECRET_ACCESS_KEY=very-safe-password +OBJECT_STORE_SECRET_ACCESS_KEY= # You will have to uncomment and configure this for production # OBJECT_STORE_BASE_URL=http://localhost:9000 # OBJECT_STORE_REGION=auto @@ -100,11 +108,11 @@ OBJECT_STORE_SECRET_ACCESS_KEY=very-safe-password # OBJECT_STORE_R2_SECRET_ACCESS_KEY= # OBJECT_STORE_R2_REGION=auto # OBJECT_STORE_R2_SERVICE=s3 -# Credentials to access the Minio dashboard at http://localhost:9001 -# - You should change these credentials and not use them for the `OBJECT_STORE_` env vars above -# - Instead, setup a non-root user with access the "packets" bucket +# Minio dashboard at http://localhost:9001 +# - The bundled Minio's root credentials default to OBJECT_STORE_ACCESS_KEY_ID / OBJECT_STORE_SECRET_ACCESS_KEY. +# - For production, set a separate root user here and create a non-root user scoped to the "packets" bucket for OBJECT_STORE_*. # MINIO_ROOT_USER=admin -# MINIO_ROOT_PASSWORD=very-safe-password +# MINIO_ROOT_PASSWORD= # Realtime streams # - Realtime streams power AI-agent token streaming and run streams diff --git a/hosting/docker/generate-secrets.sh b/hosting/docker/generate-secrets.sh new file mode 100755 index 00000000000..679dc129cfb --- /dev/null +++ b/hosting/docker/generate-secrets.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# Generate strong, unique secrets and datastore passwords into the self-hosting .env. +# +# Safe to re-run: only fills values that are missing or empty, and never overwrites +# one that is already set. Rotating a live secret would orphan encrypted data, log +# everyone out, or break an already-initialised datastore volume - so rotation is +# opt-in only, via --force (which WILL break existing data/sessions). +set -eu + +FORCE=0 +for arg in "$@"; do + case "$arg" in + -f | --force) FORCE=1 ;; + -h | --help) + echo "Usage: $0 [--force] [env-file]" + echo " --force Regenerate every secret, overwriting existing values." + echo " WARNING: rotates live secrets - breaks encrypted data, sessions," + echo " and already-initialised datastore volumes." + exit 0 + ;; + *) env_file="$arg" ;; + esac +done + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +env_file="${env_file:-$script_dir/.env}" +env_example="$script_dir/.env.example" +htpasswd_file="$script_dir/registry/auth.htpasswd" + +# openssl rand -hex 16 -> 32 hex chars: URL-safe (no @ : / etc. to break connection +# strings) and satisfies the webapp's exact-32-byte ENCRYPTION_KEY check. +gen() { openssl rand -hex 16; } + +if [ ! -f "$env_file" ]; then + cp "$env_example" "$env_file" + echo "Created $(basename "$env_file") from $(basename "$env_example")" +fi + +sed_inplace() { + if [ "$(uname)" = "Darwin" ]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + +# Value of KEY= in the env file, trailing inline comment/whitespace stripped, so +# "KEY=" and "KEY= # todo" both read as unset. +current_value() { + grep -E "^$1=" "$env_file" | head -n1 | cut -d= -f2- \ + | sed -e 's/[[:space:]]*#.*$//' -e 's/[[:space:]]*$//' +} + +set_var() { + key="$1" + value="$2" + if grep -qE "^$key=" "$env_file"; then + sed_inplace -e "s|^$key=.*|$key=$value|" "$env_file" + else + printf '%s=%s\n' "$key" "$value" >>"$env_file" + fi +} + +# Fill KEY with a fresh secret unless it already has a value (respecting --force). +# Returns 0 if it wrote, 1 if it skipped. +fill() { + key="$1" + if [ "$FORCE" -eq 0 ] && [ -n "$(current_value "$key")" ]; then + return 1 + fi + set_var "$key" "$(gen)" + echo "Generated $key" + return 0 +} + +generated=0 + +# App + control-plane secrets, and the bundled-datastore passwords. All plain +# values consumed directly (or, for datastores, woven into the connection URLs by +# docker-compose interpolation - see .env.example). +for key in \ + SESSION_SECRET MAGIC_LINK_SECRET ENCRYPTION_KEY \ + PROVIDER_SECRET COORDINATOR_SECRET MANAGED_WORKER_SECRET \ + POSTGRES_PASSWORD CLICKHOUSE_PASSWORD OBJECT_STORE_SECRET_ACCESS_KEY; do + if fill "$key"; then generated=$((generated + 1)); fi +done + +# Registry is special: the bundled registry authenticates against a bcrypt htpasswd +# file, so when we set its password we must regenerate that file to match. bcrypt is +# the only hash the registry accepts, and openssl can't produce it - use httpd's +# htpasswd via docker (already required for this stack). +if [ "$FORCE" -eq 1 ] || [ -z "$(current_value DOCKER_REGISTRY_PASSWORD)" ]; then + registry_user=$(current_value DOCKER_REGISTRY_USERNAME) + registry_user=${registry_user:-registry-user} + registry_pass=$(gen) + if ! command -v docker >/dev/null 2>&1; then + echo "ERROR: docker is required to hash the registry password (bcrypt). Install docker and re-run." >&2 + exit 1 + fi + docker run --rm httpd:2 htpasswd -Bbn "$registry_user" "$registry_pass" >"$htpasswd_file" + set_var DOCKER_REGISTRY_PASSWORD "$registry_pass" + echo "Generated DOCKER_REGISTRY_PASSWORD (and wrote $(basename "$htpasswd_file"))" + generated=$((generated + 1)) +fi + +if [ "$generated" -eq 0 ]; then + echo "All secrets already set in $(basename "$env_file"); nothing to do. Use --force to rotate." +else + echo "Wrote $generated secret(s) to $(basename "$env_file")." +fi diff --git a/hosting/docker/registry/auth.htpasswd b/hosting/docker/registry/auth.htpasswd index 3f659261e48..e69de29bb2d 100644 --- a/hosting/docker/registry/auth.htpasswd +++ b/hosting/docker/registry/auth.htpasswd @@ -1 +0,0 @@ -registry-user:$2y$05$6ingYqw0.3j13dxHY4w3neMSvKhF3pvRmc0AFifScWsVA9JpuLwNK diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 611ce4939e3..8d133cbeea5 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -54,11 +54,13 @@ services: REALTIME_STREAMS_S2_ENDPOINT: ${REALTIME_STREAMS_S2_ENDPOINT:-http://s2/v1} REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS: ${REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS:-true} REALTIME_STREAMS_S2_ACCESS_TOKEN: ${REALTIME_STREAMS_S2_ACCESS_TOKEN:-} - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} - DIRECT_URL: ${DIRECT_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} + DIRECT_URL: ${DIRECT_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} SESSION_SECRET: ${SESSION_SECRET} MAGIC_LINK_SECRET: ${MAGIC_LINK_SECRET} ENCRYPTION_KEY: ${ENCRYPTION_KEY} + PROVIDER_SECRET: ${PROVIDER_SECRET} + COORDINATOR_SECRET: ${COORDINATOR_SECRET} MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET} REDIS_HOST: redis REDIS_PORT: 6379 @@ -78,11 +80,11 @@ services: TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: bootstrap TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration - CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} + CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123?secure=false} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} - RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:password@clickhouse:8123} + RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123} RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info} # Limits # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB @@ -109,7 +111,7 @@ services: - wal_level=logical environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env - run ./generate-secrets.sh} POSTGRES_DB: ${POSTGRES_DB:-postgres} healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] @@ -144,7 +146,7 @@ services: networks: - webapp environment: - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true ELECTRIC_USAGE_REPORTING: false healthcheck: @@ -163,7 +165,7 @@ services: - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000 environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} - CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} + CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set CLICKHOUSE_PASSWORD in .env - run ./generate-secrets.sh} volumes: - clickhouse:/bitnami/clickhouse - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro @@ -181,7 +183,7 @@ services: "--user", "${CLICKHOUSE_USER:-default}", "--password", - "${CLICKHOUSE_PASSWORD:-password}", + "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1", ] @@ -224,8 +226,8 @@ services: volumes: - minio:/bitnami/minio/data environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-very-safe-password} + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-${OBJECT_STORE_ACCESS_KEY_ID:-admin}} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-${OBJECT_STORE_SECRET_ACCESS_KEY:?Set OBJECT_STORE_SECRET_ACCESS_KEY in .env - run ./generate-secrets.sh}} MINIO_DEFAULT_BUCKETS: packets MINIO_BROWSER: "on" healthcheck: diff --git a/hosting/k8s/helm/README.md b/hosting/k8s/helm/README.md index 33e64a93a23..79c49138215 100644 --- a/hosting/k8s/helm/README.md +++ b/hosting/k8s/helm/README.md @@ -20,7 +20,10 @@ helm template trigger . --dependency-update ### Installation ```bash -# Deploy with default values (testing/development only) +# A bare install works: application, control-plane, and bundled-datastore +# secrets are auto-generated on first install (and retained across upgrades) +# when you don't set them. Provide a values file to pin any of them or to +# point at external datastores. helm install trigger . # Deploy to specific namespace @@ -59,24 +62,28 @@ npx trigger.dev@latest deploy --push ### Secrets Configuration -**IMPORTANT**: The default secrets are for **TESTING ONLY** and must be changed for production. +**IMPORTANT**: The chart ships no working secret defaults. Application, control-plane, and bundled-datastore secrets (postgres/clickhouse/minio/registry) are **auto-generated per install** (and retained across upgrades) when left unset. You can still set them explicitly - e.g. to share `managedWorkerSecret` with an external supervisor, or to manage everything via `secrets.existingSecret`. Explicit values always win over generation. -#### Required Secrets +#### Auto-generated secrets -All secrets must be exactly **32 hexadecimal characters** (16 bytes): +Left unset, these are generated on first install and preserved on `helm upgrade` (rotating them would invalidate sessions and orphan encrypted data, so they are never regenerated once set): - `sessionSecret` - User authentication sessions -- `magicLinkSecret` - Passwordless login tokens +- `magicLinkSecret` - Passwordless login tokens - `encryptionKey` - Sensitive data encryption -- `managedWorkerSecret` - Worker authentication +- `managedWorkerSecret` - Worker authentication (set explicitly if an external supervisor must share it) +- `providerSecret` - Provider control-plane socket authentication +- `coordinatorSecret` - Coordinator control-plane socket authentication -#### Generate Production Secrets +#### Generating secrets yourself + +If you prefer to set them explicitly (e.g. to reuse across clusters): ```bash -for i in {1..4}; do openssl rand -hex 16; done +for i in {1..6}; do openssl rand -hex 16; done ``` -#### Configure Production Secrets +#### Configure secrets explicitly ```yaml # values-production.yaml @@ -85,6 +92,8 @@ secrets: magicLinkSecret: "your-generated-secret-2" encryptionKey: "your-generated-secret-3" managedWorkerSecret: "your-generated-secret-4" + providerSecret: "your-generated-secret-5" + coordinatorSecret: "your-generated-secret-6" objectStore: accessKeyId: "your-s3-access-key" secretAccessKey: "your-s3-secret-key" diff --git a/hosting/k8s/helm/ci/lint-values.yaml b/hosting/k8s/helm/ci/lint-values.yaml new file mode 100644 index 00000000000..967cc6c503a --- /dev/null +++ b/hosting/k8s/helm/ci/lint-values.yaml @@ -0,0 +1,26 @@ +# Throwaway credentials used ONLY to render the chart during CI (helm lint / helm template). +# The shipped values.yaml no longer contains working default credentials for the bundled +# datastores (see SEC-70): an unconfigured install now fails closed. CI must therefore +# supply dummy secrets here so template rendering can be validated. These values are never +# deployed and must never be used in a real environment. +secrets: + sessionSecret: "ci-render-only-session" + magicLinkSecret: "ci-render-only-magic-link" + encryptionKey: "ci-render-only-encryption-key" + providerSecret: "ci-render-only-provider" + coordinatorSecret: "ci-render-only-coordinator" + managedWorkerSecret: "ci-render-only-managed-worker" +postgres: + auth: + postgresPassword: "ci-render-only-postgres" + password: "ci-render-only-postgres" +clickhouse: + auth: + password: "ci-render-only-clickhouse" +s3: + auth: + rootPassword: "ci-render-only-minio" +registry: + deploy: true + auth: + password: "ci-render-only-registry" diff --git a/hosting/k8s/helm/templates/NOTES.txt b/hosting/k8s/helm/templates/NOTES.txt index e70d18dd74a..4571eb9f474 100644 --- a/hosting/k8s/helm/templates/NOTES.txt +++ b/hosting/k8s/helm/templates/NOTES.txt @@ -2,21 +2,11 @@ Thank you for installing {{ .Chart.Name }}. Your release is named {{ .Release.Name }}. -🔐 SECURITY WARNING: -{{- if or (eq .Values.secrets.sessionSecret "2818143646516f6fffd707b36f334bbb") (eq .Values.secrets.magicLinkSecret "44da78b7bbb0dfe709cf38931d25dcdd") (eq .Values.secrets.encryptionKey "f686147ab967943ebbe9ed3b496e465a") (eq .Values.secrets.managedWorkerSecret "447c29678f9eaf289e9c4b70d3dd8a7f") }} - You are using DEFAULT SECRETS which are NOT SECURE for production! - - For production deployments, generate new secrets: - 1. Run: openssl rand -hex 16 (repeat for each secret) - 2. Override in your values.yaml: - secrets: - sessionSecret: "your-new-32-char-hex-secret" - magicLinkSecret: "your-new-32-char-hex-secret" - encryptionKey: "your-new-32-char-hex-secret" - managedWorkerSecret: "your-new-32-char-hex-secret" -{{- else }} - Custom secrets detected - good for production deployment! -{{- end }} +🔐 SECURITY: + This chart ships no working secret defaults. Application and control-plane + secrets are auto-generated per install (and retained across upgrades) unless + you set them explicitly or supply secrets.existingSecret. Bundled-datastore + passwords must still be provided. To get started: diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl index 2ce273c0171..0ecc17b9893 100644 --- a/hosting/k8s/helm/templates/_helpers.tpl +++ b/hosting/k8s/helm/templates/_helpers.tpl @@ -141,7 +141,7 @@ PostgreSQL connection string (fallback when not using secrets) {{- if .Values.postgres.external.databaseUrl -}} {{ .Values.postgres.external.databaseUrl }} {{- else if .Values.postgres.deploy -}} -postgresql://{{ .Values.postgres.auth.username }}:{{ .Values.postgres.auth.password }}@{{ include "trigger-v4.postgres.hostname" . }}:5432/{{ .Values.postgres.auth.database }}?schema={{ .Values.postgres.connection.schema | default "public" }}&sslmode={{ .Values.postgres.connection.sslMode | default "prefer" }} +postgresql://{{ .Values.postgres.auth.username }}:$(POSTGRES_PASSWORD)@{{ include "trigger-v4.postgres.hostname" . }}:5432/{{ .Values.postgres.auth.database }}?schema={{ .Values.postgres.connection.schema | default "public" }}&sslmode={{ .Values.postgres.connection.sslMode | default "prefer" }} {{- end -}} {{- end }} @@ -439,7 +439,7 @@ hex-encoded password or percent-encode before storing in the Secret. {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }} +{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }} {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.external.secure -}} @@ -460,7 +460,7 @@ applies to the replication URL. {{- define "trigger-v4.clickhouse.replication.url" -}} {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}@{{ include "trigger-v4.clickhouse.hostname" . }}:8123 +{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123 {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- if .Values.clickhouse.external.existingSecret -}} @@ -511,6 +511,35 @@ Get the secrets name - either existing secret or generated name {{- end -}} {{- end }} +{{/* +Fixed name of the chart-managed datastore-credentials Secret. Fixed (not release- +scoped) because bundled Bitnami subcharts read it via `*.auth.existingSecret`, which +is set in values.yaml and cannot be templated - so the name must be a literal that +both this chart and those values agree on. One release per namespace. +*/}} +{{- define "trigger-v4.datastore.secretName" -}} +trigger-datastore +{{- end }} + +{{/* +Resolve a chart-managed secret: an explicit value wins; otherwise reuse the +value already stored in the cluster Secret so `helm upgrade` never rotates it +(rotating ENCRYPTION_KEY/SESSION_SECRET would orphan encrypted data and +invalidate every session); otherwise generate a fresh 32-hex-char value. +Note: `lookup` returns nothing during `helm template`/`--dry-run`, so those +render a throwaway value each time - only real installs/upgrades retain. +Args: dict "existingData" "key" "value" +*/}} +{{- define "trigger-v4.resolveSecret" -}} +{{- if .value -}} +{{- .value -}} +{{- else if (index .existingData .key) -}} +{{- index .existingData .key | b64dec -}} +{{- else -}} +{{- sha256sum (randAlphaNum 32) | trunc 32 -}} +{{- end -}} +{{- end }} + {{/* Registry connection details */}} diff --git a/hosting/k8s/helm/templates/datastore-secret.yaml b/hosting/k8s/helm/templates/datastore-secret.yaml new file mode 100644 index 00000000000..392a2e7af76 --- /dev/null +++ b/hosting/k8s/helm/templates/datastore-secret.yaml @@ -0,0 +1,39 @@ +{{/* +Chart-managed credentials for the bundled datastores (postgres/clickhouse/minio). +Each password is generated ONCE here (retained across upgrades via lookup) and +consumed in two places from this single source: the Bitnami subchart reads it via +its `auth.existingSecret`, and the webapp reads it via secretKeyRef + `$(VAR)` +runtime interpolation in the connection URL. Generating it once and reading it back +is what keeps the server credential and the app's URL identical. + +The registry password is NOT here - it is consumed at template render time +(htpasswd + dockerconfigjson) so it is generated and retained inside secrets.yaml. +*/}} +{{- if or .Values.postgres.deploy .Values.clickhouse.deploy .Values.s3.deploy }} +{{- $name := include "trigger-v4.datastore.secretName" . }} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace $name) | default dict }} +{{- $existingData := (get $existing "data") | default dict }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "trigger-v4.labels" . | nindent 4 }} + annotations: + # Never lose these - the password is the only copy the datastore volumes accept. + helm.sh/resource-policy: keep +type: Opaque +data: + {{- if .Values.postgres.deploy }} + {{- $pg := include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "postgres-password" "value" .Values.postgres.auth.password) }} + postgres-password: {{ $pg | b64enc | quote }} + password: {{ $pg | b64enc | quote }} + {{- end }} + {{- if .Values.clickhouse.deploy }} + clickhouse-admin-password: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "clickhouse-admin-password" "value" .Values.clickhouse.auth.password) | b64enc | quote }} + {{- end }} + {{- if .Values.s3.deploy }} + minio-root-user: {{ .Values.s3.auth.rootUser | default "admin" | b64enc | quote }} + minio-root-password: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "minio-root-password" "value" (.Values.s3.auth.rootPassword | default .Values.s3.auth.secretAccessKey)) | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/hosting/k8s/helm/templates/electric.yaml b/hosting/k8s/helm/templates/electric.yaml index ac2d7582f26..5bb1964a0e9 100644 --- a/hosting/k8s/helm/templates/electric.yaml +++ b/hosting/k8s/helm/templates/electric.yaml @@ -44,6 +44,13 @@ spec: name: {{ include "trigger-v4.postgres.external.secretName" . }} key: {{ include "trigger-v4.postgres.external.databaseUrlKey" . }} {{- else }} + {{- if .Values.postgres.deploy }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: password + {{- end }} - name: DATABASE_URL value: {{ include "trigger-v4.postgres.connectionString" . | quote }} {{- end }} diff --git a/hosting/k8s/helm/templates/s2.yaml b/hosting/k8s/helm/templates/s2.yaml index 54e75b13bc2..eccfe4c14b8 100644 --- a/hosting/k8s/helm/templates/s2.yaml +++ b/hosting/k8s/helm/templates/s2.yaml @@ -56,7 +56,10 @@ spec: {{- end }} image: "{{ .Values.s2.image.registry }}/{{ .Values.s2.image.repository }}:{{ .Values.s2.image.tag }}{{ with .Values.s2.image.digest }}@{{ . }}{{ end }}" imagePullPolicy: {{ .Values.s2.image.pullPolicy }} - command: + # args, not command: the image ENTRYPOINT is ["./s2"] and these are its + # subcommand/flags. Using command would override the entrypoint and try to + # exec "lite" directly (StartError). + args: - "lite" - "--init-file" - "/config/s2-spec.json" diff --git a/hosting/k8s/helm/templates/secrets.yaml b/hosting/k8s/helm/templates/secrets.yaml index c122f89d0b3..3c19df0d3b4 100644 --- a/hosting/k8s/helm/templates/secrets.yaml +++ b/hosting/k8s/helm/templates/secrets.yaml @@ -1,4 +1,6 @@ {{- if and .Values.secrets.enabled (not .Values.secrets.existingSecret) }} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace (printf "%s-secrets" (include "trigger-v4.fullname" .))) | default dict }} +{{- $existingData := (get $existing "data") | default dict }} apiVersion: v1 kind: Secret metadata: @@ -7,10 +9,12 @@ metadata: {{- include "trigger-v4.labels" . | nindent 4 }} type: Opaque data: - SESSION_SECRET: {{ .Values.secrets.sessionSecret | b64enc | quote }} - MAGIC_LINK_SECRET: {{ .Values.secrets.magicLinkSecret | b64enc | quote }} - ENCRYPTION_KEY: {{ .Values.secrets.encryptionKey | b64enc | quote }} - MANAGED_WORKER_SECRET: {{ .Values.secrets.managedWorkerSecret | b64enc | quote }} + SESSION_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "SESSION_SECRET" "value" .Values.secrets.sessionSecret) | b64enc | quote }} + MAGIC_LINK_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "MAGIC_LINK_SECRET" "value" .Values.secrets.magicLinkSecret) | b64enc | quote }} + ENCRYPTION_KEY: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "ENCRYPTION_KEY" "value" .Values.secrets.encryptionKey) | b64enc | quote }} + PROVIDER_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "PROVIDER_SECRET" "value" .Values.secrets.providerSecret) | b64enc | quote }} + COORDINATOR_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "COORDINATOR_SECRET" "value" .Values.secrets.coordinatorSecret) | b64enc | quote }} + MANAGED_WORKER_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "MANAGED_WORKER_SECRET" "value" .Values.secrets.managedWorkerSecret) | b64enc | quote }} {{- if and .Values.s3.external.accessKeyId (not .Values.s3.external.existingSecret) }} s3-access-key-id: {{ .Values.s3.external.accessKeyId | b64enc | quote }} s3-secret-access-key: {{ .Values.s3.external.secretAccessKey | b64enc | quote }} @@ -36,15 +40,27 @@ data: {{- end }} --- {{- if and .Values.registry.deploy .Values.registry.auth.enabled }} +{{/* Generated here (not in datastore-secret) because it is consumed at render time by + htpasswd and the dockerconfigjson below. Retained via the plaintext `password` key + on this secret; htpasswd bcrypt is one-way so it can't be the retain source. */}} +{{- $regAuthName := printf "%s-registry-auth" (include "trigger-v4.fullname" .) }} +{{- $regExisting := (lookup "v1" "Secret" .Release.Namespace $regAuthName) | default dict }} +{{- $regExistingData := (get $regExisting "data") | default dict }} +{{- $regPass := include "trigger-v4.resolveSecret" (dict "existingData" $regExistingData "key" "password" "value" .Values.registry.auth.password) }} +{{- $regAuth := printf "%s:%s" .Values.registry.auth.username $regPass | b64enc }} +{{- $regConfig := dict "auths" (dict (include "trigger-v4.registry.host" .) (dict "username" .Values.registry.auth.username "password" $regPass "auth" $regAuth)) }} apiVersion: v1 kind: Secret metadata: - name: {{ include "trigger-v4.fullname" . }}-registry-auth + name: {{ $regAuthName }} labels: {{- include "trigger-v4.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep type: Opaque data: - htpasswd: {{ htpasswd .Values.registry.auth.username .Values.registry.auth.password | trim | b64enc | quote }} + htpasswd: {{ htpasswd .Values.registry.auth.username $regPass | trim | b64enc | quote }} + password: {{ $regPass | b64enc | quote }} --- apiVersion: v1 kind: Secret @@ -54,7 +70,7 @@ metadata: {{- include "trigger-v4.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: {{ include "trigger-v4.imagePullSecret" . | b64enc }} + .dockerconfigjson: {{ $regConfig | toJson | b64enc }} {{- else if and (not .Values.registry.deploy) .Values.registry.external.auth.enabled }} apiVersion: v1 kind: Secret diff --git a/hosting/k8s/helm/templates/validate-external-config.yaml b/hosting/k8s/helm/templates/validate-external-config.yaml index 063aeecbcfd..939e699a423 100644 --- a/hosting/k8s/helm/templates/validate-external-config.yaml +++ b/hosting/k8s/helm/templates/validate-external-config.yaml @@ -7,6 +7,7 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "PostgreSQL external configuration is required when postgres.deploy=false. Please provide either postgres.external.databaseUrl or postgres.external.existingSecret" }} {{- end }} {{- end }} +{{/* When postgres.deploy=true the password is auto-generated into the datastore Secret (templates/datastore-secret.yaml) and consumed by the subchart via auth.existingSecret; set postgres.auth.password to pin it. */}} {{- if not .Values.redis.deploy }} {{- if not .Values.redis.external.host }} @@ -19,12 +20,9 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "ClickHouse external configuration is required when clickhouse.deploy=false. Please provide clickhouse.external.host and clickhouse.external.username" }} {{- end }} {{- end }} +{{/* When clickhouse.deploy=true the password is auto-generated into the datastore Secret and consumed via auth.existingSecret; set clickhouse.auth.password to pin it. */}} -{{- if .Values.s3.deploy }} -{{- if and (not .Values.s3.auth.existingSecret) (not .Values.s3.auth.accessKeyId) (not .Values.s3.auth.rootUser) }} -{{- fail "S3 auth credentials are required when s3.deploy=true. Please provide either s3.auth.accessKeyId, s3.auth.existingSecret, or s3.auth.rootUser" }} -{{- end }} -{{- else }} +{{- if not .Values.s3.deploy }} {{- if not .Values.s3.external.endpoint }} {{- fail "S3 external configuration is required when s3.deploy=false. Please provide s3.external.endpoint" }} {{- end }} @@ -32,6 +30,7 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "S3 credentials are required when s3.deploy=false. Please provide either s3.external.existingSecret or both s3.external.accessKeyId and s3.external.secretAccessKey" }} {{- end }} {{- end }} +{{/* When s3.deploy=true the MinIO root password is auto-generated into the datastore Secret and consumed via auth.existingSecret; set s3.auth.rootPassword to pin it. */}} {{- if not .Values.electric.deploy }} {{- if not .Values.electric.external.url }} @@ -48,6 +47,14 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "Registry external configuration is required when registry.deploy=false. Please provide registry.external.host" }} {{- end }} {{- end }} +{{/* When registry.deploy=true the password is auto-generated + retained inside secrets.yaml; set registry.auth.password to pin it. */}} + +{{/* +Application and control-plane secrets are auto-generated by templates/secrets.yaml +when left unset (retained across upgrades via lookup), so they need no fail-closed +guard here. The webapp still rejects previously published values at startup, even +when supplied via secrets.existingSecret. +*/}} {{/* This template produces no output but will fail the deployment if validation fails diff --git a/hosting/k8s/helm/templates/webapp.yaml b/hosting/k8s/helm/templates/webapp.yaml index f83ff0ecb84..49acee98f4a 100644 --- a/hosting/k8s/helm/templates/webapp.yaml +++ b/hosting/k8s/helm/templates/webapp.yaml @@ -221,6 +221,13 @@ spec: name: {{ include "trigger-v4.postgres.external.secretName" . }} key: {{ include "trigger-v4.postgres.external.directUrlKey" . }} {{- else }} + {{- if .Values.postgres.deploy }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: password + {{- end }} - name: DATABASE_URL value: {{ include "trigger-v4.postgres.connectionString" . | quote }} - name: DIRECT_URL @@ -306,6 +313,16 @@ spec: secretKeyRef: name: {{ include "trigger-v4.secretsName" . }} key: ENCRYPTION_KEY + - name: PROVIDER_SECRET + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.secretsName" . }} + key: PROVIDER_SECRET + - name: COORDINATOR_SECRET + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.secretsName" . }} + key: COORDINATOR_SECRET - name: MANAGED_WORKER_SECRET valueFrom: secretKeyRef: @@ -401,6 +418,12 @@ spec: secretKeyRef: name: {{ include "trigger-v4.clickhouse.external.secretName" . }} key: {{ include "trigger-v4.clickhouse.external.passwordKey" . }} + {{- else if .Values.clickhouse.deploy }} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: clickhouse-admin-password {{- end }} - name: CLICKHOUSE_URL value: {{ include "trigger-v4.clickhouse.url" . | quote }} diff --git a/hosting/k8s/helm/values-production-example.yaml b/hosting/k8s/helm/values-production-example.yaml index 65bdc0a6fc5..ee99de58a9b 100644 --- a/hosting/k8s/helm/values-production-example.yaml +++ b/hosting/k8s/helm/values-production-example.yaml @@ -7,6 +7,8 @@ secrets: magicLinkSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_002" encryptionKey: "YOUR_32_CHAR_HEX_SECRET_HERE_003" managedWorkerSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_004" + providerSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_005" + coordinatorSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_006" # Production webapp configuration webapp: @@ -42,6 +44,10 @@ webapp: # Production PostgreSQL (or use external) postgres: + auth: + # Required — no built-in default. The webapp connection string uses postgres.auth.password. + postgresPassword: "your-strong-postgres-password" + password: "your-strong-postgres-password" primary: persistence: enabled: true @@ -72,6 +78,9 @@ redis: # Production ClickHouse clickhouse: + auth: + # Required — no built-in default. The webapp connection string uses clickhouse.auth.password. + password: "your-strong-clickhouse-password" # Set to true to enable TLS/secure connections in production secure: true persistence: diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml index 968419d9871..e76a921279c 100644 --- a/hosting/k8s/helm/values.yaml +++ b/hosting/k8s/helm/values.yaml @@ -10,11 +10,10 @@ nameOverride: "" fullnameOverride: "" # Secrets configuration -# IMPORTANT: The default values below are for TESTING ONLY and should NOT be used in production -# For production deployments: -# 1. Generate new secrets using: openssl rand -hex 16 -# 2. Override these values in your values.yaml or use external secret management -# 3. Each secret must be exactly 32 hex characters (16 bytes) +# No working defaults are shipped. Leave a secret empty and the chart generates a +# strong random value on first install, retained across upgrades (so sessions and +# encrypted data survive). Set a value explicitly to control it yourself, or use +# secrets.existingSecret / external secret management. Explicit values win. secrets: # Enable/disable creation of secrets # Set to false to use external secret management (Vault, Infisical, External Secrets, etc.) @@ -27,17 +26,23 @@ secrets: # - SESSION_SECRET # - MAGIC_LINK_SECRET # - ENCRYPTION_KEY + # - PROVIDER_SECRET + # - COORDINATOR_SECRET # - MANAGED_WORKER_SECRET existingSecret: "" - # Session secret for user authentication (32 hex chars) - sessionSecret: "2818143646516f6fffd707b36f334bbb" - # Magic link secret for passwordless login (32 hex chars) - magicLinkSecret: "44da78b7bbb0dfe709cf38931d25dcdd" - # Encryption key for sensitive data (32 hex chars) - encryptionKey: "f686147ab967943ebbe9ed3b496e465a" - # Worker secret for managed worker authentication (32 hex chars) - managedWorkerSecret: "447c29678f9eaf289e9c4b70d3dd8a7f" + # Session secret for user authentication + sessionSecret: "" # Leave empty to auto-generate, or set a strong value. + # Magic link secret for passwordless login + magicLinkSecret: "" # Leave empty to auto-generate, or set a strong value. + # Encryption key for sensitive data + encryptionKey: "" # Leave empty to auto-generate, or set a strong value. + # Provider socket secret + providerSecret: "" # Leave empty to auto-generate, or set a strong value. + # Coordinator socket secret + coordinatorSecret: "" # Leave empty to auto-generate, or set a strong value. + # Worker secret for managed worker authentication + managedWorkerSecret: "" # Leave empty to auto-generate, or set a strong value. # Object store credentials moved to s3.auth and s3.external section # Webapp configuration @@ -397,10 +402,16 @@ postgres: # Bitnami PostgreSQL chart configuration (when deploy: true) auth: enablePostgresUser: true - postgresPassword: "postgres" + postgresPassword: "" # Leave empty to auto-generate into the datastore secret, or set to pin. username: "postgres" - password: "postgres" + password: "" # Leave empty to auto-generate into the datastore secret, or set to pin. database: "main" + # Read the auto-generated password from the chart-managed datastore secret + # (templates/datastore-secret.yaml). Must equal trigger-v4.datastore.secretName. + existingSecret: "trigger-datastore" + secretKeys: + adminPasswordKey: "postgres-password" + userPasswordKey: "password" primary: persistence: @@ -632,7 +643,10 @@ clickhouse: # Bitnami ClickHouse chart configuration (when deploy: true) auth: username: "default" - password: "password" + password: "" # Leave empty to auto-generate into the datastore secret, or set to pin. + # Read the auto-generated password from the chart-managed datastore secret. + existingSecret: "trigger-datastore" + existingSecretKey: "clickhouse-admin-password" # Single-node configuration (disable clustering for dev/test) keeper: @@ -709,14 +723,18 @@ s3: # MinIO provides S3-compatible storage when deployed internally auth: rootUser: "admin" - rootPassword: "very-safe-password" + rootPassword: "" # Leave empty to auto-generate into the datastore secret, or set to pin. # Webapp credentials for S3 access (defaults to root credentials if not specified) accessKeyId: "" # Defaults to rootUser if empty secretAccessKey: "" # Defaults to rootPassword if empty - # Existing secret support for webapp credentials - existingSecret: "" # If set, accessKeyId/secretAccessKey will be ignored - accessKeyIdSecretKey: "access-key-id" # Key in existingSecret containing access key ID - secretAccessKeySecretKey: "secret-access-key" # Key in existingSecret containing secret access key + # Read the auto-generated root credentials from the chart-managed datastore secret. + # The same keys serve the MinIO subchart (rootUser/rootPassword) and the webapp's + # S3 credentials, so both stay in sync. Must equal trigger-v4.datastore.secretName. + existingSecret: "trigger-datastore" + rootUserSecretKey: "minio-root-user" + rootPasswordSecretKey: "minio-root-password" + accessKeyIdSecretKey: "minio-root-user" # Key in existingSecret containing access key ID + secretAccessKeySecretKey: "minio-root-password" # Key in existingSecret containing secret access key # The required "packets" bucket is created by default. defaultBuckets: "packets" @@ -729,8 +747,8 @@ s3: # External S3 connection (when deploy: false) external: endpoint: "" # e.g., "https://s3.amazonaws.com" or "https://your-minio.com:9000" - accessKeyId: "admin" # Default for internal MinIO - change for production - secretAccessKey: "very-safe-password" # Default for internal MinIO - change for production + accessKeyId: "" # Required when s3.deploy=false — no default. Set explicitly or use existingSecret. + secretAccessKey: "" # Required when s3.deploy=false — no default. Set explicitly or use existingSecret. # # Secure credential management existingSecret: "" # Name of existing secret containing S3 credentials @@ -769,7 +787,7 @@ registry: auth: enabled: true username: "registry-user" - password: "very-secure-indeed" + password: "" # Required when registry.deploy=true and auth.enabled — no default. Set a strong value. # External Registry connection (when deploy: false) external: diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts index 2d2774df420..fc587abd696 100644 --- a/internal-packages/testcontainers/src/webapp.ts +++ b/internal-packages/testcontainers/src/webapp.ts @@ -104,6 +104,11 @@ export async function startWebapp( SESSION_SECRET: "test-session-secret-for-e2e-tests", MAGIC_LINK_SECRET: "test-magic-link-secret-32chars!!", ENCRYPTION_KEY: "test-encryption-key-for-e2e!!!!!", // exactly 32 bytes + // Control-plane auth secrets are required (no defaults) and reject the + // known-insecure placeholder strings, so supply strong test values here. + PROVIDER_SECRET: "test-provider-secret-for-e2e-tests", + COORDINATOR_SECRET: "test-coordinator-secret-for-e2e-tests", + MANAGED_WORKER_SECRET: "test-managed-worker-secret-for-e2e-tests", CLICKHOUSE_URL: "http://localhost:19123", // dummy, auth paths never connect DEPLOY_REGISTRY_HOST: "registry.example.com", // dummy, not needed for auth tests ELECTRIC_ORIGIN: "http://localhost:3060", From 677ce4cc52e7a50b55eca48612d828669989fffa Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:15:23 +0100 Subject: [PATCH 09/15] fix(webapp,core): harden CLI auth-code login, socket-auth (#53) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(webapp): gate CLI auth-code PAT minting behind explicit consent + rate limit The `/account/authorization-code/:code` loader minted and bound a Personal Access Token as a pure side effect of a GET, so any authenticated browser that merely landed on the URL (phished link, prefetch) silently issued a CLI PAT bound to an attacker-supplied code, which the attacker then harvested from the unauthenticated, unthrottled `/api/v1/token` endpoint. - Move the mint out of the loader into an `action` behind an explicit "Authorize" POST from a logged-in human. The loader now only renders a consent screen (read-only `isAuthorizationCodeMintable`). The CLI contract is unchanged: it never calls this route and keeps polling `/api/v1/token`, which already returns `{ token: null }` until consent is given. - Add IP rate limiting to `/api/v1/authorization-code` (mint) and per-code rate limiting to `/api/v1/token` (poll). The poll limiter is keyed by the code, not the IP, so the CLI's ~1/s poll loop isn't broken behind a shared NAT. - Remove both endpoints from the global rate-limit allowlist. - Shorten the unconsumed-code TTL from 10 minutes to 2. Response shapes are unchanged; the only new behavior is a 429 on limit breach. Addresses the auth-code half of GHSA-58mc (TRI-9770). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 92ef2c3cd68fc6c5d3e7b01ad553bae5fcf121b6) * fix(webapp): require ADMIN role to rename or delete a project verifyProjectMembership only proved org membership, so any MEMBER-role invitee could rename or permanently delete a project and all its runs, schedules, and environments. Gate the rename/delete intents on OrgMember.role === ADMIN. Addresses TRI-9870. Co-authored-by: Daniel Sutton * fix(webapp): keep auth-code/token endpoints allowlisted (don't break CLI login) The global apiRateLimiter keys on the Authorization header and returns 401 for any matched-but-unauthenticated /api path BEFORE the route runs. The CLI auth-code/token endpoints are intentionally unauthenticated, so removing them from the allowlist 401s them and breaks CLI login outright. Restore the allowlist entries (skipping the auth-keyed global limiter) — the dedicated authCodeRateLimiter in the route actions still provides the throttle. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit b823a8750b570802f245cbbab3d948b509db3cef) * fix(webapp): require ADMIN for org delete/rename and non-DEV key regen Add an isOrgAdmin helper (legacy OrgMember.role, independent of the RBAC ability layer so it holds in OSS deployments) and gate on it for: - organization rename/delete (settings index action) - regenerating non-development API keys (production/staging/preview are org-wide; DEV self-service is unchanged) Both previously checked only org membership, letting any MEMBER perform them. Addresses TRI-9870. Co-authored-by: Daniel Sutton * fix(webapp): bound auth-code minting even without X-Forwarded-For + record deferral - The mint rate limiter previously no-op'd when extractClientIp returned null (no X-Forwarded-For, e.g. non-ALB/direct deploys), leaving minting unbounded there. Fall back to a shared key so it's always throttled. - Add a server-changes note documenting the auth-code/org-admin hardening and recording the intentional inviteMembers-baseline deferral so it isn't lost. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit d532795a28ac76b52b14d6a0ef72ac401535c7e1) * fix(webapp): restore auth-code TTL to original 10 minutes The 2-minute TTL was imported from the third-party report's recommendation, not grounded in a real need. The consent gate is what closes the phishing vector; the TTL length is immaterial to it. Restore the original 10-minute window (keep the shared constant purely to stop the mint/read paths drifting). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 70951ddb57faf9ffb96f032110fda1668ea4ed62) * fix(cli): widen login poll window to fit the new consent click The auth-code login page now requires an explicit "Authorize" click before a PAT is minted (it no longer mints as a side effect of loading the page). The CLI/MCP poll loop was sized (~60s) for the old auto-mint flow, so a human who takes longer than ~60s to approve would get "Failed to get access token" even though the code is valid for 10 minutes. Widen the poll to ~5 minutes — within the code TTL and the per-code poll rate limit (~1/s). Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 80c39d6f0d63c89a6b50fcf8b80c02c1801472ff) * docs(webapp): correct stale CLI poll-count in auth-code rate-limiter comment The CLI poll window was widened to ~5 min, so "up to ~61 times/min / 60 retries" no longer describes it. The steady cadence is ~1/s (~60/min), still under the 100/min/code cap regardless of total poll count. Comment-only. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit 3f87584db9c7e91474977ae9431b3b8b71c2e6d2) * refactor(webapp): use findFirst over findUnique in auth-code lookups Code-review cleanup: findFirst avoids the findUnique constraint on the non-unique lookup shape used by the auth-code mint/consent path. Co-authored-by: Daniel Sutton * fix(webapp): skip auth-code mint limit when there's no trustworthy client IP The shared "no-forwarded-for" fallback collapsed every X-Forwarded-For-less request into one 30/min bucket, letting a single client DoS login for a whole non-ALB instance. Skip the limit when there's no client IP instead — matching the existing magicLinkRateLimiter pattern. Our cloud is behind an ALB so the per-IP limit always applies there; the consent gate (not this limit) is what closes the PAT-theft vector, so leaving non-proxied self-host minting unbounded is low-risk row churn rather than a login outage. Co-Authored-By: Claude Opus 4.8 (1M context) (cherry picked from commit e5b9e109384b163b397ad879c9c0ae3b7e4d1853) * docs: scope server-changes note to shipped fixes * format * fix(webapp): use conform submission.reply for ADMIN-denied 403s The ADMIN gate returned the pre-parseWithZod `submission.error[""]` shape, which no longer typechecks (TS2339) and would not surface in the form. Use `submission.reply({ formErrors })` so the denial message renders and the types line up with the parseWithZod migration on the base branch. * chore: consolidate changeset and tighten comments * fix(cli): retry auth-code polling on 429 instead of aborting The login poll loop threw an AbortError for any failed token response, which pRetry treats as fatal. A 429 from the per-code poll rate limiter would therefore abandon the whole login ("Failed to get access token") even though the auth code is still valid and the user may not have approved the consent screen yet. Expose the HTTP status on wrapZodFetch failures and, in the poll path, treat a 429 as a retryable error so the loop backs off and keeps polling. Covers both the CLI login command and the MCP auth flow, which share the same getPersonalAccessToken helper. * chore(webapp): scope changes to auth-code login * chore: align auth-code login release note --------- Co-authored-by: Daniel Sutton Co-authored-by: Claude Opus 4.8 (1M context) --- .changeset/cli-login-poll-window.md | 6 + .../route.tsx | 171 +++++++++++++----- .../app/routes/api.v1.authorization-code.ts | 25 ++- apps/webapp/app/routes/api.v1.token.ts | 18 ++ .../app/services/apiRateLimit.server.ts | 3 + .../services/authCodeRateLimiter.server.ts | 85 +++++++++ .../services/personalAccessToken.server.ts | 41 ++++- .../test/authorizationCodeConsent.test.ts | 58 ++++++ packages/cli-v3/src/commands/login.ts | 14 +- packages/cli-v3/src/mcp/auth.ts | 5 +- packages/core/src/v3/apiClient/core.ts | 8 + 11 files changed, 370 insertions(+), 64 deletions(-) create mode 100644 .changeset/cli-login-poll-window.md create mode 100644 apps/webapp/app/services/authCodeRateLimiter.server.ts create mode 100644 apps/webapp/test/authorizationCodeConsent.test.ts diff --git a/.changeset/cli-login-poll-window.md b/.changeset/cli-login-poll-window.md new file mode 100644 index 00000000000..e0453895a50 --- /dev/null +++ b/.changeset/cli-login-poll-window.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. diff --git a/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx b/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx index 5bd9550e66e..067f6af4465 100644 --- a/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx +++ b/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx @@ -1,14 +1,19 @@ import { CheckCircleIcon } from "@heroicons/react/24/solid"; -import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { Form } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson, useTypedActionData, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { AppContainer, MainCenteredContainer } from "~/components/layout/AppLayout"; +import { Button } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Header1 } from "~/components/primitives/Headers"; import { Icon } from "~/components/primitives/Icon"; import { Paragraph } from "~/components/primitives/Paragraph"; import { logger } from "~/services/logger.server"; -import { createPersonalAccessTokenFromAuthorizationCode } from "~/services/personalAccessToken.server"; +import { + createPersonalAccessTokenFromAuthorizationCode, + isAuthorizationCodeMintable, +} from "~/services/personalAccessToken.server"; import { requireUserId } from "~/services/session.server"; const ParamsSchema = z.object({ @@ -20,49 +25,57 @@ const SearchParamsSchema = z.object({ clientName: z.string().optional(), }); -export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); - +function parseParams(params: unknown) { const parsedParams = ParamsSchema.safeParse(params); - if (!parsedParams.success) { logger.info("Invalid params", { params }); - throw new Response(undefined, { - status: 400, - statusText: "Invalid params", - }); + throw new Response(undefined, { status: 400, statusText: "Invalid params" }); } + return parsedParams.data; +} +function parseSearch(request: Request) { const url = new URL(request.url); const searchObject = Object.fromEntries(url.searchParams.entries()); - const searchParams = SearchParamsSchema.safeParse(searchObject); - const source = (searchParams.success ? searchParams.data.source : undefined) ?? "cli"; const clientName = (searchParams.success ? searchParams.data.clientName : undefined) ?? "unknown"; + return { source, clientName }; +} + +// The loader only renders a consent screen; minting/binding a PAT happens in +// the `action`, behind an explicit "Authorize" POST. +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + await requireUserId(request); + + const { authorizationCode } = parseParams(params); + const { source, clientName } = parseSearch(request); + + const mintable = await isAuthorizationCodeMintable(authorizationCode); + + return typedjson({ + status: mintable ? ("consent" as const) : ("invalid" as const), + source, + clientName, + }); +}; + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + + const { authorizationCode } = parseParams(params); + const { source, clientName } = parseSearch(request); try { - const _personalAccessToken = await createPersonalAccessTokenFromAuthorizationCode( - parsedParams.data.authorizationCode, - userId - ); - return typedjson({ - success: true as const, - source, - clientName, - }); + await createPersonalAccessTokenFromAuthorizationCode(authorizationCode, userId); + return typedjson({ success: true as const, source, clientName }); } catch (error) { if (error instanceof Response) { throw error; } if (error instanceof Error) { - return typedjson({ - success: false as const, - error: error.message, - source, - clientName, - }); + return typedjson({ success: false as const, error: error.message, source, clientName }); } logger.error(JSON.stringify(error)); @@ -74,32 +87,78 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; export default function Page() { - const result = useTypedLoaderData(); + const loaderData = useTypedLoaderData(); + const actionData = useTypedActionData(); + + // After the consent POST: success or failure. + if (actionData) { + return ( + + {actionData.success ? ( +
+ + Successfully + authenticated + + + {getInstructionsForSource(actionData.source, actionData.clientName)} + +
+ ) : ( +
+ Authentication failed + + {actionData.error} + + + There was a problem authenticating you, please try logging in with your CLI again. + +
+ )} + + ); + } + + // Initial GET: invalid/expired code, or the consent prompt. + if (loaderData.status === "invalid") { + return ( + +
+ Authentication failed + + This login link is invalid or has expired. + + + Please try logging in with your CLI again to get a fresh link. + +
+ + ); + } + return ( + +
+ Authorize login + {getConsentPrompt(loaderData.source, loaderData.clientName)} +
+ +
+ + Only authorize if you started this login yourself. If you didn't, close this page. + +
+ + ); +} + +function AuthShell({ children }: { children: React.ReactNode }) { return ( -
- {result.success ? ( -
- - Successfully - authenticated - - {getInstructionsForSource(result.source, result.clientName)} -
- ) : ( -
- Authentication failed - - {result.error} - - - There was a problem authenticating you, please try logging in with your CLI again. - -
- )} -
+
{children}
 ); @@ -113,6 +172,18 @@ const prettyClientNames: Record = { "claude-ai": "Claude Desktop", }; +function getConsentPrompt(source: string, clientName: string) { + if (source === "mcp") { + const pretty = prettyClientNames[clientName] ?? clientName; + if (pretty && pretty !== "unknown") { + return `Authorize ${pretty} to access your Trigger.dev account?`; + } + return `Authorize this MCP client to access your Trigger.dev account?`; + } + + return `Authorize the Trigger.dev CLI to access your account?`; +} + function getInstructionsForSource(source: string, clientName: string) { if (source === "mcp") { if (clientName) { diff --git a/apps/webapp/app/routes/api.v1.authorization-code.ts b/apps/webapp/app/routes/api.v1.authorization-code.ts index 1a56ef04096..fa7a36673f9 100644 --- a/apps/webapp/app/routes/api.v1.authorization-code.ts +++ b/apps/webapp/app/routes/api.v1.authorization-code.ts @@ -3,7 +3,12 @@ import { json } from "@remix-run/server-runtime"; import type { CreateAuthorizationCodeResponse } from "@trigger.dev/core/v3"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { + AuthorizationCodeRateLimitError, + checkAuthorizationCodeMintRateLimit, +} from "~/services/authCodeRateLimiter.server"; import { createAuthorizationCode } from "~/services/personalAccessToken.server"; +import { extractClientIp } from "~/utils/extractClientIp.server"; /** Used to create an AuthorizationCode, that can then be used to obtain a Personal Access Token by logging in with the provided URL */ export async function action({ request }: ActionFunctionArgs) { @@ -14,8 +19,24 @@ export async function action({ request }: ActionFunctionArgs) { return { status: 405, body: "Method Not Allowed" }; } - //there is no authentication on this endpoint, anyone can create an AuthorizationCode. - //they're only used to allow a user to login, when they'll then receive a Personal Access Token + //this endpoint is unauthenticated (codes only allow a user to log in), so it's + //rate-limited per client IP. Keyed by X-Forwarded-For; if there's no trustworthy + //client IP we skip the limit rather than bucket everyone together. Self-hosters + //wanting per-IP limiting should front the app with a proxy that sets X-Forwarded-For. + const clientIp = extractClientIp(request.headers.get("x-forwarded-for")); + if (clientIp) { + try { + await checkAuthorizationCodeMintRateLimit(clientIp); + } catch (error) { + if (error instanceof AuthorizationCodeRateLimitError) { + return json( + { error: "Too many requests, please try again later." }, + { status: 429, headers: { "Retry-After": Math.ceil(error.retryAfter / 1000).toString() } } + ); + } + throw error; + } + } try { const authorizationCode = await createAuthorizationCode(); diff --git a/apps/webapp/app/routes/api.v1.token.ts b/apps/webapp/app/routes/api.v1.token.ts index 7763f9bb56e..e3c85652035 100644 --- a/apps/webapp/app/routes/api.v1.token.ts +++ b/apps/webapp/app/routes/api.v1.token.ts @@ -4,6 +4,10 @@ import type { GetPersonalAccessTokenResponse } from "@trigger.dev/core/v3"; import { GetPersonalAccessTokenRequestSchema } from "@trigger.dev/core/v3"; import { generateErrorMessage } from "zod-error"; import { logger } from "~/services/logger.server"; +import { + AuthorizationCodeRateLimitError, + checkAuthorizationCodeTokenPollRateLimit, +} from "~/services/authCodeRateLimiter.server"; import { getPersonalAccessTokenFromAuthorizationCode } from "~/services/personalAccessToken.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; @@ -25,6 +29,20 @@ export async function action({ request }: ActionFunctionArgs) { return json({ error: generateErrorMessage(body.error.issues) }, { status: 422 }); } + // Per-code rate limit (keyed by the code, not the IP, so the CLI's poll loop + // isn't broken behind a shared NAT). + try { + await checkAuthorizationCodeTokenPollRateLimit(body.data.authorizationCode); + } catch (error) { + if (error instanceof AuthorizationCodeRateLimitError) { + return json( + { error: "Too many requests, please try again later." }, + { status: 429, headers: { "Retry-After": Math.ceil(error.retryAfter / 1000).toString() } } + ); + } + throw error; + } + try { const personalAccessToken = await getPersonalAccessTokenFromAuthorizationCode( body.data.authorizationCode diff --git a/apps/webapp/app/services/apiRateLimit.server.ts b/apps/webapp/app/services/apiRateLimit.server.ts index fbd59cc74cd..837d3ee5895 100644 --- a/apps/webapp/app/services/apiRateLimit.server.ts +++ b/apps/webapp/app/services/apiRateLimit.server.ts @@ -48,6 +48,9 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({ // Allow /api/v1/tasks/:id/callback/:secret pathWhiteList: [ "/api/internal/stripe_webhooks", + // Keep allowlisted: these CLI endpoints are intentionally unauthenticated, + // so this Authorization-header-keyed limiter would 401 them. They are + // throttled separately by authCodeRateLimiter.server.ts. "/api/v1/authorization-code", "/api/v1/token", "/api/v1/usage/ingest", diff --git a/apps/webapp/app/services/authCodeRateLimiter.server.ts b/apps/webapp/app/services/authCodeRateLimiter.server.ts new file mode 100644 index 00000000000..5c88c094c2c --- /dev/null +++ b/apps/webapp/app/services/authCodeRateLimiter.server.ts @@ -0,0 +1,85 @@ +import { Ratelimit } from "@upstash/ratelimit"; +import { createHash } from "node:crypto"; +import { env } from "~/env.server"; +import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server"; +import { singleton } from "~/utils/singleton"; + +/** + * Rate limiting for the unauthenticated CLI auth-code endpoints + * (`/api/v1/authorization-code` mint + `/api/v1/token` poll). The global + * limiter keys on the Authorization header, which these endpoints don't carry, + * so it can't throttle them — this module does. + */ +export class AuthorizationCodeRateLimitError extends Error { + public readonly retryAfter: number; + + constructor(retryAfter: number) { + super("Authorization code rate limit exceeded."); + this.name = "AuthorizationCodeRateLimitError"; + this.retryAfter = retryAfter; + } +} + +function getRedisClient() { + return createRedisRateLimitClient({ + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }); +} + +// Minting is unauthenticated and a real login mints one code. Cap per IP, with +// headroom for many users behind a shared NAT. +const authorizationCodeMintIpRateLimiter = singleton( + "authorizationCodeMintIpRateLimiter", + () => + new RateLimiter({ + redisClient: getRedisClient(), + keyPrefix: "auth:authcode:mint:ip", + limiter: Ratelimit.slidingWindow(30, "1 m"), // 30 code mints / min / IP + logSuccess: false, + logFailure: true, + }) +); + +// Keyed by the code, not the IP: the CLI polls this endpoint ~1/s per login, so +// IP-keying would break logins behind a shared NAT. The ~60/min cadence stays +// under the cap. The code is hashed first so it never lands in a Redis key or log. +const authorizationCodeTokenPollRateLimiter = singleton( + "authorizationCodeTokenPollRateLimiter", + () => + new RateLimiter({ + redisClient: getRedisClient(), + keyPrefix: "auth:authcode:token:code", + limiter: Ratelimit.slidingWindow(100, "1 m"), // 100 polls / min / code (CLI polls ~60/min) + logSuccess: false, + logFailure: false, + }) +); + +function hashCode(code: string): string { + return createHash("sha256").update(code).digest("hex").slice(0, 32); +} + +export async function checkAuthorizationCodeMintRateLimit(ip: string): Promise { + const result = await authorizationCodeMintIpRateLimiter.limit(ip); + + if (!result.success) { + const retryAfter = new Date(result.reset).getTime() - Date.now(); + throw new AuthorizationCodeRateLimitError(retryAfter); + } +} + +export async function checkAuthorizationCodeTokenPollRateLimit( + authorizationCode: string +): Promise { + const result = await authorizationCodeTokenPollRateLimiter.limit(hashCode(authorizationCode)); + + if (!result.success) { + const retryAfter = new Date(result.reset).getTime() - Date.now(); + throw new AuthorizationCodeRateLimitError(retryAfter); + } +} diff --git a/apps/webapp/app/services/personalAccessToken.server.ts b/apps/webapp/app/services/personalAccessToken.server.ts index a0821d6602e..d80ca6fa3f4 100644 --- a/apps/webapp/app/services/personalAccessToken.server.ts +++ b/apps/webapp/app/services/personalAccessToken.server.ts @@ -18,6 +18,10 @@ const tokenGenerator = customAlphabet("123456789abcdefghijkmnopqrstuvwxyz", toke // staleness is fine. export const PAT_LAST_ACCESSED_THROTTLE_MS = 5 * 60 * 1000; +// How long an unconsumed CLI authorization code stays valid. Shared constant so +// the mint and read paths can't drift. +export const AUTHORIZATION_CODE_TTL_MS = 10 * 60 * 1000; + type CreatePersonalAccessTokenOptions = { name: string; userId: string; @@ -60,16 +64,16 @@ export type ObfuscatedPersonalAccessToken = Awaited< /** Gets a PersonalAccessToken from an Auth Code, this only works within 10 mins of the auth code being created */ export async function getPersonalAccessTokenFromAuthorizationCode(authorizationCode: string) { - //only allow authorization codes that were created less than 10 mins ago - const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); - const code = await prisma.authorizationCode.findUnique({ + // Only allow authorization codes created within the short consent window. + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prisma.authorizationCode.findFirst({ select: { personalAccessToken: true, }, where: { code: authorizationCode, createdAt: { - gte: tenMinutesAgo, + gte: validAfter, }, }, }); @@ -280,6 +284,27 @@ export function isPersonalAccessToken(token: string) { return token.startsWith(tokenPrefix); } +/** + * Read-only check that an authorization code is still mintable: it exists, is + * unconsumed (`personalAccessTokenId: null`), and within the TTL. Lets the + * consent-screen loader show Authorize vs expired/invalid without minting a PAT. + */ +export async function isAuthorizationCodeMintable( + authorizationCode: string, + prismaClient = prisma +): Promise { + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prismaClient.authorizationCode.findFirst({ + where: { + code: authorizationCode, + personalAccessTokenId: null, + createdAt: { gte: validAfter }, + }, + select: { id: true }, + }); + return code !== null; +} + export function createAuthorizationCode() { return prisma.authorizationCode.create({ data: { @@ -293,14 +318,14 @@ export async function createPersonalAccessTokenFromAuthorizationCode( authorizationCode: string, userId: string ) { - //only allow authorization codes that were created less than 10 mins ago - const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); - const code = await prisma.authorizationCode.findUnique({ + // Only allow authorization codes created within the short consent window. + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prisma.authorizationCode.findFirst({ where: { code: authorizationCode, personalAccessTokenId: null, createdAt: { - gte: tenMinutesAgo, + gte: validAfter, }, }, }); diff --git a/apps/webapp/test/authorizationCodeConsent.test.ts b/apps/webapp/test/authorizationCodeConsent.test.ts new file mode 100644 index 00000000000..85cda08da9b --- /dev/null +++ b/apps/webapp/test/authorizationCodeConsent.test.ts @@ -0,0 +1,58 @@ +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { + AUTHORIZATION_CODE_TTL_MS, + isAuthorizationCodeMintable, +} from "~/services/personalAccessToken.server"; + +vi.setConfig({ testTimeout: 30_000 }); + +function randomCode() { + return `code_${Math.random().toString(36).slice(2)}${Math.random().toString(36).slice(2)}`; +} + +// Lock the read-only + TTL properties `isAuthorizationCodeMintable` relies on: +// the loader must not bind a PAT, and expired codes must not be mintable. +describe("authorization code consent gate", () => { + containerTest( + "a fresh, unconsumed code is mintable and checking it does NOT bind a PAT", + async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(true); + + // The check is read-only — it must not bind a Personal Access Token. + const after = await prisma.authorizationCode.findFirst({ where: { id: created.id } }); + expect(after?.personalAccessTokenId).toBeNull(); + } + ); + + containerTest("a code older than the TTL is not mintable", async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + await prisma.authorizationCode.update({ + where: { id: created.id }, + data: { createdAt: new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS - 1_000) }, + }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(false); + }); + + containerTest( + "a code created just inside the TTL is still mintable (CLI flow not broken)", + async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + await prisma.authorizationCode.update({ + where: { id: created.id }, + data: { createdAt: new Date(Date.now() - (AUTHORIZATION_CODE_TTL_MS - 30_000)) }, + }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(true); + } + ); + + containerTest("an unknown code is not mintable", async ({ prisma }) => { + expect(await isAuthorizationCodeMintable(randomCode(), prisma)).toBe(false); + }); +}); diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index 0171286d416..83576b0d010 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -295,9 +295,10 @@ export async function login(options?: LoginOptions): Promise { const indexResult = await pRetry( () => getPersonalAccessToken(apiClient, authorizationCodeResult.authorizationCode), { - //this means we're polling, same distance between each attempt + //poll at a fixed 1s interval. ~5 min window so the user has time to + //approve the consent screen; stays within the code's 10-min validity. factor: 1, - retries: 60, + retries: 300, minTimeout: 1000, } ); @@ -404,6 +405,15 @@ export async function getPersonalAccessToken(apiClient: CliApiClient, authorizat const token = await apiClient.getPersonalAccessToken(authorizationCode); if (!token.success) { + // A 429 from the per-code poll rate limiter is transient: the auth code + // is still valid and the user may just not have approved the consent + // screen yet. Throw a regular (retryable) error so the poll loop backs + // off and keeps polling, rather than an AbortError, which pRetry treats + // as fatal and would abandon the whole login. + if (token.statusCode === 429) { + throw new Error(token.error); + } + throw new AbortError(token.error); } diff --git a/packages/cli-v3/src/mcp/auth.ts b/packages/cli-v3/src/mcp/auth.ts index b948f2cb889..fa5f62708ec 100644 --- a/packages/cli-v3/src/mcp/auth.ts +++ b/packages/cli-v3/src/mcp/auth.ts @@ -123,9 +123,10 @@ export async function mcpAuth(options: McpAuthOptions): Promise { const indexResult = await pRetry( () => getPersonalAccessToken(apiClient, authorizationCodeResult.authorizationCode), { - //this means we're polling, same distance between each attempt + //poll at a fixed 1s interval. ~5 min window so the user has time to + //approve the consent screen; stays within the code's 10-min validity. factor: 1, - retries: 60, + retries: 300, minTimeout: 1000, } ); diff --git a/packages/core/src/v3/apiClient/core.ts b/packages/core/src/v3/apiClient/core.ts index 3c9ae47cfb1..c6425e7ee37 100644 --- a/packages/core/src/v3/apiClient/core.ts +++ b/packages/core/src/v3/apiClient/core.ts @@ -725,6 +725,13 @@ export type ApiResult = | { success: false; error: string; + /** + * HTTP status code, when the failure originated from an API response + * (e.g. 429 for rate limiting). Undefined for connection/transport + * errors that never reached the server. Lets callers distinguish + * transient failures worth retrying from fatal ones. + */ + statusCode?: number; }; export async function wrapZodFetch( @@ -754,6 +761,7 @@ export async function wrapZodFetch( return { success: false, error: error.message, + statusCode: error.status, }; } else if (error instanceof Error) { return { From 7fd671a054809c09bf9e3d70dae0cb2056f4d272 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:15:31 +0100 Subject: [PATCH 10/15] fix(webapp): schedule & env-var write scoping (reject cross-project/env IDs) (#46) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(webapp): scope schedule lookups by project and environment (TRI-9865) Closes a cluster of cross-tenant IDORs in the schedule routes and services by scoping every `findFirst({friendlyId})` to the caller's projectId, and by gating the env-scoped schedule API on env-visibility so a low-trust key (e.g. dev) can't see or mutate a schedule whose instances live in a different env. Foreign environment IDs passed to CheckScheduleService now fail closed instead of being silently filtered. Three observable API changes are documented in .server-changes/tri-9865-tenant-isolation-schedules.md. Closes TRI-9865, TRI-10040, TRI-10041 (P0); TRI-9854, TRI-9869, TRI-9960, TRI-9963, TRI-9997 (P1); TRI-9859 (P2). Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(webapp): consolidate schedule env-visibility into tri-state helper Address review feedback on the TRI-9865 batch: - Promote findScheduleScopedToEnvironment to a tri-state helper getScheduleEnvVisibility returning { status: 'visible' | 'hidden' | 'missing' }. PUT can disambiguate hidden (refuse, 404) from missing (fall through to upsert's create path) without re-implementing the visibility logic inline. - All three call sites (GET / PUT / DELETE in api.v1.schedules.$id) now share the single helper. - Add test coverage for the deduplicationKey branch of scheduleWhereClause (covered the sched_-prefix branch only before). - Add DeleteTaskScheduleService test asserting DECLARATIVE schedules cannot be deleted (surfaces as a service-layer error after the visibility check passes). - Rename the misleading "devEnv" test fixture to "stagingEnv" to match its actual RuntimeEnvironment type. Co-Authored-By: Claude Opus 4.7 (1M context) * test(webapp): regression tests for schedule environment scoping Scopes schedule read/update/delete/activate to the caller's environment via a shared getScheduleEnvVisibility (visible/hidden/missing) helper. Real-Postgres tests (testcontainers) cover checkSchedule, delete, setActive, and PUT upsert. Verified RED on the shared visibility guard (cross-env detection flips); GREEN 13/13. Bundles the streamBatchItems timeout bump. * fix(webapp): reject foreign environment IDs when upserting a schedule CheckScheduleService.call previously intersected `project.environments` with the caller-supplied `environmentIds` via `.filter()`, silently dropping any ID that didn't belong to the authorized project. Downstream UpsertTaskScheduleService.#createNewSchedule then iterated the raw input and created TaskScheduleInstance rows pairing the validated projectId with whichever environmentId the caller sent — including another tenant's RuntimeEnvironment. When the schedule fires, the engine triggers against the *referenced* environment, enabling cross-tenant task execution if the victim env id is known. Replace the silent intersection with an explicit rejection: build a map of project.environments by id and throw `ServiceValidationError` on the first foreign id, so #createNewSchedule and #updateExistingSchedule never see an environmentId outside the authorized project. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(repro): remove reproducer from PR branch * test(webapp): regression test for cross-project schedule env scoping Extracts the env-scoping into a pure resolveProjectScopedEnvironments (returns a foreign-id flag or the mapped envs) and unit-tests it: all-valid resolves; foreign id rejected; foreign mixed with valid still rejected (not dropped); empty ok. Verified RED against the pre-fix silent filter-drop, GREEN with the rejection. Bundles the streamBatchItems timeout bump. * fix(webapp): enforce per-env access when creating env vars The env-vars `new` action passed user-supplied `environmentIds[]` straight to `repository.create` after a project-membership check. DEV environments are per-user (`RuntimeEnvironment.orgMember.userId`), and the dashboard loader filters other members' DEV envs out of the UI — but the action did not enforce the same filter. A project member could submit `environmentIds=[my_dev, victim_dev]` plus a key/value, and the value was injected into the victim's next task run via `resolveVariablesForEnvironment` (whose secret-store key prefix is `environmentvariable:::`). Now query `runtimeEnvironment.findMany` with the same OR clause `findEnvironmentBySlug` uses — non-DEV envs in the project, or DEV envs owned by the requesting user — and refuse the submission if any submitted ID is missing from the result. Co-Authored-By: Claude Opus 4.7 (1M context) * chore(repro): remove reproducer from PR branch * test(webapp): regression test for env-var DEV-environment ownership Extracts the per-env writability rule into a pure findUnauthorizedEnvironmentId (shared env types writable by any member; DEV only by its owner; unknown id rejected) and unit-tests it. Route now fetches the candidate envs and applies the helper. Verified RED against the pre-fix no-check behaviour, GREEN with it. Bundles the streamBatchItems timeout bump. * fix(webapp): reject mixed valid/foreign environmentIds in env-var create/edit The create()/edit() guard used `environmentIds.every((v) => !inProject(v))`, which only errors when EVERY id is foreign — a single in-project id short-circuited it, so a mixed array passed and the write loop stored values/secrets against another tenant's environment. Switch to `.some` so any foreign id rejects the whole request. Adds RED→GREEN cross-tenant regression tests and bumps the streamBatchItems CI timeout. * test(webapp): isolate schedule scoping tests from the global prisma singleton The checkSchedule/deleteTaskSchedule/setActiveOnTaskSchedule regression tests imported the full services, which pull in ~/db.server; its eager global-prisma $connect() becomes an unhandled rejection in a unit-test job with no reachable global database, failing vitest even though every test passes. Make schedules.server a leaf (type-only Prisma imports) and rewrite the three tests to exercise the project/environment scoping primitives directly against the container DB (scheduleWhereClause, scheduleUniqWhereClause, resolveProjectScopedEnvironments) instead of importing the services. * fix(webapp): enforce DEV-env ownership on env-var edit and delete The per-user DEV-env write check was only on the create route; the edit/delete value actions passed a user-controlled environmentId straight to the repository, which only checks project membership — letting a member overwrite or delete an env-var value in another member's DEV environment. Gate both actions with the same findUnauthorizedEnvironmentId check the create route uses. * format * chore: consolidate server-change and tighten comments * fix(webapp): use .some for schedule env visibility and gate activate/deactivate A schedule can be bound to several environments at once, and the schedule list surfaces a schedule for any environment it has an instance in. The per-schedule visibility check required every instance to be in the caller's environment, so a multi-environment schedule was listed but returned 404 on GET/PUT/DELETE for the same key. Match the list's semantics: a schedule is visible when it has at least one instance in the caller's environment. Also apply the same environment-visibility gate to the activate and deactivate endpoints, which previously scoped only by project and let a key scoped to one environment enable/disable a schedule that runs only in another environment of the same project. * fix(webapp): reject empty-string foreign environment id instead of dropping it The foreign-id guard used a truthiness check, so an empty-string environment id (a foreign id) was falsy and fell through to the accepted path, where it was silently dropped rather than rejected. Use an explicit undefined check so any foreign id, including the empty string, is reported and rejected. --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Chris Arderne --- .../scope-schedule-and-env-var-writes.md | 6 + apps/webapp/app/models/schedules.server.ts | 47 +++- .../route.tsx | 26 ++ .../route.tsx | 19 ++ .../route.tsx | 7 +- .../api.v1.schedules.$scheduleId.activate.ts | 21 +- ...api.v1.schedules.$scheduleId.deactivate.ts | 21 +- .../routes/api.v1.schedules.$scheduleId.ts | 35 ++- .../environmentVariablesRepository.server.ts | 9 +- .../app/v3/services/checkSchedule.server.ts | 15 +- .../v3/services/deleteTaskSchedule.server.ts | 5 +- .../resolveProjectScopedEnvironments.ts | 27 ++ .../setActiveOnTaskSchedule.server.ts | 9 +- apps/webapp/app/v3/writableEnvironments.ts | 33 +++ apps/webapp/test/checkSchedule.test.ts | 60 ++++ apps/webapp/test/deleteTaskSchedule.test.ts | 75 +++++ .../environmentVariablesRepository.test.ts | 76 +++++ .../resolveProjectScopedEnvironments.test.ts | 44 +++ .../test/schedulesPutEnvScoping.test.ts | 261 ++++++++++++++++++ .../test/setActiveOnTaskSchedule.test.ts | 149 ++++++++++ apps/webapp/test/writableEnvironments.test.ts | 55 ++++ 21 files changed, 962 insertions(+), 38 deletions(-) create mode 100644 .server-changes/scope-schedule-and-env-var-writes.md create mode 100644 apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts create mode 100644 apps/webapp/app/v3/writableEnvironments.ts create mode 100644 apps/webapp/test/checkSchedule.test.ts create mode 100644 apps/webapp/test/deleteTaskSchedule.test.ts create mode 100644 apps/webapp/test/resolveProjectScopedEnvironments.test.ts create mode 100644 apps/webapp/test/schedulesPutEnvScoping.test.ts create mode 100644 apps/webapp/test/setActiveOnTaskSchedule.test.ts create mode 100644 apps/webapp/test/writableEnvironments.test.ts diff --git a/.server-changes/scope-schedule-and-env-var-writes.md b/.server-changes/scope-schedule-and-env-var-writes.md new file mode 100644 index 00000000000..3ddcd6a1aaa --- /dev/null +++ b/.server-changes/scope-schedule-and-env-var-writes.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Scope schedule and environment-variable writes to the caller's project and environment diff --git a/apps/webapp/app/models/schedules.server.ts b/apps/webapp/app/models/schedules.server.ts index 72d08c4fb9f..b9d72665caf 100644 --- a/apps/webapp/app/models/schedules.server.ts +++ b/apps/webapp/app/models/schedules.server.ts @@ -1,4 +1,4 @@ -import type { Prisma } from "~/db.server"; +import type { Prisma, PrismaClientOrTransaction, TaskSchedule } from "@trigger.dev/database"; export function scheduleUniqWhereClause( projectId: string, @@ -35,3 +35,48 @@ export function scheduleWhereClause( deduplicationKey: scheduleId, }; } + +/** + * Resolve a schedule's visibility for an environment-scoped caller. + * + * - "visible": the schedule exists in the project and has at least one + * instance bound to `environmentId` (or has no instances yet). + * - "hidden": the schedule exists but none of its instances live in the + * caller's environment. + * - "missing": no schedule exists for the (project, scheduleId) pair. + * + * A schedule can be bound to several environments at once, so visibility + * mirrors the "some instance is in this environment" rule the schedule + * list uses: a schedule that is listed for a key must also be readable + * and mutable by that key. This still rejects cross-environment access to + * schedules the caller has no instance in, and `scheduleWhereClause` + * already confines the lookup to the caller's project. + * + * The tri-state lets PUT (upsert) disambiguate "hidden" (refuse) from + * "missing" (fall through to create). DELETE/GET treat hidden and + * missing the same way. + */ +export type ScheduleEnvVisibility = + | { status: "visible"; schedule: TaskSchedule } + | { status: "hidden" } + | { status: "missing" }; + +export async function getScheduleEnvVisibility( + prisma: PrismaClientOrTransaction, + projectId: string, + scheduleId: string, + environmentId: string +): Promise { + const schedule = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(projectId, scheduleId), + include: { instances: { select: { environmentId: true } } }, + }); + + if (!schedule) return { status: "missing" }; + + const { instances, ...rest } = schedule; + if (instances.length === 0) return { status: "visible", schedule: rest }; + const scoped = instances.some((i) => i.environmentId === environmentId); + if (!scoped) return { status: "hidden" }; + return { status: "visible", schedule: rest }; +} diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx index d552b3104fd..02d6ef00c4d 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx @@ -55,6 +55,7 @@ import { } from "~/utils/pathBuilder"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; import { EnvironmentVariableKey } from "~/v3/environmentVariables/repository"; +import { findUnauthorizedEnvironmentId } from "~/v3/writableEnvironments"; const Variable = z.object({ key: EnvironmentVariableKey, @@ -164,6 +165,31 @@ export const action = dashboardAction( return json(submission.reply({ formErrors: ["Project not found"] })); } + // The submitted `environmentIds` are user-supplied. Shared env types are + // writable by any member; a DEV env only by its owner. See + // findUnauthorizedEnvironmentId. + const submittedEnvs = await prisma.runtimeEnvironment.findMany({ + where: { + projectId: project.id, + id: { in: submission.value.environmentIds }, + }, + select: { id: true, type: true, orgMember: { select: { userId: true } } }, + }); + const unauthorizedEnvironmentId = findUnauthorizedEnvironmentId( + submittedEnvs, + submission.value.environmentIds, + userId + ); + if (unauthorizedEnvironmentId) { + return json( + submission.reply({ + fieldErrors: { + environmentIds: ["One or more of the selected environments is not writable by you."], + }, + }) + ); + } + const repository = new EnvironmentVariablesRepository(prisma); const result = await repository.create(project.id, { ...submission.value, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index b0b2a893144..cdbe364d88e 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -78,6 +78,7 @@ import { v3NewEnvironmentVariablesPath, } from "~/utils/pathBuilder"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import { findUnauthorizedEnvironmentId } from "~/v3/writableEnvironments"; import { DeleteEnvironmentVariableValue, EditEnvironmentVariableValue, @@ -267,6 +268,24 @@ export const action = dashboardAction( return json(submission.reply({ formErrors: ["Project not found"] })); } + // Per-env write gate for the mutating value actions: `environmentId` is a + // user-supplied hidden field and the repository only checks project + // membership. Mirrors the create route's check. + if (submission.value.action === "edit" || submission.value.action === "delete") { + const submittedEnvs = await prisma.runtimeEnvironment.findMany({ + where: { projectId: project.id, id: submission.value.environmentId }, + select: { id: true, type: true, orgMember: { select: { userId: true } } }, + }); + const unauthorizedEnvironmentId = findUnauthorizedEnvironmentId( + submittedEnvs, + [submission.value.environmentId], + userId + ); + if (unauthorizedEnvironmentId) { + return json(submission.reply({ formErrors: ["This environment is not writable by you."] })); + } + } + switch (submission.value.action) { case "edit": { const repository = new EnvironmentVariablesRepository(prisma); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx index cd433467c77..a3a0ef05470 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx @@ -6,7 +6,6 @@ import { z } from "zod"; import { ExitIcon } from "~/assets/icons/ExitIcon"; import { LinkButton } from "~/components/primitives/Buttons"; import { ScheduleInspector } from "~/components/schedules/ScheduleInspector"; -import { prisma } from "~/db.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; @@ -78,11 +77,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { // `_format=json` → return JSON instead of redirecting; caller stays put. const wantsJson = formData.get("_format") === "json"; - const project = await prisma.project.findFirst({ - where: { - slug: projectParam, - }, - }); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { const message = `No project found with slug ${projectParam}`; diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts index 9cc8b7173c7..99ca3159954 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -34,14 +34,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const existingSchedule = await prisma.taskSchedule.findFirst({ - where: scheduleWhereClause( - authenticationResult.environment.projectId, - parsedParams.data.scheduleId - ), - }); - - if (!existingSchedule) { + // Env-scoped API keys can only toggle schedules that have an instance in + // their own environment. Without this a key scoped to one environment + // could enable/disable a schedule that only runs in another environment + // of the same project. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { return json({ error: "Schedule not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts index eb985bea728..3c9514ef8e3 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -34,14 +34,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const existingSchedule = await prisma.taskSchedule.findFirst({ - where: scheduleWhereClause( - authenticationResult.environment.projectId, - parsedParams.data.scheduleId - ), - }); - - if (!existingSchedule) { + // Env-scoped API keys can only toggle schedules that have an instance in + // their own environment. Without this a key scoped to one environment + // could enable/disable a schedule that only runs in another environment + // of the same project. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { return json({ error: "Schedule not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts index 9da9ffcb121..4f7e8d8c164 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts @@ -5,7 +5,7 @@ import { UpdateScheduleOptions } from "@trigger.dev/core/v3"; import { z } from "zod"; import { Prisma, prisma } from "~/db.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; -import { scheduleUniqWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -38,6 +38,16 @@ export async function action({ request, params }: ActionFunctionArgs) { switch (method) { case "DELETE": { + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + try { const deletedSchedule = await prisma.taskSchedule.delete({ where: scheduleUniqWhereClause( @@ -76,6 +86,19 @@ export async function action({ request, params }: ActionFunctionArgs) { return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 }); } + // Env-scoped API keys can't see or mutate a schedule whose + // instances live in a different environment. "hidden" → refuse; + // "missing" → fall through to the upsert's create path. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status === "hidden") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + const service = new UpsertTaskScheduleService(); try { @@ -137,6 +160,16 @@ export async function loader({ request, params }: LoaderFunctionArgs) { ); } + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + const presenter = new ViewSchedulePresenter(); const result = await presenter.call({ diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index a8f2ce701ec..1a0b8247347 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -82,7 +82,10 @@ export class EnvironmentVariablesRepository implements Repository { return { success: false as const, error: "Project not found" }; } - if (options.environmentIds.every((v) => !project.environments.some((e) => e.id === v))) { + // Reject if ANY supplied environmentId is outside the caller's project. + // `.some` (not `.every`) so one in-project id can't let a mixed array + // through. + if (options.environmentIds.some((v) => !project.environments.some((e) => e.id === v))) { return { success: false as const, error: `Environment not found` }; } @@ -291,7 +294,9 @@ export class EnvironmentVariablesRepository implements Repository { return { success: false as const, error: "Project not found" }; } - if (options.values.every((v) => !project.environments.some((e) => e.id === v.environmentId))) { + // Same guard as `create()`: reject if ANY supplied environmentId is + // outside the caller's project (`.some`, not `.every`). + if (options.values.some((v) => !project.environments.some((e) => e.id === v.environmentId))) { return { success: false as const, error: `Environment not found` }; } diff --git a/apps/webapp/app/v3/services/checkSchedule.server.ts b/apps/webapp/app/v3/services/checkSchedule.server.ts index e6429a2adc1..beb5de4b596 100644 --- a/apps/webapp/app/v3/services/checkSchedule.server.ts +++ b/apps/webapp/app/v3/services/checkSchedule.server.ts @@ -1,6 +1,7 @@ import { ZodError } from "zod"; import { CronPattern } from "../schedules"; import { BaseService, ServiceValidationError } from "./baseService.server"; +import { resolveProjectScopedEnvironments } from "./resolveProjectScopedEnvironments"; import { getLimit } from "~/services/platform.v3.server"; import { getTimezones } from "~/utils/timezones.server"; import { env } from "~/env.server"; @@ -82,7 +83,19 @@ export class CheckScheduleService extends BaseService { throw new ServiceValidationError("Project not found"); } - const environments = project.environments.filter((env) => environmentIds.includes(env.id)); + // Reject (don't silently drop) any environmentId that doesn't belong to the + // authorized project. + const scopedEnvironments = resolveProjectScopedEnvironments( + environmentIds, + project.environments + ); + if (scopedEnvironments.kind === "foreign") { + throw new ServiceValidationError( + `Environment ${scopedEnvironments.foreignEnvironmentId} does not belong to this project.` + ); + } + + const environments = scopedEnvironments.environments; if (environments.some((env) => env.archivedAt)) { throw new ServiceValidationError("Can't add or edit a schedule for an archived branch"); } diff --git a/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts b/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts index a696fd1ffa9..601f7470af5 100644 --- a/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts +++ b/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts @@ -1,3 +1,4 @@ +import { scheduleWhereClause } from "~/models/schedules.server"; import { BaseService } from "./baseService.server"; type Options = { @@ -28,9 +29,7 @@ export class DeleteTaskScheduleService extends BaseService { try { const schedule = await this._prisma.taskSchedule.findFirst({ - where: { - friendlyId, - }, + where: scheduleWhereClause(projectId, friendlyId), }); if (!schedule) { diff --git a/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts b/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts new file mode 100644 index 00000000000..631eefac718 --- /dev/null +++ b/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts @@ -0,0 +1,27 @@ +// Resolve a caller-supplied list of environment ids against the environments +// that belong to the authorized project. Any id not in the project is reported +// as `foreign` so the caller can reject the request rather than silently drop +// it. Returns a discriminated result instead of throwing so it stays +// dependency-free and unit-testable. + +export type ProjectScopedEnvironmentsResult = + | { kind: "foreign"; foreignEnvironmentId: string } + | { kind: "ok"; environments: E[] }; + +export function resolveProjectScopedEnvironments( + environmentIds: string[], + projectEnvironments: ReadonlyArray +): ProjectScopedEnvironmentsResult { + const byId = new Map(projectEnvironments.map((e) => [e.id, e])); + + const foreignEnvironmentId = environmentIds.find((id) => !byId.has(id)); + // Explicit undefined check: an empty-string id is a foreign id that must be + // rejected, but it is falsy, so a truthiness test would silently drop it + // instead. + if (foreignEnvironmentId !== undefined) { + return { kind: "foreign", foreignEnvironmentId }; + } + + const environments = environmentIds.map((id) => byId.get(id)).filter((e): e is E => Boolean(e)); + return { kind: "ok", environments }; +} diff --git a/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts b/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts index 5afa96d7c1f..a78e4722e1b 100644 --- a/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts +++ b/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts @@ -1,3 +1,4 @@ +import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; import { BaseService } from "./baseService.server"; type Options = { @@ -29,9 +30,7 @@ export class SetActiveOnTaskScheduleService extends BaseService { try { const schedule = await this._prisma.taskSchedule.findFirst({ - where: { - friendlyId, - }, + where: scheduleWhereClause(projectId, friendlyId), }); if (!schedule) { @@ -43,9 +42,7 @@ export class SetActiveOnTaskScheduleService extends BaseService { } await this._prisma.taskSchedule.update({ - where: { - friendlyId, - }, + where: scheduleUniqWhereClause(projectId, friendlyId), data: { active, }, diff --git a/apps/webapp/app/v3/writableEnvironments.ts b/apps/webapp/app/v3/writableEnvironments.ts new file mode 100644 index 00000000000..af3ff12e8d9 --- /dev/null +++ b/apps/webapp/app/v3/writableEnvironments.ts @@ -0,0 +1,33 @@ +// Which environments a user may write env vars to. Shared env types +// (preview/staging/production) are writable by any project member; DEVELOPMENT +// environments are per-user and only writable by their owner. + +export type WriteCheckEnvironment = { + id: string; + type: string; + orgMember: { userId: string } | null; +}; + +const SHARED_ENV_TYPES = new Set(["PREVIEW", "STAGING", "PRODUCTION"]); + +/** + * Return the first submitted id the user may NOT write to — either it isn't one + * of the project's environments or it's a DEV env owned by someone else. + * Returns null when every submitted id is writable by `userId`. + */ +export function findUnauthorizedEnvironmentId( + projectEnvironments: ReadonlyArray, + submittedIds: ReadonlyArray, + userId: string +): string | null { + const byId = new Map(projectEnvironments.map((e) => [e.id, e])); + for (const id of submittedIds) { + const env = byId.get(id); + if (!env) return id; + const writable = + SHARED_ENV_TYPES.has(env.type) || + (env.type === "DEVELOPMENT" && env.orgMember?.userId === userId); + if (!writable) return id; + } + return null; +} diff --git a/apps/webapp/test/checkSchedule.test.ts b/apps/webapp/test/checkSchedule.test.ts new file mode 100644 index 00000000000..bd872bbcec2 --- /dev/null +++ b/apps/webapp/test/checkSchedule.test.ts @@ -0,0 +1,60 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { resolveProjectScopedEnvironments } from "~/v3/services/resolveProjectScopedEnvironments"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the environment-scoping primitive CheckScheduleService relies on +// (`resolveProjectScopedEnvironments`) with real RuntimeEnvironment rows, +// imported directly to avoid `~/db.server` and its eager global-prisma connect. + +async function seedProjectWithEnv(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: `${slug}-prod`, + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: slug.slice(0, 6), + }, + }); + return { organization, project, environment }; +} + +function projectEnvironments(prisma: PrismaClient, projectId: string) { + return prisma.runtimeEnvironment.findMany({ where: { projectId }, select: { id: true } }); +} + +describe("resolveProjectScopedEnvironments (schedule env scoping)", () => { + containerTest("rejects an environment id that belongs to another project", async ({ prisma }) => { + const a = await seedProjectWithEnv(prisma, "orga"); + const b = await seedProjectWithEnv(prisma, "orgb"); + + const result = resolveProjectScopedEnvironments( + [a.environment.id, b.environment.id], + await projectEnvironments(prisma, a.project.id) + ); + + expect(result.kind).toBe("foreign"); + expect(result).toMatchObject({ foreignEnvironmentId: b.environment.id }); + }); + + containerTest("accepts environment ids that belong to the project", async ({ prisma }) => { + const a = await seedProjectWithEnv(prisma, "orga"); + + const result = resolveProjectScopedEnvironments( + [a.environment.id], + await projectEnvironments(prisma, a.project.id) + ); + + expect(result.kind).toBe("ok"); + }); +}); diff --git a/apps/webapp/test/deleteTaskSchedule.test.ts b/apps/webapp/test/deleteTaskSchedule.test.ts new file mode 100644 index 00000000000..320b44bf4f0 --- /dev/null +++ b/apps/webapp/test/deleteTaskSchedule.test.ts @@ -0,0 +1,75 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { scheduleWhereClause } from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the project-scoping primitive DeleteTaskScheduleService relies on +// (`scheduleWhereClause`) directly against a real database, to avoid importing +// `~/db.server` and its eager global-prisma connect. + +async function seedProject(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + return { organization, project }; +} + +function seedSchedule(prisma: PrismaClient, projectId: string, friendlyId: string) { + return prisma.taskSchedule.create({ + data: { + friendlyId, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); +} + +describe("scheduleWhereClause (delete lookup scoping)", () => { + containerTest( + "a schedule from another project is not found when scoped to the caller's project", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const b = await seedProject(prisma, "orgb"); + const victim = await seedSchedule( + prisma, + b.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}` + ); + + // Scoped to A's project: the cross-tenant schedule is invisible (the + // `projectId` in the where is what prevents a cross-project delete). + const fromA = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(a.project.id, victim.friendlyId), + }); + expect(fromA).toBeNull(); + + // Scoped to its own project: found. + const fromB = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(b.project.id, victim.friendlyId), + }); + expect(fromB?.id).toBe(victim.id); + } + ); + + containerTest("the where pins projectId for both id shapes", async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + + // friendlyId shape + expect(scheduleWhereClause(a.project.id, "sched_abc")).toMatchObject({ + friendlyId: "sched_abc", + projectId: a.project.id, + }); + // deduplicationKey shape + expect(scheduleWhereClause(a.project.id, "my-dedup-key")).toMatchObject({ + projectId: a.project.id, + deduplicationKey: "my-dedup-key", + }); + }); +}); diff --git a/apps/webapp/test/environmentVariablesRepository.test.ts b/apps/webapp/test/environmentVariablesRepository.test.ts index 4025d8fd2eb..b5661c89cbe 100644 --- a/apps/webapp/test/environmentVariablesRepository.test.ts +++ b/apps/webapp/test/environmentVariablesRepository.test.ts @@ -180,4 +180,80 @@ describe("EnvironmentVariablesRepository.getVariableValuesForKeys", () => { expect(crossProjectRequest.size).toBe(0); }); + + postgresTest( + "create() rejects a mix of in-project and foreign environmentIds without writing foreign values", + async ({ prisma }) => { + const { + user, + organization, + project: projectA, + } = await createTestOrgProjectWithMember(prisma); + + const projectB = await prisma.project.create({ + data: { + name: "Project B", + slug: `proj-b-${Date.now()}`, + organizationId: organization.id, + externalRef: `ext-b-${Date.now()}`, + }, + }); + + const envA = await createRuntimeEnvironment(prisma, { + projectId: projectA.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + const envB = await createRuntimeEnvironment(prisma, { + projectId: projectB.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + // Caller scoped to projectA supplies a mixed array: an in-project env + // (envA) plus a foreign one (envB). The whole request must be refused. + const result = await repository.create(projectA.id, { + override: true, + environmentIds: [envA.id, envB.id], + variables: [{ key: "CROSS_TENANT", value: "x" }], + isSecret: false, + lastUpdatedBy: { type: "user", userId: user.id }, + }); + + expect(result.success).toBe(false); + + // No value row may have been written against the foreign environment. + const foreignValues = await prisma.environmentVariableValue.findMany({ + where: { environmentId: envB.id }, + }); + expect(foreignValues).toHaveLength(0); + } + ); + + postgresTest( + "create() still succeeds for an all-in-project environmentIds array", + async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + const result = await repository.create(project.id, { + override: true, + environmentIds: [environment.id], + variables: [{ key: "OK_KEY", value: "v" }], + isSecret: false, + lastUpdatedBy: { type: "user", userId: user.id }, + }); + + expect(result.success).toBe(true); + } + ); }); diff --git a/apps/webapp/test/resolveProjectScopedEnvironments.test.ts b/apps/webapp/test/resolveProjectScopedEnvironments.test.ts new file mode 100644 index 00000000000..f04a623d676 --- /dev/null +++ b/apps/webapp/test/resolveProjectScopedEnvironments.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { resolveProjectScopedEnvironments } from "../app/v3/services/resolveProjectScopedEnvironments.js"; + +const projectEnvs = [{ id: "env_prod" }, { id: "env_staging" }, { id: "env_dev" }]; + +// A submitted environment id that doesn't belong to the project must be +// rejected, not silently dropped. +describe("resolveProjectScopedEnvironments", () => { + it("resolves ids that all belong to the project", () => { + const r = resolveProjectScopedEnvironments(["env_prod", "env_dev"], projectEnvs); + expect(r.kind).toBe("ok"); + if (r.kind === "ok") expect(r.environments.map((e) => e.id)).toEqual(["env_prod", "env_dev"]); + }); + + it("rejects a foreign environment id (the cross-tenant vector)", () => { + const r = resolveProjectScopedEnvironments(["env_someone_elses"], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe("env_someone_elses"); + }); + + it("rejects when a foreign id is mixed in with valid ones (not silently dropped)", () => { + const r = resolveProjectScopedEnvironments(["env_prod", "env_foreign"], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe("env_foreign"); + }); + + it("returns an empty set for no ids", () => { + const r = resolveProjectScopedEnvironments([], projectEnvs); + expect(r.kind).toBe("ok"); + if (r.kind === "ok") expect(r.environments).toEqual([]); + }); + + it("rejects an empty-string id rather than dropping it (falsy edge case)", () => { + const r = resolveProjectScopedEnvironments([""], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe(""); + }); + + it("rejects an empty-string id mixed in with valid ids", () => { + const r = resolveProjectScopedEnvironments(["env_prod", ""], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe(""); + }); +}); diff --git a/apps/webapp/test/schedulesPutEnvScoping.test.ts b/apps/webapp/test/schedulesPutEnvScoping.test.ts new file mode 100644 index 00000000000..4bc937f6088 --- /dev/null +++ b/apps/webapp/test/schedulesPutEnvScoping.test.ts @@ -0,0 +1,261 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { getScheduleEnvVisibility } from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +async function seedProjectWithEnvs(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ + data: { title: slug, slug }, + }); + const project = await prisma.project.create({ + data: { + name: slug, + slug, + organizationId: organization.id, + externalRef: slug, + }, + }); + const prodEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 4)}`, + }, + }); + const stagingEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging", + type: "STAGING", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_staging_${slug}`, + pkApiKey: `pk_staging_${slug}`, + shortcode: `s${slug.slice(0, 4)}`, + }, + }); + return { organization, project, prodEnv, stagingEnv }; +} + +async function seedScheduleWithInstance( + prisma: PrismaClient, + projectId: string, + environmentId: string, + opts: { friendlyId?: string; deduplicationKey?: string } = {} +) { + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: opts.friendlyId ?? `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + ...(opts.deduplicationKey + ? { deduplicationKey: opts.deduplicationKey, userProvidedDeduplicationKey: true } + : {}), + }, + }); + await prisma.taskScheduleInstance.create({ + data: { + taskScheduleId: schedule.id, + environmentId, + projectId, + }, + }); + return schedule; +} + +describe("getScheduleEnvVisibility", () => { + containerTest( + "returns 'hidden' when an instance lives in a different env (by friendlyId)", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.stagingEnv.id + ); + expect(visibility.status).toBe("hidden"); + } + ); + + containerTest("returns 'visible' when every instance is in caller env", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(visibility.status).toBe("visible"); + if (visibility.status === "visible") { + expect(visibility.schedule.id).toBe(schedule.id); + } + }); + + containerTest("returns 'missing' when no schedule exists", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + "sched_does_not_exist", + env.prodEnv.id + ); + expect(visibility.status).toBe("missing"); + }); + + containerTest("returns 'visible' when no instances exist yet", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(visibility.status).toBe("visible"); + }); + + containerTest( + "returns 'visible' from every environment a multi-env schedule spans", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + await prisma.taskScheduleInstance.createMany({ + data: [ + { taskScheduleId: schedule.id, environmentId: env.prodEnv.id, projectId: env.project.id }, + { + taskScheduleId: schedule.id, + environmentId: env.stagingEnv.id, + projectId: env.project.id, + }, + ], + }); + + // The schedule list surfaces a schedule for any environment it has an + // instance in, so per-schedule reads/mutations must resolve the same + // way. A multi-env schedule is visible from each environment it spans. + const fromProd = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(fromProd.status).toBe("visible"); + + const fromStaging = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.stagingEnv.id + ); + expect(fromStaging.status).toBe("visible"); + } + ); + + containerTest( + "returns 'hidden' from an environment the schedule has no instance in", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + // A third environment with no instance of the schedule. + const devEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "DEVELOPMENT", + projectId: env.project.id, + organizationId: env.organization.id, + apiKey: `tr_dev_${env.project.slug}`, + pkApiKey: `pk_dev_${env.project.slug}`, + shortcode: `d${env.project.slug.slice(0, 4)}`, + }, + }); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + await prisma.taskScheduleInstance.createMany({ + data: [ + { taskScheduleId: schedule.id, environmentId: env.prodEnv.id, projectId: env.project.id }, + { + taskScheduleId: schedule.id, + environmentId: env.stagingEnv.id, + projectId: env.project.id, + }, + ], + }); + + const fromDev = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + devEnv.id + ); + expect(fromDev.status).toBe("hidden"); + } + ); + + containerTest( + "resolves by user-provided deduplicationKey (the non-sched_ prefix branch)", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const dedupKey = `my-daily-cleanup-${Math.random().toString(36).slice(2, 8)}`; + await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id, { + deduplicationKey: dedupKey, + }); + + const hiddenFromStaging = await getScheduleEnvVisibility( + prisma, + env.project.id, + dedupKey, + env.stagingEnv.id + ); + expect(hiddenFromStaging.status).toBe("hidden"); + + const visibleFromProd = await getScheduleEnvVisibility( + prisma, + env.project.id, + dedupKey, + env.prodEnv.id + ); + expect(visibleFromProd.status).toBe("visible"); + } + ); +}); diff --git a/apps/webapp/test/setActiveOnTaskSchedule.test.ts b/apps/webapp/test/setActiveOnTaskSchedule.test.ts new file mode 100644 index 00000000000..6c3b6e9ab2b --- /dev/null +++ b/apps/webapp/test/setActiveOnTaskSchedule.test.ts @@ -0,0 +1,149 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { + getScheduleEnvVisibility, + scheduleUniqWhereClause, + scheduleWhereClause, +} from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the project-scoping primitives SetActiveOnTaskScheduleService relies +// on (`scheduleWhereClause` + `scheduleUniqWhereClause`) directly against a real +// database, to avoid importing `~/db.server` and its eager global-prisma connect. + +async function seedProject(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + return { organization, project }; +} + +function seedSchedule( + prisma: PrismaClient, + projectId: string, + friendlyId: string, + active: boolean = true +) { + return prisma.taskSchedule.create({ + data: { + friendlyId, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + active, + }, + }); +} + +describe("schedule scoping (enable/disable)", () => { + containerTest( + "toggling scoped to another project does not touch the victim schedule", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const b = await seedProject(prisma, "orgb"); + const victim = await seedSchedule( + prisma, + b.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}`, + true + ); + + // Project A cannot toggle B's schedule: the where pins projectId, so the + // update matches zero rows. + const result = await prisma.taskSchedule.updateMany({ + where: scheduleWhereClause(a.project.id, victim.friendlyId), + data: { active: false }, + }); + expect(result.count).toBe(0); + + const unchanged = await prisma.taskSchedule.findUnique({ where: { id: victim.id } }); + expect(unchanged?.active).toBe(true); + + // The owning project can toggle it. + const owned = await prisma.taskSchedule.updateMany({ + where: scheduleWhereClause(b.project.id, victim.friendlyId), + data: { active: false }, + }); + expect(owned.count).toBe(1); + } + ); + + containerTest("the unique-where pins projectId", async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + expect(scheduleUniqWhereClause(a.project.id, "sched_abc")).toMatchObject({ + friendlyId: "sched_abc", + projectId: a.project.id, + }); + }); + + // The public activate/deactivate endpoints gate on getScheduleEnvVisibility + // before toggling `active`. A key scoped to one environment must not be able + // to enable/disable a schedule that only runs in another environment of the + // same project. + containerTest( + "an env-scoped caller cannot toggle a schedule that only runs in another env", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const prodEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: a.project.id, + organizationId: a.organization.id, + apiKey: `tr_prod_${a.project.slug}`, + pkApiKey: `pk_prod_${a.project.slug}`, + shortcode: `p${a.project.slug.slice(0, 4)}`, + }, + }); + const stagingEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging", + type: "STAGING", + projectId: a.project.id, + organizationId: a.organization.id, + apiKey: `tr_staging_${a.project.slug}`, + pkApiKey: `pk_staging_${a.project.slug}`, + shortcode: `s${a.project.slug.slice(0, 4)}`, + }, + }); + const schedule = await seedSchedule( + prisma, + a.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}`, + true + ); + // The schedule only runs in staging. + await prisma.taskScheduleInstance.create({ + data: { + taskScheduleId: schedule.id, + environmentId: stagingEnv.id, + projectId: a.project.id, + }, + }); + + // A prod-scoped key is refused (the route returns 404 and never updates). + const fromProd = await getScheduleEnvVisibility( + prisma, + a.project.id, + schedule.friendlyId, + prodEnv.id + ); + expect(fromProd.status).toBe("hidden"); + + // The staging-scoped key that owns an instance can toggle it. + const fromStaging = await getScheduleEnvVisibility( + prisma, + a.project.id, + schedule.friendlyId, + stagingEnv.id + ); + expect(fromStaging.status).toBe("visible"); + } + ); +}); diff --git a/apps/webapp/test/writableEnvironments.test.ts b/apps/webapp/test/writableEnvironments.test.ts new file mode 100644 index 00000000000..ed26fcde905 --- /dev/null +++ b/apps/webapp/test/writableEnvironments.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { + findUnauthorizedEnvironmentId, + type WriteCheckEnvironment, +} from "../app/v3/writableEnvironments.js"; + +const prod: WriteCheckEnvironment = { id: "env_prod", type: "PRODUCTION", orgMember: null }; +const staging: WriteCheckEnvironment = { id: "env_staging", type: "STAGING", orgMember: null }; +const myDev: WriteCheckEnvironment = { + id: "env_dev_me", + type: "DEVELOPMENT", + orgMember: { userId: "user_me" }, +}; +const otherDev: WriteCheckEnvironment = { + id: "env_dev_other", + type: "DEVELOPMENT", + orgMember: { userId: "user_other" }, +}; +const projectEnvs = [prod, staging, myDev, otherDev]; + +// Shared env types are writable by any project member; a DEVELOPMENT env only by +// its owner; an id not in the project is never writable. +describe("findUnauthorizedEnvironmentId", () => { + it("allows shared env types for any member", () => { + expect( + findUnauthorizedEnvironmentId(projectEnvs, ["env_prod", "env_staging"], "user_me") + ).toBeNull(); + }); + + it("allows a caller's own DEV env", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_dev_me"], "user_me")).toBeNull(); + }); + + it("rejects another user's DEV env (the cross-user injection vector)", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_dev_other"], "user_me")).toBe( + "env_dev_other" + ); + }); + + it("rejects a foreign DEV env even when mixed with allowed ones", () => { + expect( + findUnauthorizedEnvironmentId( + projectEnvs, + ["env_prod", "env_dev_me", "env_dev_other"], + "user_me" + ) + ).toBe("env_dev_other"); + }); + + it("rejects an id that isn't one of the project's environments", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_not_in_project"], "user_me")).toBe( + "env_not_in_project" + ); + }); +}); From 5a5833edaff013da4993ba4e09485e5e95d5e604 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:28:59 +0100 Subject: [PATCH 11/15] fix(webapp): scope deployment lookup by environment to prevent cross-env cancel IDOR (#70) * fix(webapp): scope deployment lookup by environment to prevent cross-env cancel IDOR The private getDeployment helper in deployment.server.ts resolved worker deployments scoped only by projectId. Because a single API key authenticates to one environment but projectId is shared across all environments of a project, a lower-trust environment key (dev/preview/CI) could cancel, progress, or request registry credentials for another environment's (e.g. prod) in-progress deployment if it knew the target's friendlyId. Scope getDeployment by environmentId instead of projectId, and update the three call sites (cancelDeployment, progressDeployment, generateRegistryCredentials) to pass the environment id. The registry-credentials platform call stays project+region scoped by design. The dashboard cancel route (resources.$projectId.deployments.$deploymentShortCode.cancel) is project-scoped by design via RBAC; it now passes the located deployment's own environmentId so its behavior is preserved. GHSA-4672-hwv6-gq62 * fix(webapp): remove unused project ID from deployment cancellation * chore(webapp): format deployment cancellation call --- .../api.v1.deployments.$deploymentId.cancel.ts | 2 +- ...ctId.deployments.$deploymentShortCode.cancel.ts | 7 ++----- apps/webapp/app/v3/services/deployment.server.ts | 14 +++++++------- .../app/v3/services/initializeDeployment.server.ts | 2 +- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts index 56b93fe17e3..d14f7a8bc22 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts @@ -45,7 +45,7 @@ export async function action({ request, params }: ActionFunctionArgs) { const deploymentService = new DeploymentService(); return await deploymentService - .cancelDeployment(authenticatedEnv, deploymentId, { + .cancelDeployment({ id: authenticatedEnv.id }, deploymentId, { canceledReason: body.data.reason, }) .match( diff --git a/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts b/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts index dd1c452c51c..239d0982f9e 100644 --- a/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts +++ b/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts @@ -75,7 +75,7 @@ export const action = dashboardAction( prisma.workerDeployment.findUnique({ select: { friendlyId: true, - projectId: true, + environmentId: true, }, where: { projectId_shortCode: { @@ -96,10 +96,7 @@ export const action = dashboardAction( const result = await verifyProjectMembership() .andThen(findDeploymentFriendlyId) .andThen((deployment) => - deploymentService.cancelDeployment( - { projectId: deployment.projectId }, - deployment.friendlyId - ) + deploymentService.cancelDeployment({ id: deployment.environmentId }, deployment.friendlyId) ); if (result.isErr()) { diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 71292f4093b..f726bba3d6d 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -169,7 +169,7 @@ export class DeploymentService extends BaseService { }) ); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen((deployment) => { if (deployment.status === "PENDING") { @@ -191,7 +191,7 @@ export class DeploymentService extends BaseService { * @param data Cancelation reason. */ public cancelDeployment( - authenticatedEnv: Pick, + authenticatedEnv: Pick, friendlyId: string, data?: Partial> ) { @@ -246,7 +246,7 @@ export class DeploymentService extends BaseService { cause: error, })); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) .andThen(({ deployment }) => @@ -278,7 +278,7 @@ export class DeploymentService extends BaseService { * @param friendlyId The friendly deployment ID. */ public generateRegistryCredentials( - authenticatedEnv: Pick, + authenticatedEnv: Pick, friendlyId: string ) { const validateDeployment = ( @@ -326,7 +326,7 @@ export class DeploymentService extends BaseService { }); }); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(getDeploymentRegion) .andThen(generateCredentials); @@ -472,12 +472,12 @@ export class DeploymentService extends BaseService { ); } - private getDeployment(projectId: string, friendlyId: string) { + private getDeployment(environmentId: string, friendlyId: string) { return fromPromise( this._prisma.workerDeployment.findFirst({ where: { friendlyId, - projectId, + environmentId, }, select: { status: true, diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index 1eef2e12bcd..abb99082dd6 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -286,7 +286,7 @@ export class InitializeDeploymentService extends BaseService { }); return deploymentService - .cancelDeployment(environment, deployment.friendlyId, { + .cancelDeployment({ id: environment.id }, deployment.friendlyId, { canceledReason: "Failed to enqueue build, please try again shortly.", }) .orTee((cancelError) => From ee4f190867a608a62a0813e279477adf45132733 Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:29:06 +0100 Subject: [PATCH 12/15] fix(webapp): require private auth to initialize the session out stream (#67) --- .server-changes/session-out-stream-private-auth.md | 6 ++++++ .../app/routes/realtime.v1.sessions.$session.$io.ts | 10 ++++++++++ 2 files changed, 16 insertions(+) create mode 100644 .server-changes/session-out-stream-private-auth.md diff --git a/.server-changes/session-out-stream-private-auth.md b/.server-changes/session-out-stream-private-auth.md new file mode 100644 index 00000000000..bfc95c662f8 --- /dev/null +++ b/.server-changes/session-out-stream-private-auth.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Require secret-key authentication to initialize the session out (agent→client) stream, matching the append route. diff --git a/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts b/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts index e846e851be8..b4ff0ba9500 100644 --- a/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts +++ b/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts @@ -34,6 +34,16 @@ const { action } = createActionApiRoute( }, }, async ({ params, authentication }) => { + // `.out` is the agent→client channel. Only PRIVATE (secret key) auth — + // i.e. the agent run itself — may initialize it. Session-scoped JWTs carry + // `write:sessions:` for `.in`; without this gate they could obtain + // credentials to forge assistant chunks on their own session's `.out`. + if (params.io === "out" && authentication.type !== "PRIVATE") { + return new Response("Initializing the out channel requires secret key authentication", { + status: 403, + }); + } + // Row-optional addressing. The agent calls PUT initialize as part // of `session.out.writer()`, by which time it has already created // the row at bind, so a missing row here is an unusual case From 0c2d9ce22a1af5e2ae8d1809cba38051b4acfb1a Mon Sep 17 00:00:00 2001 From: Chris Arderne Date: Mon, 20 Jul 2026 20:29:14 +0100 Subject: [PATCH 13/15] fix(core): block prototype-pollution via run metadata operation keys (#65) * fix(core): block prototype-pollution via run metadata operation keys * chore: remove test, add changeset * respond to devin comments * fix(core): block prototype pollution in unflattenAttributes via dangerous key segments * fix(core): preserve safe constructor attributes * format * chore(core): combine prototype pollution changeset --- .changeset/block-metadata-proto-pollution.md | 5 ++++ .../core/src/v3/runMetadata/operations.ts | 9 +++++- packages/core/src/v3/schemas/common.ts | 29 +++++++++++++++---- .../core/src/v3/utils/flattenAttributes.ts | 15 ++++++++-- 4 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 .changeset/block-metadata-proto-pollution.md diff --git a/.changeset/block-metadata-proto-pollution.md b/.changeset/block-metadata-proto-pollution.md new file mode 100644 index 00000000000..aa38beab4a4 --- /dev/null +++ b/.changeset/block-metadata-proto-pollution.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Prevent prototype pollution when applying run metadata operations or reconstructing nested telemetry attributes, while preserving legitimate `constructor` and `prototype` fields. diff --git a/packages/core/src/v3/runMetadata/operations.ts b/packages/core/src/v3/runMetadata/operations.ts index b1cc68672c2..fff98c50bc9 100644 --- a/packages/core/src/v3/runMetadata/operations.ts +++ b/packages/core/src/v3/runMetadata/operations.ts @@ -1,5 +1,5 @@ import { JSONHeroPath } from "@jsonhero/path"; -import type { RunMetadataChangeOperation } from "../schemas/common.js"; +import { isSafeMetadataKey, type RunMetadataChangeOperation } from "../schemas/common.js"; import { dequal } from "dequal"; export type ApplyOperationResult = { @@ -16,6 +16,13 @@ export function applyMetadataOperations( let newMetadata: Record = structuredClone(currentMetadata); for (const operation of Array.isArray(operations) ? operations : [operations]) { + // Prevent unsafe JSON paths and direct __proto__ assignments from changing Object.prototype. + // ("update" carries no key.) + if (operation.type !== "update" && !isSafeMetadataKey(operation.key)) { + unappliedOperations.push(operation); + continue; + } + switch (operation.type) { case "set": { if (operation.key.startsWith("$.")) { diff --git a/packages/core/src/v3/schemas/common.ts b/packages/core/src/v3/schemas/common.ts index 4de2ddb5802..a6ec4b1002d 100644 --- a/packages/core/src/v3/schemas/common.ts +++ b/packages/core/src/v3/schemas/common.ts @@ -4,6 +4,25 @@ import type { RuntimeEnvironmentType as DBRuntimeEnvironmentType } from "@trigge export type Enum = { [K in T]: K }; +const DANGEROUS_METADATA_KEY_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]); + +/** + * Prototype-pollution guard for run metadata operation keys. JSON paths are applied via + * JSONHeroPath, so dangerous path segments must be rejected. Literal keys are assigned directly, + * where only __proto__ can change the target object's prototype. + */ +export function isSafeMetadataKey(key: string): boolean { + if (!key.startsWith("$.")) { + return key !== "__proto__"; + } + + return !key.split(/[.[\]'"]+/).some((segment) => DANGEROUS_METADATA_KEY_SEGMENTS.has(segment)); +} + +const MetadataOperationKey = z.string().refine(isSafeMetadataKey, { + message: "Metadata key may not reference __proto__, constructor, or prototype", +}); + export const RunMetadataUpdateOperation = z.object({ type: z.literal("update"), value: z.record(z.unknown()), @@ -13,7 +32,7 @@ export type RunMetadataUpdateOperation = z.infer; export const RunMetadataAppendKeyOperation = z.object({ type: z.literal("append"), - key: z.string(), + key: MetadataOperationKey, value: DeserializedJsonSchema, }); @@ -36,7 +55,7 @@ export type RunMetadataAppendKeyOperation = z.infer /^\d+$/.test(k))) { + // Convert the result to an array if all top-level keys are numeric indices. + // Guard against an empty result (e.g. every key was skipped as unsafe), which + // would otherwise produce Array(-Infinity) and throw. + if (Object.keys(result).length > 0 && Object.keys(result).every((k) => /^\d+$/.test(k))) { const maxIndex = Math.max(...Object.keys(result).map((k) => parseInt(k))); const arrayResult = Array(maxIndex + 1); for (const key in result) { From dcc4b40df1cb5aaf62496abbe9363f74128e9008 Mon Sep 17 00:00:00 2001 From: Daniel Sutton <45313566+d-cs@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:29:23 +0100 Subject: [PATCH 14/15] fix(webapp): harden Electric sync routes against SQLi and add org scoping (#28) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(webapp): harden Electric sync routes against SQL injection and cross-tenant leak sync.traces.\$traceId.ts and sync.traces.runs.\$traceId.ts interpolated params.traceId raw into the Electric `where` clause, and that clause had no organizationId predicate. An authenticated user can craft a traceparent header so the runtime SDK persists a malicious traceId into their own org's row, request /sync/traces/, pass the membership check on that very row, and then watch Electric stream rows from any tenant whose traceId matches the crafted SQL. Closes the vector at four layers: - OtelTraceIdSchema validates the canonical 32-lowercase-hex format at the Zod layer; non-conforming input 404s with no error echo. - buildElectricTraceWhereClause emits `"traceId"='…' AND "organizationId"='…'` so cross-tenant collision doesn't matter even if input validation is ever bypassed. Both inputs are re-checked at the function boundary. - RESERVED_ELECTRIC_SHAPE_PARAMS strips `where`/`table`/`columns` from the incoming searchParams before forwarding to Electric so callers can't override the server-set values. - validateRealtimeTags + a matching Zod allowlist in realtime.v1.runs.ts reject tag values that could break out of the `"runTags" @> ARRAY['',…]` SQL string and steer the jumpHash-derived shard key. Closes TRI-9903, TRI-9904, TRI-9905, TRI-9906 (4 P0). Also closes TRI-9913 (Shape param allowlist) and TRI-9985 (realtime tag interpolation). 19 unit tests cover the regex, the where-clause builder, the reserved- param set, and the tag allowlist. Co-Authored-By: Claude Opus 4.7 (1M context) * test(webapp): regression tests for Electric sync SQLi + scoping hardening Centralizes the hardening in a pure electricShape.server module: OtelTraceIdSchema (32-hex), buildElectricTraceWhereClause (org-scoped, validated), and realtime tag sanitization (reject unsafe chars, escape quotes), plus reserved-param stripping. Pure test (23 cases). Verified RED with the trace-id validation and tag sanitizer neutered (12 injection tests flip), GREEN with them. Bundles the streamBatchItems timeout bump. * fix(webapp): scope TaskRun trace sync by projectId, not nullable organizationId TaskRun.organizationId is nullable and the table is far too large to backfill, so AND-scoping the trace-runs Electric shape by organizationId silently dropped legacy NULL-org runs from the trace view. Scope by the non-null projectId instead: buildElectricTraceWhereClause now takes a typed tenant column (org for the TaskEvent shape, project for TaskRun). Tenant-safe — membership is verified against the project's org and a trace's runs share one project, and the column is a fixed union, never user input. * docs: reword server-changes note to comply with release-note conventions * chore(webapp): remove electricShape test and shorten comments * fix(webapp): preserve project scoping after primary trace lookup * widen test --------- Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: Chris Arderne --- .../tighten-realtime-and-trace-sync.md | 6 ++ apps/webapp/app/routes/realtime.v1.runs.ts | 15 ++++ .../webapp/app/routes/sync.traces.$traceId.ts | 32 ++++++++- .../app/routes/sync.traces.runs.$traceId.ts | 32 +++++++-- .../app/services/realtimeClient.server.ts | 6 +- apps/webapp/app/v3/electricShape.server.ts | 69 +++++++++++++++++++ .../test/spanTraceRoutes.replicaLag.test.ts | 5 +- 7 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 .server-changes/tighten-realtime-and-trace-sync.md create mode 100644 apps/webapp/app/v3/electricShape.server.ts diff --git a/.server-changes/tighten-realtime-and-trace-sync.md b/.server-changes/tighten-realtime-and-trace-sync.md new file mode 100644 index 00000000000..51b4b2d416b --- /dev/null +++ b/.server-changes/tighten-realtime-and-trace-sync.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Live run and trace subscriptions now validate their identifiers more strictly and only return data from your own organization. diff --git a/apps/webapp/app/routes/realtime.v1.runs.ts b/apps/webapp/app/routes/realtime.v1.runs.ts index ca2998f2972..465e7ab4f41 100644 --- a/apps/webapp/app/routes/realtime.v1.runs.ts +++ b/apps/webapp/app/routes/realtime.v1.runs.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { resolveRealtimeStreamClient } from "~/services/realtime/resolveRealtimeStreamClient.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { UNSAFE_REALTIME_TAG_CHARS } from "~/v3/electricShape.server"; const SearchParamsSchema = z.object({ tags: z @@ -9,6 +10,20 @@ const SearchParamsSchema = z.object({ .optional() .transform((value) => { return value ? value.split(",") : undefined; + }) + .superRefine((tags, ctx) => { + if (!tags) return; + for (const tag of tags) { + // Mirror the runtime sanitiser's reject list so the API returns 400 + // instead of a 500. Single quotes are allowed — escaped downstream. + if (UNSAFE_REALTIME_TAG_CHARS.test(tag) || tag.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid tag: ${JSON.stringify(tag)}`, + }); + return; + } + } }), createdAt: z.string().optional(), }); diff --git a/apps/webapp/app/routes/sync.traces.$traceId.ts b/apps/webapp/app/routes/sync.traces.$traceId.ts index 1a06dd48854..d7695014b56 100644 --- a/apps/webapp/app/routes/sync.traces.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.$traceId.ts @@ -1,15 +1,33 @@ import type { LoaderFunctionArgs } from "@remix-run/node"; +import { z } from "zod"; import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { getUserId } from "~/services/session.server"; import { longPollingFetch } from "~/utils/longPollingFetch"; +import { + OtelTraceIdSchema, + RESERVED_ELECTRIC_SHAPE_PARAMS, + buildElectricTraceWhereClause, +} from "~/v3/electricShape.server"; + +const Params = z.object({ + traceId: OtelTraceIdSchema, +}); export async function loader({ params, request }: LoaderFunctionArgs) { try { const userId = await getUserId(request); - logger.log(`/sync/traces/${params.traceId}`, { userId }); + const parsedParams = Params.safeParse(params); + if (!parsedParams.success) { + // Treat a malformed traceId as not-found rather than 400 to avoid + // signalling the validator. + return new Response("Not found", { status: 404 }); + } + const { traceId } = parsedParams.data; + + logger.log(`/sync/traces/${traceId}`, { userId }); if (!userId) { return new Response("No user found in cookie", { status: 401 }); @@ -20,7 +38,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { organizationId: true, }, where: { - traceId: params.traceId, + traceId, }, }); @@ -41,11 +59,19 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const url = new URL(request.url); const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskEvent"`); + // Strip params we set ourselves so the caller can't override them. url.searchParams.forEach((value, key) => { + if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; originUrl.searchParams.set(key, value); }); - originUrl.searchParams.set("where", `"traceId"='${params.traceId}'`); + originUrl.searchParams.set( + "where", + buildElectricTraceWhereClause({ + traceId, + scope: { column: "organizationId", id: trace.organizationId }, + }) + ); const finalUrl = originUrl.toString(); diff --git a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts index 5fd1f7c65c0..24870cf2f49 100644 --- a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts @@ -5,17 +5,27 @@ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { getUserId } from "~/services/session.server"; import { longPollingFetch } from "~/utils/longPollingFetch"; -import { runStore } from "~/v3/runStore.server"; +import { + OtelTraceIdSchema, + RESERVED_ELECTRIC_SHAPE_PARAMS, + buildElectricTraceWhereClause, +} from "~/v3/electricShape.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { runStore } from "~/v3/runStore.server"; const Params = z.object({ - traceId: z.string(), + traceId: OtelTraceIdSchema, }); export async function loader({ params, request }: LoaderFunctionArgs) { try { const userId = await getUserId(request); - const { traceId } = Params.parse(params); + + const parsedParams = Params.safeParse(params); + if (!parsedParams.success) { + return new Response("Not found", { status: 404 }); + } + const { traceId } = parsedParams.data; logger.log(`/sync/runs/${traceId}`, { userId }); @@ -29,6 +39,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { }, { select: { + projectId: true, runtimeEnvironmentId: true, }, }, @@ -40,7 +51,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { // primary before 404ing so a live run's realtime trace feed isn't spuriously not-found. run = await runStore.findRunOnPrimary( { traceId }, - { select: { runtimeEnvironmentId: true } } + { select: { projectId: true, runtimeEnvironmentId: true } } ); } @@ -67,11 +78,22 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const url = new URL(request.url); const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskRun"`); + // Strip params we set ourselves so the caller can't override them. url.searchParams.forEach((value, key) => { + if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; originUrl.searchParams.set(key, value); }); - originUrl.searchParams.set("where", `"traceId"='${traceId}'`); + originUrl.searchParams.set( + "where", + // Scope by non-null projectId, not the nullable organizationId (legacy + // rows would vanish). Tenant-safe: membership was verified against this + // project's org and a trace's runs all live in one project. + buildElectricTraceWhereClause({ + traceId, + scope: { column: "projectId", id: run.projectId }, + }) + ); const finalUrl = originUrl.toString(); diff --git a/apps/webapp/app/services/realtimeClient.server.ts b/apps/webapp/app/services/realtimeClient.server.ts index 9797eeebf40..bf86340641b 100644 --- a/apps/webapp/app/services/realtimeClient.server.ts +++ b/apps/webapp/app/services/realtimeClient.server.ts @@ -15,6 +15,7 @@ import { RedisCacheStore } from "./unkey/redisCacheStore.server"; import { env } from "~/env.server"; import type { API_VERSIONS } from "~/api/versions"; import { CURRENT_API_VERSION } from "~/api/versions"; +import { sanitizeRealtimeTagsForSql } from "~/v3/electricShape.server"; export interface CachedLimitProvider { getCachedLimit: (organizationId: string, defaultValue: number) => Promise; @@ -171,7 +172,10 @@ export class RealtimeClient { const whereClauses: string[] = [`"runtimeEnvironmentId"='${environment.id}'`]; if (params.tags) { - whereClauses.push(`"runTags" @> ARRAY[${params.tags.map((t) => `'${t}'`).join(",")}]`); + // Reject unsafe chars and escape single quotes so tag values can't + // break out of the Electric SQL string literal. + const safeTags = sanitizeRealtimeTagsForSql(params.tags); + whereClauses.push(`"runTags" @> ARRAY[${safeTags.map((t) => `'${t}'`).join(",")}]`); } const createdAtFilter = await this.#calculateCreatedAtFilter(url, params.createdAt); diff --git a/apps/webapp/app/v3/electricShape.server.ts b/apps/webapp/app/v3/electricShape.server.ts new file mode 100644 index 00000000000..c430cc0273c --- /dev/null +++ b/apps/webapp/app/v3/electricShape.server.ts @@ -0,0 +1,69 @@ +import { z } from "zod"; + +/** + * OTel trace IDs are 32 lowercase hex chars. The traceparent parser only + * checks the dash-delimited format, so crafted ids can be persisted and later + * interpolated into shape `where` clauses. Validate here to close the SQLi vector. + */ +export const OtelTraceIdSchema = z + .string() + .regex(/^[0-9a-f]{32}$/, "traceId must be 32 lowercase hex characters"); + +/** Params the sync routes set themselves; stripped from incoming requests. */ +export const RESERVED_ELECTRIC_SHAPE_PARAMS = new Set(["where", "table", "columns"]); + +const CUID_LIKE = /^[a-z][a-z0-9_]*$/i; + +/** + * Tenant column a trace shape is scoped by. TaskEvent scopes by non-null + * organizationId; TaskRun scopes by non-null projectId (its organizationId is + * nullable). The column is from this fixed union, never user input, so it's + * safe to interpolate. + */ +export type TraceScope = + | { column: "organizationId"; id: string } + | { column: "projectId"; id: string }; + +/** + * Build the Electric Shape `where` clause for the trace sync routes. Both ids + * are re-validated as defense-in-depth so a missed call site can't bypass scope. + */ +export function buildElectricTraceWhereClause(args: { + traceId: string; + scope: TraceScope; +}): string { + const { traceId, scope } = args; + if (!OtelTraceIdSchema.safeParse(traceId).success) { + throw new Error("buildElectricTraceWhereClause: unsafe traceId"); + } + if (!CUID_LIKE.test(scope.id)) { + throw new Error("buildElectricTraceWhereClause: unsafe scope id"); + } + return `"traceId"='${traceId}' AND "${scope.column}"='${scope.id}'`; +} + +/** + * Characters rejected in realtime tag values — the single source of truth + * shared by the apiBuilder Zod refine (`realtime.v1.runs.ts`) and the runtime + * sanitiser. Rejects control chars/DEL, backslash, and double-quote. Single + * quotes are allowed and escaped (`'` → `''`) in `sanitizeRealtimeTagForSql`. + */ +export const UNSAFE_REALTIME_TAG_CHARS = /[\x00-\x1f\x7f\\"]/; + +/** + * Sanitise a tag value for interpolation into an Electric Shape `where` clause: + * reject unsafe chars, escape single quotes per SQL standard. + */ +export function sanitizeRealtimeTagForSql(tag: string): string { + if (typeof tag !== "string" || tag.length === 0) { + throw new Error("Invalid realtime tag: empty"); + } + if (UNSAFE_REALTIME_TAG_CHARS.test(tag)) { + throw new Error(`Invalid realtime tag: ${JSON.stringify(tag)} — contains unsafe character`); + } + return tag.replace(/'/g, "''"); +} + +export function sanitizeRealtimeTagsForSql(tags: string[]): string[] { + return tags.map(sanitizeRealtimeTagForSql); +} diff --git a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts index f3df2d2ac7d..6b8dac27932 100644 --- a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts +++ b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts @@ -475,7 +475,7 @@ describe("run-trace/span-detail route loaders under a lagging replica", () => { const seed = await seedTenant(prisma14, suffix); const runId = `run_${CUID_25}`; const friendlyId = `run_${suffix}`; - const traceId = `trace_${suffix}`; + const traceId = "a".repeat(32); const userId = `user_${suffix}`; // The dashboard user, joined to the org so the route's real orgMember check passes. @@ -538,7 +538,8 @@ describe("run-trace/span-detail route loaders under a lagging replica", () => { holder.resolvedEnv = { organizationId: seed.organization.id }; holder.replicaMarker = { orgMember: prisma14.orgMember }; - const res = (await syncTraceRunsLoader(syncRequest("trace_does_not_exist"))) as Response; + const res = (await syncTraceRunsLoader(syncRequest("b".repeat(32)))) as Response; + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); expect(res.status).toBe(404); } ); From fc392b1fcaa0b929ead6ff8caef2e160f782109e Mon Sep 17 00:00:00 2001 From: nicktrn <55853254+nicktrn@users.noreply.github.com> Date: Mon, 20 Jul 2026 20:32:24 +0100 Subject: [PATCH 15/15] feat: source worker.id telemetry from the deployment friendlyId (#78) * feat(supervisor,cli): source worker.id telemetry from the deployment friendlyId The runner stamps the OTel worker.id from TRIGGER_DEPLOYMENT_ID, which the supervisor may set to an opaque token. Have the supervisor also pass the plain friendlyId in TRIGGER_DEPLOYMENT_FRIENDLY_ID, and have the runner prefer it for the telemetry worker.id (falling back to TRIGGER_DEPLOYMENT_ID for older supervisors). The auth header keeps using TRIGGER_DEPLOYMENT_ID unchanged. New deploys then report a stable deployment id in telemetry with no decode - including to customers' own external exporters. Refs TRI-11974 * fix(cli): redact TRIGGER_DEPLOYMENT_ID from managed run debug logs --- .changeset/worker-id-deployment-friendly-id.md | 5 +++++ apps/supervisor/src/workloadManager/compute.ts | 2 ++ apps/supervisor/src/workloadManager/docker.ts | 2 ++ apps/supervisor/src/workloadManager/kubernetes.ts | 5 +++++ packages/cli-v3/src/entryPoints/managed/controller.ts | 2 +- packages/cli-v3/src/entryPoints/managed/env.ts | 11 +++++++++++ packages/cli-v3/src/entryPoints/managed/execution.ts | 2 +- .../src/entryPoints/managed/taskRunProcessProvider.ts | 5 ++++- 8 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 .changeset/worker-id-deployment-friendly-id.md diff --git a/.changeset/worker-id-deployment-friendly-id.md b/.changeset/worker-id-deployment-friendly-id.md new file mode 100644 index 00000000000..2e62fee4b97 --- /dev/null +++ b/.changeset/worker-id-deployment-friendly-id.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Deployed task telemetry now reports the deployment identifier (e.g. `deployment_abc123`) in the `worker.id` attribute, instead of an opaque internal value. Upgrade to get the readable identifier in your own OpenTelemetry exporters. diff --git a/apps/supervisor/src/workloadManager/compute.ts b/apps/supervisor/src/workloadManager/compute.ts index 66e4a482ad4..89360437dd8 100644 --- a/apps/supervisor/src/workloadManager/compute.ts +++ b/apps/supervisor/src/workloadManager/compute.ts @@ -152,6 +152,8 @@ export class ComputeWorkloadManager implements WorkloadManager { TRIGGER_POD_SCHEDULED_AT_MS: String(Date.now()), TRIGGER_ENV_ID: opts.envId, TRIGGER_DEPLOYMENT_ID: opts.deploymentToken ?? opts.deploymentFriendlyId, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + TRIGGER_DEPLOYMENT_FRIENDLY_ID: opts.deploymentFriendlyId, TRIGGER_DEPLOYMENT_VERSION: opts.deploymentVersion, TRIGGER_RUN_ID: opts.runFriendlyId, TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId, diff --git a/apps/supervisor/src/workloadManager/docker.ts b/apps/supervisor/src/workloadManager/docker.ts index 70ca02654f6..c8049873b64 100644 --- a/apps/supervisor/src/workloadManager/docker.ts +++ b/apps/supervisor/src/workloadManager/docker.ts @@ -73,6 +73,8 @@ export class DockerWorkloadManager implements WorkloadManager { `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`, `TRIGGER_ENV_ID=${opts.envId}`, `TRIGGER_DEPLOYMENT_ID=${opts.deploymentToken ?? opts.deploymentFriendlyId}`, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + `TRIGGER_DEPLOYMENT_FRIENDLY_ID=${opts.deploymentFriendlyId}`, `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`, `TRIGGER_RUN_ID=${opts.runFriendlyId}`, `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`, diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index 5e9940c0e3a..0263b229e13 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -160,6 +160,11 @@ export class KubernetesWorkloadManager implements WorkloadManager { name: "TRIGGER_DEPLOYMENT_ID", value: opts.deploymentToken ?? opts.deploymentFriendlyId, }, + { + // Plain friendlyId for telemetry (worker.id), not the opaque token in DEPLOYMENT_ID. + name: "TRIGGER_DEPLOYMENT_FRIENDLY_ID", + value: opts.deploymentFriendlyId, + }, { name: "TRIGGER_DEPLOYMENT_VERSION", value: opts.deploymentVersion, diff --git a/packages/cli-v3/src/entryPoints/managed/controller.ts b/packages/cli-v3/src/entryPoints/managed/controller.ts index 5c63a4082d7..ae1f1d22a4a 100644 --- a/packages/cli-v3/src/entryPoints/managed/controller.ts +++ b/packages/cli-v3/src/entryPoints/managed/controller.ts @@ -76,7 +76,7 @@ export class ManagedRunController { }); const properties = { - ...env.raw, + ...env.rawForLogging, TRIGGER_POD_SCHEDULED_AT_MS: env.TRIGGER_POD_SCHEDULED_AT_MS.toISOString(), TRIGGER_DEQUEUED_AT_MS: env.TRIGGER_DEQUEUED_AT_MS.toISOString(), }; diff --git a/packages/cli-v3/src/entryPoints/managed/env.ts b/packages/cli-v3/src/entryPoints/managed/env.ts index 4da8361e863..825e83e1572 100644 --- a/packages/cli-v3/src/entryPoints/managed/env.ts +++ b/packages/cli-v3/src/entryPoints/managed/env.ts @@ -27,6 +27,9 @@ const Env = z.object({ // Set at runtime TRIGGER_DEPLOYMENT_ID: z.string(), + // Plain deployment friendlyId for telemetry. Optional: older supervisors don't set it, in which + // case we fall back to TRIGGER_DEPLOYMENT_ID. + TRIGGER_DEPLOYMENT_FRIENDLY_ID: z.string().optional(), TRIGGER_DEPLOYMENT_VERSION: z.string(), TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`), TRIGGER_ENV_ID: z.string(), @@ -74,6 +77,11 @@ export class RunnerEnv { return this.env; } + // TRIGGER_DEPLOYMENT_ID carries the deployment token; redact it before logging. + get rawForLogging() { + return { ...this.env, TRIGGER_DEPLOYMENT_ID: "[redacted]" }; + } + // Base environment variables get NODE_ENV() { return this.env.NODE_ENV; @@ -93,6 +101,9 @@ export class RunnerEnv { get TRIGGER_DEPLOYMENT_ID() { return this.env.TRIGGER_DEPLOYMENT_ID; } + get TRIGGER_DEPLOYMENT_FRIENDLY_ID() { + return this.env.TRIGGER_DEPLOYMENT_FRIENDLY_ID; + } get TRIGGER_DEPLOYMENT_VERSION() { return this.env.TRIGGER_DEPLOYMENT_VERSION; } diff --git a/packages/cli-v3/src/entryPoints/managed/execution.ts b/packages/cli-v3/src/entryPoints/managed/execution.ts index 8dca48fe279..c5bb4875bce 100644 --- a/packages/cli-v3/src/entryPoints/managed/execution.ts +++ b/packages/cli-v3/src/entryPoints/managed/execution.ts @@ -951,7 +951,7 @@ export class RunExecution { this.sendDebugLog(`[override] processing: ${reason}`, { overrides, - currentEnv: this.env.raw, + currentEnv: this.env.rawForLogging, }); // Override the env with the new values diff --git a/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts b/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts index 4e0a561220b..aaf09862f1d 100644 --- a/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts +++ b/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts @@ -252,11 +252,14 @@ export class TaskRunProcessProvider { workerManifest: this.workerManifest, env: processEnv, serverWorker: { - id: this.env.TRIGGER_DEPLOYMENT_ID, + // Telemetry-only (becomes OTel worker.id). Prefer the plain friendlyId; fall back to + // DEPLOYMENT_ID (which may be an opaque token) only when an older supervisor didn't set it. + id: this.env.TRIGGER_DEPLOYMENT_FRIENDLY_ID ?? this.env.TRIGGER_DEPLOYMENT_ID, contentHash: this.env.TRIGGER_CONTENT_HASH, version: this.env.TRIGGER_DEPLOYMENT_VERSION, engine: "V2", }, + machineResources: { cpu: Number(this.env.TRIGGER_MACHINE_CPU), memory: Number(this.env.TRIGGER_MACHINE_MEMORY),